Try using PMAP-SOURCES everywhere.

This bins sources into one of 16 buckets for parallelization
purposes.

Author: xach

dist-cache.lisp

@@ -187,7 +187,7 @@ if needed."
 	    (directory (merge-pathnames "contrib/*.asd" base))))
       (dolist (file contrib-system-files)
         (setf (gethash (pathname-name file) table) file)))
-    (map-sources
+    (pmap-sources
      (lambda (source)
        (let ((base (ensure-cached-build-directory source))
              (system-files (system-files source)))

misc.lisp

@@ -232,7 +232,7 @@
 (defun ensure-what-wins-you-can ()
   (call-with-skipping
    (lambda ()
-     (map-sources
+     (pmap-sources
       (lambda (source)
         (format t "~&Checking ~S~%" (project-name source))
         ;;(clear-fasl-cache)

recrank.lisp

@@ -4,7 +4,7 @@
 
 (defun recrank (&key (update t) (report t) (feeds t)
 		  (publish-failure-report t)
-                  parallel
+                  (parallel t)
                 (file #p"quicklisp:tmp;update-failures.txt"))
   (clear-fasl-cache)
   (preflight)

upstream.lisp

@@ -150,11 +150,24 @@
 		     (push source result))))
     (sort result 'string< :key 'name)))
 
-(defun pmap-sources (fun &key (parallel-key #'source-host)
+
+(defun source-bucket (source)
+  "Return a string suitable for binning a source for parallel work."
+  (subseq (string-digest (first-line-of (source-file source)))
+          0 1))
+
+(defun fasl-directory (source)
+  (translate-logical-pathname
+   (make-pathname :host "quicklisp-controller"
+                  :directory (list :absolute "dist" "fasls"
+                                   (source-bucket source)))))
+
+(defun pmap-sources (fun &key (parallel-key 'source-bucket)
                            (test #'identity))
   (let ((dependency-tree (lparallel:make-ptree))
         (parallel-key-dependency (make-hash-table :test 'equal))
-        (i 0))
+        (i 0)
+        (result '()))
     (map-sources (lambda (source)
                    (let ((testp (funcall test source))
                          (pkey (funcall parallel-key source)))
@@ -163,15 +176,22 @@
                                                       parallel-key-dependency)
                                            (lambda (&optional arg)
                                              (declare (ignore arg))
-                                             (map-source fun source))
+                                             (multiple-value-bind (result error)
+                                                 (ignore-errors
+                                                  (map-source fun source))
+                                               (when error
+                                                 (format *trace-output* "; ERROR: ~A -> ~%;; ~A~%"
+                                                         source error)
+                                                 (push (cons source error)
+                                                       result))))
                                            dependency-tree)
                        (setf (gethash pkey parallel-key-dependency)
                              (list i))
                        (incf i)))))
     (lparallel:ptree-fn 'everything (loop for j below i collect j)
                         (constantly nil) dependency-tree)
     (lparallel:call-ptree 'everything dependency-tree)
-    nil))
+    (values nil result)))
 
 (defun project-name-source-file (project-name)
   (merge-pathnames