Switch to thread pool for pulling projects

app/jobs/clone_projects_job.rb

@@ -3,26 +3,36 @@ class CloneProjectsJob < ApplicationJob
 
   def perform
     submissions = YAML.load_file(Rails.root.join("submissions.yml"))
-
-    submissions["projects"].each do |url|
+    urls = submissions["projects"].map do |url|
       url = url.chomp("/")
       url = "#{url}.git" unless url.end_with?(".git")
+      url
+    end
 
-      org, repo = url.gsub("https://github.com/", "").gsub(".git", "").split("/")
-      org = org.downcase
-      repo = repo.downcase
+    queue = Queue.new
+    urls.each { |url| queue << url }
 
-      target_dir = File.join(Rails.root, "content", "projects", org, repo)
+    threads = 5.times.map do
+      Thread.new do
+        while url = queue.pop(true) rescue nil
+          org, repo = url.gsub("https://github.com/", "").gsub(".git", "").split("/")
+          org = org.downcase
+          repo = repo.downcase
+          target_dir = File.join(Rails.root, "content", "projects", org, repo)
 
-      if Dir.exist?(target_dir) && Dir.exist?(File.join(target_dir, ".git"))
-        Rails.logger.info "Pulling latest for #{org}/#{repo}..."
-        system("cd '#{target_dir}' && git pull")
-      else
-        Rails.logger.info "Cloning #{org}/#{repo}..."
-        system("git clone #{url} '#{target_dir}'")
+          if Dir.exist?(target_dir) && Dir.exist?(File.join(target_dir, ".git"))
+            Rails.logger.info "Pulling latest for #{org}/#{repo}..."
+            system("cd '#{target_dir}' && git pull")
+          else
+            Rails.logger.info "Cloning #{org}/#{repo}..."
+            system("git clone #{url} '#{target_dir}'")
+          end
+        end
       end
     end
 
+    threads.each(&:join)
+
     Rails.logger.info "Done cloning and updating all projects!"
   end
 end