diff --git a/app/jobs/clone_projects_job.rb b/app/jobs/clone_projects_job.rb index 3acb810..767639b 100644 --- a/app/jobs/clone_projects_job.rb +++ b/app/jobs/clone_projects_job.rb @@ -3,26 +3,36 @@ class CloneProjectsJob < ApplicationJob def perform submissions = YAML.load_file(Rails.root.join("submissions.yml")) - - submissions["projects"].each do |url| + urls = submissions["projects"].map do |url| url = url.chomp("/") url = "#{url}.git" unless url.end_with?(".git") + url + end - org, repo = url.gsub("https://github.com/", "").gsub(".git", "").split("/") - org = org.downcase - repo = repo.downcase + queue = Queue.new + urls.each { |url| queue << url } - target_dir = File.join(Rails.root, "content", "projects", org, repo) + threads = 5.times.map do + Thread.new do + while url = queue.pop(true) rescue nil + org, repo = url.gsub("https://github.com/", "").gsub(".git", "").split("/") + org = org.downcase + repo = repo.downcase + target_dir = File.join(Rails.root, "content", "projects", org, repo) - if Dir.exist?(target_dir) && Dir.exist?(File.join(target_dir, ".git")) - Rails.logger.info "Pulling latest for #{org}/#{repo}..." - system("cd '#{target_dir}' && git pull") - else - Rails.logger.info "Cloning #{org}/#{repo}..." - system("git clone #{url} '#{target_dir}'") + if Dir.exist?(target_dir) && Dir.exist?(File.join(target_dir, ".git")) + Rails.logger.info "Pulling latest for #{org}/#{repo}..." + system("cd '#{target_dir}' && git pull") + else + Rails.logger.info "Cloning #{org}/#{repo}..." + system("git clone #{url} '#{target_dir}'") + end + end end end + threads.each(&:join) + Rails.logger.info "Done cloning and updating all projects!" end end