Update submodules in parallel to speed up bootstrap

This runs 8 `git submodule update` processes in parallel, speeding up
bootstrap from 2 minutes to 30 seconds for me. (Obviously this is
dependent on bandwidth.)
This commit is contained in:
Adam Roben
2014-12-19 15:24:19 -05:00
parent ff678642e6
commit 6a2bf3fd2f
2 changed files with 68 additions and 1 deletions

View File

@@ -9,4 +9,4 @@ bundle config --local path vendor/gems
bundle check > /dev/null 2>&1 || bundle install
git submodule sync --quiet
git submodule update --init
script/fast-submodule-update

67
script/fast-submodule-update Executable file
View File

@@ -0,0 +1,67 @@
#!/usr/bin/env ruby
require "thread"
ROOT = File.expand_path("../..", __FILE__).freeze
Dir.chdir(ROOT)
SUBMODULES = `git config --list --file .gitmodules`.lines.grep(/\.path=/).map { |line| line.chomp.split("=", 2).last }.freeze
SLOW_SUBMODULES = %w[
grammar_sources/factor
grammar_sources/fsharpbinding
grammar_sources/ioke-outdated
]
class TaskResult < Struct.new(:submodule, :output, :success?); end
def run_process(*args)
read, write = IO.pipe
pid = Process.spawn(*args, in: :close, out: write, err: [:child, :out])
write.close
output = read.read
read.close
Process.wait(pid)
[output, $?.success?]
end
def update_submodule(submodule)
output, success = run_process("git", "submodule", "update", "--init", "--", submodule)
TaskResult.new(submodule, output, success)
end
def run_thread(submodules, results)
loop do
begin
submodule = submodules.pop(true)
rescue ThreadError
# The queue is empty.
break
end
results.push(update_submodule(submodule))
end
end
submodules = Queue.new
results = Queue.new
# Update the slow submodules first so they can update in the background while
# the fast ones run.
SUBMODULES.partition { |submodule| SLOW_SUBMODULES.include?(submodule) }.flatten.each do |submodule|
submodules.push(submodule)
end
8.times do
Thread.new { run_thread(submodules, results) }
end
success = true
SUBMODULES.each do
result = results.pop
unless result.success?
success = false
puts "Error updating #{result.submodule}"
end
puts result.output if result.output =~ /\S/
end
exit success ? 0 : 1