mirror of
https://github.com/KevinMidboe/linguist.git
synced 2025-10-29 09:40:21 +00:00
Update submodules in parallel to speed up bootstrap
This runs 8 `git submodule update` processes in parallel, speeding up bootstrap from 2 minutes to 30 seconds for me. (Obviously this is dependent on bandwidth.)
This commit is contained in:
@@ -9,4 +9,4 @@ bundle config --local path vendor/gems
|
||||
bundle check > /dev/null 2>&1 || bundle install
|
||||
|
||||
git submodule sync --quiet
|
||||
git submodule update --init
|
||||
script/fast-submodule-update
|
||||
|
||||
67
script/fast-submodule-update
Executable file
67
script/fast-submodule-update
Executable file
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env ruby
|
||||
|
||||
require "thread"
|
||||
|
||||
ROOT = File.expand_path("../..", __FILE__).freeze
|
||||
Dir.chdir(ROOT)
|
||||
|
||||
SUBMODULES = `git config --list --file .gitmodules`.lines.grep(/\.path=/).map { |line| line.chomp.split("=", 2).last }.freeze
|
||||
SLOW_SUBMODULES = %w[
|
||||
grammar_sources/factor
|
||||
grammar_sources/fsharpbinding
|
||||
grammar_sources/ioke-outdated
|
||||
]
|
||||
|
||||
class TaskResult < Struct.new(:submodule, :output, :success?); end
|
||||
|
||||
def run_process(*args)
|
||||
read, write = IO.pipe
|
||||
pid = Process.spawn(*args, in: :close, out: write, err: [:child, :out])
|
||||
write.close
|
||||
output = read.read
|
||||
read.close
|
||||
Process.wait(pid)
|
||||
[output, $?.success?]
|
||||
end
|
||||
|
||||
def update_submodule(submodule)
|
||||
output, success = run_process("git", "submodule", "update", "--init", "--", submodule)
|
||||
TaskResult.new(submodule, output, success)
|
||||
end
|
||||
|
||||
def run_thread(submodules, results)
|
||||
loop do
|
||||
begin
|
||||
submodule = submodules.pop(true)
|
||||
rescue ThreadError
|
||||
# The queue is empty.
|
||||
break
|
||||
end
|
||||
|
||||
results.push(update_submodule(submodule))
|
||||
end
|
||||
end
|
||||
|
||||
submodules = Queue.new
|
||||
results = Queue.new
|
||||
|
||||
# Update the slow submodules first so they can update in the background while
|
||||
# the fast ones run.
|
||||
SUBMODULES.partition { |submodule| SLOW_SUBMODULES.include?(submodule) }.flatten.each do |submodule|
|
||||
submodules.push(submodule)
|
||||
end
|
||||
|
||||
8.times do
|
||||
Thread.new { run_thread(submodules, results) }
|
||||
end
|
||||
|
||||
success = true
|
||||
SUBMODULES.each do
|
||||
result = results.pop
|
||||
unless result.success?
|
||||
success = false
|
||||
puts "Error updating #{result.submodule}"
|
||||
end
|
||||
puts result.output if result.output =~ /\S/
|
||||
end
|
||||
exit success ? 0 : 1
|
||||
Reference in New Issue
Block a user