mirror of
https://github.com/KevinMidboe/linguist.git
synced 2025-12-08 20:38:47 +00:00
Update submodules in parallel to speed up bootstrap
This runs 8 `git submodule update` processes in parallel, speeding up bootstrap from 2 minutes to 30 seconds for me. (Obviously this is dependent on bandwidth.)
This commit is contained in:
@@ -9,4 +9,4 @@ bundle config --local path vendor/gems
|
|||||||
bundle check > /dev/null 2>&1 || bundle install
|
bundle check > /dev/null 2>&1 || bundle install
|
||||||
|
|
||||||
git submodule sync --quiet
|
git submodule sync --quiet
|
||||||
git submodule update --init
|
script/fast-submodule-update
|
||||||
|
|||||||
67
script/fast-submodule-update
Executable file
67
script/fast-submodule-update
Executable file
@@ -0,0 +1,67 @@
|
|||||||
|
#!/usr/bin/env ruby
|
||||||
|
|
||||||
|
require "thread"
|
||||||
|
|
||||||
|
ROOT = File.expand_path("../..", __FILE__).freeze
|
||||||
|
Dir.chdir(ROOT)
|
||||||
|
|
||||||
|
SUBMODULES = `git config --list --file .gitmodules`.lines.grep(/\.path=/).map { |line| line.chomp.split("=", 2).last }.freeze
|
||||||
|
SLOW_SUBMODULES = %w[
|
||||||
|
grammar_sources/factor
|
||||||
|
grammar_sources/fsharpbinding
|
||||||
|
grammar_sources/ioke-outdated
|
||||||
|
]
|
||||||
|
|
||||||
|
class TaskResult < Struct.new(:submodule, :output, :success?); end
|
||||||
|
|
||||||
|
def run_process(*args)
|
||||||
|
read, write = IO.pipe
|
||||||
|
pid = Process.spawn(*args, in: :close, out: write, err: [:child, :out])
|
||||||
|
write.close
|
||||||
|
output = read.read
|
||||||
|
read.close
|
||||||
|
Process.wait(pid)
|
||||||
|
[output, $?.success?]
|
||||||
|
end
|
||||||
|
|
||||||
|
def update_submodule(submodule)
|
||||||
|
output, success = run_process("git", "submodule", "update", "--init", "--", submodule)
|
||||||
|
TaskResult.new(submodule, output, success)
|
||||||
|
end
|
||||||
|
|
||||||
|
def run_thread(submodules, results)
|
||||||
|
loop do
|
||||||
|
begin
|
||||||
|
submodule = submodules.pop(true)
|
||||||
|
rescue ThreadError
|
||||||
|
# The queue is empty.
|
||||||
|
break
|
||||||
|
end
|
||||||
|
|
||||||
|
results.push(update_submodule(submodule))
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
submodules = Queue.new
|
||||||
|
results = Queue.new
|
||||||
|
|
||||||
|
# Update the slow submodules first so they can update in the background while
|
||||||
|
# the fast ones run.
|
||||||
|
SUBMODULES.partition { |submodule| SLOW_SUBMODULES.include?(submodule) }.flatten.each do |submodule|
|
||||||
|
submodules.push(submodule)
|
||||||
|
end
|
||||||
|
|
||||||
|
8.times do
|
||||||
|
Thread.new { run_thread(submodules, results) }
|
||||||
|
end
|
||||||
|
|
||||||
|
success = true
|
||||||
|
SUBMODULES.each do
|
||||||
|
result = results.pop
|
||||||
|
unless result.success?
|
||||||
|
success = false
|
||||||
|
puts "Error updating #{result.submodule}"
|
||||||
|
end
|
||||||
|
puts result.output if result.output =~ /\S/
|
||||||
|
end
|
||||||
|
exit success ? 0 : 1
|
||||||
Reference in New Issue
Block a user