From 78a0030d4606a2259e64db388e883abd10610419 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Mon, 22 Dec 2014 17:20:25 -0500 Subject: [PATCH 1/8] download-grammars -> convert-grammars Downloading is only a small part of what this script does. The main thing it does is convert grammars to JSON. --- CONTRIBUTING.md | 2 +- Rakefile | 2 +- script/{download-grammars => convert-grammars} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename script/{download-grammars => convert-grammars} (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 69081657..b6b682f6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,7 +23,7 @@ Great! You'll need to: 0. Add an entry for your language to [`languages.yml`][languages]. 0. Add a grammar for your language. Please only add grammars that have a license that permits redistribution. 0. Add your grammar as a submodule: `git submodule add https://github.com/JaneSmith/MyGrammar vendor/grammars/MyGrammar`. - 0. Add your grammar to [`grammars.yml`][grammars] by running `script/download-grammars --add vendor/grammars/MyGrammar`. + 0. Add your grammar to [`grammars.yml`][grammars] by running `script/convert-grammars --add vendor/grammars/MyGrammar`. 0. Add samples for your language to the [samples directory][samples]. In addition, if your new language defines an extension that's already listed in [`languages.yml`][languages] (such as `.foo`) then sometimes a few more steps will need to be taken: diff --git a/Rakefile b/Rakefile index 068af0bb..b38486c0 100644 --- a/Rakefile +++ b/Rakefile @@ -48,7 +48,7 @@ end task :build_grammars_gem do rm_rf "grammars" - sh "script/download-grammars" + sh "script/convert-grammars" sh "gem", "build", "github-linguist-grammars.gemspec" end diff --git a/script/download-grammars b/script/convert-grammars similarity index 100% rename from script/download-grammars rename to script/convert-grammars From ada6f6882a7b9e239231ef6e1c813d84e8843c07 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Mon, 22 Dec 2014 16:35:01 -0500 Subject: [PATCH 2/8] Test that grammars.yml lists the right scopes for each submodule convert-grammars now supports a few flags that we can use to make it dump out the YAML just for the local grammar submodules. We can then compare this to the YAML that's actually in grammars.yml to check that they're the same. If they aren't, grammars.yml needs to be updated. This will help catch mistakes like using the wrong scope name. --- script/convert-grammars | 60 ++++++++++++++++++++++++++++++++++------- test/test_grammars.rb | 13 +++++++++ 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/script/convert-grammars b/script/convert-grammars index 3792f451..4f67edd5 100755 --- a/script/convert-grammars +++ b/script/convert-grammars @@ -2,6 +2,7 @@ require 'json' require 'net/http' +require 'optparse' require 'plist' require 'set' require 'tmpdir' @@ -13,6 +14,13 @@ GRAMMARS_PATH = File.join(ROOT, "grammars") SOURCES_FILE = File.join(ROOT, "grammars.yml") CSONC = File.join(ROOT, "node_modules", ".bin", "csonc") +$options = { + :add => false, + :install => true, + :output => SOURCES_FILE, + :remote => true, +} + class SingleFile def initialize(path) @path = path @@ -148,8 +156,9 @@ def load_grammar(path) end end -def install_grammar(tmp_dir, source, all_scopes) +def load_grammars(tmp_dir, source, all_scopes) is_url = source.start_with?("http:", "https:") + return [] if is_url && !$options[:remote] is_single_file = source.end_with?('.tmLanguage', '.plist') p = if !is_url @@ -172,9 +181,7 @@ def install_grammar(tmp_dir, source, all_scopes) raise "Unsupported source: #{source}" unless p - installed = [] - - p.fetch(tmp_dir).each do |path| + p.fetch(tmp_dir).map do |path| grammar = load_grammar(path) scope = grammar['scopeName'] @@ -184,9 +191,17 @@ def install_grammar(tmp_dir, source, all_scopes) " Previous package: #{all_scopes[scope]}" next end - - File.write(File.join(GRAMMARS_PATH, "#{scope}.json"), JSON.pretty_generate(grammar)) all_scopes[scope] = p.url + grammar + end +end + +def install_grammars(grammars) + installed = [] + + grammars.each do |grammar| + scope = grammar['scopeName'] + File.write(File.join(GRAMMARS_PATH, "#{scope}.json"), JSON.pretty_generate(grammar)) installed << scope end @@ -206,7 +221,8 @@ def run_thread(queue, all_scopes) dir = "#{tmpdir}/#{index}" Dir.mkdir(dir) - install_grammar(dir, source, all_scopes) + grammars = load_grammars(dir, source, all_scopes) + install_grammars(grammars) if $options[:install] end end end @@ -232,9 +248,9 @@ def main(sources) all_scopes = {} - if ARGV[0] == '--add' + if $options[:add] Dir.mktmpdir do |tmpdir| - install_grammar(tmpdir, ARGV[1], all_scopes) + install_grammar(tmpdir, ARGV[0], all_scopes) end generate_yaml(all_scopes, sources) else @@ -252,12 +268,36 @@ def main(sources) end end +OptionParser.new do |opts| + opts.banner = "Usage: #{$0} [options]" + + opts.on("--add GRAMMAR", "Add a new grammar. GRAMMAR may be a file path or URL.") do |a| + $options[:add] = a + end + + opts.on("--[no-]install", "Install grammars into grammars/ directory.") do |i| + $options[:install] = i + end + + opts.on("--output FILE", "Write output to FILE. Use - for stdout.") do |o| + $options[:output] = o == "-" ? $stdout : o + end + + opts.on("--[no-]remote", "Download remote grammars.") do |r| + $options[:remote] = r + end +end.parse! + sources = File.open(SOURCES_FILE) do |file| YAML.load(file) end yaml = main(sources) -File.write(SOURCES_FILE, YAML.dump(yaml)) +if $options[:output].is_a?(IO) + $options[:output].write(YAML.dump(yaml)) +else + File.write($options[:output], YAML.dump(yaml)) +end $stderr.puts("Done") diff --git a/test/test_grammars.rb b/test/test_grammars.rb index 57483348..871ba195 100644 --- a/test/test_grammars.rb +++ b/test/test_grammars.rb @@ -36,4 +36,17 @@ class TestGrammars < Minitest::Test assert nonexistent_submodules.empty? && unlisted_submodules.empty?, message end + + def test_local_scopes_are_in_sync + actual = YAML.load(`"#{File.join(ROOT, "script", "convert-grammars")}" --output - --no-install --no-remote 2>/dev/null`) + assert_predicate $?, :success? + + # We're not checking remote grammars. That can take a long time and make CI + # flaky if network conditions are poor. + @grammars.delete_if { |k, v| k.start_with?("http:", "https:") } + + @grammars.each do |k, v| + assert_equal v, actual[k], "The scopes listed for #{k} in grammars.yml don't match the scopes found in that repository" + end + end end From 5fa85f268ff849caf5c63350449f0d0894c0efe6 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Mon, 22 Dec 2014 17:28:25 -0500 Subject: [PATCH 3/8] Ruby 1.9 compatibility --- test/test_grammars.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_grammars.rb b/test/test_grammars.rb index 871ba195..0ea12081 100644 --- a/test/test_grammars.rb +++ b/test/test_grammars.rb @@ -39,7 +39,7 @@ class TestGrammars < Minitest::Test def test_local_scopes_are_in_sync actual = YAML.load(`"#{File.join(ROOT, "script", "convert-grammars")}" --output - --no-install --no-remote 2>/dev/null`) - assert_predicate $?, :success? + assert $?.success?, "script/convert-grammars failed" # We're not checking remote grammars. That can take a long time and make CI # flaky if network conditions are poor. From a4286ba950e6b09dffbfa0e03a150f3a93f8a527 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Tue, 30 Dec 2014 07:31:00 -0500 Subject: [PATCH 4/8] Show stderr output when running convert-grammars This will let us see why it's failing. --- test/test_grammars.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_grammars.rb b/test/test_grammars.rb index 0ea12081..a80e690d 100644 --- a/test/test_grammars.rb +++ b/test/test_grammars.rb @@ -38,7 +38,7 @@ class TestGrammars < Minitest::Test end def test_local_scopes_are_in_sync - actual = YAML.load(`"#{File.join(ROOT, "script", "convert-grammars")}" --output - --no-install --no-remote 2>/dev/null`) + actual = YAML.load(`"#{File.join(ROOT, "script", "convert-grammars")}" --output - --no-install --no-remote`) assert $?.success?, "script/convert-grammars failed" # We're not checking remote grammars. That can take a long time and make CI From 581dc36c47a95ed01c0beae55b74e63ad12dfbab Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Tue, 30 Dec 2014 07:36:28 -0500 Subject: [PATCH 5/8] Ruby 1.9 compatibility for convert-grammars --- script/convert-grammars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/convert-grammars b/script/convert-grammars index 4f67edd5..e03acd71 100755 --- a/script/convert-grammars +++ b/script/convert-grammars @@ -233,7 +233,7 @@ def generate_yaml(all_scopes, base) out[value] << key end - yaml = yaml.sort.to_h + yaml = Hash[yaml.sort] yaml.each { |k, v| v.sort! } yaml end From 917bc48348406c07c07a008881330db4aee63fa7 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Tue, 6 Jan 2015 13:31:26 -0500 Subject: [PATCH 6/8] Use File.directory? to test for local directories Some of our submodules end in ".tmLanguage", which we used to think meant it was a file rather than a directory. --- script/convert-grammars | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/script/convert-grammars b/script/convert-grammars index e03acd71..5ed47a61 100755 --- a/script/convert-grammars +++ b/script/convert-grammars @@ -159,15 +159,14 @@ end def load_grammars(tmp_dir, source, all_scopes) is_url = source.start_with?("http:", "https:") return [] if is_url && !$options[:remote] - is_single_file = source.end_with?('.tmLanguage', '.plist') p = if !is_url - if is_single_file - SingleFile.new(source) - else + if File.directory?(source) DirectoryPackage.new(source) + else + SingleFile.new(source) end - elsif is_single_file + elsif source.end_with?('.tmLanguage', '.plist') SingleGrammar.new(source) elsif source.start_with?('https://github.com') GitHubPackage.new(source) From 8f3779c94c3139da5a98d24f1e53753127a3aa74 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Tue, 6 Jan 2015 13:32:28 -0500 Subject: [PATCH 7/8] Update grammars.yml to match submodules I should have done this in 2c2c4740a89402e755cc7f8eed278e70fa3c5a0b. --- grammars.yml | 7 ++++--- lib/linguist/languages.yml | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/grammars.yml b/grammars.yml index d695e95d..8bfbf4dc 100644 --- a/grammars.yml +++ b/grammars.yml @@ -47,9 +47,9 @@ vendor/grammars/LiveScript.tmbundle: vendor/grammars/NSIS: - source.nsis vendor/grammars/NimLime: -- source.nimrod -- source.nimrod_filter -- source.nimrodcfg +- source.nim +- source.nim_filter +- source.nimcfg vendor/grammars/PHP-Twig.tmbundle: - text.html.twig vendor/grammars/RDoc.tmbundle: @@ -259,6 +259,7 @@ vendor/grammars/language-javascript: vendor/grammars/language-python: - source.python - source.regexp.python +- text.python.console - text.python.traceback vendor/grammars/language-shellscript: - source.shell diff --git a/lib/linguist/languages.yml b/lib/linguist/languages.yml index 7890be55..9005eabb 100644 --- a/lib/linguist/languages.yml +++ b/lib/linguist/languages.yml @@ -1891,6 +1891,7 @@ Nimrod: - .nim - .nimrod ace_mode: text + tm_scope: source.nim Ninja: type: data From 7edddec9209a615532ed34d3664e1fd2a0cd74fc Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Tue, 6 Jan 2015 13:39:18 -0500 Subject: [PATCH 8/8] Make convert-grammars a little quieter No need to say "Done" at the end. We're obviously done; the script is exiting. --- script/convert-grammars | 2 -- 1 file changed, 2 deletions(-) diff --git a/script/convert-grammars b/script/convert-grammars index 5ed47a61..a9b2bfea 100755 --- a/script/convert-grammars +++ b/script/convert-grammars @@ -298,5 +298,3 @@ if $options[:output].is_a?(IO) else File.write($options[:output], YAML.dump(yaml)) end - -$stderr.puts("Done")