diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 69081657..b6b682f6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,7 +23,7 @@ Great! You'll need to: 0. Add an entry for your language to [`languages.yml`][languages]. 0. Add a grammar for your language. Please only add grammars that have a license that permits redistribution. 0. Add your grammar as a submodule: `git submodule add https://github.com/JaneSmith/MyGrammar vendor/grammars/MyGrammar`. - 0. Add your grammar to [`grammars.yml`][grammars] by running `script/download-grammars --add vendor/grammars/MyGrammar`. + 0. Add your grammar to [`grammars.yml`][grammars] by running `script/convert-grammars --add vendor/grammars/MyGrammar`. 0. Add samples for your language to the [samples directory][samples]. In addition, if your new language defines an extension that's already listed in [`languages.yml`][languages] (such as `.foo`) then sometimes a few more steps will need to be taken: diff --git a/Rakefile b/Rakefile index 068af0bb..b38486c0 100644 --- a/Rakefile +++ b/Rakefile @@ -48,7 +48,7 @@ end task :build_grammars_gem do rm_rf "grammars" - sh "script/download-grammars" + sh "script/convert-grammars" sh "gem", "build", "github-linguist-grammars.gemspec" end diff --git a/grammars.yml b/grammars.yml index d695e95d..8bfbf4dc 100644 --- a/grammars.yml +++ b/grammars.yml @@ -47,9 +47,9 @@ vendor/grammars/LiveScript.tmbundle: vendor/grammars/NSIS: - source.nsis vendor/grammars/NimLime: -- source.nimrod -- source.nimrod_filter -- source.nimrodcfg +- source.nim +- source.nim_filter +- source.nimcfg vendor/grammars/PHP-Twig.tmbundle: - text.html.twig vendor/grammars/RDoc.tmbundle: @@ -259,6 +259,7 @@ vendor/grammars/language-javascript: vendor/grammars/language-python: - source.python - source.regexp.python +- text.python.console - text.python.traceback vendor/grammars/language-shellscript: - source.shell diff --git a/lib/linguist/languages.yml b/lib/linguist/languages.yml index 7890be55..9005eabb 100644 --- a/lib/linguist/languages.yml +++ b/lib/linguist/languages.yml @@ -1891,6 +1891,7 @@ Nimrod: - .nim - .nimrod ace_mode: text + tm_scope: source.nim Ninja: type: data diff --git a/script/download-grammars b/script/convert-grammars similarity index 79% rename from script/download-grammars rename to script/convert-grammars index 3792f451..a9b2bfea 100755 --- a/script/download-grammars +++ b/script/convert-grammars @@ -2,6 +2,7 @@ require 'json' require 'net/http' +require 'optparse' require 'plist' require 'set' require 'tmpdir' @@ -13,6 +14,13 @@ GRAMMARS_PATH = File.join(ROOT, "grammars") SOURCES_FILE = File.join(ROOT, "grammars.yml") CSONC = File.join(ROOT, "node_modules", ".bin", "csonc") +$options = { + :add => false, + :install => true, + :output => SOURCES_FILE, + :remote => true, +} + class SingleFile def initialize(path) @path = path @@ -148,17 +156,17 @@ def load_grammar(path) end end -def install_grammar(tmp_dir, source, all_scopes) +def load_grammars(tmp_dir, source, all_scopes) is_url = source.start_with?("http:", "https:") - is_single_file = source.end_with?('.tmLanguage', '.plist') + return [] if is_url && !$options[:remote] p = if !is_url - if is_single_file - SingleFile.new(source) - else + if File.directory?(source) DirectoryPackage.new(source) + else + SingleFile.new(source) end - elsif is_single_file + elsif source.end_with?('.tmLanguage', '.plist') SingleGrammar.new(source) elsif source.start_with?('https://github.com') GitHubPackage.new(source) @@ -172,9 +180,7 @@ def install_grammar(tmp_dir, source, all_scopes) raise "Unsupported source: #{source}" unless p - installed = [] - - p.fetch(tmp_dir).each do |path| + p.fetch(tmp_dir).map do |path| grammar = load_grammar(path) scope = grammar['scopeName'] @@ -184,9 +190,17 @@ def install_grammar(tmp_dir, source, all_scopes) " Previous package: #{all_scopes[scope]}" next end - - File.write(File.join(GRAMMARS_PATH, "#{scope}.json"), JSON.pretty_generate(grammar)) all_scopes[scope] = p.url + grammar + end +end + +def install_grammars(grammars) + installed = [] + + grammars.each do |grammar| + scope = grammar['scopeName'] + File.write(File.join(GRAMMARS_PATH, "#{scope}.json"), JSON.pretty_generate(grammar)) installed << scope end @@ -206,7 +220,8 @@ def run_thread(queue, all_scopes) dir = "#{tmpdir}/#{index}" Dir.mkdir(dir) - install_grammar(dir, source, all_scopes) + grammars = load_grammars(dir, source, all_scopes) + install_grammars(grammars) if $options[:install] end end end @@ -217,7 +232,7 @@ def generate_yaml(all_scopes, base) out[value] << key end - yaml = yaml.sort.to_h + yaml = Hash[yaml.sort] yaml.each { |k, v| v.sort! } yaml end @@ -232,9 +247,9 @@ def main(sources) all_scopes = {} - if ARGV[0] == '--add' + if $options[:add] Dir.mktmpdir do |tmpdir| - install_grammar(tmpdir, ARGV[1], all_scopes) + install_grammar(tmpdir, ARGV[0], all_scopes) end generate_yaml(all_scopes, sources) else @@ -252,12 +267,34 @@ def main(sources) end end +OptionParser.new do |opts| + opts.banner = "Usage: #{$0} [options]" + + opts.on("--add GRAMMAR", "Add a new grammar. GRAMMAR may be a file path or URL.") do |a| + $options[:add] = a + end + + opts.on("--[no-]install", "Install grammars into grammars/ directory.") do |i| + $options[:install] = i + end + + opts.on("--output FILE", "Write output to FILE. Use - for stdout.") do |o| + $options[:output] = o == "-" ? $stdout : o + end + + opts.on("--[no-]remote", "Download remote grammars.") do |r| + $options[:remote] = r + end +end.parse! + sources = File.open(SOURCES_FILE) do |file| YAML.load(file) end yaml = main(sources) -File.write(SOURCES_FILE, YAML.dump(yaml)) - -$stderr.puts("Done") +if $options[:output].is_a?(IO) + $options[:output].write(YAML.dump(yaml)) +else + File.write($options[:output], YAML.dump(yaml)) +end diff --git a/test/test_grammars.rb b/test/test_grammars.rb index 57483348..a80e690d 100644 --- a/test/test_grammars.rb +++ b/test/test_grammars.rb @@ -36,4 +36,17 @@ class TestGrammars < Minitest::Test assert nonexistent_submodules.empty? && unlisted_submodules.empty?, message end + + def test_local_scopes_are_in_sync + actual = YAML.load(`"#{File.join(ROOT, "script", "convert-grammars")}" --output - --no-install --no-remote`) + assert $?.success?, "script/convert-grammars failed" + + # We're not checking remote grammars. That can take a long time and make CI + # flaky if network conditions are poor. + @grammars.delete_if { |k, v| k.start_with?("http:", "https:") } + + @grammars.each do |k, v| + assert_equal v, actual[k], "The scopes listed for #{k} in grammars.yml don't match the scopes found in that repository" + end + end end