Merge pull request #1909 from github/scopes-test

Test that grammars.yml lists the right scopes for each submodule

CONTRIBUTING.md

@@ -23,7 +23,7 @@ Great! You'll need to:
 0. Add an entry for your language to [`languages.yml`][languages].
 0. Add a grammar for your language. Please only add grammars that have a license that permits redistribution.
     0. Add your grammar as a submodule: `git submodule add https://github.com/JaneSmith/MyGrammar vendor/grammars/MyGrammar`.
-    0. Add your grammar to [`grammars.yml`][grammars] by running `script/download-grammars --add vendor/grammars/MyGrammar`.
+    0. Add your grammar to [`grammars.yml`][grammars] by running `script/convert-grammars --add vendor/grammars/MyGrammar`.
 0. Add samples for your language to the [samples directory][samples].
 
 In addition, if your new language defines an extension that's already listed in [`languages.yml`][languages] (such as `.foo`) then sometimes a few more steps will need to be taken:

Rakefile

@@ -48,7 +48,7 @@ end
 
 task :build_grammars_gem do
   rm_rf "grammars"
-  sh "script/download-grammars"
+  sh "script/convert-grammars"
   sh "gem", "build", "github-linguist-grammars.gemspec"
 end
 

grammars.yml

@@ -47,9 +47,9 @@ vendor/grammars/LiveScript.tmbundle:
 vendor/grammars/NSIS:
 - source.nsis
 vendor/grammars/NimLime:
-- source.nimrod
+- source.nim
-- source.nimrod_filter
+- source.nim_filter
-- source.nimrodcfg
+- source.nimcfg
 vendor/grammars/PHP-Twig.tmbundle:
 - text.html.twig
 vendor/grammars/RDoc.tmbundle:
@@ -259,6 +259,7 @@ vendor/grammars/language-javascript:
 vendor/grammars/language-python:
 - source.python
 - source.regexp.python
+- text.python.console
 - text.python.traceback
 vendor/grammars/language-shellscript:
 - source.shell

lib/linguist/languages.yml

@@ -1891,6 +1891,7 @@ Nimrod:
   - .nim
   - .nimrod
   ace_mode: text
+  tm_scope: source.nim
 
 Ninja:
   type: data

script/convert-grammars

@@ -2,6 +2,7 @@
 
 require 'json'
 require 'net/http'
+require 'optparse'
 require 'plist'
 require 'set'
 require 'tmpdir'
@@ -13,6 +14,13 @@ GRAMMARS_PATH = File.join(ROOT, "grammars")
 SOURCES_FILE = File.join(ROOT, "grammars.yml")
 CSONC = File.join(ROOT, "node_modules", ".bin", "csonc")
 
+$options = {
+  :add => false,
+  :install => true,
+  :output => SOURCES_FILE,
+  :remote => true,
+}
+
 class SingleFile
   def initialize(path)
     @path = path
@@ -148,17 +156,17 @@ def load_grammar(path)
   end
 end
 
-def install_grammar(tmp_dir, source, all_scopes)
+def load_grammars(tmp_dir, source, all_scopes)
   is_url = source.start_with?("http:", "https:")
-  is_single_file = source.end_with?('.tmLanguage', '.plist')
+  return [] if is_url && !$options[:remote]
 
   p = if !is_url
-        if is_single_file
+        if File.directory?(source)
-          SingleFile.new(source)
-        else
           DirectoryPackage.new(source)
+        else
+          SingleFile.new(source)
         end
-      elsif is_single_file
+      elsif source.end_with?('.tmLanguage', '.plist')
         SingleGrammar.new(source)
       elsif source.start_with?('https://github.com')
         GitHubPackage.new(source)
@@ -172,9 +180,7 @@ def install_grammar(tmp_dir, source, all_scopes)
 
   raise "Unsupported source: #{source}" unless p
 
-  installed = []
+  p.fetch(tmp_dir).map do |path|
-
-  p.fetch(tmp_dir).each do |path|
     grammar = load_grammar(path)
     scope = grammar['scopeName']
 
@@ -184,9 +190,17 @@ def install_grammar(tmp_dir, source, all_scopes)
       "  Previous package: #{all_scopes[scope]}"
       next
     end
-
-    File.write(File.join(GRAMMARS_PATH, "#{scope}.json"), JSON.pretty_generate(grammar))
     all_scopes[scope] = p.url
+    grammar
+  end
+end
+
+def install_grammars(grammars)
+  installed = []
+
+  grammars.each do |grammar|
+    scope = grammar['scopeName']
+    File.write(File.join(GRAMMARS_PATH, "#{scope}.json"), JSON.pretty_generate(grammar))
     installed << scope
   end
 
@@ -206,7 +220,8 @@ def run_thread(queue, all_scopes)
       dir = "#{tmpdir}/#{index}"
       Dir.mkdir(dir)
 
-      install_grammar(dir, source, all_scopes)
+      grammars = load_grammars(dir, source, all_scopes)
+      install_grammars(grammars) if $options[:install]
     end
   end
 end
@@ -217,7 +232,7 @@ def generate_yaml(all_scopes, base)
     out[value] << key
   end
 
-  yaml = yaml.sort.to_h
+  yaml = Hash[yaml.sort]
   yaml.each { |k, v| v.sort! }
   yaml
 end
@@ -232,9 +247,9 @@ def main(sources)
 
   all_scopes = {}
 
-  if ARGV[0] == '--add'
+  if $options[:add]
     Dir.mktmpdir do |tmpdir|
-      install_grammar(tmpdir, ARGV[1], all_scopes)
+      install_grammar(tmpdir, ARGV[0], all_scopes)
     end
     generate_yaml(all_scopes, sources)
   else
@@ -252,12 +267,34 @@ def main(sources)
   end
 end
 
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+
+  opts.on("--add GRAMMAR", "Add a new grammar. GRAMMAR may be a file path or URL.") do |a|
+    $options[:add] = a
+  end
+
+  opts.on("--[no-]install", "Install grammars into grammars/ directory.") do |i|
+    $options[:install] = i
+  end
+
+  opts.on("--output FILE", "Write output to FILE. Use - for stdout.") do |o|
+    $options[:output] = o == "-" ? $stdout : o
+  end
+
+  opts.on("--[no-]remote", "Download remote grammars.") do |r|
+    $options[:remote] = r
+  end
+end.parse!
+
 sources = File.open(SOURCES_FILE) do |file|
   YAML.load(file)
 end
 
 yaml = main(sources)
 
-File.write(SOURCES_FILE, YAML.dump(yaml))
+if $options[:output].is_a?(IO)
-
+  $options[:output].write(YAML.dump(yaml))
-$stderr.puts("Done")
+else
+  File.write($options[:output], YAML.dump(yaml))
+end

test/test_grammars.rb

@@ -36,4 +36,17 @@ class TestGrammars < Minitest::Test
 
     assert nonexistent_submodules.empty? && unlisted_submodules.empty?, message
   end
+
+  def test_local_scopes_are_in_sync
+    actual = YAML.load(`"#{File.join(ROOT, "script", "convert-grammars")}" --output - --no-install --no-remote`)
+    assert $?.success?, "script/convert-grammars failed"
+
+    # We're not checking remote grammars. That can take a long time and make CI
+    # flaky if network conditions are poor.
+    @grammars.delete_if { |k, v| k.start_with?("http:", "https:") }
+
+    @grammars.each do |k, v|
+      assert_equal v, actual[k], "The scopes listed for #{k} in grammars.yml don't match the scopes found in that repository"
+    end
+  end
 end