Improve grammar scripts (#3350)

* Remove trailing spaces

* Setup Bundler in some scripts

* Update grammar index

* Make prune-grammars script to be callable in a script directory

* Prune unused xquery grammar repo

source.xq by language-jsoniq is actual tm_scope for XQuery.

* Remove xquery submodule

git submodule deinit vendor/grammars/xquery/
git rm vendor/grammars/xquery/

* Fix invocation of script/list-grammars

This fixes #3339.

* Make add-grammars script to be callable in a script directory

* Generate samples.json before running list-grammars

list-grammars requires linguist.

.gitmodules

@@ -776,9 +776,6 @@
 [submodule "vendor/grammars/vhdl"]
 	path = vendor/grammars/vhdl
 	url = https://github.com/textmate/vhdl.tmbundle
-[submodule "vendor/grammars/xquery"]
-	path = vendor/grammars/xquery
-	url = https://github.com/textmate/xquery.tmbundle
 [submodule "vendor/grammars/language-rpm-spec"]
 	path = vendor/grammars/language-rpm-spec
 	url = https://github.com/waveclaw/language-rpm-spec

grammars.yml

@@ -653,7 +653,5 @@ vendor/grammars/xc.tmbundle:
 vendor/grammars/xml.tmbundle:
 - text.xml
 - text.xml.xsl
-vendor/grammars/xquery:
-- source.xquery
 vendor/grammars/zephir-sublime:
 - source.php.zephir

script/add-grammar

@@ -78,6 +78,8 @@ https    = "https://#{parts[:host]}/#{parts[:user]}/#{parts[:repo]}"
 repo_new = "vendor/grammars/#{parts[:repo]}"
 repo_old = parse_submodule($replace) if $replace
 
+Dir.chdir(ROOT)
+
 if repo_old
   log "Deregistering: #{repo_old}"
   `git submodule deinit #{repo_old}`
@@ -93,4 +95,5 @@ log "Confirming license"
 `script/licensed --module "#{repo_new}"`
 
 log "Updating grammar documentation in vendor/REAEDME.md"
-`script list-grammars`
+`bundle exec rake samples`
+`script/list-grammars`

script/convert-grammars

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
 
+require 'bundler/setup'
 require 'json'
 require 'net/http'
 require 'optparse'

script/list-grammars

@@ -1,19 +1,20 @@
 #!/usr/bin/env ruby
 
+require "bundler/setup"
 require "linguist"
 require "json"
 require "yaml"
 
 class GrammarList
-  
+
   ROOT = File.expand_path "../../", __FILE__
-  
+
   def initialize
     @submodules     = load_submodules()
     @sources        = load_sources()
     @language_names = load_languages()
   end
-  
+
   # Load .gitmodules
   def load_submodules
     submodules = {}
@@ -29,14 +30,14 @@ class GrammarList
     end
     submodules
   end
-  
+
   # Grab the name of each language, sorted case-insensitively
   def load_languages
     Linguist::Language.all.map(&:name).sort do |a, b|
       a.downcase() <=> b.downcase()
     end
   end
-  
+
   # Load grammars.yml
   def load_sources
     sources = {}
@@ -46,7 +47,7 @@ class GrammarList
     end
     sources
   end
-  
+
   # Shorten a repository URL
   def shorten(url)
     if url =~ /^https?:\/\/(?:www\.)?github\.com\/([^\/]+\/[^\/]+)/i
@@ -57,7 +58,7 @@ class GrammarList
       url.replace(/^https?:\/\/(?:www\.)?/i, "")
     end
   end
-  
+
   # Markdown: Generate grammar list
   def to_markdown
     markdown = ""
@@ -87,7 +88,7 @@ class GrammarList
 
     markdown
   end
-  
+
   # Update the file displaying the reader-friendly list of grammar repos
   def update_readme
     readme = "#{ROOT}/vendor/README.md"

script/prune-grammars

@@ -1,10 +1,13 @@
 #!/usr/bin/env ruby
 
+require "bundler/setup"
 require "json"
 require "linguist"
 require "set"
 require "yaml"
 
+ROOT = File.expand_path("../../", __FILE__)
+
 def find_includes(json)
   case json
   when Hash
@@ -32,7 +35,7 @@ def transitive_includes(scope, includes)
 end
 
 includes = {}
-Dir["grammars/*.json"].each do |path|
+Dir[File.join(ROOT, "grammars/*.json")].each do |path|
   scope = File.basename(path).sub(/\.json/, '')
   json = JSON.load(File.read(path))
   incs = find_includes(json)
@@ -41,7 +44,7 @@ Dir["grammars/*.json"].each do |path|
   includes[scope] += incs
 end
 
-yaml = YAML.load(File.read("grammars.yml"))
+yaml = YAML.load(File.read(File.join(ROOT, "grammars.yml")))
 language_scopes = Linguist::Language.all.map(&:tm_scope).to_set
 
 # The set of used scopes is the scopes for each language, plus all the scopes
@@ -54,4 +57,4 @@ puts "Unused grammar repos"
 puts unused.map { |repo, scopes| sprintf("%-100s %s", repo, scopes.join(", ")) }.sort.join("\n")
 
 yaml.delete_if { |k| unused.key?(k) }
-File.write("grammars.yml", YAML.dump(yaml))
+File.write(File.join(ROOT, "grammars.yml"), YAML.dump(yaml))

script/set-language-ids

@@ -1,4 +1,6 @@
 #!/usr/bin/env ruby
+
+require 'bundler/setup'
 require 'yaml'
 require 'pry'
 

vendor/README.md

@@ -6,6 +6,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 <!-- Everything below this line is auto-generated by script/list-grammars. Manual edits will be lost -->
 - **1C Enterprise:** [xDrivenDevelopment/atom-language-1c-bsl](https://github.com/xDrivenDevelopment/atom-language-1c-bsl)
 - **ABAP:** [pvl/abap.tmbundle](https://github.com/pvl/abap.tmbundle)
+- **ABNF:** [sanssecours/ABNF.tmbundle](https://github.com/sanssecours/ABNF.tmbundle)
 - **ActionScript:** [simongregory/actionscript3-tmbundle](https://github.com/simongregory/actionscript3-tmbundle)
 - **Ada:** [textmate/ada.tmbundle](https://github.com/textmate/ada.tmbundle)
 - **Agda:** [mokus0/Agda.tmbundle](https://github.com/mokus0/Agda.tmbundle)
@@ -91,6 +92,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **DTrace:** [textmate/c.tmbundle](https://github.com/textmate/c.tmbundle)
 - **Dylan:** [textmate/dylan.tmbundle](https://github.com/textmate/dylan.tmbundle)
 - **Eagle:** [textmate/xml.tmbundle](https://github.com/textmate/xml.tmbundle)
+- **EBNF:** [sanssecours/EBNF.tmbundle](https://github.com/sanssecours/EBNF.tmbundle)
 - **eC:** [ecere/ec.tmbundle](https://github.com/ecere/ec.tmbundle)
 - **Ecere Projects:** [textmate/json.tmbundle](https://github.com/textmate/json.tmbundle)
 - **ECLiPSe:** [alnkpa/sublimeprolog](https://github.com/alnkpa/sublimeprolog)
@@ -264,6 +266,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **Puppet:** [russCloak/SublimePuppet](https://github.com/russCloak/SublimePuppet)
 - **PureScript:** [purescript-contrib/atom-language-purescript](https://github.com/purescript-contrib/atom-language-purescript)
 - **Python:** [MagicStack/MagicPython](https://github.com/MagicStack/MagicPython)
+- **Python console:** [atom/language-python](https://github.com/atom/language-python)
 - **Python traceback:** [atom/language-python](https://github.com/atom/language-python)
 - **QMake:** [textmate/cpp-qt.tmbundle](https://github.com/textmate/cpp-qt.tmbundle)
 - **QML:** [skozlovf/Sublime-QML](https://github.com/skozlovf/Sublime-QML)