Improve grammar scripts (#3350)

* Remove trailing spaces

* Setup Bundler in some scripts

* Update grammar index

* Make prune-grammars script to be callable in a script directory

* Prune unused xquery grammar repo

source.xq by language-jsoniq is actual tm_scope for XQuery.

* Remove xquery submodule

git submodule deinit vendor/grammars/xquery/
git rm vendor/grammars/xquery/

* Fix invocation of script/list-grammars

This fixes #3339.

* Make add-grammars script to be callable in a script directory

* Generate samples.json before running list-grammars

list-grammars requires linguist.

script/add-grammar

@@ -78,6 +78,8 @@ https    = "https://#{parts[:host]}/#{parts[:user]}/#{parts[:repo]}"
 repo_new = "vendor/grammars/#{parts[:repo]}"
 repo_old = parse_submodule($replace) if $replace
 
+Dir.chdir(ROOT)
+
 if repo_old
   log "Deregistering: #{repo_old}"
   `git submodule deinit #{repo_old}`
@@ -93,4 +95,5 @@ log "Confirming license"
 `script/licensed --module "#{repo_new}"`
 
 log "Updating grammar documentation in vendor/REAEDME.md"
-`script list-grammars`
+`bundle exec rake samples`
+`script/list-grammars`

script/convert-grammars

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
 
+require 'bundler/setup'
 require 'json'
 require 'net/http'
 require 'optparse'

script/list-grammars

@@ -1,19 +1,20 @@
 #!/usr/bin/env ruby
 
+require "bundler/setup"
 require "linguist"
 require "json"
 require "yaml"
 
 class GrammarList
-  
+
   ROOT = File.expand_path "../../", __FILE__
-  
+
   def initialize
     @submodules     = load_submodules()
     @sources        = load_sources()
     @language_names = load_languages()
   end
-  
+
   # Load .gitmodules
   def load_submodules
     submodules = {}
@@ -29,14 +30,14 @@ class GrammarList
     end
     submodules
   end
-  
+
   # Grab the name of each language, sorted case-insensitively
   def load_languages
     Linguist::Language.all.map(&:name).sort do |a, b|
       a.downcase() <=> b.downcase()
     end
   end
-  
+
   # Load grammars.yml
   def load_sources
     sources = {}
@@ -46,7 +47,7 @@ class GrammarList
     end
     sources
   end
-  
+
   # Shorten a repository URL
   def shorten(url)
     if url =~ /^https?:\/\/(?:www\.)?github\.com\/([^\/]+\/[^\/]+)/i
@@ -57,7 +58,7 @@ class GrammarList
       url.replace(/^https?:\/\/(?:www\.)?/i, "")
     end
   end
-  
+
   # Markdown: Generate grammar list
   def to_markdown
     markdown = ""
@@ -87,7 +88,7 @@ class GrammarList
 
     markdown
   end
-  
+
   # Update the file displaying the reader-friendly list of grammar repos
   def update_readme
     readme = "#{ROOT}/vendor/README.md"

script/prune-grammars

@@ -1,10 +1,13 @@
 #!/usr/bin/env ruby
 
+require "bundler/setup"
 require "json"
 require "linguist"
 require "set"
 require "yaml"
 
+ROOT = File.expand_path("../../", __FILE__)
+
 def find_includes(json)
   case json
   when Hash
@@ -32,7 +35,7 @@ def transitive_includes(scope, includes)
 end
 
 includes = {}
-Dir["grammars/*.json"].each do |path|
+Dir[File.join(ROOT, "grammars/*.json")].each do |path|
   scope = File.basename(path).sub(/\.json/, '')
   json = JSON.load(File.read(path))
   incs = find_includes(json)
@@ -41,7 +44,7 @@ Dir["grammars/*.json"].each do |path|
   includes[scope] += incs
 end
 
-yaml = YAML.load(File.read("grammars.yml"))
+yaml = YAML.load(File.read(File.join(ROOT, "grammars.yml")))
 language_scopes = Linguist::Language.all.map(&:tm_scope).to_set
 
 # The set of used scopes is the scopes for each language, plus all the scopes
@@ -54,4 +57,4 @@ puts "Unused grammar repos"
 puts unused.map { |repo, scopes| sprintf("%-100s %s", repo, scopes.join(", ")) }.sort.join("\n")
 
 yaml.delete_if { |k| unused.key?(k) }
-File.write("grammars.yml", YAML.dump(yaml))
+File.write(File.join(ROOT, "grammars.yml"), YAML.dump(yaml))

script/set-language-ids

@@ -1,4 +1,6 @@
 #!/usr/bin/env ruby
+
+require 'bundler/setup'
 require 'yaml'
 require 'pry'