Merge branch 'master' into grammar-list

script/add-grammar

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+
+require "optparse"
+
+ROOT = File.expand_path("../../", __FILE__)
+
+
+# Break a repository URL into its separate components
+def parse_url(input)
+  hosts = "github\.com|bitbucket\.org|gitlab\.com"
+  
+  # HTTPS/HTTP link pointing to recognised hosts
+  if input =~ /^(?:https?:\/\/)?(?:[^.@]+@)?(?:www\.)?(#{hosts})\/([^\/]+)\/([^\/]+)/i
+    { host: $1.downcase(), user: $2, repo: $3.sub(/\.git$/, "") }
+  # SSH
+  elsif input =~ /^git@(#{hosts}):([^\/]+)\/([^\/]+)\.git$/i
+    { host: $1.downcase(), user: $2, repo: $3 }
+  # provider:user/repo
+  elsif input =~ /^(github|bitbucket|gitlab):\/?([^\/]+)\/([^\/]+)\/?$/i
+    { host: $1.downcase(), user: $2, repo: $3 }
+  # user/repo - Common GitHub shorthand
+  elsif input =~ /^\/?([^\/]+)\/([^\/]+)\/?$/
+    { host: "github.com", user: $1, repo: $2 }
+  else
+    raise "Unsupported URL: #{input}"
+  end
+end
+
+# Isolate the vendor-name component of a submodule path
+def parse_submodule(name)
+  name =~ /^(?:.*(?:vendor\/)?grammars\/)?([^\/]+)/i
+  path = "vendor/grammars/#{$1}"
+  unless File.exist?("#{ROOT}/" + path)
+    warn "Submodule '#{path}' does not exist. Aborting."
+    exit 1
+  end
+  path
+end
+
+# Print debugging feedback to STDOUT if running with --verbose
+def log(msg)
+  puts msg if $verbose
+end
+
+
+usage = """Usage:
+  #{$0} [-v|--verbose] [--replace grammar] url
+Examples:
+  #{$0} https://github.com/Alhadis/language-roff
+  #{$0} --replace sublime-apl https://github.com/Alhadis/language-apl
+"""
+
+$replace = nil
+$verbose = false
+
+OptionParser.new do |opts|
+  opts.banner = usage
+  opts.on("-v", "--verbose", "Print verbose feedback to STDOUT") do
+    $verbose = true
+  end
+  opts.on("-rSUBMODULE", "--replace=SUBMODDULE", "Replace an existing grammar submodule.") do |name|
+    $replace = name
+  end
+end.parse!
+
+
+$url = ARGV[0]
+
+# No URL? Print a usage message and bail.
+unless $url
+  warn usage
+  exit 1;
+end
+
+# Ensure the given URL is an HTTPS link
+parts    = parse_url $url
+https    = "https://#{parts[:host]}/#{parts[:user]}/#{parts[:repo]}"
+repo_new = "vendor/grammars/#{parts[:repo]}"
+repo_old = parse_submodule($replace) if $replace
+
+if repo_old
+  log "Deregistering: #{repo_old}"
+  `git submodule deinit #{repo_old}`
+  `git rm -rf #{repo_old}`
+end
+
+log "Registering new submodule: #{repo_new}"
+`git submodule add -f #{https} #{repo_new}`
+exit 1 if $?.exitstatus > 0
+`script/convert-grammars --add #{repo_new}`
+
+log "Confirming license"
+`script/licensed --module "#{repo_new}"`

script/convert-grammars

@@ -42,7 +42,7 @@ class DirectoryPackage
       case File.extname(path.downcase)
       when '.plist'
         path.split('/')[-2] == 'Syntaxes'
-      when '.tmlanguage', '.yaml-tmlanguage', '.sublime-syntax'
+      when '.tmlanguage', '.yaml-tmlanguage'
         true
       when '.cson', '.json'
         path.split('/')[-2] == 'grammars'
@@ -114,7 +114,7 @@ class SVNPackage
   def fetch(tmp_dir)
     `svn export -q "#{url}/Syntaxes" "#{tmp_dir}/Syntaxes"`
     raise "Failed to export SVN repository: #{url}: #{$?.to_s}" unless $?.success?
-    Dir["#{tmp_dir}/Syntaxes/*.{plist,tmLanguage,tmlanguage,YAML-tmLanguage,sublime-syntax}"]
+    Dir["#{tmp_dir}/Syntaxes/*.{plist,tmLanguage,tmlanguage,YAML-tmLanguage}"]
   end
 end
 
@@ -148,7 +148,7 @@ def load_grammar(path)
   case File.extname(path.downcase)
   when '.plist', '.tmlanguage'
     Plist::parse_xml(path)
-  when '.yaml-tmlanguage', '.sublime-syntax'
+  when '.yaml-tmlanguage'
     content = File.read(path)
     # Attempt to parse YAML file even if it has a YAML 1.2 header
     if content.lines[0] =~ /^%YAML[ :]1\.2/
@@ -180,7 +180,7 @@ def load_grammars(tmp_dir, source, all_scopes)
         else
           SingleFile.new(source)
         end
-      elsif source.end_with?('.tmLanguage', '.plist', '.YAML-tmLanguage', '.sublime-syntax')
+      elsif source.end_with?('.tmLanguage', '.plist', '.YAML-tmLanguage')
         SingleGrammar.new(source)
       elsif source.start_with?('https://github.com')
         GitHubPackage.new(source)

script/licensed

@@ -4,6 +4,7 @@
 
 require "bundler/setup"
 require "licensed/cli"
+require "optparse"
 
 module Licensed
   module Source
@@ -32,7 +33,14 @@ module Licensed
   end
 end
 
-source = Licensed::Source::Filesystem.new("vendor/grammars/*/", type: "grammar")
+module_path = nil
+OptionParser.new do |opts|
+  opts.on("-mPATH", "--module=PATH", "Cache license file for specific grammar") do |p|
+    module_path = p
+  end
+end.parse!
+
+source = Licensed::Source::Filesystem.new(module_path || "vendor/grammars/*/", type: "grammar")
 config = Licensed::Configuration.new
 config.sources << source
 
@@ -43,4 +51,5 @@ else
 end
 
 command.run
+`git checkout -- vendor/licenses/grammar/` if module_path
 exit command.success?

script/set-language-ids

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+require 'yaml'
+require 'pry'
+
+header = <<-EOF
+# Defines all Languages known to GitHub.
+#
+# type              - Either data, programming, markup, prose, or nil
+# aliases           - An Array of additional aliases (implicitly
+#                     includes name.downcase)
+# ace_mode          - A String name of the Ace Mode used for highlighting whenever
+#                     a file is edited. This must match one of the filenames in http://git.io/3XO_Cg.
+#                     Use "text" if a mode does not exist.
+# wrap              - Boolean wrap to enable line wrapping (default: false)
+# extensions        - An Array of associated extensions (the first one is
+#                     considered the primary extension, the others should be
+#                     listed alphabetically)
+# interpreters      - An Array of associated interpreters
+# searchable        - Boolean flag to enable searching (defaults to true)
+# search_term       - Deprecated: Some languages may be indexed under a
+#                     different alias. Avoid defining new exceptions.
+# language_id       - Integer used as a language-name-independent indexed field so that we can rename
+#                     languages in Linguist without reindexing all the code on GitHub. Must not be 
+#                     changed for existing languages without the explicit permission of GitHub staff.
+# color             - CSS hex color to represent the language.
+# tm_scope          - The TextMate scope that represents this programming
+#                     language. This should match one of the scopes listed in
+#                     the grammars.yml file. Use "none" if there is no grammar
+#                     for this language.
+# group             - Name of the parent language. Languages in a group are counted
+#                     in the statistics as the parent language.
+#
+# Any additions or modifications (even trivial) should have corresponding
+# test changes in `test/test_blob.rb`.
+#
+# Please keep this list alphabetized. Capitalization comes before lowercase.
+
+EOF
+
+generated = true if ARGV[0] == "--force"
+update = true if ARGV[0] == "--update"
+
+if generated
+  puts "You're regenerating all of the language_id attributes for all Linguist "
+  puts "languages defined in languages.yml. This is almost certainly NOT what"
+  puts "you meant to do!"
+
+  language_index = 0
+
+  languages = YAML.load(File.read("lib/linguist/languages.yml"))
+  languages.each do |name, vals|
+    vals.merge!('language_id' => language_index)
+    language_index += 1
+  end
+
+  File.write("lib/linguist/languages.yml", header + YAML.dump(languages))
+elsif update
+  puts "Adding new language_id attributes to languages.yml that don't have one set"
+  languages = YAML.load(File.read("lib/linguist/languages.yml"))
+
+  # First grab the maximum language_id
+  language_ids = []
+  languages.each { |name, vals| language_ids << vals['language_id'] if vals.has_key?('language_id')}
+  max_language_id = language_ids.max
+  puts "Current maximum language_id is #{max_language_id}"
+
+  missing_count = 0
+  language_index = max_language_id
+
+  languages.each do |name, vals|
+    unless vals.has_key?('language_id')
+      language_index += 1
+      missing_count += 1
+      vals.merge!('language_id' => language_index)
+    end
+  end
+
+  File.write("lib/linguist/languages.yml", header + YAML.dump(languages))
+  puts "Updated language_id attributes for #{missing_count} languages"
+else
+  puts "Whatever you want me to do, I can't figure it out. Giving up..."
+end