Merge branch 'master' into grammar-scripts

script/convert-grammars

@@ -42,7 +42,7 @@ class DirectoryPackage
       case File.extname(path.downcase)
       when '.plist'
         path.split('/')[-2] == 'Syntaxes'
-      when '.tmlanguage', '.yaml-tmlanguage', '.sublime-syntax'
+      when '.tmlanguage', '.yaml-tmlanguage'
         true
       when '.cson', '.json'
         path.split('/')[-2] == 'grammars'
@@ -114,7 +114,7 @@ class SVNPackage
   def fetch(tmp_dir)
     `svn export -q "#{url}/Syntaxes" "#{tmp_dir}/Syntaxes"`
     raise "Failed to export SVN repository: #{url}: #{$?.to_s}" unless $?.success?
-    Dir["#{tmp_dir}/Syntaxes/*.{plist,tmLanguage,tmlanguage,YAML-tmLanguage,sublime-syntax}"]
+    Dir["#{tmp_dir}/Syntaxes/*.{plist,tmLanguage,tmlanguage,YAML-tmLanguage}"]
   end
 end
 
@@ -148,7 +148,7 @@ def load_grammar(path)
   case File.extname(path.downcase)
   when '.plist', '.tmlanguage'
     Plist::parse_xml(path)
-  when '.yaml-tmlanguage', '.sublime-syntax'
+  when '.yaml-tmlanguage'
     content = File.read(path)
     # Attempt to parse YAML file even if it has a YAML 1.2 header
     if content.lines[0] =~ /^%YAML[ :]1\.2/
@@ -180,7 +180,7 @@ def load_grammars(tmp_dir, source, all_scopes)
         else
           SingleFile.new(source)
         end
-      elsif source.end_with?('.tmLanguage', '.plist', '.YAML-tmLanguage', '.sublime-syntax')
+      elsif source.end_with?('.tmLanguage', '.plist', '.YAML-tmLanguage')
         SingleGrammar.new(source)
       elsif source.start_with?('https://github.com')
         GitHubPackage.new(source)

script/set-language-ids

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+require 'yaml'
+require 'pry'
+
+header = <<-EOF
+# Defines all Languages known to GitHub.
+#
+# type              - Either data, programming, markup, prose, or nil
+# aliases           - An Array of additional aliases (implicitly
+#                     includes name.downcase)
+# ace_mode          - A String name of the Ace Mode used for highlighting whenever
+#                     a file is edited. This must match one of the filenames in http://git.io/3XO_Cg.
+#                     Use "text" if a mode does not exist.
+# wrap              - Boolean wrap to enable line wrapping (default: false)
+# extensions        - An Array of associated extensions (the first one is
+#                     considered the primary extension, the others should be
+#                     listed alphabetically)
+# interpreters      - An Array of associated interpreters
+# searchable        - Boolean flag to enable searching (defaults to true)
+# search_term       - Deprecated: Some languages may be indexed under a
+#                     different alias. Avoid defining new exceptions.
+# language_id       - Integer used as a language-name-independent indexed field so that we can rename
+#                     languages in Linguist without reindexing all the code on GitHub. Must not be 
+#                     changed for existing languages without the explicit permission of GitHub staff.
+# color             - CSS hex color to represent the language.
+# tm_scope          - The TextMate scope that represents this programming
+#                     language. This should match one of the scopes listed in
+#                     the grammars.yml file. Use "none" if there is no grammar
+#                     for this language.
+# group             - Name of the parent language. Languages in a group are counted
+#                     in the statistics as the parent language.
+#
+# Any additions or modifications (even trivial) should have corresponding
+# test changes in `test/test_blob.rb`.
+#
+# Please keep this list alphabetized. Capitalization comes before lowercase.
+
+EOF
+
+generated = true if ARGV[0] == "--force"
+update = true if ARGV[0] == "--update"
+
+if generated
+  puts "You're regenerating all of the language_id attributes for all Linguist "
+  puts "languages defined in languages.yml. This is almost certainly NOT what"
+  puts "you meant to do!"
+
+  language_index = 0
+
+  languages = YAML.load(File.read("lib/linguist/languages.yml"))
+  languages.each do |name, vals|
+    vals.merge!('language_id' => language_index)
+    language_index += 1
+  end
+
+  File.write("lib/linguist/languages.yml", header + YAML.dump(languages))
+elsif update
+  puts "Adding new language_id attributes to languages.yml that don't have one set"
+  languages = YAML.load(File.read("lib/linguist/languages.yml"))
+
+  # First grab the maximum language_id
+  language_ids = []
+  languages.each { |name, vals| language_ids << vals['language_id'] if vals.has_key?('language_id')}
+  max_language_id = language_ids.max
+  puts "Current maximum language_id is #{max_language_id}"
+
+  missing_count = 0
+  language_index = max_language_id
+
+  languages.each do |name, vals|
+    unless vals.has_key?('language_id')
+      language_index += 1
+      missing_count += 1
+      vals.merge!('language_id' => language_index)
+    end
+  end
+
+  File.write("lib/linguist/languages.yml", header + YAML.dump(languages))
+  puts "Updated language_id attributes for #{missing_count} languages"
+else
+  puts "Whatever you want me to do, I can't figure it out. Giving up..."
+end