Merge pull request #3205 from github/language-id

Language ids

CONTRIBUTING.md

@@ -33,6 +33,7 @@ To add support for a new language:
   0. Add your grammar to [`grammars.yml`][grammars] by running `script/convert-grammars --add vendor/grammars/MyGrammar`.
   0. Download the license for the grammar: `script/licensed`. Be careful to only commit the file for the new grammar, as this script may update licenses for other grammars as well.
 0. Add samples for your language to the [samples directory][samples] in the correct subdirectory.
+0. Add a `language_id` for your language. See `script/set-language-ids` for more information. **You should only ever need to run `script/set-language-ids --update`. Anything other than this risks breaking GitHub search :cry:**
 0. Open a pull request, linking to a [GitHub search result](https://github.com/search?utf8=%E2%9C%93&q=extension%3Aboot+NOT+nothack&type=Code&ref=searchresults) showing in-the-wild usage.
 
 In addition, if your new language defines an extension that's already listed in [`languages.yml`][languages] (such as `.foo`) then sometimes a few more steps will need to be taken:
@@ -84,7 +85,7 @@ Linguist is maintained with :heart: by:
 - @arfon (GitHub Staff)
 - @larsbrinkhoff
 - @pchaigno
- 
+
 As Linguist is a production dependency for GitHub we have a couple of workflow restrictions:
 
 - Anyone with commit rights can merge Pull Requests provided that there is a :+1: from a GitHub member of staff

lib/linguist/language.rb

@@ -20,10 +20,11 @@ module Linguist
   #
   # Languages are defined in `lib/linguist/languages.yml`.
   class Language
-    @languages       = []
-    @index           = {}
-    @name_index      = {}
-    @alias_index     = {}
+    @languages          = []
+    @index              = {}
+    @name_index         = {}
+    @alias_index        = {}
+    @language_id_index  = {}
 
     @extension_index          = Hash.new { |h,k| h[k] = [] }
     @interpreter_index        = Hash.new { |h,k| h[k] = [] }
@@ -84,6 +85,8 @@ module Linguist
         @filename_index[filename] << language
       end
 
+      @language_id_index[language.language_id] = language
+
       language
     end
 
@@ -193,6 +196,19 @@ module Linguist
       @interpreter_index[interpreter]
     end
 
+    # Public: Look up Languages by its language_id.
+    #
+    # language_id - Integer of language_id
+    #
+    # Examples
+    #
+    #   Language.find_by_id(100)
+    #   # => [#<Language name="Elixir">]
+    #
+    # Returns the matching Language
+    def self.find_by_id(language_id)
+      @language_id_index[language_id.to_i]
+    end
 
     # Public: Look up Language by its name.
     #
@@ -289,6 +305,9 @@ module Linguist
       # Set legacy search term
       @search_term = attributes[:search_term] || default_alias_name
 
+      # Set the language_id 
+      @language_id = attributes[:language_id]
+
       # Set extensions or default to [].
       @extensions = attributes[:extensions] || []
       @interpreters = attributes[:interpreters]   || []
@@ -351,6 +370,17 @@ module Linguist
     # Returns the name String
     attr_reader :search_term
 
+    # Public: Get language_id (used in GitHub search)
+    #
+    # Examples
+    #
+    #   # => "1"
+    #   # => "2"
+    #   # => "3"
+    #
+    # Returns the integer language_id
+    attr_reader :language_id
+
     # Public: Get the name of a TextMate-compatible scope
     #
     # Returns the scope
@@ -547,6 +577,7 @@ module Linguist
       :group_name        => options['group'],
       :searchable        => options.fetch('searchable', true),
       :search_term       => options['search_term'],
+      :language_id       => options['language_id'],
       :extensions        => Array(options['extensions']),
       :interpreters      => options['interpreters'].sort,
       :filenames         => options['filenames'],

script/set-language-ids

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+require 'yaml'
+require 'pry'
+
+header = <<-EOF
+# Defines all Languages known to GitHub.
+#
+# type              - Either data, programming, markup, prose, or nil
+# aliases           - An Array of additional aliases (implicitly
+#                     includes name.downcase)
+# ace_mode          - A String name of the Ace Mode used for highlighting whenever
+#                     a file is edited. This must match one of the filenames in http://git.io/3XO_Cg.
+#                     Use "text" if a mode does not exist.
+# wrap              - Boolean wrap to enable line wrapping (default: false)
+# extensions        - An Array of associated extensions (the first one is
+#                     considered the primary extension, the others should be
+#                     listed alphabetically)
+# interpreters      - An Array of associated interpreters
+# searchable        - Boolean flag to enable searching (defaults to true)
+# search_term       - Deprecated: Some languages may be indexed under a
+#                     different alias. Avoid defining new exceptions.
+# language_id       - Integer used as a language-name-independent indexed field so that we can rename
+#                     languages in Linguist without reindexing all the code on GitHub. Must not be 
+#                     changed for existing languages without the explicit permission of GitHub staff.
+# color             - CSS hex color to represent the language.
+# tm_scope          - The TextMate scope that represents this programming
+#                     language. This should match one of the scopes listed in
+#                     the grammars.yml file. Use "none" if there is no grammar
+#                     for this language.
+# group             - Name of the parent language. Languages in a group are counted
+#                     in the statistics as the parent language.
+#
+# Any additions or modifications (even trivial) should have corresponding
+# test changes in `test/test_blob.rb`.
+#
+# Please keep this list alphabetized. Capitalization comes before lowercase.
+
+EOF
+
+generated = true if ARGV[0] == "--force"
+update = true if ARGV[0] == "--update"
+
+if generated
+  puts "You're regenerating all of the language_id attributes for all Linguist "
+  puts "languages defined in languages.yml. This is almost certainly NOT what"
+  puts "you meant to do!"
+
+  language_index = 0
+
+  languages = YAML.load(File.read("lib/linguist/languages.yml"))
+  languages.each do |name, vals|
+    vals.merge!('language_id' => language_index)
+    language_index += 1
+  end
+
+  File.write("lib/linguist/languages.yml", header + YAML.dump(languages))
+elsif update
+  puts "Adding new language_id attributes to languages.yml that don't have one set"
+  languages = YAML.load(File.read("lib/linguist/languages.yml"))
+
+  # First grab the maximum language_id
+  language_ids = []
+  languages.each { |name, vals| language_ids << vals['language_id'] if vals.has_key?('language_id')}
+  max_language_id = language_ids.max
+  puts "Current maximum language_id is #{max_language_id}"
+
+  missing_count = 0
+  language_index = max_language_id
+
+  languages.each do |name, vals|
+    unless vals.has_key?('language_id')
+      language_index += 1
+      missing_count += 1
+      vals.merge!('language_id' => language_index)
+    end
+  end
+
+  File.write("lib/linguist/languages.yml", header + YAML.dump(languages))
+  puts "Updated language_id attributes for #{missing_count} languages"
+else
+  puts "Whatever you want me to do, I can't figure it out. Giving up..."
+end

test/test_language.rb

@@ -67,6 +67,22 @@ class TestLanguage < Minitest::Test
     assert_nil Language.find_by_alias(nil)
   end
 
+  # Note these are set by script/set-language-ids. If these tests fail then someone
+  # has changed the language_id fields set in languages.yml which is almost certainly
+  # not what you want to happen (these fields are used in GitHub's search indexes)
+  def test_language_ids
+    assert_equal 4, Language['ANTLR'].language_id
+    assert_equal 54, Language['Ceylon'].language_id
+    assert_equal 326, Language['Ruby'].language_id
+    assert_equal 421, Language['xBase'].language_id
+  end
+
+  def test_find_by_id
+    assert_equal Language['Elixir'], Language.find_by_id(100)
+    assert_equal Language['Ruby'], Language.find_by_id(326)
+    assert_equal Language['xBase'], Language.find_by_id(421)
+  end
+
   def test_groups
     # Test a couple identity cases
     assert_equal Language['Perl'], Language['Perl'].group
@@ -386,6 +402,22 @@ class TestLanguage < Minitest::Test
     assert missing.empty?, message
   end
 
+  def test_all_languages_have_a_language_id_set
+    missing = Language.all.select { |language| language.language_id.nil? }
+
+    message = "The following languages do not have a language_id listed in languages.yml. Please add language_id fields for all new languages.\n"
+    missing.each { |language| message << "#{language.name}\n" }
+    assert missing.empty?, message
+  end
+
+  def test_all_language_id_are_unique
+    duplicates = Language.all.group_by{ |language| language.language_id }.select { |k, v| v.size > 1 }.map(&:first)
+
+    message = "The following language_id are used several times in languages.yml. Please use script/set-language-ids --update as per the contribution guidelines.\n"
+    duplicates.each { |language_id| message << "#{language_id}\n" }
+    assert duplicates.empty?, message
+  end
+
   def test_all_languages_have_a_valid_ace_mode
     ace_fixture_path = File.join('test', 'fixtures', 'ace_modes.json')
     skip("No ace_modes.json file") unless File.exist?(ace_fixture_path)