Linguist 4.8.15

Convert from mode names to mimetypes for better usage.

.gitignore

@@ -1,5 +1,6 @@
 /Gemfile.lock
 .bundle/
+.idea
 benchmark/
 lib/linguist/samples.json
 /grammars

.gitmodules

@@ -12,7 +12,7 @@
 	url = https://github.com/Drako/SublimeBrainfuck
 [submodule "vendor/grammars/awk-sublime"]
 	path = vendor/grammars/awk-sublime
-	url = https://github.com/JohnNilsson/awk-sublime
+	url = https://github.com/github-linguist/awk-sublime
 [submodule "vendor/grammars/Sublime-SQF-Language"]
 	path = vendor/grammars/Sublime-SQF-Language
 	url = https://github.com/JonBons/Sublime-SQF-Language
@@ -22,15 +22,15 @@
 [submodule "vendor/grammars/Sublime-REBOL"]
 	path = vendor/grammars/Sublime-REBOL
 	url = https://github.com/Oldes/Sublime-REBOL
-[submodule "vendor/grammars/Sublime-VimL"]
+[submodule "vendor/grammars/language-viml"]
-	path = vendor/grammars/Sublime-VimL
+	path = vendor/grammars/language-viml
-	url = https://github.com/SalGnt/Sublime-VimL
+	url = https://github.com/Alhadis/language-viml
 [submodule "vendor/grammars/ColdFusion"]
 	path = vendor/grammars/ColdFusion
 	url = https://github.com/SublimeText/ColdFusion
 [submodule "vendor/grammars/NSIS"]
 	path = vendor/grammars/NSIS
-	url = https://github.com/SublimeText/NSIS
+	url = https://github.com/github-linguist/NSIS
 [submodule "vendor/grammars/NimLime"]
 	path = vendor/grammars/NimLime
 	url = https://github.com/Varriount/NimLime
@@ -667,7 +667,7 @@
 	url = https://github.com/goblindegook/sublime-text-pig-latin
 [submodule "vendor/grammars/sourcepawn"]
 	path = vendor/grammars/sourcepawn
-	url = https://github.com/austinwagner/sublime-sourcepawn
+	url = https://github.com/github-linguist/sublime-sourcepawn
 [submodule "vendor/grammars/gdscript"]
 	path = vendor/grammars/gdscript
 	url = https://github.com/beefsack/GDScript-sublime
@@ -791,3 +791,6 @@
 [submodule "vendor/grammars/language-babel"]
 	path = vendor/grammars/language-babel
 	url = https://github.com/github-linguist/language-babel
+[submodule "vendor/CodeMirror"]
+	path = vendor/CodeMirror
+	url = https://github.com/codemirror/CodeMirror

CONTRIBUTING.md

@@ -33,6 +33,7 @@ To add support for a new language:
   0. Add your grammar to [`grammars.yml`][grammars] by running `script/convert-grammars --add vendor/grammars/MyGrammar`.
   0. Download the license for the grammar: `script/licensed`. Be careful to only commit the file for the new grammar, as this script may update licenses for other grammars as well.
 0. Add samples for your language to the [samples directory][samples] in the correct subdirectory.
+0. Add a `language_id` for your language. See `script/set-language-ids` for more information. **You should only ever need to run `script/set-language-ids --update`. Anything other than this risks breaking GitHub search :cry:**
 0. Open a pull request, linking to a [GitHub search result](https://github.com/search?utf8=%E2%9C%93&q=extension%3Aboot+NOT+nothack&type=Code&ref=searchresults) showing in-the-wild usage.
 
 In addition, if your new language defines an extension that's already listed in [`languages.yml`][languages] (such as `.foo`) then sometimes a few more steps will need to be taken:
@@ -84,7 +85,7 @@ Linguist is maintained with :heart: by:
 - @arfon (GitHub Staff)
 - @larsbrinkhoff
 - @pchaigno
- 
+
 As Linguist is a production dependency for GitHub we have a couple of workflow restrictions:
 
 - Anyone with commit rights can merge Pull Requests provided that there is a :+1: from a GitHub member of staff

grammars.yml

@@ -20,7 +20,7 @@ vendor/grammars/ColdFusion:
 - text.html.cfm
 vendor/grammars/Docker.tmbundle:
 - source.dockerfile
-vendor/grammars/Elm:
+vendor/grammars/Elm/Syntaxes:
 - source.elm
 - text.html.mediawiki.elm-build-output
 - text.html.mediawiki.elm-documentation
@@ -103,8 +103,6 @@ vendor/grammars/Sublime-SQF-Language:
 vendor/grammars/Sublime-Text-2-OpenEdge-ABL:
 - source.abl
 - text.html.abl
-vendor/grammars/Sublime-VimL:
-- source.viml
 vendor/grammars/SublimeBrainfuck:
 - source.bf
 vendor/grammars/SublimeClarion:
@@ -414,6 +412,8 @@ vendor/grammars/language-toc-wow:
 - source.toc
 vendor/grammars/language-turing:
 - source.turing
+vendor/grammars/language-viml:
+- source.viml
 vendor/grammars/language-wavefront:
 - source.wavefront.mtl
 - source.wavefront.obj

lib/linguist/heuristics.rb

@@ -144,7 +144,7 @@ module Linguist
       end
     end
 
-    fortran_rx = /^([c*][^abd-z]|      (subroutine|program|end)\s|\s*!)/i
+    fortran_rx = /^([c*][^abd-z]|      (subroutine|program|end|data)\s|\s*!)/i
 
     disambiguate ".f" do |data|
       if /^: /.match(data)
@@ -202,6 +202,8 @@ module Linguist
     disambiguate ".inc" do |data|
       if /^<\?(?:php)?/.match(data)
         Language["PHP"]
+      elsif /^\s*#(declare|local|macro|while)\s/.match(data)
+        Language["POV-Ray SDL"]
       end
     end
 

lib/linguist/language.rb

@@ -20,10 +20,11 @@ module Linguist
   #
   # Languages are defined in `lib/linguist/languages.yml`.
   class Language
-    @languages       = []
+    @languages          = []
-    @index           = {}
+    @index              = {}
-    @name_index      = {}
+    @name_index         = {}
-    @alias_index     = {}
+    @alias_index        = {}
+    @language_id_index  = {}
 
     @extension_index          = Hash.new { |h,k| h[k] = [] }
     @interpreter_index        = Hash.new { |h,k| h[k] = [] }
@@ -84,6 +85,8 @@ module Linguist
         @filename_index[filename] << language
       end
 
+      @language_id_index[language.language_id] = language
+
       language
     end
 
@@ -193,6 +196,19 @@ module Linguist
       @interpreter_index[interpreter]
     end
 
+    # Public: Look up Languages by its language_id.
+    #
+    # language_id - Integer of language_id
+    #
+    # Examples
+    #
+    #   Language.find_by_id(100)
+    #   # => [#<Language name="Elixir">]
+    #
+    # Returns the matching Language
+    def self.find_by_id(language_id)
+      @language_id_index[language_id.to_i]
+    end
 
     # Public: Look up Language by its name.
     #
@@ -251,6 +267,7 @@ module Linguist
     # Returns an Array of Languages.
     def self.ace_modes
       warn "This method will be deprecated in a future 5.x release. Every language now has an `ace_mode` set."
+      warn caller
       @ace_modes ||= all.select(&:ace_mode).sort_by { |lang| lang.name.downcase }
     end
 
@@ -284,11 +301,16 @@ module Linguist
       end
 
       @ace_mode = attributes[:ace_mode]
+      @codemirror_mode = attributes[:codemirror_mode]
+      @codemirror_mime_type = attributes[:codemirror_mime_type]
       @wrap = attributes[:wrap] || false
 
       # Set legacy search term
       @search_term = attributes[:search_term] || default_alias_name
 
+      # Set the language_id
+      @language_id = attributes[:language_id]
+
       # Set extensions or default to [].
       @extensions = attributes[:extensions] || []
       @interpreters = attributes[:interpreters]   || []
@@ -351,6 +373,17 @@ module Linguist
     # Returns the name String
     attr_reader :search_term
 
+    # Public: Get language_id (used in GitHub search)
+    #
+    # Examples
+    #
+    #   # => "1"
+    #   # => "2"
+    #   # => "3"
+    #
+    # Returns the integer language_id
+    attr_reader :language_id
+
     # Public: Get the name of a TextMate-compatible scope
     #
     # Returns the scope
@@ -367,6 +400,31 @@ module Linguist
     # Returns a String name or nil
     attr_reader :ace_mode
 
+    # Public: Get CodeMirror mode
+    #
+    # Maps to a directory in the `mode/` source code.
+    #   https://github.com/codemirror/CodeMirror/tree/master/mode
+    #
+    # Examples
+    #
+    #  # => "nil"
+    #  # => "javascript"
+    #  # => "clike"
+    #
+    # Returns a String name or nil
+    attr_reader :codemirror_mode
+
+    # Public: Get CodeMirror MIME type mode
+    #
+    # Examples
+    #
+    #  # => "nil"
+    #  # => "text/x-javascript"
+    #  # => "text/x-csrc"
+    #
+    # Returns a String name or nil
+    attr_reader :codemirror_mime_type
+
     # Public: Should language lines be wrapped
     #
     # Returns true or false
@@ -543,10 +601,13 @@ module Linguist
       :aliases           => options['aliases'],
       :tm_scope          => options['tm_scope'],
       :ace_mode          => options['ace_mode'],
+      :codemirror_mode   => options['codemirror_mode'],
+      :codemirror_mime_type => options['codemirror_mime_type'],
       :wrap              => options['wrap'],
       :group_name        => options['group'],
       :searchable        => options.fetch('searchable', true),
       :search_term       => options['search_term'],
+      :language_id       => options['language_id'],
       :extensions        => Array(options['extensions']),
       :interpreters      => options['interpreters'].sort,
       :filenames         => options['filenames'],

lib/linguist/strategy/modeline.rb

@@ -1,19 +1,98 @@
 module Linguist
   module Strategy
     class Modeline
-      EMACS_MODELINE = /-\*-\s*(?:(?!mode)[\w-]+\s*:\s*(?:[\w+-]+)\s*;?\s*)*(?:mode\s*:)?\s*([\w+-]+)\s*(?:;\s*(?!mode)[\w-]+\s*:\s*[\w+-]+\s*)*;?\s*-\*-/i
+      EMACS_MODELINE = /
+        -\*-
+        (?:
+          # Short form: `-*- ruby -*-`
+          \s* (?= [^:;\s]+ \s* -\*-)
+          |
+          # Longer form: `-*- foo:bar; mode: ruby; -*-`
+          (?:
+            .*?       # Preceding variables: `-*- foo:bar bar:baz;`
+            [;\s]     # Which are delimited by spaces or semicolons
+            |
+            (?<=-\*-) # Not preceded by anything: `-*-mode:ruby-*-`
+          )
+          mode        # Major mode indicator
+          \s*:\s*     # Allow whitespace around colon: `mode : ruby`
+        )
+        ([^:;\s]+)    # Name of mode
 
-      # First form vim modeline
+        # Ensure the mode is terminated correctly
-      # [text]{white}{vi:|vim:|ex:}[white]{options}
+        (?=
-      # ex: 'vim: syntax=ruby'
+          # Followed by semicolon or whitespace
-      VIM_MODELINE_1 = /(?:vim|vi|ex):\s*(?:ft|filetype|syntax)=(\w+)\s?/i
+          [\s;]
+          |
+          # Touching the ending sequence: `ruby-*-`
+          (?<![-*])   # Don't allow stuff like `ruby--*-` to match; it'll invalidate the mode
+          -\*-        # Emacs has no problems reading `ruby --*-`, however.
+        )
+        .*?           # Anything between a cleanly-terminated mode and the ending -*-
+        -\*-
+      /xi
 
-      # Second form vim modeline (compatible with some versions of Vi)
+      VIM_MODELINE   = /
-      # [text]{white}{vi:|vim:|Vim:|ex:}[white]se[t] {options}:[text]
-      # ex: 'vim set syntax=ruby:'
-      VIM_MODELINE_2 = /(?:vim|vi|Vim|ex):\s*se(?:t)?.*\s(?:ft|filetype|syntax)=(\w+)\s?.*:/i
 
-      MODELINES = [EMACS_MODELINE, VIM_MODELINE_1, VIM_MODELINE_2]
+        # Start modeline. Could be `vim:`, `vi:` or `ex:`
+        (?:
+          (?:\s|^)
+          vi
+          (?:m[<=>]?\d+|m)? # Version-specific modeline
+          |
+          [\t\x20] # `ex:` requires whitespace, because "ex:" might be short for "example:"
+          ex
+        )
+
+        # If the option-list begins with `set ` or `se `, it indicates an alternative
+        # modeline syntax partly-compatible with older versions of Vi. Here, the colon
+        # serves as a terminator for an option sequence, delimited by whitespace.
+        (?=
+          # So we have to ensure the modeline ends with a colon
+          : (?=\s* set? \s [^\n:]+ :) |
+
+          # Otherwise, it isn't valid syntax and should be ignored
+          : (?!\s* set? \s)
+        )
+
+        # Possible (unrelated) `option=value` pairs to skip past
+        (?:
+          # Option separator. Vim uses whitespace or colons to separate options (except if
+          # the alternate "vim: set " form is used, where only whitespace is used)
+          (?:
+            \s
+            |
+            \s* : \s* # Note that whitespace around colons is accepted too:
+          )           # vim: noai :  ft=ruby:noexpandtab
+
+          # Option's name. All recognised Vim options have an alphanumeric form.
+          \w*
+
+          # Possible value. Not every option takes an argument.
+          (?:
+            # Whitespace between name and value is allowed: `vim: ft   =ruby`
+            \s*=
+
+            # Option's value. Might be blank; `vim: ft= ` says "use no filetype".
+            (?:
+              [^\\\s] # Beware of escaped characters: titlestring=\ ft=ruby
+              |       # will be read by Vim as { titlestring: " ft=ruby" }.
+              \\.
+            )*
+          )?
+        )*
+
+        # The actual filetype declaration
+        [\s:] (?:filetype|ft|syntax) \s*=
+
+        # Language's name
+        (\w+)
+
+        # Ensure it's followed by a legal separator
+        (?=\s|:|$)
+      /xi
+
+      MODELINES = [EMACS_MODELINE, VIM_MODELINE]
 
       # Scope of the search for modelines
       # Number of lines to check at the beginning and at the end of the file

lib/linguist/vendor.yml

@@ -165,7 +165,7 @@
 # Chart.js
 - (^|/)Chart\.js$
 
-# Codemirror
+# CodeMirror
 - (^|/)[Cc]ode[Mm]irror/(\d+\.\d+/)?(lib|mode|theme|addon|keymap|demo)
 
 # SyntaxHighlighter - http://alexgorbatchev.com/

lib/linguist/version.rb

@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "4.8.10"
+  VERSION = "4.8.15"
 end

samples/Emacs Lisp/filenames/.abbrev_defs

@@ -0,0 +1,6 @@
+(define-abbrev-table 'c-mode-abbrev-table '(
+	))
+(define-abbrev-table 'fundamental-mode-abbrev-table '(
+	("TM" "™" nil 0)
+	("(R)" "®" nil 0)
+	("C=" "€" nil 0)))

samples/Emacs Lisp/filenames/.gnus

@@ -0,0 +1,20 @@
+(setq user-full-name "Alhadis")
+(setq user-mail-address "fake.account@gmail.com")
+
+(auto-image-file-mode)
+(setq mm-inline-large-images t)
+(add-to-list 'mm-attachment-override-types "image/*")
+
+(setq gnus-select-method 
+	  '(nnimap "gmail"
+		(nnimap-address "imap.gmail.com")
+		(nnimap-server-port 777)
+		(nnimap-stream ssl)))
+
+(setq message-send-mail-function 'smtpmail-send-it
+	  smtpmail-starttls-credentials '(("smtp.gmail.com" 600 nil nil))
+	  smtpmail-auth-credentials '(("smtp.gmail.com" 700 "me@lisp.com" nil))
+	  smtpmail-default-smtp-server "smtp.gmail.com"
+	  smtpmail-smtp-server "smtp.gmail.com"
+	  smtpmail-smtp-service 800
+	  setq gnus-ignored-from-addresses "^from\\.Telstra[ \t\r\n]+Thanks")

samples/Emacs Lisp/filenames/.viper

@@ -0,0 +1,10 @@
+(setq viper-inhibit-startup-message 't)
+(setq viper-expert-level '5)
+
+; Key bindings
+(define-key viper-vi-global-user-map "\C-d" 'end-of-line)
+
+; Return to top of window
+(defun my-viper-return-to-top ()
+  (interactive)
+  (beginning-of-buffer))

samples/Emacs Lisp/filenames/Project.ede

@@ -0,0 +1,34 @@
+;; Object EDE
+(ede-proj-project "Linguist"
+  :name "Linguist"
+  :version "4.9"
+  :file "Project.ede"
+  :targets (list 
+   (ede-proj-target-elisp-autoloads "autoloads"
+    :name "autoloads"
+    :path "test/samples/Emacs Lisp"
+    :autoload-file "dude.el"
+    )
+   (ede-proj-target-elisp "init"
+    :name "init"
+    :path ""
+    :source '("ede-load.el" "wait-what.el")
+    :compiler 'ede-emacs-preload-compiler
+    :pre-load-packages '("sample-names")
+    )
+   (ede-proj-target-elisp "what"
+    :name "the"
+    :path ""
+    :source '("h.el" "am-i-writing.el")
+    :versionsource '("hell.el")
+    :compiler 'ede-emacs-preload-compiler
+    :aux-packages '("what" "the" "hell-files" "am-i-writing")
+    )
+   )
+  :web-site-url "https://github.com/github/linguist"
+  :web-site-directory "../"
+  :web-site-file "CONTRIBUTING.md"
+  :ftp-upload-site "/ftp@git.hub.com:/madeup"
+  :configuration-variables 'nil
+  :metasubproject 't
+  )

samples/Emacs Lisp/filenames/_emacs

@@ -0,0 +1,70 @@
+;; UTF-8 support
+;; (set-language-environment "UTF-8")
+(setenv "LANG" "en_AU.UTF-8")
+(setenv "LC_ALL" "en_AU.UTF-8")
+(setq default-tab-width 4)
+
+
+;;; Function to load all ".el" files in ~/.emacs.d/config
+(defun load-directory (directory)
+  "Recursively load all Emacs Lisp files in a directory."
+  (dolist (element (directory-files-and-attributes directory nil nil nil))
+    (let* ((path (car element))
+           (fullpath (concat directory "/" path))
+           (isdir (car (cdr element)))
+           (ignore-dir (or (string= path ".") (string= path ".."))))
+      (cond
+       ((and (eq isdir t) (not ignore-dir))
+        (load-directory fullpath))
+       ((and (eq isdir nil) (string= (substring path -3) ".el"))
+        (load (file-name-sans-extension fullpath)))))))
+
+;; Tell Emacs we'd like to use Hunspell for spell-checking
+(setq ispell-program-name (executable-find "hunspell"))
+
+;; Load Homebrew-installed packages
+(let ((default-directory "/usr/local/share/emacs/site-lisp/"))
+  (normal-top-level-add-subdirs-to-load-path))
+(load "aggressive-indent")
+(add-hook 'emacs-lisp-mode-hook #'aggressive-indent-mode)
+(autoload 'rust-mode "rust-mode" nil t)
+(add-to-list 'auto-mode-alist '("\\.rs\\'" . rust-mode))
+
+;; Load Git-related syntax highlighting
+(add-to-list 'load-path "~/.emacs.d/lisp/")
+(load "git-modes")
+(load "git-commit")
+
+;; Keybindings
+(global-set-key (kbd "C-u") (lambda ()
+                             (interactive)
+                             (kill-line 0)))
+
+;; Show cursor's current column number
+(setq column-number-mode t)
+
+;; Disable autosave
+(setq auto-save-default nil)
+
+;; Use a single directory for storing backup files
+(setq backup-directory-alist `(("." . "~/.emacs.d/auto-save-list")))
+(setq backup-by-copying t)
+(setq delete-old-versions t
+      kept-new-versions 6
+      kept-old-versions 2
+      version-control t)
+
+(custom-set-variables
+ ;; custom-set-variables was added by Custom.
+ ;; If you edit it by hand, you could mess it up, so be careful.
+ ;; Your init file should contain only one such instance.
+ ;; If there is more than one, they won't work right.
+ '(blink-cursor-mode nil)
+ '(column-number-mode t)
+ '(show-paren-mode t))
+(custom-set-faces
+ ;; custom-set-faces was added by Custom.
+ ;; If you edit it by hand, you could mess it up, so be careful.
+ ;; Your init file should contain only one such instance.
+ ;; If there is more than one, they won't work right.
+ )

samples/Emacs Lisp/filenames/abbrev_defs

@@ -0,0 +1,8 @@
+(define-abbrev-table 'fundamental-mode-abbrev-table '(
+	("cat" "Concatenate" nil 0)
+	("WTF" "World Trade Federation " nil 0)
+	("rtbtm" "Read that back to me" nil 0)))
+
+(define-abbrev-table 'shell-script-mode-abbrev-table '(
+	("brake", "bundle rake exec" nil 0)
+	("pls", "warning: setting Encoding.default_external")))

samples/FORTRAN/bug-185631.f

@@ -0,0 +1,6 @@
+! Codes/HYCOM/hycom/ATLb2.00/src_2.0.01_22_one/
+      real onemu, twomu
+      data onemu/0.0098/
+      data twomu/1./
+      data threemu/0.e9/
+      end

script/convert-grammars

@@ -42,7 +42,7 @@ class DirectoryPackage
       case File.extname(path.downcase)
       when '.plist'
         path.split('/')[-2] == 'Syntaxes'
-      when '.tmlanguage', '.yaml-tmlanguage', '.sublime-syntax'
+      when '.tmlanguage', '.yaml-tmlanguage'
         true
       when '.cson', '.json'
         path.split('/')[-2] == 'grammars'
@@ -114,7 +114,7 @@ class SVNPackage
   def fetch(tmp_dir)
     `svn export -q "#{url}/Syntaxes" "#{tmp_dir}/Syntaxes"`
     raise "Failed to export SVN repository: #{url}: #{$?.to_s}" unless $?.success?
-    Dir["#{tmp_dir}/Syntaxes/*.{plist,tmLanguage,tmlanguage,YAML-tmLanguage,sublime-syntax}"]
+    Dir["#{tmp_dir}/Syntaxes/*.{plist,tmLanguage,tmlanguage,YAML-tmLanguage}"]
   end
 end
 
@@ -148,7 +148,7 @@ def load_grammar(path)
   case File.extname(path.downcase)
   when '.plist', '.tmlanguage'
     Plist::parse_xml(path)
-  when '.yaml-tmlanguage', '.sublime-syntax'
+  when '.yaml-tmlanguage'
     content = File.read(path)
     # Attempt to parse YAML file even if it has a YAML 1.2 header
     if content.lines[0] =~ /^%YAML[ :]1\.2/
@@ -180,7 +180,7 @@ def load_grammars(tmp_dir, source, all_scopes)
         else
           SingleFile.new(source)
         end
-      elsif source.end_with?('.tmLanguage', '.plist', '.YAML-tmLanguage', '.sublime-syntax')
+      elsif source.end_with?('.tmLanguage', '.plist', '.YAML-tmLanguage')
         SingleGrammar.new(source)
       elsif source.start_with?('https://github.com')
         GitHubPackage.new(source)

script/set-language-ids

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+require 'yaml'
+require 'pry'
+
+header = <<-EOF
+# Defines all Languages known to GitHub.
+#
+# type              - Either data, programming, markup, prose, or nil
+# aliases           - An Array of additional aliases (implicitly
+#                     includes name.downcase)
+# ace_mode          - A String name of the Ace Mode used for highlighting whenever
+#                     a file is edited. This must match one of the filenames in http://git.io/3XO_Cg.
+#                     Use "text" if a mode does not exist.
+# wrap              - Boolean wrap to enable line wrapping (default: false)
+# extensions        - An Array of associated extensions (the first one is
+#                     considered the primary extension, the others should be
+#                     listed alphabetically)
+# interpreters      - An Array of associated interpreters
+# searchable        - Boolean flag to enable searching (defaults to true)
+# search_term       - Deprecated: Some languages may be indexed under a
+#                     different alias. Avoid defining new exceptions.
+# language_id       - Integer used as a language-name-independent indexed field so that we can rename
+#                     languages in Linguist without reindexing all the code on GitHub. Must not be 
+#                     changed for existing languages without the explicit permission of GitHub staff.
+# color             - CSS hex color to represent the language.
+# tm_scope          - The TextMate scope that represents this programming
+#                     language. This should match one of the scopes listed in
+#                     the grammars.yml file. Use "none" if there is no grammar
+#                     for this language.
+# group             - Name of the parent language. Languages in a group are counted
+#                     in the statistics as the parent language.
+#
+# Any additions or modifications (even trivial) should have corresponding
+# test changes in `test/test_blob.rb`.
+#
+# Please keep this list alphabetized. Capitalization comes before lowercase.
+
+EOF
+
+generated = true if ARGV[0] == "--force"
+update = true if ARGV[0] == "--update"
+
+if generated
+  puts "You're regenerating all of the language_id attributes for all Linguist "
+  puts "languages defined in languages.yml. This is almost certainly NOT what"
+  puts "you meant to do!"
+
+  language_index = 0
+
+  languages = YAML.load(File.read("lib/linguist/languages.yml"))
+  languages.each do |name, vals|
+    vals.merge!('language_id' => language_index)
+    language_index += 1
+  end
+
+  File.write("lib/linguist/languages.yml", header + YAML.dump(languages))
+elsif update
+  puts "Adding new language_id attributes to languages.yml that don't have one set"
+  languages = YAML.load(File.read("lib/linguist/languages.yml"))
+
+  # First grab the maximum language_id
+  language_ids = []
+  languages.each { |name, vals| language_ids << vals['language_id'] if vals.has_key?('language_id')}
+  max_language_id = language_ids.max
+  puts "Current maximum language_id is #{max_language_id}"
+
+  missing_count = 0
+  language_index = max_language_id
+
+  languages.each do |name, vals|
+    unless vals.has_key?('language_id')
+      language_index += 1
+      missing_count += 1
+      vals.merge!('language_id' => language_index)
+    end
+  end
+
+  File.write("lib/linguist/languages.yml", header + YAML.dump(languages))
+  puts "Updated language_id attributes for #{missing_count} languages"
+else
+  puts "Whatever you want me to do, I can't figure it out. Giving up..."
+end

test/fixtures/Data/Modelines/iamjs.pl

@@ -0,0 +1,3 @@
+# vim: noexpandtab: ft=javascript
+
+"It's JavaScript, baby";

test/fixtures/Data/Modelines/iamjs2.pl

@@ -0,0 +1,4 @@
+# vim:noexpandtab titlestring=hi\|there\\\ ft=perl ts=4
+# vim:noexpandtab titlestring=hi|there\\ ft=javascript ts=4
+
+"Still JavaScript, bruh";

test/fixtures/Data/Modelines/ruby10

@@ -0,0 +1,3 @@
+ ex: noexpandtab: ft=ruby
+
+# Still Ruby

test/fixtures/Data/Modelines/ruby11

@@ -0,0 +1,3 @@
+# vim600: ft=ruby
+
+# Targets Vim 6.0 or later

test/fixtures/Data/Modelines/ruby12

@@ -0,0 +1,3 @@
+vim<520: ft=ruby
+
+# Targets Vim 5.20 and earlier

test/fixtures/Data/Modelines/seeplusplusEmacs10

@@ -0,0 +1,3 @@
+// -*- foo-bar mode: c++ -*-
+
+"Malformed modeline, but still understood by Emacs to be C++."

test/fixtures/Data/Modelines/seeplusplusEmacs11

@@ -0,0 +1 @@
+/* -*- mode: c++ -------*- */

test/fixtures/Data/Modelines/seeplusplusEmacs12

@@ -0,0 +1 @@
+-*--------- foo:bar mode: c++ -*-

test/helper.rb

@@ -35,3 +35,11 @@ def sample_blob_memory(name)
   content = File.read(filepath)
   Linguist::Blob.new(name, content)
 end
+
+def silence_warnings
+  original_verbosity = $VERBOSE
+  $VERBOSE = nil
+  yield
+ensure
+  $VERBOSE = original_verbosity
+end

test/test_blob.rb

@@ -4,14 +4,18 @@ class TestBlob < Minitest::Test
   include Linguist
 
   def setup
-    # git blobs are normally loaded as ASCII-8BIT since they may contain data
+    silence_warnings do
-    # with arbitrary encoding not known ahead of time
+      # git blobs are normally loaded as ASCII-8BIT since they may contain data
-    @original_external = Encoding.default_external
+      # with arbitrary encoding not known ahead of time
-    Encoding.default_external = Encoding.find("ASCII-8BIT")
+      @original_external = Encoding.default_external
+      Encoding.default_external = Encoding.find("ASCII-8BIT")
+    end
   end
 
   def teardown
-    Encoding.default_external = @original_external
+    silence_warnings do
+      Encoding.default_external = @original_external
+    end
   end
 
   def script_blob(name)

test/test_file_blob.rb

@@ -3,15 +3,27 @@ require_relative "./helper"
 class TestFileBlob < Minitest::Test
   include Linguist
 
+  def silence_warnings
+    original_verbosity = $VERBOSE
+    $VERBOSE = nil
+    yield
+  ensure
+    $VERBOSE = original_verbosity
+  end
+
   def setup
-    # git blobs are normally loaded as ASCII-8BIT since they may contain data
+    silence_warnings do
-    # with arbitrary encoding not known ahead of time
+      # git blobs are normally loaded as ASCII-8BIT since they may contain data
-    @original_external = Encoding.default_external
+      # with arbitrary encoding not known ahead of time
-    Encoding.default_external = Encoding.find("ASCII-8BIT")
+      @original_external = Encoding.default_external
+      Encoding.default_external = Encoding.find("ASCII-8BIT")
+    end
   end
 
   def teardown
-    Encoding.default_external = @original_external
+    silence_warnings do
+      Encoding.default_external = @original_external
+    end
   end
 
   def script_blob(name)
@@ -305,7 +317,7 @@ class TestFileBlob < Minitest::Test
     assert sample_blob("some/vendored/path/Chart.js").vendored?
     assert !sample_blob("some/vendored/path/chart.js").vendored?
 
-    # Codemirror deps
+    # CodeMirror deps
     assert sample_blob("codemirror/mode/blah.js").vendored?
     assert sample_blob("codemirror/5.0/mode/blah.js").vendored?
 

test/test_grammars.rb

@@ -5,8 +5,11 @@ class TestGrammars < Minitest::Test
 
   # List of projects that are allowed without licenses
   PROJECT_WHITELIST = [
-    # Dual MIT and GPL license
+    "vendor/grammars/factor",
+    "vendor/grammars/go-tmbundle",
+    "vendor/grammars/jflex.tmbundle",
     "vendor/grammars/language-csharp",
+    "vendor/grammars/language-viml",
     "vendor/grammars/sublimeassembly"
   ].freeze
 
@@ -123,7 +126,7 @@ class TestGrammars < Minitest::Test
   private
 
   def submodule_paths
-    @submodule_paths ||= `git config --list --file "#{File.join(ROOT, ".gitmodules")}"`.lines.grep(/\.path=/).map { |line| line.chomp.split("=", 2).last }
+    @submodule_paths ||= `git config --list --file "#{File.join(ROOT, ".gitmodules")}"`.lines.grep(/\.path=/).map { |line| line.chomp.split("=", 2).last }.reject { |path| path =~ /CodeMirror/ }
   end
 
   # Returns a hash of submodules in the form of submodule_path => license
@@ -151,7 +154,7 @@ class TestGrammars < Minitest::Test
 
     # Neither Licensee nor our own regex was able to detect the license, let's check the readme
     files = Dir[File.join(ROOT, submodule, "*")]
-    if readme = files.find { |path| File.basename(path) =~ /\Areadme\b/i }
+    if readme = files.find { |file| File.basename(file) =~ /\Areadme\b/i }
       classify_license(readme)
     end
   end

test/test_heuristics.rb

@@ -122,9 +122,12 @@ class TestHeuristcs < Minitest::Test
     })
   end
 
+  # Candidate languages = ["Assembly", "C++", "HTML", "PAWN", "PHP",
+  #                        "POV-Ray SDL", "Pascal", "SQL", "SourcePawn"]
   def test_inc_by_heuristics
     assert_heuristics({
-      "PHP" => all_fixtures("PHP", "*.inc")
+      "PHP" => all_fixtures("PHP", "*.inc"),
+      "POV-Ray SDL" => all_fixtures("POV-Ray SDL", "*.inc")
     })
   end
 

test/test_language.rb

@@ -67,6 +67,22 @@ class TestLanguage < Minitest::Test
     assert_nil Language.find_by_alias(nil)
   end
 
+  # Note these are set by script/set-language-ids. If these tests fail then someone
+  # has changed the language_id fields set in languages.yml which is almost certainly
+  # not what you want to happen (these fields are used in GitHub's search indexes)
+  def test_language_ids
+    assert_equal 4, Language['ANTLR'].language_id
+    assert_equal 54, Language['Ceylon'].language_id
+    assert_equal 326, Language['Ruby'].language_id
+    assert_equal 421, Language['xBase'].language_id
+  end
+
+  def test_find_by_id
+    assert_equal Language['Elixir'], Language.find_by_id(100)
+    assert_equal Language['Ruby'], Language.find_by_id(326)
+    assert_equal Language['xBase'], Language.find_by_id(421)
+  end
+
   def test_groups
     # Test a couple identity cases
     assert_equal Language['Perl'], Language['Perl'].group
@@ -329,8 +345,24 @@ class TestLanguage < Minitest::Test
   end
 
   def test_ace_modes
-    assert Language.ace_modes.include?(Language['Ruby'])
+    silence_warnings do
-    assert Language.ace_modes.include?(Language['FORTRAN'])
+      assert Language.ace_modes.include?(Language['Ruby'])
+      assert Language.ace_modes.include?(Language['FORTRAN'])
+    end
+  end
+
+  def test_codemirror_mode
+    assert_equal 'ruby', Language['Ruby'].codemirror_mode
+    assert_equal 'javascript', Language['JavaScript'].codemirror_mode
+    assert_equal 'clike', Language['C'].codemirror_mode
+    assert_equal 'clike', Language['C++'].codemirror_mode
+  end
+
+  def test_codemirror_mime_type
+    assert_equal 'text/x-ruby', Language['Ruby'].codemirror_mime_type
+    assert_equal 'text/javascript', Language['JavaScript'].codemirror_mime_type
+    assert_equal 'text/x-csrc', Language['C'].codemirror_mime_type
+    assert_equal 'text/x-c++src', Language['C++'].codemirror_mime_type
   end
 
   def test_wrap
@@ -386,6 +418,22 @@ class TestLanguage < Minitest::Test
     assert missing.empty?, message
   end
 
+  def test_all_languages_have_a_language_id_set
+    missing = Language.all.select { |language| language.language_id.nil? }
+
+    message = "The following languages do not have a language_id listed in languages.yml. Please add language_id fields for all new languages.\n"
+    missing.each { |language| message << "#{language.name}\n" }
+    assert missing.empty?, message
+  end
+
+  def test_all_language_id_are_unique
+    duplicates = Language.all.group_by{ |language| language.language_id }.select { |k, v| v.size > 1 }.map(&:first)
+
+    message = "The following language_id are used several times in languages.yml. Please use script/set-language-ids --update as per the contribution guidelines.\n"
+    duplicates.each { |language_id| message << "#{language_id}\n" }
+    assert duplicates.empty?, message
+  end
+
   def test_all_languages_have_a_valid_ace_mode
     ace_fixture_path = File.join('test', 'fixtures', 'ace_modes.json')
     skip("No ace_modes.json file") unless File.exist?(ace_fixture_path)
@@ -404,6 +452,40 @@ class TestLanguage < Minitest::Test
     assert missing.empty?, message
   end
 
+  def test_codemirror_modes_present
+    Language.all.each do |language|
+      if language.codemirror_mode || language.codemirror_mime_type
+        assert language.codemirror_mode, "#{language.inspect} missing CodeMirror mode"
+        assert language.codemirror_mime_type, "#{language.inspect} missing CodeMirror MIME mode"
+      end
+    end
+  end
+
+  def test_valid_codemirror_mode
+    Language.all.each do |language|
+      if mode = language.codemirror_mode
+        assert File.exist?(File.expand_path("../../vendor/CodeMirror/mode/#{mode}", __FILE__)), "#{mode} isn't a valid CodeMirror mode"
+      end
+    end
+  end
+
+  def test_codemirror_mode_and_mime_defined_by_meta_mapping
+    meta = File.read(File.expand_path("../../vendor/CodeMirror/mode/meta.js", __FILE__))
+    Language.all.each do |language|
+      next unless language.codemirror_mode && language.codemirror_mime_type
+      assert meta.match(/^.+#{Regexp.escape(language.codemirror_mime_type)}.+#{Regexp.escape(language.codemirror_mode)}.+$/), "#{language.inspect}: #{language.codemirror_mime_type} not defined under #{language.codemirror_mode}"
+    end
+  end
+
+  def test_codemirror_mime_declared_in_mode_file
+    Language.all.each do |language|
+      next unless language.codemirror_mode && language.codemirror_mime_type
+      filename = File.expand_path("../../vendor/CodeMirror/mode/#{language.codemirror_mode}/#{language.codemirror_mode}.js", __FILE__)
+      assert File.exist?(filename), "#{filename} does not exist"
+      assert File.read(filename).match(language.codemirror_mime_type), "#{language.inspect}: #{language.codemirror_mime_type} not defined in #{filename}"
+    end
+  end
+
   def test_all_popular_languages_exist
     popular = YAML.load(File.read(File.expand_path("../../lib/linguist/popular.yml", __FILE__)))
 

test/test_modelines.rb

@@ -17,6 +17,9 @@ class TestModelines < Minitest::Test
     assert_modeline Language["Ruby"], fixture_blob("Data/Modelines/ruby7")
     assert_modeline Language["Ruby"], fixture_blob("Data/Modelines/ruby8")
     assert_modeline Language["Ruby"], fixture_blob("Data/Modelines/ruby9")
+    assert_modeline Language["Ruby"], fixture_blob("Data/Modelines/ruby10")
+    assert_modeline Language["Ruby"], fixture_blob("Data/Modelines/ruby11")
+    assert_modeline Language["Ruby"], fixture_blob("Data/Modelines/ruby12")
     assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplus")
     assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs1")
     assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs2")
@@ -27,9 +30,14 @@ class TestModelines < Minitest::Test
     assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs7")
     assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs8")
     assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs9")
+    assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs10")
+    assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs11")
+    assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs12")
     assert_modeline Language["Text"], fixture_blob("Data/Modelines/fundamentalEmacs.c")
     assert_modeline Language["Prolog"], fixture_blob("Data/Modelines/not_perl.pl")
     assert_modeline Language["Smalltalk"], fixture_blob("Data/Modelines/example_smalltalk.md")
+    assert_modeline Language["JavaScript"], fixture_blob("Data/Modelines/iamjs.pl")
+    assert_modeline Language["JavaScript"], fixture_blob("Data/Modelines/iamjs2.pl")
     assert_modeline Language["PHP"], fixture_blob("Data/Modelines/iamphp.inc")
     assert_modeline nil, sample_blob("C/main.c")
   end
@@ -48,9 +56,14 @@ class TestModelines < Minitest::Test
     assert_equal Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs7").language
     assert_equal Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs8").language
     assert_equal Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs9").language
+    assert_equal Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs10").language
+    assert_equal Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs11").language
+    assert_equal Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs12").language
     assert_equal Language["Text"], fixture_blob("Data/Modelines/fundamentalEmacs.c").language
     assert_equal Language["Prolog"], fixture_blob("Data/Modelines/not_perl.pl").language
     assert_equal Language["Smalltalk"], fixture_blob("Data/Modelines/example_smalltalk.md").language
+    assert_equal Language["JavaScript"], fixture_blob("Data/Modelines/iamjs.pl").language
+    assert_equal Language["JavaScript"], fixture_blob("Data/Modelines/iamjs2.pl").language
     assert_equal Language["PHP"], fixture_blob("Data/Modelines/iamphp.inc").language
   end
 end

vendor/licenses/grammar/language-viml.txt

@@ -1,11 +1,12 @@
 ---
 type: grammar
-name: Sublime-VimL
+name: language-viml
 license: mit
 ---
 The MIT License (MIT)
 
-Copyright (c) 2014 Max Vasiliev, Salvatore Gentile
+Copyright (c) 2014-2016 Evan Hahn
+Copyright (c) 2016 John Gardner
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -14,13 +15,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in all
+The above copyright notice and this permission notice shall be included in
-copies or substantial portions of the Software.
+all copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-SOFTWARE.
+THE SOFTWARE.