Release v5.0.0 (#3388)

* bumping version to v5.0.0

* updating license information

* reverting accidental changes to language ids cc @twp

* Updating grammars

.gitmodules

@@ -325,9 +325,6 @@
 [submodule "vendor/grammars/nemerle.tmbundle"]
 	path = vendor/grammars/nemerle.tmbundle
 	url = https://github.com/textmate/nemerle.tmbundle
-[submodule "vendor/grammars/ninja.tmbundle"]
-	path = vendor/grammars/ninja.tmbundle
-	url = https://github.com/textmate/ninja.tmbundle
 [submodule "vendor/grammars/objective-c.tmbundle"]
 	path = vendor/grammars/objective-c.tmbundle
 	url = https://github.com/textmate/objective-c.tmbundle
@@ -355,9 +352,6 @@
 [submodule "vendor/grammars/r.tmbundle"]
 	path = vendor/grammars/r.tmbundle
 	url = https://github.com/textmate/r.tmbundle
-[submodule "vendor/grammars/ruby-haml.tmbundle"]
-	path = vendor/grammars/ruby-haml.tmbundle
-	url = https://github.com/textmate/ruby-haml.tmbundle
 [submodule "vendor/grammars/scheme.tmbundle"]
 	path = vendor/grammars/scheme.tmbundle
 	url = https://github.com/textmate/scheme.tmbundle
@@ -776,9 +770,6 @@
 [submodule "vendor/grammars/vhdl"]
 	path = vendor/grammars/vhdl
 	url = https://github.com/textmate/vhdl.tmbundle
-[submodule "vendor/grammars/xquery"]
-	path = vendor/grammars/xquery
-	url = https://github.com/textmate/xquery.tmbundle
 [submodule "vendor/grammars/language-rpm-spec"]
 	path = vendor/grammars/language-rpm-spec
 	url = https://github.com/waveclaw/language-rpm-spec
@@ -803,3 +794,19 @@
 [submodule "vendor/grammars/EBNF.tmbundle"]
 	path = vendor/grammars/EBNF.tmbundle
 	url = https://github.com/sanssecours/EBNF.tmbundle
+[submodule "vendor/grammars/language-haml"]
+	path = vendor/grammars/language-haml
+	url = https://github.com/ezekg/language-haml
+[submodule "vendor/grammars/language-ninja"]
+	path = vendor/grammars/language-ninja
+	url = https://github.com/khyo/language-ninja
+[submodule "vendor/grammars/language-fontforge"]
+	path = vendor/grammars/language-fontforge
+	url = https://github.com/Alhadis/language-fontforge
+[submodule "vendor/grammars/language-gn"]
+	path = vendor/grammars/language-gn
+	url = https://github.com/devoncarew/language-gn
+[submodule "vendor/grammars/rascal-syntax-highlighting"]
+	path = vendor/grammars/rascal-syntax-highlighting
+	url = https://github.com/usethesource/rascal-syntax-highlighting
+

CONTRIBUTING.md

@@ -79,9 +79,13 @@ Here's our current build status: [![Build Status](https://api.travis-ci.org/gith
 
 Linguist is maintained with :heart: by:
 
-- **@arfon** (GitHub Staff)
 - **@Alhadis**
+- **@arfon**
+- **@brandonblack** (GitHub staff)
 - **@larsbrinkhoff**
+- **@lildude** (GitHub staff)
+- **@lizzhale** (GitHub staff)
+- **@mikemcquaid** (GitHub staff)
 - **@pchaigno**
 
 As Linguist is a production dependency for GitHub we have a couple of workflow restrictions:

github-linguist.gemspec

@@ -27,5 +27,4 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'color-proximity', '~> 0.2.1'
   s.add_development_dependency 'licensed'
   s.add_development_dependency 'licensee', '>= 8.6.0'
-
 end

grammars.yml

@@ -362,10 +362,19 @@ vendor/grammars/language-csound:
 - source.csound-score
 vendor/grammars/language-emacs-lisp:
 - source.emacs.lisp
+vendor/grammars/language-fontforge:
+- source.fontforge
+- source.opentype
+- text.sfd
 vendor/grammars/language-gfm:
 - source.gfm
+vendor/grammars/language-gn:
+- source.gn
 vendor/grammars/language-graphql:
 - source.graphql
+vendor/grammars/language-haml:
+- text.haml
+- text.hamlc
 vendor/grammars/language-haskell:
 - hint.haskell
 - hint.message.haskell
@@ -381,7 +390,6 @@ vendor/grammars/language-inform7:
 - source.inform7
 vendor/grammars/language-javascript:
 - source.js
-- source.js.embedded.html
 - source.js.regexp
 - source.js.regexp.replacement
 vendor/grammars/language-jsoniq:
@@ -393,6 +401,8 @@ vendor/grammars/language-maxscript:
 - source.maxscript
 vendor/grammars/language-ncl:
 - source.ncl
+vendor/grammars/language-ninja:
+- source.ninja
 vendor/grammars/language-povray:
 - source.pov-ray sdl
 vendor/grammars/language-python:
@@ -474,8 +484,6 @@ vendor/grammars/nemerle.tmbundle:
 - source.nemerle
 vendor/grammars/nesC:
 - source.nesc
-vendor/grammars/ninja.tmbundle:
-- source.ninja
 vendor/grammars/nix:
 - source.nix
 vendor/grammars/nu.tmbundle:
@@ -531,8 +539,8 @@ vendor/grammars/python-django.tmbundle:
 vendor/grammars/r.tmbundle:
 - source.r
 - text.tex.latex.rd
-vendor/grammars/ruby-haml.tmbundle:
-- text.haml
+vendor/grammars/rascal-syntax-highlighting:
+- source.rascal
 vendor/grammars/ruby-slim.tmbundle:
 - text.slim
 vendor/grammars/ruby.tmbundle:
@@ -653,7 +661,5 @@ vendor/grammars/xc.tmbundle:
 vendor/grammars/xml.tmbundle:
 - text.xml
 - text.xml.xsl
-vendor/grammars/xquery:
-- source.xquery
 vendor/grammars/zephir-sublime:
 - source.php.zephir

lib/linguist.rb

@@ -59,8 +59,9 @@ class << Linguist
   # Strategies are called in turn until a single Language is returned.
   STRATEGIES = [
     Linguist::Strategy::Modeline,
-    Linguist::Shebang,
     Linguist::Strategy::Filename,
+    Linguist::Shebang,
+    Linguist::Strategy::Extension,
     Linguist::Heuristics,
     Linguist::Classifier
   ]

lib/linguist/blob.rb

@@ -63,7 +63,7 @@ module Linguist
     #
     # Returns an Array
     def extensions
-      _, *segments = name.downcase.split(".")
+      _, *segments = name.downcase.split(".", -1)
 
       segments.map.with_index do |segment, index|
         "." + segments[index..-1].join(".")

lib/linguist/generated.rb

@@ -275,16 +275,14 @@ module Linguist
       return lines[0].include?("Generated by the protocol buffer compiler.  DO NOT EDIT!")
     end
 
-    APACHE_THRIFT_EXTENSIONS = ['.rb', '.py', '.go', '.js', '.m', '.java', '.h', '.cc', '.cpp']
+    APACHE_THRIFT_EXTENSIONS = ['.rb', '.py', '.go', '.js', '.m', '.java', '.h', '.cc', '.cpp', '.php']
 
     # Internal: Is the blob generated by Apache Thrift compiler?
     #
     # Returns true or false
     def generated_apache_thrift?
       return false unless APACHE_THRIFT_EXTENSIONS.include?(extname)
-      return false unless lines.count > 1
-
-      return lines[0].include?("Autogenerated by Thrift Compiler") || lines[1].include?("Autogenerated by Thrift Compiler")
+      return lines.first(6).any? { |l| l.include?("Autogenerated by Thrift Compiler") }
     end
 
     # Internal: Is the blob a C/C++ header generated by the Java JNI tool javah?

lib/linguist/heuristics.rb

@@ -110,6 +110,12 @@ module Linguist
       end
     end
 
+    disambiguate ".cls" do |data|
+      if /\\\w+{/.match(data)
+        Language["TeX"]
+      end
+    end
+
     disambiguate ".cs" do |data|
       if /![\w\s]+methodsFor: /.match(data)
         Language["Smalltalk"]

lib/linguist/language.rb

@@ -11,6 +11,7 @@ require 'linguist/samples'
 require 'linguist/file_blob'
 require 'linguist/blob_helper'
 require 'linguist/strategy/filename'
+require 'linguist/strategy/extension'
 require 'linguist/strategy/modeline'
 require 'linguist/shebang'
 
@@ -90,17 +91,6 @@ module Linguist
       language
     end
 
-    # Public: Detects the Language of the blob.
-    #
-    # blob - an object that includes the Linguist `BlobHelper` interface;
-    #       see Linguist::LazyBlob and Linguist::FileBlob for examples
-    #
-    # Returns Language or nil.
-    def self.detect(blob)
-      warn "[DEPRECATED] `Linguist::Language.detect` is deprecated. Use `Linguist.detect`. #{caller[0]}"
-      Linguist.detect(blob)
-    end
-
     # Public: Get all Languages
     #
     # Returns an Array of Languages
@@ -140,46 +130,46 @@ module Linguist
 
     # Public: Look up Languages by filename.
     #
+    # The behaviour of this method recently changed.
+    # See the second example below.
+    #
     # filename - The path String.
     #
     # Examples
     #
+    #   Language.find_by_filename('Cakefile')
+    #   # => [#<Language name="CoffeeScript">]
     #   Language.find_by_filename('foo.rb')
-    #   # => [#<Language name="Ruby">]
+    #   # => []
     #
     # Returns all matching Languages or [] if none were found.
     def self.find_by_filename(filename)
       basename = File.basename(filename)
-
-      # find the first extension with language definitions
-      extname = FileBlob.new(filename).extensions.detect do |e|
-        !@extension_index[e].empty?
-      end
-
-      (@filename_index[basename] + @extension_index[extname]).compact.uniq
+      @filename_index[basename]
     end
 
     # Public: Look up Languages by file extension.
     #
-    # extname - The extension String.
+    # The behaviour of this method recently changed.
+    # See the second example below.
+    #
+    # filename - The path String.
     #
     # Examples
     #
-    #   Language.find_by_extension('.rb')
+    #   Language.find_by_extension('dummy.rb')
     #   # => [#<Language name="Ruby">]
-    #
     #   Language.find_by_extension('rb')
-    #   # => [#<Language name="Ruby">]
+    #   # => []
     #
     # Returns all matching Languages or [] if none were found.
-    def self.find_by_extension(extname)
-      extname = ".#{extname}" unless extname.start_with?(".")
-      @extension_index[extname.downcase]
-    end
+    def self.find_by_extension(filename)
+      # find the first extension with language definitions
+      extname = FileBlob.new(filename.downcase).extensions.detect do |e|
+        !@extension_index[e].empty?
+      end
 
-    # DEPRECATED
-    def self.find_by_shebang(data)
-      @interpreter_index[Shebang.interpreter(data)]
+      @extension_index[extname]
     end
 
     # Public: Look up Languages by interpreter.
@@ -259,18 +249,6 @@ module Linguist
       @colors ||= all.select(&:color).sort_by { |lang| lang.name.downcase }
     end
 
-    # Public: A List of languages compatible with Ace.
-    #
-    # TODO: Remove this method in a 5.x release. Every language now needs an ace_mode
-    # key, so this function isn't doing anything unique anymore.
-    #
-    # Returns an Array of Languages.
-    def self.ace_modes
-      warn "This method will be deprecated in a future 5.x release. Every language now has an `ace_mode` set."
-      warn caller
-      @ace_modes ||= all.select(&:ace_mode).sort_by { |lang| lang.name.downcase }
-    end
-
     # Internal: Initialize a new Language
     #
     # attributes - A hash of attributes
@@ -362,17 +340,6 @@ module Linguist
     # Returns an Array of String names
     attr_reader :aliases
 
-    # Deprecated: Get code search term
-    #
-    # Examples
-    #
-    #   # => "ruby"
-    #   # => "python"
-    #   # => "perl"
-    #
-    # Returns the name String
-    attr_reader :search_term
-
     # Public: Get language_id (used in GitHub search)
     #
     # Examples
@@ -457,22 +424,6 @@ module Linguist
     # Returns the extensions Array
     attr_reader :filenames
 
-    # Deprecated: Get primary extension
-    #
-    # Defaults to the first extension but can be overridden
-    # in the languages.yml.
-    #
-    # The primary extension can not be nil. Tests should verify this.
-    #
-    # This method is only used by app/helpers/gists_helper.rb for creating
-    # the language dropdown. It really should be using `name` instead.
-    # Would like to drop primary extension.
-    #
-    # Returns the extension String.
-    def primary_extension
-      extensions.first
-    end
-
     # Public: Get URL escaped name.
     #
     # Examples

lib/linguist/languages.yml

@@ -14,12 +14,10 @@
 #                     listed alphabetically)
 # interpreters      - An Array of associated interpreters
 # searchable        - Boolean flag to enable searching (defaults to true)
-# search_term       - Deprecated: Some languages may be indexed under a
-#                     different alias. Avoid defining new exceptions.
 # language_id       - Integer used as a language-name-independent indexed field so that we can rename
 #                     languages in Linguist without reindexing all the code on GitHub. Must not be
 #                     changed for existing languages without the explicit permission of GitHub staff.
-# color             - CSS hex color to represent the language. Only used if type is "programming" or "prose"
+# color             - CSS hex color to represent the language. Only used if type is "programming" or "prose".
 # tm_scope          - The TextMate scope that represents this programming
 #                     language. This should match one of the scopes listed in
 #                     the grammars.yml file. Use "none" if there is no grammar
@@ -121,7 +119,6 @@ ASN.1:
 ASP:
   type: programming
   color: "#6a40fd"
-  search_term: aspx-vb
   tm_scope: text.html.asp
   aliases:
   - aspx
@@ -154,7 +151,6 @@ ActionScript:
   type: programming
   tm_scope: source.actionscript.3
   color: "#882B0F"
-  search_term: as3
   aliases:
   - actionscript 3
   - actionscript3
@@ -291,7 +287,6 @@ AspectJ:
 Assembly:
   type: programming
   color: "#6E4C13"
-  search_term: nasm
   aliases:
   - nasm
   extensions:
@@ -349,7 +344,6 @@ Awk:
   language_id: 28
 Batchfile:
   type: programming
-  search_term: bat
   aliases:
   - bat
   - batch
@@ -474,7 +468,6 @@ C#:
   codemirror_mode: clike
   codemirror_mime_type: text/x-csharp
   tm_scope: source.cs
-  search_term: csharp
   color: "#178600"
   aliases:
   - csharp
@@ -489,7 +482,6 @@ C++:
   ace_mode: c_cpp
   codemirror_mode: clike
   codemirror_mime_type: text/x-c++src
-  search_term: cpp
   color: "#f34b7d"
   aliases:
   - cpp
@@ -719,7 +711,6 @@ ColdFusion:
   type: programming
   ace_mode: coldfusion
   color: "#ed2cd6"
-  search_term: cfm
   aliases:
   - cfm
   - cfml
@@ -733,7 +724,6 @@ ColdFusion CFC:
   type: programming
   group: ColdFusion
   ace_mode: coldfusion
-  search_term: cfc
   aliases:
   - cfc
   extensions:
@@ -854,16 +844,6 @@ Csound Score:
   tm_scope: source.csound-score
   ace_mode: text
   language_id: 75
-Cucumber:
-  type: programming
-  extensions:
-  - ".feature"
-  tm_scope: text.gherkin.feature
-  aliases:
-  - gherkin
-  ace_mode: text
-  color: "#5B2063"
-  language_id: 76
 Cuda:
   type: programming
   extensions:
@@ -956,7 +936,6 @@ DTrace:
   language_id: 85
 Darcs Patch:
   type: data
-  search_term: dpatch
   aliases:
   - dpatch
   extensions:
@@ -1141,9 +1120,9 @@ Emacs Lisp:
   - ".gnus"
   - ".spacemacs"
   - ".viper"
-  - "Project.ede"
-  - "_emacs"
-  - "abbrev_defs"
+  - Project.ede
+  - _emacs
+  - abbrev_defs
   extensions:
   - ".el"
   - ".emacs"
@@ -1187,7 +1166,6 @@ Erlang:
 F#:
   type: programming
   color: "#b845fc"
-  search_term: fsharp
   aliases:
   - fsharp
   extensions:
@@ -1208,23 +1186,6 @@ FLUX:
   tm_scope: none
   ace_mode: text
   language_id: 106
-FORTRAN:
-  type: programming
-  color: "#4d41b1"
-  extensions:
-  - ".f90"
-  - ".f"
-  - ".f03"
-  - ".f08"
-  - ".f77"
-  - ".f95"
-  - ".for"
-  - ".fpp"
-  tm_scope: source.fortran.modern
-  ace_mode: text
-  codemirror_mode: fortran
-  codemirror_mime_type: text/x-fortran
-  language_id: 107
 Factor:
   type: programming
   color: "#636746"
@@ -1294,6 +1255,23 @@ Forth:
   codemirror_mode: forth
   codemirror_mime_type: text/x-forth
   language_id: 114
+Fortran:
+  type: programming
+  color: "#4d41b1"
+  extensions:
+  - ".f90"
+  - ".f"
+  - ".f03"
+  - ".f08"
+  - ".f77"
+  - ".f95"
+  - ".for"
+  - ".fpp"
+  tm_scope: source.fortran.modern
+  ace_mode: text
+  codemirror_mode: fortran
+  codemirror_mime_type: text/x-fortran
+  language_id: 107
 FreeMarker:
   type: programming
   color: "#0050b2"
@@ -1339,15 +1317,6 @@ GAP:
   tm_scope: source.gap
   ace_mode: text
   language_id: 119
-GAS:
-  type: programming
-  group: Assembly
-  extensions:
-  - ".s"
-  - ".ms"
-  tm_scope: source.assembly
-  ace_mode: assembly_x86
-  language_id: 120
 GCC Machine Description:
   type: programming
   extensions:
@@ -1393,6 +1362,18 @@ GLSL:
   - ".vshader"
   ace_mode: glsl
   language_id: 124
+GN:
+  type: data
+  extensions:
+  - ".gn"
+  - ".gni"
+  interpreters:
+  - gn
+  tm_scope: source.gn
+  ace_mode: python
+  codemirror_mode: python
+  codemirror_mime_type: text/x-python
+  language_id: 302957008
 Game Maker Language:
   type: programming
   color: "#8fb200"
@@ -1437,7 +1418,6 @@ Gentoo Eclass:
   language_id: 128
 Gettext Catalog:
   type: prose
-  search_term: pot
   searchable: false
   aliases:
   - pot
@@ -1447,6 +1427,16 @@ Gettext Catalog:
   tm_scope: source.po
   ace_mode: text
   language_id: 129
+Gherkin:
+  type: programming
+  extensions:
+  - ".feature"
+  tm_scope: text.gherkin.feature
+  aliases:
+  - cucumber
+  ace_mode: text
+  color: "#5B2063"
+  language_id: 76
 Glyph:
   type: programming
   color: "#e4cc98"
@@ -1473,6 +1463,8 @@ Gnuplot:
 Go:
   type: programming
   color: "#375eab"
+  aliases:
+  - golang
   extensions:
   - ".go"
   ace_mode: golang
@@ -1548,45 +1540,6 @@ Graphviz (DOT):
   - ".gv"
   ace_mode: text
   language_id: 140
-Groff:
-  type: markup
-  color: "#ecdebe"
-  extensions:
-  - ".man"
-  - ".1"
-  - ".1in"
-  - ".1m"
-  - ".1x"
-  - ".2"
-  - ".3"
-  - ".3in"
-  - ".3m"
-  - ".3qt"
-  - ".3x"
-  - ".4"
-  - ".5"
-  - ".6"
-  - ".7"
-  - ".8"
-  - ".9"
-  - ".l"
-  - ".me"
-  - ".ms"
-  - ".n"
-  - ".rno"
-  - ".roff"
-  - ".tmac"
-  filenames:
-  - mmn
-  - mmt
-  tm_scope: text.roff
-  aliases:
-  - nroff
-  - troff
-  ace_mode: text
-  codemirror_mode: troff
-  codemirror_mime_type: text/troff
-  language_id: 141
 Groovy:
   type: programming
   ace_mode: groovy
@@ -1659,13 +1612,16 @@ HTML+Django:
   tm_scope: text.html.django
   group: HTML
   extensions:
-  - ".mustache"
   - ".jinja"
+  - ".mustache"
+  - ".njk"
   aliases:
   - django
   - html+django/jinja
   - html+jinja
   - htmldjango
+  - njk
+  - nunjucks
   ace_mode: django
   codemirror_mode: django
   codemirror_mime_type: text/x-django
@@ -1844,7 +1800,6 @@ INI:
   language_id: 163
 IRC log:
   type: data
-  search_term: irc
   aliases:
   - irc
   - irc logs
@@ -1959,6 +1914,8 @@ JSON5:
   type: data
   extensions:
   - ".json5"
+  filenames:
+  - ".babelrc"
   tm_scope: source.js
   ace_mode: javascript
   codemirror_mode: javascript
@@ -1992,17 +1949,6 @@ JSX:
   codemirror_mode: jsx
   codemirror_mime_type: text/jsx
   language_id: 178
-Jade:
-  group: HTML
-  type: markup
-  extensions:
-  - ".jade"
-  - ".pug"
-  tm_scope: text.jade
-  ace_mode: jade
-  codemirror_mode: pug
-  codemirror_mime_type: text/x-pug
-  language_id: 179
 Jasmin:
   type: programming
   ace_mode: java
@@ -2022,7 +1968,6 @@ Java:
 Java Server Pages:
   type: programming
   group: Java
-  search_term: jsp
   aliases:
   - jsp
   extensions:
@@ -2279,7 +2224,6 @@ Literate CoffeeScript:
   group: CoffeeScript
   ace_mode: text
   wrap: true
-  search_term: litcoffee
   aliases:
   - litcoffee
   extensions:
@@ -2288,7 +2232,6 @@ Literate CoffeeScript:
 Literate Haskell:
   type: programming
   group: Haskell
-  search_term: lhs
   aliases:
   - lhaskell
   - lhs
@@ -2550,7 +2493,6 @@ Max:
   aliases:
   - max/msp
   - maxmsp
-  search_term: max/msp
   extensions:
   - ".maxpat"
   - ".maxhelp"
@@ -2602,7 +2544,6 @@ MiniD:
   language_id: 231
 Mirah:
   type: programming
-  search_term: mirah
   color: "#c7a938"
   extensions:
   - ".druby"
@@ -2759,7 +2700,7 @@ Nginx:
   codemirror_mime_type: text/x-nginx-conf
   color: "#9469E9"
   language_id: 248
-Nimrod:
+Nim:
   type: programming
   color: "#37775b"
   extensions:
@@ -2954,6 +2895,15 @@ OpenSCAD:
   tm_scope: none
   ace_mode: scad
   language_id: 266
+OpenType Feature File:
+  type: data
+  aliases:
+  - AFDKO
+  extensions:
+  - ".fea"
+  tm_scope: source.opentype
+  ace_mode: text
+  language_id: 374317347
 Org:
   type: prose
   wrap: true
@@ -3017,6 +2967,8 @@ PHP:
   - ".phps"
   - ".phpt"
   filenames:
+  - ".php_cs"
+  - ".php_cs.dist"
   - Phakefile
   interpreters:
   - php
@@ -3171,8 +3123,8 @@ Perl6:
   language_id: 283
 Pic:
   type: markup
-  group: Groff
-  tm_scope: "source.pic"
+  group: Roff
+  tm_scope: source.pic
   extensions:
   - ".pic"
   - ".chem"
@@ -3325,6 +3277,17 @@ Public Key:
   codemirror_mode: asciiarmor
   codemirror_mime_type: application/pgp
   language_id: 298
+Pug:
+  group: HTML
+  type: markup
+  extensions:
+  - ".jade"
+  - ".pug"
+  tm_scope: text.jade
+  ace_mode: jade
+  codemirror_mode: pug
+  codemirror_mime_type: text/x-pug
+  language_id: 179
 Puppet:
   type: programming
   color: "#302B6D"
@@ -3376,7 +3339,9 @@ Python:
   - ".cgi"
   - ".fcgi"
   - ".gyp"
+  - ".gypi"
   - ".lmi"
+  - ".py3"
   - ".pyde"
   - ".pyp"
   - ".pyt"
@@ -3387,6 +3352,7 @@ Python:
   - ".wsgi"
   - ".xpy"
   filenames:
+  - ".gclient"
   - BUCK
   - BUILD
   - SConscript
@@ -3554,7 +3520,7 @@ Racket:
   tm_scope: source.racket
   ace_mode: lisp
   language_id: 316
-Ragel in Ruby Host:
+Ragel:
   type: programming
   color: "#9d5200"
   extensions:
@@ -3565,9 +3531,16 @@ Ragel in Ruby Host:
   tm_scope: none
   ace_mode: text
   language_id: 317
+Rascal:
+  type: programming
+  color: "#fffaa0"
+  extensions:
+  - ".rsc"
+  tm_scope: source.rascal
+  ace_mode: text
+  language_id: 173616037
 Raw token data:
   type: data
-  search_term: raw
   aliases:
   - raw
   extensions:
@@ -3630,6 +3603,44 @@ RobotFramework:
   tm_scope: text.robot
   ace_mode: text
   language_id: 324
+Roff:
+  type: markup
+  color: "#ecdebe"
+  extensions:
+  - ".man"
+  - ".1"
+  - ".1in"
+  - ".1m"
+  - ".1x"
+  - ".2"
+  - ".3"
+  - ".3in"
+  - ".3m"
+  - ".3qt"
+  - ".3x"
+  - ".4"
+  - ".5"
+  - ".6"
+  - ".7"
+  - ".8"
+  - ".9"
+  - ".l"
+  - ".me"
+  - ".ms"
+  - ".n"
+  - ".rno"
+  - ".roff"
+  - ".tmac"
+  filenames:
+  - mmn
+  - mmt
+  tm_scope: text.roff
+  aliases:
+  - nroff
+  ace_mode: text
+  codemirror_mode: troff
+  codemirror_mime_type: text/troff
+  language_id: 141
 Rouge:
   type: programming
   ace_mode: clojure
@@ -3685,6 +3696,7 @@ Ruby:
   - Berksfile
   - Brewfile
   - Buildfile
+  - Dangerfile
   - Deliverfile
   - Fastfile
   - Gemfile
@@ -3913,7 +3925,6 @@ Self:
   language_id: 345
 Shell:
   type: programming
-  search_term: bash
   color: "#89e051"
   aliases:
   - sh
@@ -4026,6 +4037,13 @@ SourcePawn:
   tm_scope: source.sp
   ace_mode: text
   language_id: 354
+Spline Font Database:
+  type: data
+  extensions:
+  - ".sfd"
+  tm_scope: text.sfd
+  ace_mode: yaml
+  language_id: 767169629
 Squirrel:
   type: programming
   color: "#800000"
@@ -4256,6 +4274,7 @@ Text:
   - ".no"
   filenames:
   - COPYING
+  - FONTLOG
   - INSTALL
   - LICENSE
   - NEWS
@@ -4351,6 +4370,15 @@ Unity3D Asset:
   - ".unity"
   tm_scope: source.yaml
   language_id: 380
+Unix Assembly:
+  type: programming
+  group: Assembly
+  extensions:
+  - ".s"
+  - ".ms"
+  tm_scope: source.assembly
+  ace_mode: assembly_x86
+  language_id: 120
 Uno:
   type: programming
   extensions:
@@ -4423,13 +4451,13 @@ Verilog:
   codemirror_mode: verilog
   codemirror_mime_type: text/x-verilog
   language_id: 387
-VimL:
+Vim script:
   type: programming
   color: "#199f4b"
-  search_term: vim
   tm_scope: source.viml
   aliases:
   - vim
+  - viml
   - nvim
   extensions:
   - ".vim"
@@ -4782,7 +4810,6 @@ desktop:
 eC:
   type: programming
   color: "#913960"
-  search_term: ec
   extensions:
   - ".ec"
   - ".eh"
@@ -4832,7 +4859,6 @@ ooc:
 reStructuredText:
   type: prose
   wrap: true
-  search_term: rst
   aliases:
   - rst
   extensions:

lib/linguist/popular.yml

@@ -26,4 +26,4 @@
 - Shell
 - Swift
 - TeX
-- VimL
+- Vim script

lib/linguist/strategy/extension.rb

@@ -0,0 +1,10 @@
+module Linguist
+  module Strategy
+    # Detects language based on extension
+    class Extension
+      def self.call(blob, _)
+        Language.find_by_extension(blob.name.to_s)
+      end
+    end
+  end
+end

lib/linguist/strategy/filename.rb

@@ -1,9 +1,10 @@
 module Linguist
   module Strategy
-    # Detects language based on filename and/or extension
+    # Detects language based on filename
     class Filename
       def self.call(blob, _)
-        Language.find_by_filename(blob.name.to_s)
+        name = blob.name.to_s
+        Language.find_by_filename(name)
       end
     end
   end

lib/linguist/vendor.yml

@@ -50,6 +50,9 @@
 # Go dependencies
 - Godeps/_workspace/
 
+# GNU indent profiles
+- .indent.pro
+
 # Minified JavaScript and CSS
 - (\.|-)min\.(js|css)$
 

lib/linguist/version.rb

@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "4.8.16"
+  VERSION = "5.0.0"
 end

samples/GN/BUILD.2.gn

@@ -0,0 +1,59 @@
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("../gni/isolate.gni")
+
+group("gn_all") {
+  testonly = true
+
+  if (v8_test_isolation_mode != "noop") {
+    deps = [
+      ":check-static-initializers_run",
+      ":jsfunfuzz_run",
+      ":run-deopt-fuzzer_run",
+      ":run-gcmole_run",
+      ":run-valgrind_run",
+    ]
+  }
+}
+
+v8_isolate_run("check-static-initializers") {
+  deps = [
+    "..:d8_run",
+  ]
+
+  isolate = "check-static-initializers.isolate"
+}
+
+v8_isolate_run("jsfunfuzz") {
+  deps = [
+    "..:d8_run",
+  ]
+
+  isolate = "jsfunfuzz/jsfunfuzz.isolate"
+}
+
+v8_isolate_run("run-deopt-fuzzer") {
+  deps = [
+    "..:d8_run",
+  ]
+
+  isolate = "run-deopt-fuzzer.isolate"
+}
+
+v8_isolate_run("run-gcmole") {
+  deps = [
+    "..:d8_run",
+  ]
+
+  isolate = "gcmole/run-gcmole.isolate"
+}
+
+v8_isolate_run("run-valgrind") {
+  deps = [
+    "..:d8_run",
+  ]
+
+  isolate = "run-valgrind.isolate"
+}

samples/GN/clang.gni

@@ -0,0 +1,13 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/toolchain/toolchain.gni")
+
+declare_args() {
+  # Indicates if the build should use the Chrome-specific plugins for enforcing
+  # coding guidelines, etc. Only used when compiling with Clang.
+  clang_use_chrome_plugins = is_clang && !is_nacl && !use_xcode_clang
+
+  clang_base_path = "//third_party/llvm-build/Release+Asserts"
+}

samples/GN/filenames/.gn

@@ -0,0 +1,25 @@
+# This file is used by the GN meta build system to find the root of the source
+# tree and to set startup options. For documentation on the values set in this
+# file, run "gn help dotfile" at the command line.
+
+import("//build/dotfile_settings.gni")
+
+# The location of the build configuration file.
+buildconfig = "//build/config/BUILDCONFIG.gn"
+
+# The secondary source root is a parallel directory tree where
+# GN build files are placed when they can not be placed directly
+# in the source tree, e.g. for third party source trees.
+secondary_source = "//build/secondary/"
+
+# These are the targets to check headers for by default. The files in targets
+# matching these patterns (see "gn help label_pattern" for format) will have
+# their includes checked for proper dependencies when you run either
+# "gn check" or "gn gen --check".
+check_targets = []
+
+# These are the list of GN files that run exec_script. This whitelist exists
+# to force additional review for new uses of exec_script, which is strongly
+# discouraged except for gypi_to_gn calls.
+exec_script_whitelist =
+    build_dotfile_settings.exec_script_whitelist + [ "//test/test262/BUILD.gn" ]

samples/GN/gcc_toolchain.gni

@@ -0,0 +1,503 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/config/android/config.gni")
+import("//build/config/clang/clang.gni")
+import("//build/config/nacl/config.gni")
+import("//build/config/sanitizers/sanitizers.gni")
+import("//build/config/v8_target_cpu.gni")
+import("//build/toolchain/cc_wrapper.gni")
+import("//build/toolchain/goma.gni")
+import("//build/toolchain/toolchain.gni")
+
+# This template defines a toolchain for something that works like gcc
+# (including clang).
+#
+# It requires the following variables specifying the executables to run:
+#  - ar
+#  - cc
+#  - cxx
+#  - ld
+#
+# Optional parameters that control the tools:
+#
+#  - extra_cflags
+#      Extra flags to be appended when compiling C files (but not C++ files).
+#  - extra_cppflags
+#      Extra flags to be appended when compiling both C and C++ files. "CPP"
+#      stands for "C PreProcessor" in this context, although it can be
+#      used for non-preprocessor flags as well. Not to be confused with
+#      "CXX" (which follows).
+#  - extra_cxxflags
+#      Extra flags to be appended when compiling C++ files (but not C files).
+#  - extra_ldflags
+#      Extra flags to be appended when linking
+#
+#  - libs_section_prefix
+#  - libs_section_postfix
+#      The contents of these strings, if specified, will be placed around
+#      the libs section of the linker line. It allows one to inject libraries
+#      at the beginning and end for all targets in a toolchain.
+#  - solink_libs_section_prefix
+#  - solink_libs_section_postfix
+#      Same as libs_section_{pre,post}fix except used for solink instead of link.
+#  - link_outputs
+#      The content of this array, if specified, will be added to the list of
+#      outputs from the link command. This can be useful in conjunction with
+#      the post_link parameter.
+#  - post_link
+#      The content of this string, if specified, will be run as a separate
+#      command following the the link command.
+#  - deps
+#      Just forwarded to the toolchain definition.
+#  - executable_extension
+#      If this string is specified it will be used for the file extension
+#      for an executable, rather than using no extension; targets will
+#      still be able to override the extension using the output_extension
+#      variable.
+#  - rebuild_define
+#      The contents of this string, if specified, will be passed as a #define
+#      to the toolchain. It can be used to force recompiles whenever a
+#      toolchain is updated.
+#  - shlib_extension
+#      If this string is specified it will be used for the file extension
+#      for a shared library, rather than default value specified in
+#      toolchain.gni
+#  - strip
+#      Location of the strip executable. When specified, strip will be run on
+#      all shared libraries and executables as they are built. The pre-stripped
+#      artifacts will be put in lib.unstripped/ and exe.unstripped/.
+template("gcc_toolchain") {
+  toolchain(target_name) {
+    assert(defined(invoker.ar), "gcc_toolchain() must specify a \"ar\" value")
+    assert(defined(invoker.cc), "gcc_toolchain() must specify a \"cc\" value")
+    assert(defined(invoker.cxx), "gcc_toolchain() must specify a \"cxx\" value")
+    assert(defined(invoker.ld), "gcc_toolchain() must specify a \"ld\" value")
+
+    # This define changes when the toolchain changes, forcing a rebuild.
+    # Nothing should ever use this define.
+    if (defined(invoker.rebuild_define)) {
+      rebuild_string = "-D" + invoker.rebuild_define + " "
+    } else {
+      rebuild_string = ""
+    }
+
+    # GN's syntax can't handle more than one scope dereference at once, like
+    # "invoker.toolchain_args.foo", so make a temporary to hold the toolchain
+    # args so we can do "invoker_toolchain_args.foo".
+    assert(defined(invoker.toolchain_args),
+           "Toolchains must specify toolchain_args")
+    invoker_toolchain_args = invoker.toolchain_args
+    assert(defined(invoker_toolchain_args.current_cpu),
+           "toolchain_args must specify a current_cpu")
+    assert(defined(invoker_toolchain_args.current_os),
+           "toolchain_args must specify a current_os")
+
+    # When invoking this toolchain not as the default one, these args will be
+    # passed to the build. They are ignored when this is the default toolchain.
+    toolchain_args = {
+      # Populate toolchain args from the invoker.
+      forward_variables_from(invoker_toolchain_args, "*")
+
+      # The host toolchain value computed by the default toolchain's setup
+      # needs to be passed through unchanged to all secondary toolchains to
+      # ensure that it's always the same, regardless of the values that may be
+      # set on those toolchains.
+      host_toolchain = host_toolchain
+
+      if (!defined(invoker_toolchain_args.v8_current_cpu)) {
+        v8_current_cpu = invoker_toolchain_args.current_cpu
+      }
+    }
+
+    # When the invoker has explicitly overridden use_goma or cc_wrapper in the
+    # toolchain args, use those values, otherwise default to the global one.
+    # This works because the only reasonable override that toolchains might
+    # supply for these values are to force-disable them.
+    if (defined(toolchain_args.use_goma)) {
+      toolchain_uses_goma = toolchain_args.use_goma
+    } else {
+      toolchain_uses_goma = use_goma
+    }
+    if (defined(toolchain_args.cc_wrapper)) {
+      toolchain_cc_wrapper = toolchain_args.cc_wrapper
+    } else {
+      toolchain_cc_wrapper = cc_wrapper
+    }
+
+    # Compute the compiler prefix.
+    if (toolchain_uses_goma) {
+      assert(toolchain_cc_wrapper == "",
+             "Goma and cc_wrapper can't be used together.")
+      compiler_prefix = "$goma_dir/gomacc "
+    } else if (toolchain_cc_wrapper != "") {
+      compiler_prefix = toolchain_cc_wrapper + " "
+    } else {
+      compiler_prefix = ""
+    }
+
+    cc = compiler_prefix + invoker.cc
+    cxx = compiler_prefix + invoker.cxx
+    ar = invoker.ar
+    ld = invoker.ld
+    if (defined(invoker.readelf)) {
+      readelf = invoker.readelf
+    } else {
+      readelf = "readelf"
+    }
+    if (defined(invoker.nm)) {
+      nm = invoker.nm
+    } else {
+      nm = "nm"
+    }
+
+    if (defined(invoker.shlib_extension)) {
+      default_shlib_extension = invoker.shlib_extension
+    } else {
+      default_shlib_extension = shlib_extension
+    }
+
+    if (defined(invoker.executable_extension)) {
+      default_executable_extension = invoker.executable_extension
+    } else {
+      default_executable_extension = ""
+    }
+
+    # Bring these into our scope for string interpolation with default values.
+    if (defined(invoker.libs_section_prefix)) {
+      libs_section_prefix = invoker.libs_section_prefix
+    } else {
+      libs_section_prefix = ""
+    }
+
+    if (defined(invoker.libs_section_postfix)) {
+      libs_section_postfix = invoker.libs_section_postfix
+    } else {
+      libs_section_postfix = ""
+    }
+
+    if (defined(invoker.solink_libs_section_prefix)) {
+      solink_libs_section_prefix = invoker.solink_libs_section_prefix
+    } else {
+      solink_libs_section_prefix = ""
+    }
+
+    if (defined(invoker.solink_libs_section_postfix)) {
+      solink_libs_section_postfix = invoker.solink_libs_section_postfix
+    } else {
+      solink_libs_section_postfix = ""
+    }
+
+    if (defined(invoker.extra_cflags) && invoker.extra_cflags != "") {
+      extra_cflags = " " + invoker.extra_cflags
+    } else {
+      extra_cflags = ""
+    }
+
+    if (defined(invoker.extra_cppflags) && invoker.extra_cppflags != "") {
+      extra_cppflags = " " + invoker.extra_cppflags
+    } else {
+      extra_cppflags = ""
+    }
+
+    if (defined(invoker.extra_cxxflags) && invoker.extra_cxxflags != "") {
+      extra_cxxflags = " " + invoker.extra_cxxflags
+    } else {
+      extra_cxxflags = ""
+    }
+
+    if (defined(invoker.extra_ldflags) && invoker.extra_ldflags != "") {
+      extra_ldflags = " " + invoker.extra_ldflags
+    } else {
+      extra_ldflags = ""
+    }
+
+    # These library switches can apply to all tools below.
+    lib_switch = "-l"
+    lib_dir_switch = "-L"
+
+    # Object files go in this directory.
+    object_subdir = "{{target_out_dir}}/{{label_name}}"
+
+    tool("cc") {
+      depfile = "{{output}}.d"
+      command = "$cc -MMD -MF $depfile ${rebuild_string}{{defines}} {{include_dirs}} {{cflags}} {{cflags_c}}${extra_cppflags}${extra_cflags} -c {{source}} -o {{output}}"
+      depsformat = "gcc"
+      description = "CC {{output}}"
+      outputs = [
+        # The whitelist file is also an output, but ninja does not
+        # currently support multiple outputs for tool("cc").
+        "$object_subdir/{{source_name_part}}.o",
+      ]
+      if (enable_resource_whitelist_generation) {
+        compile_wrapper =
+            rebase_path("//build/toolchain/gcc_compile_wrapper.py",
+                        root_build_dir)
+        command = "$python_path \"$compile_wrapper\" --resource-whitelist=\"{{output}}.whitelist\" $command"
+      }
+    }
+
+    tool("cxx") {
+      depfile = "{{output}}.d"
+      command = "$cxx -MMD -MF $depfile ${rebuild_string}{{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}}${extra_cppflags}${extra_cxxflags} -c {{source}} -o {{output}}"
+      depsformat = "gcc"
+      description = "CXX {{output}}"
+      outputs = [
+        # The whitelist file is also an output, but ninja does not
+        # currently support multiple outputs for tool("cxx").
+        "$object_subdir/{{source_name_part}}.o",
+      ]
+      if (enable_resource_whitelist_generation) {
+        compile_wrapper =
+            rebase_path("//build/toolchain/gcc_compile_wrapper.py",
+                        root_build_dir)
+        command = "$python_path \"$compile_wrapper\" --resource-whitelist=\"{{output}}.whitelist\" $command"
+      }
+    }
+
+    tool("asm") {
+      # For GCC we can just use the C compiler to compile assembly.
+      depfile = "{{output}}.d"
+      command = "$cc -MMD -MF $depfile ${rebuild_string}{{defines}} {{include_dirs}} {{asmflags}} -c {{source}} -o {{output}}"
+      depsformat = "gcc"
+      description = "ASM {{output}}"
+      outputs = [
+        "$object_subdir/{{source_name_part}}.o",
+      ]
+    }
+
+    tool("alink") {
+      rspfile = "{{output}}.rsp"
+      whitelist_flag = " "
+      if (enable_resource_whitelist_generation) {
+        whitelist_flag = " --resource-whitelist=\"{{output}}.whitelist\""
+      }
+
+      # This needs a Python script to avoid using simple sh features in this
+      # command, in case the host does not use a POSIX shell (e.g. compiling
+      # POSIX-like toolchains such as NaCl on Windows).
+      ar_wrapper =
+          rebase_path("//build/toolchain/gcc_ar_wrapper.py", root_build_dir)
+      command = "$python_path \"$ar_wrapper\"$whitelist_flag --output={{output}} --ar=\"$ar\" {{arflags}} rcsD @\"$rspfile\""
+      description = "AR {{output}}"
+      rspfile_content = "{{inputs}}"
+      outputs = [
+        "{{output_dir}}/{{target_output_name}}{{output_extension}}",
+      ]
+
+      # Shared libraries go in the target out directory by default so we can
+      # generate different targets with the same name and not have them collide.
+      default_output_dir = "{{target_out_dir}}"
+      default_output_extension = ".a"
+      output_prefix = "lib"
+    }
+
+    tool("solink") {
+      soname = "{{target_output_name}}{{output_extension}}"  # e.g. "libfoo.so".
+      sofile = "{{output_dir}}/$soname"  # Possibly including toolchain dir.
+      rspfile = sofile + ".rsp"
+      pool = "//build/toolchain:link_pool($default_toolchain)"
+      whitelist_flag = " "
+      if (enable_resource_whitelist_generation) {
+        whitelist_file = "$sofile.whitelist"
+        whitelist_flag = " --resource-whitelist=\"$whitelist_file\""
+      }
+
+      if (defined(invoker.strip)) {
+        unstripped_sofile = "{{root_out_dir}}/lib.unstripped/$soname"
+      } else {
+        unstripped_sofile = sofile
+      }
+
+      # These variables are not built into GN but are helpers that
+      # implement (1) linking to produce a .so, (2) extracting the symbols
+      # from that file (3) if the extracted list differs from the existing
+      # .TOC file, overwrite it, otherwise, don't change it.
+      tocfile = sofile + ".TOC"
+
+      link_command = "$ld -shared {{ldflags}}${extra_ldflags} -o \"$unstripped_sofile\" -Wl,-soname=\"$soname\" @\"$rspfile\""
+
+      assert(defined(readelf), "to solink you must have a readelf")
+      assert(defined(nm), "to solink you must have an nm")
+      strip_switch = ""
+      if (defined(invoker.strip)) {
+        strip_switch = "--strip=${invoker.strip}"
+      }
+
+      # This needs a Python script to avoid using a complex shell command
+      # requiring sh control structures, pipelines, and POSIX utilities.
+      # The host might not have a POSIX shell and utilities (e.g. Windows).
+      solink_wrapper = rebase_path("//build/toolchain/gcc_solink_wrapper.py")
+      command = "$python_path \"$solink_wrapper\" --readelf=\"$readelf\" --nm=\"$nm\" $strip_switch --sofile=\"$unstripped_sofile\" --tocfile=\"$tocfile\" --output=\"$sofile\"$whitelist_flag -- $link_command"
+
+      rspfile_content = "-Wl,--whole-archive {{inputs}} {{solibs}} -Wl,--no-whole-archive $solink_libs_section_prefix {{libs}} $solink_libs_section_postfix"
+
+      description = "SOLINK $sofile"
+
+      # Use this for {{output_extension}} expansions unless a target manually
+      # overrides it (in which case {{output_extension}} will be what the target
+      # specifies).
+      default_output_extension = default_shlib_extension
+
+      default_output_dir = "{{root_out_dir}}"
+      if (shlib_subdir != ".") {
+        default_output_dir += "/$shlib_subdir"
+      }
+
+      output_prefix = "lib"
+
+      # Since the above commands only updates the .TOC file when it changes, ask
+      # Ninja to check if the timestamp actually changed to know if downstream
+      # dependencies should be recompiled.
+      restat = true
+
+      # Tell GN about the output files. It will link to the sofile but use the
+      # tocfile for dependency management.
+      outputs = [
+        sofile,
+        tocfile,
+      ]
+      if (enable_resource_whitelist_generation) {
+        outputs += [ whitelist_file ]
+      }
+      if (sofile != unstripped_sofile) {
+        outputs += [ unstripped_sofile ]
+      }
+      link_output = sofile
+      depend_output = tocfile
+    }
+
+    tool("solink_module") {
+      soname = "{{target_output_name}}{{output_extension}}"  # e.g. "libfoo.so".
+      sofile = "{{output_dir}}/$soname"
+      rspfile = sofile + ".rsp"
+      pool = "//build/toolchain:link_pool($default_toolchain)"
+
+      if (defined(invoker.strip)) {
+        unstripped_sofile = "{{root_out_dir}}/lib.unstripped/$soname"
+      } else {
+        unstripped_sofile = sofile
+      }
+
+      command = "$ld -shared {{ldflags}}${extra_ldflags} -o \"$unstripped_sofile\" -Wl,-soname=\"$soname\" @\"$rspfile\""
+
+      if (defined(invoker.strip)) {
+        strip_command = "${invoker.strip} --strip-unneeded -o \"$sofile\" \"$unstripped_sofile\""
+        command += " && " + strip_command
+      }
+      rspfile_content = "-Wl,--whole-archive {{inputs}} {{solibs}} -Wl,--no-whole-archive $solink_libs_section_prefix {{libs}} $solink_libs_section_postfix"
+
+      description = "SOLINK_MODULE $sofile"
+
+      # Use this for {{output_extension}} expansions unless a target manually
+      # overrides it (in which case {{output_extension}} will be what the target
+      # specifies).
+      if (defined(invoker.loadable_module_extension)) {
+        default_output_extension = invoker.loadable_module_extension
+      } else {
+        default_output_extension = default_shlib_extension
+      }
+
+      default_output_dir = "{{root_out_dir}}"
+      if (shlib_subdir != ".") {
+        default_output_dir += "/$shlib_subdir"
+      }
+
+      output_prefix = "lib"
+
+      outputs = [
+        sofile,
+      ]
+      if (sofile != unstripped_sofile) {
+        outputs += [ unstripped_sofile ]
+      }
+    }
+
+    tool("link") {
+      exename = "{{target_output_name}}{{output_extension}}"
+      outfile = "{{output_dir}}/$exename"
+      rspfile = "$outfile.rsp"
+      unstripped_outfile = outfile
+      pool = "//build/toolchain:link_pool($default_toolchain)"
+
+      # Use this for {{output_extension}} expansions unless a target manually
+      # overrides it (in which case {{output_extension}} will be what the target
+      # specifies).
+      default_output_extension = default_executable_extension
+
+      default_output_dir = "{{root_out_dir}}"
+
+      if (defined(invoker.strip)) {
+        unstripped_outfile = "{{root_out_dir}}/exe.unstripped/$exename"
+      }
+
+      command = "$ld {{ldflags}}${extra_ldflags} -o \"$unstripped_outfile\" -Wl,--start-group @\"$rspfile\" {{solibs}} -Wl,--end-group $libs_section_prefix {{libs}} $libs_section_postfix"
+      if (defined(invoker.strip)) {
+        link_wrapper =
+            rebase_path("//build/toolchain/gcc_link_wrapper.py", root_build_dir)
+        command = "$python_path \"$link_wrapper\" --strip=\"${invoker.strip}\" --unstripped-file=\"$unstripped_outfile\" --output=\"$outfile\" -- $command"
+      }
+      description = "LINK $outfile"
+      rspfile_content = "{{inputs}}"
+      outputs = [
+        outfile,
+      ]
+      if (outfile != unstripped_outfile) {
+        outputs += [ unstripped_outfile ]
+      }
+      if (defined(invoker.link_outputs)) {
+        outputs += invoker.link_outputs
+      }
+    }
+
+    # These two are really entirely generic, but have to be repeated in
+    # each toolchain because GN doesn't allow a template to be used here.
+    # See //build/toolchain/toolchain.gni for details.
+    tool("stamp") {
+      command = stamp_command
+      description = stamp_description
+    }
+    tool("copy") {
+      command = copy_command
+      description = copy_description
+    }
+
+    forward_variables_from(invoker, [ "deps" ])
+  }
+}
+
+# This is a shorthand for gcc_toolchain instances based on the Chromium-built
+# version of Clang. Only the toolchain_cpu and toolchain_os variables need to
+# be specified by the invoker, and optionally toolprefix if it's a
+# cross-compile case. Note that for a cross-compile case this toolchain
+# requires a config to pass the appropriate -target option, or else it will
+# actually just be doing a native compile. The invoker can optionally override
+# use_gold too.
+template("clang_toolchain") {
+  if (defined(invoker.toolprefix)) {
+    toolprefix = invoker.toolprefix
+  } else {
+    toolprefix = ""
+  }
+
+  gcc_toolchain(target_name) {
+    prefix = rebase_path("$clang_base_path/bin", root_build_dir)
+    cc = "$prefix/clang"
+    cxx = "$prefix/clang++"
+    ld = cxx
+
+    readelf = "${toolprefix}readelf"
+    ar = "${toolprefix}ar"
+    nm = "${toolprefix}nm"
+
+    forward_variables_from(invoker, [ "strip" ])
+
+    toolchain_args = {
+      if (defined(invoker.toolchain_args)) {
+        forward_variables_from(invoker.toolchain_args, "*")
+      }
+      is_clang = true
+    }
+  }
+}

samples/GN/icu.gn

@@ -0,0 +1,235 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/config/linux/pkg_config.gni")
+import("//build/shim_headers.gni")
+
+group("icu") {
+  public_deps = [
+    ":icui18n",
+    ":icuuc",
+  ]
+}
+
+config("icu_config") {
+  defines = [
+    "USING_SYSTEM_ICU=1",
+    "ICU_UTIL_DATA_IMPL=ICU_UTIL_DATA_STATIC",
+  ]
+}
+
+pkg_config("system_icui18n") {
+  packages = [ "icu-i18n" ]
+}
+
+pkg_config("system_icuuc") {
+  packages = [ "icu-uc" ]
+}
+
+source_set("icui18n") {
+  deps = [
+    ":icui18n_shim",
+  ]
+  public_configs = [
+    ":icu_config",
+    ":system_icui18n",
+  ]
+}
+
+source_set("icuuc") {
+  deps = [
+    ":icuuc_shim",
+  ]
+  public_configs = [
+    ":icu_config",
+    ":system_icuuc",
+  ]
+}
+
+shim_headers("icui18n_shim") {
+  root_path = "source/i18n"
+  headers = [
+    # This list can easily be updated using the command below:
+    # find third_party/icu/source/i18n/unicode \
+    # -iname '*.h' -printf '"%p",\n' | \
+    # sed -e 's|third_party/icu/i18n/common/||' | sort -u
+    "unicode/alphaindex.h",
+    "unicode/basictz.h",
+    "unicode/calendar.h",
+    "unicode/choicfmt.h",
+    "unicode/coleitr.h",
+    "unicode/coll.h",
+    "unicode/compactdecimalformat.h",
+    "unicode/curramt.h",
+    "unicode/currpinf.h",
+    "unicode/currunit.h",
+    "unicode/datefmt.h",
+    "unicode/dcfmtsym.h",
+    "unicode/decimfmt.h",
+    "unicode/dtfmtsym.h",
+    "unicode/dtitvfmt.h",
+    "unicode/dtitvinf.h",
+    "unicode/dtptngen.h",
+    "unicode/dtrule.h",
+    "unicode/fieldpos.h",
+    "unicode/fmtable.h",
+    "unicode/format.h",
+    "unicode/fpositer.h",
+    "unicode/gender.h",
+    "unicode/gregocal.h",
+    "unicode/locdspnm.h",
+    "unicode/measfmt.h",
+    "unicode/measunit.h",
+    "unicode/measure.h",
+    "unicode/msgfmt.h",
+    "unicode/numfmt.h",
+    "unicode/numsys.h",
+    "unicode/plurfmt.h",
+    "unicode/plurrule.h",
+    "unicode/rbnf.h",
+    "unicode/rbtz.h",
+    "unicode/regex.h",
+    "unicode/region.h",
+    "unicode/reldatefmt.h",
+    "unicode/scientificnumberformatter.h",
+    "unicode/search.h",
+    "unicode/selfmt.h",
+    "unicode/simpletz.h",
+    "unicode/smpdtfmt.h",
+    "unicode/sortkey.h",
+    "unicode/stsearch.h",
+    "unicode/tblcoll.h",
+    "unicode/timezone.h",
+    "unicode/tmunit.h",
+    "unicode/tmutamt.h",
+    "unicode/tmutfmt.h",
+    "unicode/translit.h",
+    "unicode/tzfmt.h",
+    "unicode/tznames.h",
+    "unicode/tzrule.h",
+    "unicode/tztrans.h",
+    "unicode/ucal.h",
+    "unicode/ucol.h",
+    "unicode/ucoleitr.h",
+    "unicode/ucsdet.h",
+    "unicode/ucurr.h",
+    "unicode/udat.h",
+    "unicode/udateintervalformat.h",
+    "unicode/udatpg.h",
+    "unicode/udisplaycontext.h",
+    "unicode/ufieldpositer.h",
+    "unicode/uformattable.h",
+    "unicode/ugender.h",
+    "unicode/uldnames.h",
+    "unicode/ulocdata.h",
+    "unicode/umsg.h",
+    "unicode/unirepl.h",
+    "unicode/unum.h",
+    "unicode/unumsys.h",
+    "unicode/upluralrules.h",
+    "unicode/uregex.h",
+    "unicode/uregion.h",
+    "unicode/usearch.h",
+    "unicode/uspoof.h",
+    "unicode/utmscale.h",
+    "unicode/utrans.h",
+    "unicode/vtzone.h",
+  ]
+}
+
+shim_headers("icuuc_shim") {
+  root_path = "source/common"
+  headers = [
+    # This list can easily be updated using the command below:
+    # find third_party/icu/source/common/unicode \
+    # -iname '*.h' -printf '"%p",\n' | \
+    # sed -e 's|third_party/icu/source/common/||' | sort -u
+    "unicode/appendable.h",
+    "unicode/brkiter.h",
+    "unicode/bytestream.h",
+    "unicode/bytestrie.h",
+    "unicode/bytestriebuilder.h",
+    "unicode/caniter.h",
+    "unicode/chariter.h",
+    "unicode/dbbi.h",
+    "unicode/docmain.h",
+    "unicode/dtintrv.h",
+    "unicode/enumset.h",
+    "unicode/errorcode.h",
+    "unicode/filteredbrk.h",
+    "unicode/icudataver.h",
+    "unicode/icuplug.h",
+    "unicode/idna.h",
+    "unicode/listformatter.h",
+    "unicode/localpointer.h",
+    "unicode/locid.h",
+    "unicode/messagepattern.h",
+    "unicode/normalizer2.h",
+    "unicode/normlzr.h",
+    "unicode/parseerr.h",
+    "unicode/parsepos.h",
+    "unicode/platform.h",
+    "unicode/ptypes.h",
+    "unicode/putil.h",
+    "unicode/rbbi.h",
+    "unicode/rep.h",
+    "unicode/resbund.h",
+    "unicode/schriter.h",
+    "unicode/std_string.h",
+    "unicode/strenum.h",
+    "unicode/stringpiece.h",
+    "unicode/stringtriebuilder.h",
+    "unicode/symtable.h",
+    "unicode/ubidi.h",
+    "unicode/ubrk.h",
+    "unicode/ucasemap.h",
+    "unicode/ucat.h",
+    "unicode/uchar.h",
+    "unicode/ucharstrie.h",
+    "unicode/ucharstriebuilder.h",
+    "unicode/uchriter.h",
+    "unicode/uclean.h",
+    "unicode/ucnv.h",
+    "unicode/ucnv_cb.h",
+    "unicode/ucnv_err.h",
+    "unicode/ucnvsel.h",
+    "unicode/uconfig.h",
+    "unicode/udata.h",
+    "unicode/uenum.h",
+    "unicode/uidna.h",
+    "unicode/uiter.h",
+    "unicode/ulistformatter.h",
+    "unicode/uloc.h",
+    "unicode/umachine.h",
+    "unicode/umisc.h",
+    "unicode/unifilt.h",
+    "unicode/unifunct.h",
+    "unicode/unimatch.h",
+    "unicode/uniset.h",
+    "unicode/unistr.h",
+    "unicode/unorm.h",
+    "unicode/unorm2.h",
+    "unicode/uobject.h",
+    "unicode/urename.h",
+    "unicode/urep.h",
+    "unicode/ures.h",
+    "unicode/uscript.h",
+    "unicode/uset.h",
+    "unicode/usetiter.h",
+    "unicode/ushape.h",
+    "unicode/usprep.h",
+    "unicode/ustring.h",
+    "unicode/ustringtrie.h",
+    "unicode/utext.h",
+    "unicode/utf.h",
+    "unicode/utf16.h",
+    "unicode/utf32.h",
+    "unicode/utf8.h",
+    "unicode/utf_old.h",
+    "unicode/utrace.h",
+    "unicode/utypes.h",
+    "unicode/uvernum.h",
+    "unicode/uversion.h",
+  ]
+}

samples/GN/isolate.gni

@@ -0,0 +1,193 @@
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/config/sanitizers/sanitizers.gni")
+import("//third_party/icu/config.gni")
+import("v8.gni")
+
+declare_args() {
+  # Sets the test isolation mode (noop|prepare|check).
+  v8_test_isolation_mode = "noop"
+}
+
+template("v8_isolate_run") {
+  forward_variables_from(invoker,
+                         "*",
+                         [
+                           "deps",
+                           "isolate",
+                         ])
+
+  # Remember target name as within the action scope the target name will be
+  # different.
+  name = target_name
+
+  assert(defined(invoker.deps))
+  assert(defined(invoker.isolate))
+
+  if (name != "" && v8_test_isolation_mode != "noop") {
+    action(name + "_run") {
+      testonly = true
+
+      deps = invoker.deps
+
+      script = "//tools/isolate_driver.py"
+
+      sources = [
+        invoker.isolate,
+      ]
+
+      inputs = [
+        # Files that are known to be involved in this step.
+        "//tools/swarming_client/isolate.py",
+        "//tools/swarming_client/run_isolated.py",
+      ]
+
+      if (v8_test_isolation_mode == "prepare") {
+        outputs = [
+          "$root_out_dir/$name.isolated.gen.json",
+        ]
+      } else if (v8_test_isolation_mode == "check") {
+        outputs = [
+          "$root_out_dir/$name.isolated",
+          "$root_out_dir/$name.isolated.state",
+        ]
+      }
+
+      # Translate gn to gyp variables.
+      if (is_asan) {
+        asan = "1"
+      } else {
+        asan = "0"
+      }
+      if (is_msan) {
+        msan = "1"
+      } else {
+        msan = "0"
+      }
+      if (is_tsan) {
+        tsan = "1"
+      } else {
+        tsan = "0"
+      }
+      if (is_cfi) {
+        cfi_vptr = "1"
+      } else {
+        cfi_vptr = "0"
+      }
+      if (target_cpu == "x86") {
+        target_arch = "ia32"
+      } else {
+        target_arch = target_cpu
+      }
+      if (is_debug) {
+        configuration_name = "Debug"
+      } else {
+        configuration_name = "Release"
+      }
+      if (is_component_build) {
+        component = "shared_library"
+      } else {
+        component = "static_library"
+      }
+      if (icu_use_data_file) {
+        icu_use_data_file_flag = "1"
+      } else {
+        icu_use_data_file_flag = "0"
+      }
+      if (v8_enable_inspector) {
+        enable_inspector = "1"
+      } else {
+        enable_inspector = "0"
+      }
+      if (v8_use_external_startup_data) {
+        use_external_startup_data = "1"
+      } else {
+        use_external_startup_data = "0"
+      }
+      if (v8_use_snapshot) {
+        use_snapshot = "true"
+      } else {
+        use_snapshot = "false"
+      }
+      if (v8_has_valgrind) {
+        has_valgrind = "1"
+      } else {
+        has_valgrind = "0"
+      }
+      if (v8_gcmole) {
+        gcmole = "1"
+      } else {
+        gcmole = "0"
+      }
+
+      # Note, all paths will be rebased in isolate_driver.py to be relative to
+      # the isolate file.
+      args = [
+        v8_test_isolation_mode,
+        "--isolated",
+        rebase_path("$root_out_dir/$name.isolated", root_build_dir),
+        "--isolate",
+        rebase_path(invoker.isolate, root_build_dir),
+
+        # Path variables are used to replace file paths when loading a .isolate
+        # file
+        "--path-variable",
+        "DEPTH",
+        rebase_path("//", root_build_dir),
+        "--path-variable",
+        "PRODUCT_DIR",
+        rebase_path(root_out_dir, root_build_dir),
+
+        # TODO(machenbach): Set variables for remaining features.
+        "--config-variable",
+        "CONFIGURATION_NAME=$configuration_name",
+        "--config-variable",
+        "OS=$target_os",
+        "--config-variable",
+        "asan=$asan",
+        "--config-variable",
+        "cfi_vptr=$cfi_vptr",
+        "--config-variable",
+        "gcmole=$gcmole",
+        "--config-variable",
+        "has_valgrind=$has_valgrind",
+        "--config-variable",
+        "icu_use_data_file_flag=$icu_use_data_file_flag",
+        "--config-variable",
+        "is_gn=1",
+        "--config-variable",
+        "msan=$msan",
+        "--config-variable",
+        "tsan=$tsan",
+        "--config-variable",
+        "coverage=0",
+        "--config-variable",
+        "sanitizer_coverage=0",
+        "--config-variable",
+        "component=$component",
+        "--config-variable",
+        "target_arch=$target_arch",
+        "--config-variable",
+        "v8_enable_inspector=$enable_inspector",
+        "--config-variable",
+        "v8_use_external_startup_data=$use_external_startup_data",
+        "--config-variable",
+        "v8_use_snapshot=$use_snapshot",
+      ]
+
+      if (is_win) {
+        args += [
+          "--config-variable",
+          "msvs_version=2015",
+        ]
+      } else {
+        args += [
+          "--config-variable",
+          "msvs_version=0",
+        ]
+      }
+    }
+  }
+}

samples/HTML+Django/nunjucks.njk

@@ -0,0 +1,48 @@
+{% from "forms.html" import label as description %}
+
+
+{% macro field(name, value='', type='text') %}
+    <div class="field">
+        <input type="{{ type }}" name="{{ name }}"
+                value="{{ value | escape }}" />
+    </div>
+{% endmacro %}
+
+<html>
+<head>
+    {% extends "head.html" %}
+</head>
+<body>
+{% if horse %}
+    Chuck Norris once kicked a horse in the chin. Its descendants are known today as Giraffes.
+{% elif optimus %}
+    Chuck Norris once urinated in a semi truck's gas tank as a joke....that truck is now known as Optimus Prime.
+{% else %}
+    Chuck Norris threw a grenade and killed 50 people, then the grenade exploded.
+{% endif %}
+
+{% block left %}
+    This is the left side!
+{% endblock %}
+
+{% block right %}
+    This is the right side!
+{% endblock %}
+
+{{ description('Username') }}
+{{ field('user') }}
+{{ field('pass', type='password') }}
+
+<h1>Posts</h1>
+<ul>
+    {% for item in items %}
+        <li>{{ item.title }}</li>
+    {% else %}
+        <li>This would display if the 'item' collection were empty</li>
+    {% endfor %}
+</ul>
+
+{# Don't escape foo #}
+{{ foo | safe }}
+</body>
+</html>

samples/JSON5/filenames/.babelrc

@@ -0,0 +1,6 @@
+{
+  "presets": [
+    "es2015",
+    "es2016"
+  ]
+}

samples/PHP/ThriftGenerated.php

@@ -0,0 +1,93 @@
+<?php
+namespace github\com;
+
+/**
+ * Autogenerated by Thrift Compiler (0.9.3)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ *  @generated
+ */
+use Thrift\Base\TBase;
+use Thrift\Type\TType;
+use Thrift\Type\TMessageType;
+use Thrift\Exception\TException;
+use Thrift\Exception\TProtocolException;
+use Thrift\Protocol\TProtocol;
+use Thrift\Protocol\TBinaryProtocolAccelerated;
+use Thrift\Exception\TApplicationException;
+
+
+class PullRequest {
+  static $_TSPEC;
+
+  /**
+   * @var string
+   */
+  public $title = null;
+
+  public function __construct($vals=null) {
+    if (!isset(self::$_TSPEC)) {
+      self::$_TSPEC = array(
+        1 => array(
+          'var' => 'title',
+          'type' => TType::STRING,
+          ),
+        );
+    }
+    if (is_array($vals)) {
+      if (isset($vals['title'])) {
+        $this->title = $vals['title'];
+      }
+    }
+  }
+
+  public function getName() {
+    return 'PullRequest';
+  }
+
+  public function read($input)
+  {
+    $xfer = 0;
+    $fname = null;
+    $ftype = 0;
+    $fid = 0;
+    $xfer += $input->readStructBegin($fname);
+    while (true)
+    {
+      $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+      if ($ftype == TType::STOP) {
+        break;
+      }
+      switch ($fid)
+      {
+        case 1:
+          if ($ftype == TType::STRING) {
+            $xfer += $input->readString($this->title);
+          } else {
+            $xfer += $input->skip($ftype);
+          }
+          break;
+        default:
+          $xfer += $input->skip($ftype);
+          break;
+      }
+      $xfer += $input->readFieldEnd();
+    }
+    $xfer += $input->readStructEnd();
+    return $xfer;
+  }
+
+  public function write($output) {
+    $xfer = 0;
+    $xfer += $output->writeStructBegin('PullRequest');
+    if ($this->title !== null) {
+      $xfer += $output->writeFieldBegin('title', TType::STRING, 1);
+      $xfer += $output->writeString($this->title);
+      $xfer += $output->writeFieldEnd();
+    }
+    $xfer += $output->writeFieldStop();
+    $xfer += $output->writeStructEnd();
+    return $xfer;
+  }
+
+}

samples/PHP/filenames/.php_cs

@@ -0,0 +1,37 @@
+<?php
+
+$header = <<<'EOF'
+This file is part of PHP CS Fixer.
+
+(c) Fabien Potencier <fabien@symfony.com>
+    Dariusz Rumiński <dariusz.ruminski@gmail.com>
+
+This source file is subject to the MIT license that is bundled
+with this source code in the file LICENSE.
+EOF;
+
+return PhpCsFixer\Config::create()
+    ->setRiskyAllowed(true)
+    ->setRules(array(
+        '@Symfony' => true,
+        '@Symfony:risky' => true,
+        'combine_consecutive_unsets' => true,
+        'header_comment' => array('header' => $header),
+        'array_syntax' => array('syntax' => 'long'),
+        'no_extra_consecutive_blank_lines' => array('break', 'continue', 'extra', 'return', 'throw', 'use', 'parenthesis_brace_block', 'square_brace_block', 'curly_brace_block'),
+        'no_useless_else' => true,
+        'no_useless_return' => true,
+        'ordered_class_elements' => true,
+        'ordered_imports' => true,
+        'php_unit_strict' => true,
+        'phpdoc_add_missing_param_annotation' => true,
+        'psr4' => true,
+        'strict_comparison' => true,
+        'strict_param' => true,
+    ))
+    ->setFinder(
+        PhpCsFixer\Finder::create()
+            ->exclude('tests/Fixtures')
+            ->in(__DIR__)
+    )
+;

samples/PHP/filenames/.php_cs.dist

@@ -0,0 +1,37 @@
+<?php
+
+$header = <<<'EOF'
+This file is part of PHP CS Fixer.
+
+(c) Fabien Potencier <fabien@symfony.com>
+    Dariusz Rumiński <dariusz.ruminski@gmail.com>
+
+This source file is subject to the MIT license that is bundled
+with this source code in the file LICENSE.
+EOF;
+
+return PhpCsFixer\Config::create()
+    ->setRiskyAllowed(true)
+    ->setRules(array(
+        '@Symfony' => true,
+        '@Symfony:risky' => true,
+        'combine_consecutive_unsets' => true,
+        'header_comment' => array('header' => $header),
+        'array_syntax' => array('syntax' => 'long'),
+        'no_extra_consecutive_blank_lines' => array('break', 'continue', 'extra', 'return', 'throw', 'use', 'parenthesis_brace_block', 'square_brace_block', 'curly_brace_block'),
+        'no_useless_else' => true,
+        'no_useless_return' => true,
+        'ordered_class_elements' => true,
+        'ordered_imports' => true,
+        'php_unit_strict' => true,
+        'phpdoc_add_missing_param_annotation' => true,
+        'psr4' => true,
+        'strict_comparison' => true,
+        'strict_param' => true,
+    ))
+    ->setFinder(
+        PhpCsFixer\Finder::create()
+            ->exclude('tests/Fixtures')
+            ->in(__DIR__)
+    )
+;

samples/Python/filenames/.gclient

@@ -0,0 +1,9 @@
+solutions = [
+  {
+    "url": "https://chromium.googlesource.com/v8/v8.git",
+    "managed": False,
+    "name": "v8",
+    "deps_file": "DEPS",
+    "custom_deps": {},
+  },
+]

samples/Python/py3.py3

@@ -0,0 +1,20 @@
+import random
+
+guesses = 0
+
+number = random.randint(1, 20)
+
+print("Guess the number between 1 and 20! You have 6 tries.")
+while guesses < 6:
+    guess = int(input("Is it... "))
+
+    if guess == number:
+        print("Hooray! You guessed it right!")
+        break
+    elif guess < number:
+        print("It's bigger...")
+    elif guess > number:
+        print("It's not so big.")
+    guesses += 1
+if guesses == 6:
+    print("You've ran out of tries.")

samples/Rascal/Analyze.rsc

@@ -0,0 +1,10 @@
+module Analyze
+
+import Syntax;
+
+set[Id] unreachable(Machine m) {
+  r = { <q1,q2> | (State)`state <Id q1> <Trans* ts>` <- m.states, 
+				  (Trans)`<Id _>: <Id q2>` <- ts }+;
+  qs = [ q.name | /State q := m ];
+  return { q | q <- qs, q notin r[qs[0]] };
+}

samples/Rascal/Compile.rsc

@@ -0,0 +1,18 @@
+module Compile
+
+import Syntax;
+
+str compile(Machine m) =
+  "while (true) {
+  '  event = input.next();
+  '  switch (current) { 
+  '    <for (q <- m.states) {>
+  '    case \"<q.name>\":
+  '      <for (t <- q.out) {>
+  '      if (event.equals(\"<t.event>\"))
+  '        current = \"<t.to>\";
+  '      <}>
+  '      break;
+  '    <}>
+  '  }
+  '}"; 

samples/Rascal/Rascal.rsc

@@ -0,0 +1,887 @@
+ @license{
+  Copyright (c) 2009-2015 CWI
+  All rights reserved. This program and the accompanying materials
+  are made available under the terms of the Eclipse Public License v1.0
+  which accompanies this distribution, and is available at
+  http://www.eclipse.org/legal/epl-v10.html
+}
+@contributor{Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI}
+@contributor{Tijs van der Storm - Tijs.van.der.Storm@cwi.nl}
+@contributor{Paul Klint - Paul.Klint@cwi.nl - CWI}
+@contributor{Arnold Lankamp - Arnold.Lankamp@cwi.nl}
+@contributor{Michael Steindorfer - Michael.Steindorfer@cwi.nl - CWI}
+@doc{The syntax definition of Rascal, excluding concrete syntax fragments}
+module lang::rascal::\syntax::Rascal
+
+lexical BooleanLiteral
+	= "true" 
+	| "false" ;
+
+syntax Literal
+	= integer: IntegerLiteral integerLiteral 
+	| regExp: RegExpLiteral regExpLiteral 
+	| \real: RealLiteral realLiteral 
+	| boolean: BooleanLiteral booleanLiteral 
+	| string: StringLiteral stringLiteral 
+	| dateTime: DateTimeLiteral dateTimeLiteral 
+	| location: LocationLiteral locationLiteral
+	| rational: RationalLiteral rationalLiteral
+	;
+
+syntax Expression = concrete: Concrete concrete;
+syntax Pattern    = concrete: Concrete concrete;
+
+lexical Concrete 
+  = typed: "(" LAYOUTLIST l1 Sym symbol LAYOUTLIST l2 ")" LAYOUTLIST l3 "`" ConcretePart* parts "`";
+
+lexical ConcretePart
+  = @category="MetaSkipped" text   : ![`\<\>\\\n]+ !>> ![`\<\>\\\n]
+  | newline: "\n" [\ \t \u00A0 \u1680 \u2000-\u200A \u202F \u205F \u3000]* "\'"
+  | @category="MetaVariable" hole : ConcreteHole hole
+  | @category="MetaSkipped" lt: "\\\<"
+  | @category="MetaSkipped" gt: "\\\>"
+  | @category="MetaSkipped" bq: "\\`"
+  | @category="MetaSkipped" bs: "\\\\"
+  ;
+  
+syntax ConcreteHole 
+  = \one: "\<" Sym symbol Name name "\>"
+  ;
+  
+start syntax Module
+	= \default: Header header Body body ;
+
+syntax ModuleParameters
+	= \default: "[" {TypeVar ","}+ parameters "]" ;
+
+lexical DateAndTime
+	= "$" DatePart "T" TimePartNoTZ !>> [+\-] "$"
+	| "$" DatePart "T" TimePartNoTZ TimeZonePart "$";
+
+syntax Strategy
+	= topDownBreak: "top-down-break" 
+	| topDown: "top-down" 
+	| bottomUp: "bottom-up" 
+	| bottomUpBreak: "bottom-up-break" 
+	| outermost: "outermost" 
+	| innermost: "innermost" ;
+
+lexical UnicodeEscape
+	  = utf16: "\\" [u] [0-9 A-F a-f] [0-9 A-F a-f] [0-9 A-F a-f] [0-9 A-F a-f] 
+    | utf32: "\\" [U] (("0" [0-9 A-F a-f]) | "10") [0-9 A-F a-f] [0-9 A-F a-f] [0-9 A-F a-f] [0-9 A-F a-f] // 24 bits 
+    | ascii: "\\" [a] [0-7] [0-9A-Fa-f]
+    ;
+    
+syntax Variable
+	= initialized: Name name "=" Expression initial 
+	| unInitialized: Name name ;
+
+lexical OctalIntegerLiteral
+	= [0] [0-7]+ !>> [0-9 A-Z _ a-z] ;
+
+syntax TypeArg
+	= \default: Type type 
+	| named: Type type Name name ;
+
+syntax Renaming
+	= \default: Name from "=\>" Name to ;
+
+syntax Catch
+	= \default: "catch" ":" Statement body 
+	| binding: "catch" Pattern pattern ":" Statement body ;
+
+lexical PathChars
+	= URLChars [|] ;
+
+syntax Signature
+	= withThrows: FunctionModifiers modifiers Type type  Name name Parameters parameters "throws" {Type ","}+ exceptions 
+	| noThrows: FunctionModifiers modifiers Type type  Name name Parameters parameters ;
+
+syntax Sym
+// named non-terminals
+	= nonterminal: Nonterminal nonterminal !>> "["
+	| parameter: "&" Nonterminal nonterminal 
+	| parametrized: Nonterminal nonterminal >> "[" "[" {Sym ","}+ parameters "]"
+	| \start: "start" "[" Nonterminal nonterminal "]"
+	| labeled: Sym symbol NonterminalLabel label
+// literals 
+	| characterClass: Class charClass 
+	| literal: StringConstant string 
+	| caseInsensitiveLiteral: CaseInsensitiveStringConstant cistring
+// regular expressions
+	| iter: Sym symbol "+" 
+	| iterStar: Sym symbol "*" 
+	| iterSep: "{" Sym symbol Sym sep "}" "+" 
+	| iterStarSep: "{" Sym symbol Sym sep "}" "*" 
+	| optional: Sym symbol "?" 
+	| alternative: "(" Sym first "|" {Sym "|"}+ alternatives ")"
+	| sequence: "(" Sym first Sym+ sequence ")"
+	// TODO: MinimalIter: Sym symbol IntegerConstant minimal "+"
+	// TODO: MinimalIterSep: "{" Sym symbol Symbol sep "}" IntegerConstant minimal "+"
+	// TODO | Permutation: "(" Sym first "~" {Sym "~"}+ participants ")"
+	// TODO | Combination: "(" Sym first "#" {Sym "#"}+ elements ")"
+	| empty: "(" ")"
+// conditionals
+	| column: Sym symbol "@" IntegerLiteral column 
+	| endOfLine: Sym symbol "$" 
+	| startOfLine: "^" Sym symbol
+	| except:   Sym symbol "!" NonterminalLabel label
+	>  
+	assoc ( 
+	  left  ( follow:     Sym symbol  "\>\>" Sym match
+	        | notFollow:  Sym symbol "!\>\>" Sym match
+	        )
+	  | 
+	  right ( precede:    Sym match "\<\<" Sym symbol 
+	        | notPrecede: Sym match "!\<\<" Sym symbol
+	        )
+	)
+	> 
+	left unequal:  Sym symbol "\\" Sym match
+	;
+
+lexical TimePartNoTZ
+	= [0-2] [0-9] [0-5] [0-9] [0-5] [0-9] ([, .] [0-9] ([0-9] [0-9]?)?)? 
+	| [0-2] [0-9] ":" [0-5] [0-9] ":" [0-5] [0-9] ([, .] [0-9] ([0-9] [0-9]?)?)? 
+	;
+
+syntax Header
+	= parameters: Tags tags "module" QualifiedName name ModuleParameters params Import* imports 
+	| \default: Tags tags "module" QualifiedName name Import* imports ;
+
+lexical Name
+    // Names are surrounded by non-alphabetical characters, i.e. we want longest match.
+	=  ([A-Z a-z _] !<< [A-Z _ a-z] [0-9 A-Z _ a-z]* !>> [0-9 A-Z _ a-z]) \ RascalKeywords 
+	| [\\] [A-Z _ a-z] [\- 0-9 A-Z _ a-z]* !>> [\- 0-9 A-Z _ a-z] 
+	;
+
+syntax SyntaxDefinition
+	=  @Foldable \layout  : Visibility vis "layout"  Sym defined "=" Prod production ";" 
+	|  @Foldable \lexical : "lexical" Sym defined "=" Prod production ";" 
+	|  @Foldable \keyword : "keyword" Sym defined "=" Prod production ";"
+	|  @Foldable language: Start start "syntax" Sym defined "=" Prod production ";" ;
+
+syntax Kind
+	= function: "function" 
+	| variable: "variable" 
+	| \all: "all" 
+	| \anno: "anno" 
+	| \data: "data" 
+	| view: "view" 
+	| \alias: "alias" 
+	| \module: "module" 
+	| \tag: "tag" ;
+
+syntax ImportedModule
+	= \default: QualifiedName name 
+	| actualsRenaming: QualifiedName name ModuleActuals actuals Renamings renamings 
+	| renamings: QualifiedName name Renamings renamings 
+	| actuals: QualifiedName name ModuleActuals actuals 
+	;
+
+syntax Target
+	= empty: 
+	| labeled: Name name ;
+
+syntax IntegerLiteral
+	= /*prefer()*/ decimalIntegerLiteral: DecimalIntegerLiteral decimal 
+	| /*prefer()*/ hexIntegerLiteral: HexIntegerLiteral hex 
+	| /*prefer()*/ octalIntegerLiteral: OctalIntegerLiteral octal ;
+
+syntax FunctionBody
+	= \default: "{" Statement* statements "}" ;
+    
+syntax Expression
+	= nonEmptyBlock  : "{" Statement+ statements "}" 
+	| bracket \bracket: "(" Expression expression ")" 
+	| closure        : Type type Parameters parameters "{" Statement+ statements "}" 
+	| stepRange      : "[" Expression first "," Expression second ".." Expression last "]" 
+	| voidClosure    : Parameters parameters "{" Statement* statements0 "}" 
+	| \visit          : Label label Visit visit 
+	| reducer        : "(" Expression init "|" Expression result "|" {Expression ","}+ generators ")" 
+	| reifiedType    : "type" "(" Expression symbol "," Expression definitions ")"  
+	| callOrTree     : Expression!transitiveClosure!transitiveReflexiveClosure!isDefined expression "(" {Expression ","}* arguments KeywordArguments[Expression] keywordArguments ")"
+	| literal        : Literal literal 
+	| \any            : "any" "(" {Expression ","}+ generators ")" 
+	| \all            : "all" "(" {Expression ","}+ generators ")" 
+	| comprehension  : Comprehension comprehension 
+	| \set            : "{" {Expression ","}* elements0 "}" 
+	| \list           : "[" {Expression ","}* elements0 "]"
+	| reifyType      : "#" Type type !>> "[" !selector
+	| range          : "[" Expression first ".." Expression last "]"
+	| \tuple          : "\<" {Expression ","}+ elements "\>" 
+	| \map            : "(" {Mapping[Expression] ","}* mappings ")" 
+	| \it             : [A-Z a-z _] !<< "it" !>> [A-Z a-z _]
+	| qualifiedName  : QualifiedName qualifiedName 
+	| subscript    : Expression expression!transitiveClosure!transitiveReflexiveClosure!isDefined "[" {Expression ","}+ subscripts "]"
+	| slice        : Expression expression!transitiveClosure!transitiveReflexiveClosure!isDefined "[" OptionalExpression optFirst ".." OptionalExpression optLast "]" 
+    | sliceStep    : Expression expression!transitiveClosure!transitiveReflexiveClosure!isDefined "[" OptionalExpression optFirst "," Expression second ".." OptionalExpression optLast "]" 
+	| fieldAccess  : Expression expression "." Name field 
+	| fieldUpdate  : Expression expression "[" Name key "=" Expression replacement "]" 
+	| fieldProject : Expression expression!transitiveClosure!transitiveReflexiveClosure!isDefined "\<" {Field ","}+ fields "\>" 
+	| setAnnotation: Expression expression "[" "@" Name name "=" Expression value "]" 
+    | getAnnotation: Expression expression >> "@" "@" Name name 
+	| is           : Expression expression "is" Name name
+	| has          : Expression expression "has" Name name
+	| transitiveClosure: Expression argument "+" !>> "="
+    | transitiveReflexiveClosure: Expression argument "*" !>> "=" 
+	> isDefined    : Expression argument "?" 
+	> negation     : "!" Expression!match!noMatch argument 
+	| negative     : "-" Expression argument 
+	| non-assoc splice : "*" Expression argument
+	| asType       : "[" Type type "]" Expression!match!noMatch argument
+	> left composition: Expression lhs "o" Expression rhs 
+	> left ( product: Expression lhs "*" () !>> "*" Expression!noMatch!match rhs  
+		   | \join   : Expression lhs "join" Expression rhs 
+	       | remainder: Expression lhs "%" Expression rhs
+		   | division: Expression lhs "/" Expression rhs 
+	     )
+	> left intersection: Expression lhs "&" !>> "&" Expression rhs 
+	> left ( addition   : Expression lhs "+" Expression!noMatch!match rhs  
+		   | subtraction: Expression!transitiveClosure!transitiveReflexiveClosure lhs "-" Expression rhs
+		   | appendAfter: Expression lhs "\<\<" !>> "=" Expression rhs
+		   | insertBefore: Expression lhs "\>\>" Expression rhs 
+	       )
+	> left modulo: Expression lhs "mod" Expression rhs
+	> non-assoc ( notIn: Expression lhs "notin" Expression rhs  
+		        | \in: Expression lhs "in" Expression rhs 
+	)
+	> non-assoc ( greaterThanOrEq: Expression lhs "\>=" Expression rhs  
+		        | lessThanOrEq   : Expression lhs "\<=" Expression rhs 
+		        | lessThan       : Expression lhs "\<" !>> "-" Expression rhs 
+		        | greaterThan    : Expression lhs "\>" Expression rhs 
+	            )
+	> non-assoc ( equals         : Expression lhs "==" Expression rhs
+	            | nonEquals      : Expression lhs "!=" Expression rhs 
+	            )
+	> non-assoc ifDefinedOtherwise: Expression lhs "?" Expression rhs
+	> non-assoc ( noMatch: Pattern pattern "!:=" Expression expression  
+		        | match: Pattern pattern ":=" Expression expression 
+		        | enumerator: Pattern pattern "\<-" Expression expression 
+	            ) 
+	> non-assoc ( implication: Expression lhs "==\>" Expression rhs  
+		        | equivalence: Expression lhs "\<==\>" Expression rhs 
+	            )
+	> left and: Expression lhs "&&" Expression rhs 
+	> left or: Expression lhs "||" Expression rhs 
+	> right ifThenElse: Expression condition "?" Expression thenExp ":" Expression elseExp
+	; 
+
+syntax OptionalExpression 
+  = expression: Expression expression
+  | noExpression: ()
+  ;
+    
+syntax UserType
+	= name: QualifiedName name 
+	| parametric: QualifiedName name >> "[" "[" {Type ","}+ parameters "]" ;
+
+syntax Import
+	= \extend: "extend" ImportedModule module ";" 
+	| \default: "import" ImportedModule module ";"
+	| \external: "import" QualifiedName name "=" LocationLiteral at ";"
+	| \syntax: SyntaxDefinition syntax ;
+
+syntax Body
+	= toplevels: Toplevel* toplevels ;
+
+lexical URLChars
+	= ![\t-\n \r \  \< |]* ;
+
+lexical TimeZonePart
+	= [+ \-] [0-1] [0-9] ":" [0-5] [0-9] 
+	| "Z" 
+	| [+ \-] [0-1] [0-9] 
+	| [+ \-] [0-1] [0-9] [0-5] [0-9] 
+	;
+
+syntax ProtocolPart
+	= nonInterpolated: ProtocolChars protocolChars 
+	| interpolated: PreProtocolChars pre Expression expression ProtocolTail tail ;
+
+syntax StringTemplate
+	= ifThen    : "if"    "(" {Expression ","}+ conditions ")" "{" Statement* preStats StringMiddle body Statement* postStats "}" 
+	| ifThenElse: "if"    "(" {Expression ","}+ conditions ")" "{" Statement* preStatsThen StringMiddle thenString Statement* postStatsThen "}" "else" "{" Statement* preStatsElse StringMiddle elseString Statement* postStatsElse "}" 
+	| \for       : "for"   "(" {Expression ","}+ generators ")" "{" Statement* preStats StringMiddle body Statement* postStats "}" 
+	| doWhile   : "do"    "{" Statement* preStats StringMiddle body Statement* postStats "}" "while" "(" Expression condition ")" 
+	| \while     : "while" "(" Expression condition ")" "{" Statement* preStats StringMiddle body Statement* postStats "}" ;
+
+lexical PreStringChars
+	= @category="Constant" [\"] StringCharacter* [\<] ;
+
+lexical CaseInsensitiveStringConstant
+	= @category="Constant" "\'" StringCharacter* chars "\'" ;
+
+lexical Backslash
+	= [\\] !>> [/ \< \> \\] ;
+
+syntax Label
+	= \default: Name name ":" 
+	| empty: ;
+
+lexical MidProtocolChars
+	= "\>" URLChars "\<" ;
+
+lexical NamedBackslash
+	= [\\] !>> [\< \> \\] ;
+
+syntax Field
+	= index: IntegerLiteral fieldIndex 
+	| name: Name fieldName ;
+
+lexical JustDate
+	= "$" DatePart "$";
+
+lexical PostPathChars
+	=  "\>" URLChars "|" ;
+
+syntax PathPart
+	= nonInterpolated: PathChars pathChars 
+	| interpolated: PrePathChars pre Expression expression PathTail tail ;
+
+lexical DatePart
+	= [0-9] [0-9] [0-9] [0-9] "-" [0-1] [0-9] "-" [0-3] [0-9] 
+	| [0-9] [0-9] [0-9] [0-9] [0-1] [0-9] [0-3] [0-9] ;
+
+syntax FunctionModifier
+	= java: "java" 
+	| \test: "test" 
+	| \default: "default";
+
+syntax Assignment
+	= ifDefined: "?=" 
+	| division: "/=" 
+	| product: "*=" 
+	| intersection: "&=" 
+	| subtraction: "-=" 
+	| \default: "=" 
+	| addition: "+=" 
+	| \append: "\<\<="
+	;
+
+syntax Assignable
+	= bracket \bracket   : "(" Assignable arg ")"
+	| variable          : QualifiedName qualifiedName
+    | subscript         : Assignable receiver "[" Expression subscript "]" 
+    | slice             : Assignable receiver "[" OptionalExpression optFirst ".." OptionalExpression optLast "]" 
+    | sliceStep         : Assignable receiver "[" OptionalExpression optFirst "," Expression second ".." OptionalExpression optLast "]"     
+	| fieldAccess       : Assignable receiver "." Name field 
+	| ifDefinedOrDefault: Assignable receiver "?" Expression defaultExpression 
+	| constructor       : Name name "(" {Assignable ","}+ arguments ")"  
+	| \tuple             : "\<" {Assignable ","}+ elements "\>" 
+	| annotation        : Assignable receiver "@" Name annotation  ;
+
+lexical StringConstant
+	= @category="Constant" "\"" StringCharacter* chars "\"" ;
+
+
+
+syntax Assoc
+	= associative: "assoc" 
+	| left: "left" 
+	| nonAssociative: "non-assoc" 
+	| right: "right" ;
+
+syntax Replacement
+	= unconditional: Expression replacementExpression 
+	| conditional: Expression replacementExpression "when" {Expression ","}+ conditions ;
+
+syntax DataTarget
+	= empty: 
+	| labeled: Name label ":" ;
+
+lexical StringCharacter
+	= "\\" [\" \' \< \> \\ b f n r t] 
+	| UnicodeEscape 
+	| ![\" \' \< \> \\]
+	| [\n][\ \t \u00A0 \u1680 \u2000-\u200A \u202F \u205F \u3000]* [\'] // margin 
+	;
+
+lexical JustTime
+	= "$T" TimePartNoTZ !>> [+\-] "$"
+	| "$T" TimePartNoTZ TimeZonePart "$"
+	;
+
+lexical MidStringChars
+	= @category="Constant" [\>] StringCharacter* [\<] ;
+
+lexical ProtocolChars
+	= [|] URLChars "://" !>> [\t-\n \r \ \u00A0 \u1680 \u2000-\u200A \u202F \u205F \u3000];
+
+lexical RegExpModifier
+	= [d i m s]* ;
+
+syntax CommonKeywordParameters 
+  = absent: ()
+  | present: "(" {KeywordFormal ","}+ keywordFormalList ")"
+  ;
+    
+syntax Parameters
+	= \default: "(" Formals formals KeywordFormals keywordFormals ")" 
+	| varArgs: "(" Formals formals "..." KeywordFormals keywordFormals ")" ;
+
+lexical OptionalComma = \default: ","? ;
+
+syntax KeywordFormals
+    = \default: OptionalComma optionalComma [,\ (\t\n] << {KeywordFormal ","}+ keywordFormalList
+    | none: ()
+    ;
+    
+syntax KeywordFormal 
+    = \default: Type type Name name "=" Expression expression
+    ;
+    
+syntax KeywordArguments[&T]
+    = \default:  OptionalComma optionalComma [,\ (\t\n] << {KeywordArgument[&T] ","}+ keywordArgumentList
+    | none: ()
+    ;
+    
+syntax KeywordArgument[&T] = \default: Name name "=" &T expression ;
+
+lexical RegExp
+	= ![/ \< \> \\] 
+	| "\<" Name "\>" 
+	| [\\] [/ \< \> \\] 
+	| "\<" Name ":" NamedRegExp* "\>" 
+	| Backslash 
+	// | @category="MetaVariable" [\<]  Expression expression [\>] TODO: find out why this production existed 
+	;
+	
+
+layout LAYOUTLIST
+	= LAYOUT* !>> [\u0009-\u000D \u0020 \u0085 \u00A0 \u1680 \u180E \u2000-\u200A \u2028 \u2029 \u202F \u205F \u3000] !>> "//" !>> "/*";
+
+syntax LocalVariableDeclaration
+	= \default: Declarator declarator 
+	| \dynamic: "dynamic" Declarator declarator ;
+
+lexical RealLiteral
+	= [0-9]+ [D F d f] 
+	| [0-9]+ [E e] [+ \-]? [0-9]+ [D F d f]?
+	| [0-9]+ "." !>> "." [0-9]* [D F d f]?  
+	| [0-9]+ "." [0-9]* [E e] [+ \-]? [0-9]+ [D F d f]? 
+	| [.] !<< "." [0-9]+ [D F d f]? 
+	| [.] !<< "." [0-9]+ [E e] [+ \-]? [0-9]+ [D F d f]? 
+	;
+
+syntax Range
+	= fromTo: Char start "-" Char end 
+	| character: Char character ;
+
+syntax LocationLiteral
+	= \default: ProtocolPart protocolPart PathPart pathPart ;
+
+syntax ShellCommand
+	= setOption: "set" QualifiedName name Expression expression 
+	| undeclare: "undeclare" QualifiedName name 
+	| help: "help" 
+	| edit: "edit" QualifiedName name 
+	| unimport: "unimport" QualifiedName name 
+	| listDeclarations: "declarations" 
+	| quit: "quit" 
+	| history: "history" 
+	| \test: "test" 
+	| listModules: "modules" 
+	| clear: "clear";
+
+syntax StringMiddle
+	= mid: MidStringChars mid 
+	| template: MidStringChars mid StringTemplate template StringMiddle tail 
+	| interpolated: MidStringChars mid Expression expression StringMiddle tail ;
+
+syntax QualifiedName
+	= \default: {Name "::"}+ names !>> "::" ;
+
+lexical RationalLiteral
+   = [0-9][0-9]* [r]
+   | [1-9][0-9]* [r] [0-9][0-9]* !>> [0-9 A-Z _ a-z]
+   ;
+
+lexical DecimalIntegerLiteral
+	= "0" !>> [0-9 A-Z _ a-z] 
+	| [1-9] [0-9]* !>> [0-9 A-Z _ a-z] ;
+
+syntax DataTypeSelector
+	= selector: QualifiedName sort "." Name production ;
+
+syntax StringTail
+	= midInterpolated: MidStringChars mid Expression expression StringTail tail 
+	| post: PostStringChars post 
+	| midTemplate: MidStringChars mid StringTemplate template StringTail tail ;
+
+syntax PatternWithAction
+	= replacing: Pattern pattern "=\>" Replacement replacement 
+	| arbitrary: Pattern pattern ":" Statement statement ;
+
+lexical LAYOUT
+	= Comment 
+	// all the white space chars defined in Unicode 6.0 
+	| [\u0009-\u000D \u0020 \u0085 \u00A0 \u1680 \u180E \u2000-\u200A \u2028 \u2029 \u202F \u205F \u3000] 
+	;
+
+syntax Visit
+	= givenStrategy: Strategy strategy "visit" "(" Expression subject ")" "{" Case+ cases "}" 
+	| defaultStrategy: "visit" "(" Expression subject ")" "{" Case+ cases "}" ;
+
+start syntax Commands
+	= \commandlist: EvalCommand+ commands
+	;
+
+start syntax EvalCommand
+  = declaration: Declaration declaration  
+  | statement: Statement!variableDeclaration!functionDeclaration!visit statement 
+  | \import: Import imported
+  | output: Output
+  ;
+ 
+lexical Output   
+  = @category="Result" resultOutput: "⇨" ![\n\r]* [\n] 
+  | @category="StdOut" stdoutOutput: ^ "≫" ![\n\r]* [\n]
+  | @category="StdErr" stderrOutput: ^ "⚠" ![\n\r]* [\n]
+  ;
+  
+start syntax Command
+	= expression: Expression!nonEmptyBlock expression 
+	| declaration: Declaration declaration 
+	| shell: ":" ShellCommand command 
+	| statement: Statement!variableDeclaration!functionDeclaration!visit statement 
+	| \import: Import imported ;
+
+lexical TagString
+	= "\\" !<< "{" ( ![{}] | ("\\" [{}]) | TagString)* contents "\\" !<< "}";
+
+syntax ProtocolTail
+	= mid: MidProtocolChars mid Expression expression ProtocolTail tail 
+	| post: PostProtocolChars post ;
+
+lexical Nonterminal
+	= ([A-Z] !<< [A-Z] [0-9 A-Z _ a-z]* !>> [0-9 A-Z _ a-z]) \ RascalKeywords;
+
+syntax PathTail
+	= mid: MidPathChars mid Expression expression PathTail tail 
+	| post: PostPathChars post ;
+
+syntax Visibility
+	= \private: "private" 
+	| \default: 
+	| \public: "public" ;
+
+syntax StringLiteral
+	= template: PreStringChars pre StringTemplate template StringTail tail 
+	| interpolated: PreStringChars pre Expression expression StringTail tail 
+	| nonInterpolated: StringConstant constant ;
+
+lexical Comment
+	= @category="Comment" "/*" (![*] | [*] !>> [/])* "*/" 
+	| @category="Comment" "//" ![\n]* !>> [\ \t\r \u00A0 \u1680 \u2000-\u200A \u202F \u205F \u3000] $ // the restriction helps with parsing speed
+	;
+	
+
+syntax Renamings
+	= \default: "renaming" {Renaming ","}+ renamings ;
+
+syntax Tags
+	= \default: Tag* tags ;
+
+syntax Formals
+	= \default: {Pattern ","}* formals ;
+
+lexical PostProtocolChars
+	= "\>" URLChars "://" ;
+
+syntax Start
+	= absent: 
+	| present: "start" ;
+
+syntax Statement
+	= @breakable \assert: "assert" Expression expression ";" 
+	| @breakable assertWithMessage: "assert" Expression expression ":" Expression message ";" 
+	| @breakable expression: Expression!visit!nonEmptyBlock expression ";" 
+	| @breakable \visit: Label label Visit visit 
+	| @breakable \while: Label label "while" "(" {Expression ","}+ conditions ")" Statement!variableDeclaration!functionDeclaration body 
+	| @breakable doWhile: Label label "do" Statement body "while" "(" Expression condition ")" ";" 
+	| @breakable @breakable{generators} \for: Label label "for" "(" {Expression ","}+ generators ")" Statement body 
+	| @breakable ifThen: Label label "if" "(" {Expression ","}+ conditions ")" Statement!variableDeclaration!functionDeclaration thenStatement () !>> "else" 
+	| @breakable ifThenElse: Label label "if" "(" {Expression ","}+ conditions ")" Statement thenStatement "else" Statement!variableDeclaration!functionDeclaration elseStatement 
+	| @breakable \switch: Label label "switch" "(" Expression expression ")" "{" Case+ cases "}" 
+	| @breakable \fail: "fail" Target target ";" 
+	| @breakable \break: "break" Target target ";" 
+	| @breakable \continue: "continue" Target target ";" 
+    | @breakable \filter: "filter" ";"
+	| @breakable \solve: "solve" "(" {QualifiedName ","}+ variables Bound bound ")" Statement!variableDeclaration!functionDeclaration body 
+	| @breakable non-assoc \try: "try" Statement body Catch+ handlers 
+	| @breakable tryFinally: "try" Statement body Catch+ handlers "finally" Statement!variableDeclaration!functionDeclaration finallyBody 
+	| nonEmptyBlock: Label label "{" Statement+ statements "}" 
+	| emptyStatement: ";" 
+	| @breakable globalDirective: "global" Type type {QualifiedName ","}+ names ";" 
+	| @breakable assignment: Assignable assignable Assignment operator Statement!functionDeclaration!variableDeclaration statement
+	| non-assoc  ( 
+		          @breakable \return    : "return" Statement!functionDeclaration!variableDeclaration statement  
+		        | @breakable \throw     : "throw" Statement!functionDeclaration!variableDeclaration statement 
+		        | @breakable \insert    : "insert" DataTarget dataTarget Statement!functionDeclaration!variableDeclaration statement 
+		        | @breakable \append    : "append" DataTarget dataTarget Statement!functionDeclaration!variableDeclaration statement 
+	            )
+    | @breakable functionDeclaration: FunctionDeclaration functionDeclaration 
+	| @breakable variableDeclaration: LocalVariableDeclaration declaration ";"
+	; 
+	
+    
+syntax StructuredType
+	= \default: BasicType basicType "[" {TypeArg ","}+ arguments "]" ;
+
+lexical NonterminalLabel
+	= [a-z] [0-9 A-Z _ a-z]* !>> [0-9 A-Z _ a-z] ;
+
+syntax FunctionType
+	= typeArguments: Type type "(" {TypeArg ","}* arguments ")" ;
+
+syntax Case
+	= @Foldable patternWithAction: "case" PatternWithAction patternWithAction 
+	| @Foldable \default: "default" ":" Statement statement ;
+
+syntax Declarator
+	= \default: Type type {Variable ","}+ variables ;
+
+syntax Bound
+	= \default: ";" Expression expression 
+	| empty: ;
+
+keyword RascalKeywords
+	= "o"
+	| "syntax"
+	| "keyword"
+	| "lexical"
+	| "int"
+	| "break"
+	| "continue"
+	| "rat" 
+	| "true" 
+	| "bag" 
+	| "num" 
+	| "node" 
+	| "finally" 
+	| "private" 
+	| "real" 
+	| "list" 
+	| "fail" 
+	| "filter" 
+	| "if" 
+	| "tag" 
+	| BasicType
+	| "extend" 
+	| "append" 
+	| "rel" 
+	| "lrel"
+	| "void" 
+	| "non-assoc" 
+	| "assoc" 
+	| "test" 
+	| "anno" 
+	| "layout" 
+	| "data" 
+	| "join" 
+	| "it" 
+	| "bracket" 
+	| "in" 
+	| "import" 
+	| "false" 
+	| "all" 
+	| "dynamic" 
+	| "solve" 
+	| "type" 
+	| "try" 
+	| "catch" 
+	| "notin" 
+	| "else" 
+	| "insert" 
+	| "switch" 
+	| "return" 
+	| "case" 
+	| "while" 
+	| "str" 
+	| "throws" 
+	| "visit" 
+	| "tuple" 
+	| "for" 
+	| "assert" 
+	| "loc" 
+	| "default" 
+	| "map" 
+	| "alias" 
+	| "any" 
+	| "module" 
+	| "mod"
+	| "bool" 
+	| "public" 
+	| "one" 
+	| "throw" 
+	| "set" 
+	| "start"
+	| "datetime" 
+	| "value" 
+	;
+
+syntax Type
+	= bracket \bracket: "(" Type type ")" 
+	| user: UserType user
+	| function: FunctionType function 
+	| structured: StructuredType structured 
+	| basic: BasicType basic 
+	| selector: DataTypeSelector selector 
+	| variable: TypeVar typeVar 
+	| symbol: Sym!nonterminal!labeled!parametrized!parameter symbol
+	;
+
+syntax Declaration
+	= variable    : Tags tags Visibility visibility Type type {Variable ","}+ variables ";" 
+	| annotation  : Tags tags Visibility visibility "anno" Type annoType Type onType "@" Name name ";" 
+	| \alias       : Tags tags Visibility visibility "alias" UserType user "=" Type base ";" 
+	| \tag         : Tags tags Visibility visibility "tag" Kind kind Name name "on" {Type ","}+ types ";" 
+	| dataAbstract: Tags tags Visibility visibility "data" UserType user CommonKeywordParameters commonKeywordParameters ";" 
+	| @Foldable \data : Tags tags Visibility visibility "data" UserType user CommonKeywordParameters commonKeywordParameters"=" {Variant "|"}+ variants ";"
+	| function       : FunctionDeclaration functionDeclaration 
+	;
+
+syntax Class
+	= simpleCharclass: "[" Range* ranges "]" 
+	| complement: "!" Class charClass 
+	> left difference: Class lhs "-" Class rhs 
+	> left intersection: Class lhs "&&" Class rhs 
+	> left union: Class lhs "||" Class rhs 
+	| bracket \bracket: "(" Class charclass ")" ;
+
+lexical RegExpLiteral
+	= "/" RegExp* "/" RegExpModifier ;
+
+syntax FunctionModifiers
+	= \modifierlist: FunctionModifier* modifiers ;
+
+syntax Comprehension
+	= @breakable{results,generators} \set: "{" {Expression ","}+ results "|" {Expression ","}+ generators "}" 
+	| @breakable{from,to,generators} \map: "(" Expression from ":" Expression to "|" {Expression ","}+ generators ")" 
+	| @breakable{results,generators} \list: "[" {Expression ","}+ results "|" {Expression ","}+ generators "]" ;
+
+syntax Variant
+	= nAryConstructor: Name name "(" {TypeArg ","}* arguments  KeywordFormals keywordArguments ")" ;
+
+syntax FunctionDeclaration
+	= abstract: Tags tags Visibility visibility Signature signature ";" 
+	| @Foldable @breakable{expression} expression: Tags tags Visibility visibility Signature signature "=" Expression expression ";"
+	| @Foldable @breakable{expression,conditions} conditional: Tags tags Visibility visibility Signature signature "=" Expression expression "when" {Expression ","}+ conditions ";"
+	| @Foldable \default: Tags tags Visibility visibility Signature signature FunctionBody body ;
+
+lexical PreProtocolChars
+	= "|" URLChars "\<" ;
+
+lexical NamedRegExp
+	= "\<" Name "\>" 
+	| [\\] [/ \< \> \\] 
+	| NamedBackslash 
+	| ![/ \< \> \\] ;
+
+syntax ProdModifier
+	= associativity: Assoc associativity 
+	| \bracket: "bracket" 
+	| \tag: Tag tag;
+
+syntax Toplevel
+	= givenVisibility: Declaration declaration ;
+
+lexical PostStringChars
+	= @category="Constant" [\>] StringCharacter* [\"] ;
+
+lexical HexIntegerLiteral
+	= [0] [X x] [0-9 A-F a-f]+ !>> [0-9 A-Z _ a-z] ;
+
+syntax TypeVar
+	= free: "&" Name name 
+	| bounded: "&" Name name "\<:" Type bound ;
+
+
+
+syntax BasicType
+	= \value: "value" 
+	| \loc: "loc" 
+	| \node: "node" 
+	| \num: "num" 
+	| \type: "type" 
+	| \bag: "bag" 
+	| \int: "int"
+	| rational: "rat" 
+	| relation: "rel" 
+	| listRelation: "lrel"
+	| \real: "real" 
+	| \tuple: "tuple" 
+	| string: "str" 
+	| \bool: "bool" 
+	| \void: "void" 
+	| dateTime: "datetime" 
+	| \set: "set" 
+	| \map: "map" 
+	| \list: "list" 
+	;
+
+lexical Char
+	= @category="Constant" "\\" [\  \" \' \- \< \> \[ \\ \] b f n r t] 
+	| @category="Constant" ![\  \" \' \- \< \> \[ \\ \]] 
+	| @category="Constant" UnicodeEscape 
+    ; 
+    
+syntax Prod
+	= reference: ":" Name referenced
+	| labeled: ProdModifier* modifiers Name name ":" Sym* syms 
+	| others: "..." 
+	| unlabeled: ProdModifier* modifiers Sym* syms
+	| @Foldable associativityGroup: Assoc associativity "(" Prod group ")" 
+	// | TODO add bracket rule for easy readability
+	> left \all   : Prod lhs "|" Prod rhs 
+	> left first : Prod lhs "\>" !>> "\>" Prod rhs
+	;
+
+syntax DateTimeLiteral
+	= /*prefer()*/ dateLiteral: JustDate date 
+	| /*prefer()*/ timeLiteral: JustTime time 
+	| /*prefer()*/ dateAndTimeLiteral: DateAndTime dateAndTime ;
+
+lexical PrePathChars
+	= URLChars "\<" ;
+
+syntax Mapping[&T]
+	= \default: &T!ifDefinedOtherwise from ":" &T to 
+	;
+
+lexical MidPathChars
+	= "\>" URLChars "\<" ;
+
+/*
+  Note that Pattern must closely follow the definitions of Expression because eventually
+  these two non-terminals will be fused just before AST generation.
+*/
+syntax Pattern
+	= \set                 : "{" {Pattern ","}* elements0 "}" 
+	| \list                : "[" {Pattern ","}* elements0 "]" 
+	| qualifiedName       : QualifiedName qualifiedName 
+	| multiVariable       : QualifiedName qualifiedName "*"
+	| splice              : "*" Pattern argument
+	| splicePlus          : "+" Pattern argument 
+	| negative            : "-" Pattern argument
+	| literal             : Literal literal 
+	| \tuple               : "\<" {Pattern ","}+ elements "\>" 
+	| typedVariable       : Type type Name name 
+	| \map                 : "(" {Mapping[Pattern] ","}* mappings ")" 
+	| reifiedType         : "type" "(" Pattern symbol "," Pattern definitions ")" 
+	| callOrTree          : Pattern expression "(" {Pattern ","}* arguments KeywordArguments[Pattern] keywordArguments ")" 
+	> variableBecomes     : Name name ":" Pattern pattern
+	| asType              : "[" Type type "]" Pattern argument 
+	| descendant          : "/" Pattern pattern 
+	| anti                : "!" Pattern pattern 
+	| typedVariableBecomes: Type type Name name ":" Pattern pattern 
+    ;
+    
+syntax Tag
+	= @Folded @category="Comment" \default   : "@" Name name TagString contents 
+	| @Folded @category="Comment" empty     : "@" Name name 
+	| @Folded @category="Comment" expression: "@" Name name "=" Expression expression !>> "@";
+
+syntax ModuleActuals
+	= \default: "[" {Type ","}+ types "]" ;

samples/Rascal/Syntax.rsc

@@ -0,0 +1,8 @@
+module Syntax
+
+extend lang::std::Layout;
+extend lang::std::Id;
+
+start syntax Machine = machine: State+ states;
+syntax State = @Foldable state: "state" Id name Trans* out;
+syntax Trans = trans: Id event ":" Id to;

samples/Ruby/filenames/Dangerfile

@@ -0,0 +1,75 @@
+# Sometimes its a README fix, or something like that - which isn't relevant for
+# including in a CHANGELOG for example
+# From https://github.com/danger/danger/blob/master/Dangerfile
+
+has_app_changes = !git.modified_files.grep(/lib/).empty?
+has_test_changes = !git.modified_files.grep(/spec/).empty?
+is_version_bump = git.modified_files.sort == ["CHANGELOG.md", "lib/danger/version.rb"].sort
+
+if has_app_changes && !has_test_changes && !is_version_bump
+  warn("Tests were not updated", sticky: false)
+end
+
+# Thanks other people!
+message(":tada:") if is_version_bump && github.pr_author != "orta"
+
+# Make a note about contributors not in the organization
+unless github.api.organization_member?('danger', github.pr_author)
+  message "@#{github.pr_author} is not a contributor yet, would you like to join the Danger org?"
+
+  # Pay extra attention if they modify the gemspec
+  if git.modified_files.include?("*.gemspec")
+    warn "External contributor has edited the Gemspec"
+  end
+end
+
+# Mainly to encourage writing up some reasoning about the PR, rather than
+# just leaving a title
+if github.pr_body.length < 5
+  fail "Please provide a summary in the Pull Request description"
+end
+
+# Let people say that this isn't worth a CHANGELOG entry in the PR if they choose
+declared_trivial = (github.pr_title + github.pr_body).include?("#trivial") || !has_app_changes
+
+if !git.modified_files.include?("CHANGELOG.md") && !declared_trivial
+  fail("Please include a CHANGELOG entry. \nYou can find it at [CHANGELOG.md](https://github.com/danger/danger/blob/master/CHANGELOG.md).", sticky: false)
+end
+
+# Docs are critical, so let's re-run the docs part of the specs and show any issues:
+core_plugins_docs = `bundle exec danger plugins lint lib/danger/danger_core/plugins/*.rb --warnings-as-errors`
+
+# If it failed, fail the build, and include markdown with the output error.
+unless $?.success?
+  # We want to strip ANSI colors for our markdown, and make paths relative
+  colourless_error = core_plugins_docs.gsub(/\e\[(\d+)(;\d+)*m/, "")
+  markdown("### Core Docs Errors \n\n#{colourless_error}")
+  fail("Failing due to documentation issues, see below.", sticky: false)
+end
+
+# Oddly enough, it's quite possible to do some testing of Danger, inside Danger
+# So, you can ignore these, if you're looking at the Dangerfile to get ideas.
+#
+# If these are all empty something has gone wrong, better to raise it in a comment
+if git.modified_files.empty? && git.added_files.empty? && git.deleted_files.empty?
+  fail "This PR has no changes at all, this is likely an issue during development."
+end
+
+# This comes from `./danger_plugins/protect_files.rb` which is automatically parsed by Danger
+files.protect_files(path: "danger.gemspec", message: ".gemspec modified", fail_build: false)
+
+# Ensure that our core plugins all have 100% documentation
+core_plugins = Dir.glob("lib/danger/danger_core/plugins/*.rb")
+core_lint_output = `bundle exec yard stats #{core_plugins.join ' '} --list-undoc --tag tags`
+
+if !core_lint_output.include?("100.00%")
+  fail "The core plugins are not at 100% doc'd - see below:", sticky: false
+  markdown "```\n#{core_lint_output}```"
+elsif core_lint_output.include? "warning"
+  warn "The core plugins are have yard warnings - see below", sticky: false
+  markdown "```\n#{core_lint_output}```"
+end
+
+junit.parse "junit-results.xml"
+junit.headers = [:file, :name]
+junit.report

script/add-grammar

@@ -78,6 +78,8 @@ https    = "https://#{parts[:host]}/#{parts[:user]}/#{parts[:repo]}"
 repo_new = "vendor/grammars/#{parts[:repo]}"
 repo_old = parse_submodule($replace) if $replace
 
+Dir.chdir(ROOT)
+
 if repo_old
   log "Deregistering: #{repo_old}"
   `git submodule deinit #{repo_old}`
@@ -93,4 +95,5 @@ log "Confirming license"
 `script/licensed --module "#{repo_new}"`
 
 log "Updating grammar documentation in vendor/REAEDME.md"
-`script list-grammars`
+`bundle exec rake samples`
+`script/list-grammars`

script/convert-grammars

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
 
+require 'bundler/setup'
 require 'json'
 require 'net/http'
 require 'optparse'

script/list-grammars

@@ -1,19 +1,20 @@
 #!/usr/bin/env ruby
 
+require "bundler/setup"
 require "linguist"
 require "json"
 require "yaml"
 
 class GrammarList
-  
+
   ROOT = File.expand_path "../../", __FILE__
-  
+
   def initialize
     @submodules     = load_submodules()
     @sources        = load_sources()
     @language_names = load_languages()
   end
-  
+
   # Load .gitmodules
   def load_submodules
     submodules = {}
@@ -29,14 +30,14 @@ class GrammarList
     end
     submodules
   end
-  
+
   # Grab the name of each language, sorted case-insensitively
   def load_languages
     Linguist::Language.all.map(&:name).sort do |a, b|
       a.downcase() <=> b.downcase()
     end
   end
-  
+
   # Load grammars.yml
   def load_sources
     sources = {}
@@ -46,7 +47,7 @@ class GrammarList
     end
     sources
   end
-  
+
   # Shorten a repository URL
   def shorten(url)
     if url =~ /^https?:\/\/(?:www\.)?github\.com\/([^\/]+\/[^\/]+)/i
@@ -57,7 +58,7 @@ class GrammarList
       url.replace(/^https?:\/\/(?:www\.)?/i, "")
     end
   end
-  
+
   # Markdown: Generate grammar list
   def to_markdown
     markdown = ""
@@ -87,7 +88,7 @@ class GrammarList
 
     markdown
   end
-  
+
   # Update the file displaying the reader-friendly list of grammar repos
   def update_readme
     readme = "#{ROOT}/vendor/README.md"

script/prune-grammars

@@ -1,10 +1,13 @@
 #!/usr/bin/env ruby
 
+require "bundler/setup"
 require "json"
 require "linguist"
 require "set"
 require "yaml"
 
+ROOT = File.expand_path("../../", __FILE__)
+
 def find_includes(json)
   case json
   when Hash
@@ -32,7 +35,7 @@ def transitive_includes(scope, includes)
 end
 
 includes = {}
-Dir["grammars/*.json"].each do |path|
+Dir[File.join(ROOT, "grammars/*.json")].each do |path|
   scope = File.basename(path).sub(/\.json/, '')
   json = JSON.load(File.read(path))
   incs = find_includes(json)
@@ -41,7 +44,7 @@ Dir["grammars/*.json"].each do |path|
   includes[scope] += incs
 end
 
-yaml = YAML.load(File.read("grammars.yml"))
+yaml = YAML.load(File.read(File.join(ROOT, "grammars.yml")))
 language_scopes = Linguist::Language.all.map(&:tm_scope).to_set
 
 # The set of used scopes is the scopes for each language, plus all the scopes
@@ -54,4 +57,4 @@ puts "Unused grammar repos"
 puts unused.map { |repo, scopes| sprintf("%-100s %s", repo, scopes.join(", ")) }.sort.join("\n")
 
 yaml.delete_if { |k| unused.key?(k) }
-File.write("grammars.yml", YAML.dump(yaml))
+File.write(File.join(ROOT, "grammars.yml"), YAML.dump(yaml))

script/set-language-ids

@@ -1,4 +1,6 @@
 #!/usr/bin/env ruby
+
+require 'bundler/setup'
 require 'yaml'
 require 'pry'
 
@@ -11,18 +13,18 @@ header = <<-EOF
 # ace_mode          - A String name of the Ace Mode used for highlighting whenever
 #                     a file is edited. This must match one of the filenames in http://git.io/3XO_Cg.
 #                     Use "text" if a mode does not exist.
+# codemirror_mode   - A String name of the CodeMirror Mode used for highlighting whenever a file is edited.
+#                     This must match a mode from https://git.io/vi9Fx
 # wrap              - Boolean wrap to enable line wrapping (default: false)
 # extensions        - An Array of associated extensions (the first one is
 #                     considered the primary extension, the others should be
 #                     listed alphabetically)
 # interpreters      - An Array of associated interpreters
 # searchable        - Boolean flag to enable searching (defaults to true)
-# search_term       - Deprecated: Some languages may be indexed under a
-#                     different alias. Avoid defining new exceptions.
 # language_id       - Integer used as a language-name-independent indexed field so that we can rename
-#                     languages in Linguist without reindexing all the code on GitHub. Must not be 
+#                     languages in Linguist without reindexing all the code on GitHub. Must not be
 #                     changed for existing languages without the explicit permission of GitHub staff.
-# color             - CSS hex color to represent the language.
+# color             - CSS hex color to represent the language. Only used if type is "programming" or "prose".
 # tm_scope          - The TextMate scope that represents this programming
 #                     language. This should match one of the scopes listed in
 #                     the grammars.yml file. Use "none" if there is no grammar
@@ -36,21 +38,23 @@ header = <<-EOF
 # Please keep this list alphabetized. Capitalization comes before lowercase.
 
 EOF
+require 'digest'
 
 generated = true if ARGV[0] == "--force"
 update = true if ARGV[0] == "--update"
 
+def generate_language_id(language)
+  Digest::SHA256.hexdigest(language).to_i(16) % (2**30 - 1)
+end
+
 if generated
   puts "You're regenerating all of the language_id attributes for all Linguist "
   puts "languages defined in languages.yml. This is almost certainly NOT what"
   puts "you meant to do!"
 
-  language_index = 0
-
   languages = YAML.load(File.read("lib/linguist/languages.yml"))
   languages.each do |name, vals|
-    vals.merge!('language_id' => language_index)
-    language_index += 1
+    vals.merge!('language_id' => generate_language_id(name))
   end
 
   File.write("lib/linguist/languages.yml", header + YAML.dump(languages))
@@ -58,20 +62,12 @@ elsif update
   puts "Adding new language_id attributes to languages.yml that don't have one set"
   languages = YAML.load(File.read("lib/linguist/languages.yml"))
 
-  # First grab the maximum language_id
-  language_ids = []
-  languages.each { |name, vals| language_ids << vals['language_id'] if vals.has_key?('language_id')}
-  max_language_id = language_ids.max
-  puts "Current maximum language_id is #{max_language_id}"
-
   missing_count = 0
-  language_index = max_language_id
 
   languages.each do |name, vals|
     unless vals.has_key?('language_id')
-      language_index += 1
       missing_count += 1
-      vals.merge!('language_id' => language_index)
+      vals.merge!('language_id' => generate_language_id(name))
     end
   end
 

test/fixtures/CMake/CMakeLists.txt

@@ -0,0 +1,29 @@
+cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
+
+project("To do list")
+
+enable_testing()
+
+if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" OR
+    "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+    set(warnings "-Wall -Wextra -Werror")
+elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
+    set(warnings "/W4 /WX /EHsc")	
+endif()
+
+set(optimize "-O2")
+
+if (NOT CONFIGURED_ONCE)
+    set(CMAKE_CXX_FLAGS "${warnings} ${optimize}"
+        CACHE STRING "Flags used by the compiler during all build types." FORCE)
+    set(CMAKE_C_FLAGS   "${warnings} ${optimize}"
+        CACHE STRING "Flags used by the compiler during all build types." FORCE)
+endif()
+
+
+add_executable(toDo main.cpp ToDo.cpp)
+
+add_test(toDoTest toDo)
+
+set(CONFIGURED_ONCE TRUE CACHE INTERNAL
+    "A flag showing that CMake has configured at least once.")

test/fixtures/CoffeeScript/Cakefile

@@ -0,0 +1,200 @@
+http = require 'http'
+https = require 'https'
+fs = require 'fs'
+path = require 'path'
+{spawn, exec} = require 'child_process'
+semver = require 'semver'
+AdmZip = require('adm-zip')
+GitHubApi = require 'github'
+
+github = new GitHubApi(version: '3.0.0')
+
+# ----------------
+# Server / Builder
+# ----------------
+
+option '-P', '--production', 'run server in production mode'
+option null, '--port [PORT]', 'listen on specified port (default 3333)'
+
+LOCAL_BRUNCH = path.join('.', 'node_modules', '.bin', 'brunch')
+
+spawnBrunch = (flags, env) ->
+  if fs.existsSync(LOCAL_BRUNCH)
+    brunch = spawn LOCAL_BRUNCH, flags, env
+  else
+    console.error 'Warning, using global brunch. Run `npm install`.'
+    brunch = spawn 'brunch', flags, env
+
+  brunch.stdout.on 'data', (data) -> console.log data.toString().trim()
+  brunch.stderr.on 'data', (data) -> console.log data.toString().trim()
+
+runBrunchWatch = (options, shouldStartServer) ->
+  flags = ['w']
+  flags.push '-s' if shouldStartServer
+
+  if options.production?
+    flags.push('-P')
+    process.env.BRUNCH_ENV = 'production'
+
+  if options.port?
+    flags.push '-p'
+    flags.push options.port
+
+  spawnBrunch flags, process.env
+
+task 'server', 'start the brunch server in development', (options) ->
+  runBrunchWatch(options, true)
+
+task 'watch', 'build the app continuously without a server', (options) ->
+  runBrunchWatch(options, false)
+
+task 'build', 'build for production', ->
+  process.env.BRUNCH_ENV = 'production'
+  spawnBrunch ['b', '-P'], process.env
+
+task 'test', 'run brunch in the test environment', ->
+  flags = ['w', '-s']
+  process.env.BRUNCH_ENV = 'test'
+  spawnBrunch flags, process.env
+
+# -------------
+# Tapas Updates
+# -------------
+updateMessage = 'update Tapas to latest (Cakefile, package.json, portkey.json,
+ config.coffee, generators/*)'
+task 'tapas:update', updateMessage, (options) ->
+  url = 'https://codeload.github.com/mutewinter/tapas-with-ember/zip/master'
+  filesToUpdate = [
+    'Cakefile'
+    'package.json'
+    'portkey.json'
+    'config.coffee'
+    'generators/'
+    'testem.json'
+    'bower.json'
+  ]
+  https.get url, (res) ->
+    data = []
+    dataLen = 0
+
+    res.on('data', (chunk) ->
+      data.push(chunk)
+      dataLen += chunk.length
+    ).on('end', ->
+      buf = new Buffer(dataLen)
+
+      pos = 0
+      for dataItem in data
+        dataItem.copy(buf, pos)
+        pos += dataItem.length
+
+      zip = new AdmZip(buf)
+
+      filesToUpdate.forEach (file) ->
+        targetFile = "tapas-with-ember-master/#{file}"
+        if /\/$/.test(file)
+          zip.extractEntryTo(targetFile, file, false, true)
+        else
+          zip.extractEntryTo(targetFile, '', false, true)
+    )
+
+# --------------
+# Script Updates
+# --------------
+
+EMBER_BASE_URL = 'http://builds.emberjs.com'
+GITHUB_API_URL = 'https://api.github.com'
+EMBER = {}
+EMBER_DATA = {}
+['release', 'beta', 'canary'].forEach (build) ->
+  EMBER[build] =
+    prod: "#{EMBER_BASE_URL}/#{build}/ember.prod.js"
+    dev: "#{EMBER_BASE_URL}/#{build}/ember.js"
+  EMBER_DATA[build] =
+    prod: "#{EMBER_BASE_URL}/#{build}/ember-data.prod.js"
+    dev: "#{EMBER_BASE_URL}/#{build}/ember-data.js"
+
+EMBER['tag'] =
+  prod: "#{EMBER_BASE_URL}/tags/{{tag}}/ember.prod.js"
+  dev: "#{EMBER_BASE_URL}/tags/{{tag}}/ember.js"
+
+EMBER_DATA['tag'] =
+  prod: "#{EMBER_BASE_URL}/tags/{{tag}}/ember-data.prod.js"
+  dev: "#{EMBER_BASE_URL}/tags/{{tag}}/ember-data.js"
+
+downloadFile = (src, dest) ->
+  console.log('Downloading ' + src + ' to ' + dest)
+  data = ''
+  request = http.get src, (response) ->
+    response.on('data', (chunk) ->
+      data += chunk
+    )
+    response.on('end', ->
+      fs.writeFileSync(dest, data)
+    )
+
+downloadEmberFile = (src, dest) ->
+  downloadFile(src, "vendor/ember/#{dest}")
+
+listTags = (user, repo, since, name, command) ->
+  github.repos.getTags(user: user, repo: repo, (resp, tags) ->
+    for tag in tags
+      if semver.valid(tag.name) and !semver.lt(tag.name, since)
+        firstTag = tag.name unless firstTag
+        console.log "  #{tag.name}"
+    console.log "Install with cake -t \"#{firstTag}\" #{command}"
+  )
+
+installEmberFiles = (project, filename, options) ->
+  if 'tag' of options
+    # Download a Tag
+    tag = options.tag
+    tag = "v#{tag}" unless /^v/.test(tag)
+    downloadEmberFile(project['tag'].dev.replace(/{{tag}}/, tag),
+      "development/#{filename}")
+    downloadEmberFile(project['tag'].prod.replace(/{{tag}}/, tag),
+      "production/#{filename}")
+  else
+    # Download a Channel
+    channel = options.channel ? 'release'
+    downloadEmberFile project[channel].dev, "development/#{filename}"
+    downloadEmberFile project[channel].prod, "production/#{filename}"
+
+# Channel
+option '-c', '--channel "[CHANNEL_NAME]"',
+  'relase, beta, or canary (http://emberjs.com/builds)'
+
+# Tag
+option '-t', '--tag "[TAG_NAME]"',
+  'a tagged release to install. Run cake ember:list to see known tags'
+
+# -----
+# Ember
+# -----
+task 'ember:install', 'install latest Ember', (options) ->
+  installEmberFiles(EMBER, 'ember.js', options)
+
+task 'ember:list', 'list tagged relases of Ember since v1.0.0', (options) ->
+  listTags 'emberjs', 'ember.js', 'v1.0.0', 'Ember', 'ember:install'
+
+# ----------
+# Ember Data
+# ----------
+task 'ember-data:install', 'install latest Ember Data', (options) ->
+  options.channel or= 'beta'
+  installEmberFiles(EMBER_DATA, 'ember-data.js', options)
+
+task 'ember-data:list', 'list tagged relases of Ember Data', (options) ->
+  listTags 'emberjs', 'data', 'v0.0.1', 'Ember Data',
+    'ember-data:install'
+
+# -----------
+# Ember Model
+# -----------
+EMBER_MODEL =
+  dev: 'http://builds.erikbryn.com/ember-model/ember-model-latest.js'
+  prod: 'http://builds.erikbryn.com/ember-model/ember-model-latest.prod.js'
+
+task 'ember-model:install', 'install latest Ember Model', (options) ->
+  downloadEmberFile EMBER_MODEL.dev, 'development/ember-model.js'
+  downloadEmberFile EMBER_MODEL.prod, 'production/ember-model.js'

test/fixtures/Dockerfile/Dockerfile

@@ -0,0 +1,97 @@
+FROM ubuntu:14.04
+
+MAINTAINER Wesley Hales <wesleyhales@gmail.com>
+
+# Install.
+RUN \
+  sed -i 's/# \(.*multiverse$\)/\1/g' /etc/apt/sources.list && \
+  apt-get update && \
+  apt-get -y upgrade && \
+  apt-get install -y build-essential && \
+  apt-get install -y software-properties-common && \
+  apt-get install -y byobu curl git htop man unzip vim wget && \
+  rm -rf /var/lib/apt/lists/*
+
+# Set environment variables.
+ENV HOME /root
+
+# Define working directory.
+WORKDIR /root
+
+# Install Java.
+RUN \
+  echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \
+  add-apt-repository -y ppa:webupd8team/java && \
+  apt-get update && \
+  apt-get install -y oracle-java7-installer && \
+  rm -rf /var/lib/apt/lists/* \
+  echo "done"
+
+# Install Phantom2 build requirements    (Won't build on systems < 2GB ram)
+RUN \
+  sudo apt-get update && apt-get -y install g++ flex bison gperf ruby perl \
+  libsqlite3-dev libfontconfig1-dev libicu-dev libfreetype6 libssl-dev libjpeg-dev libqt5webkit5-dev
+
+#####################################build latest phantom
+######################################+++++ only do this in dev when needed
+
+#RUN rm -rf phantomjs
+
+#RUN git clone git://github.com/ariya/phantomjs.git
+
+#RUN cd /root/phantomjs/ && ./build.sh --confirm
+
+#RUN ln -s /root/phantomjs/bin/phantomjs /usr/bin/phantomjs
+######################################+++++ END only do this in dev when needed
+
+######################################+++++ comment out when building new version of phantomjs
+ADD phantomjs /root/phantomjs
+
+RUN ln -s /root/phantomjs /usr/bin/phantomjs
+######################################+++++ END comment out when building new version of phantomjs
+
+RUN git clone git://github.com/wesleyhales/speedgun.git
+
+#RUN mkdir /root/speedgun/core/reports
+
+#VOLUME ["/root/speedgun/core/reports"]
+
+RUN cd speedgun/core && phantomjs --ssl-protocol=any --ignore-ssl-errors=yes speedgun.js http://www.google.com performance csv
+
+RUN cd /root && wget https://dl.dropboxusercontent.com/u/12278845/server.tar
+
+RUN cd /root && tar -xvf server.tar
+
+#RUN echo "cd /root/jboss-as-7.1.1.Final-fluxui/ && ./bin/standalone.sh --server-config=standalone-full.xml -b 0.0.0.0" >> /root/.bashrc
+
+# install maven
+RUN sudo apt-get update && apt-get install -y maven
+
+ADD src /root/src
+ADD pom.xml /root/pom.xml
+RUN mvn clean install
+
+#RUN cp -rf /root/target/speedgun.war /root/jboss-as-7.1.1.Final-fluxui/standalone/deployments/
+
+RUN ln -s /root/target/speedgun /root/jboss-as-7.1.1.Final-fluxui/standalone/deployments/speedgun.war
+
+RUN touch /root/jboss-as-7.1.1.Final-fluxui/standalone/deployments/speedgun.war.dodeploy
+
+# Cleanup old JMS queue
+RUN rm -rf /root/jboss-as-7.1.1.Final-fluxui/standalone/tmp/ /root/jboss-as-7.1.1.Final-fluxui/standalone/data/*
+
+RUN mkdir /root/jboss-as-7.1.1.Final-fluxui/speedgun
+RUN cd /root/jboss-as-7.1.1.Final-fluxui/speedgun && curl -O https://raw.githubusercontent.com/wesleyhales/speedgun/master/core/speedgun.js
+RUN cd /root/jboss-as-7.1.1.Final-fluxui/speedgun && curl -O https://raw.githubusercontent.com/wesleyhales/speedgun/master/core/config.json
+
+COPY server-entrypoint.sh /
+
+ENTRYPOINT ["/server-entrypoint.sh"]
+
+RUN apt-get install -y postgresql-client
+
+COPY speedgun.sql /
+
+EXPOSE 3306 8080 8443
+
+#CMD ["postgres"]

test/fixtures/Makefile/Makefile

@@ -0,0 +1,5 @@
+SUBDIRS:=components test
+.PHONY: ${SUBDIRS} clean
+all:${SUBDIRS}
+${SUBDIRS}:
+	${MAKE} -C $@ all

test/fixtures/Maven POM/pom.xml

@@ -0,0 +1,57 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+<groupId>awilbur.personal</groupId>
+ <artifactId>hudsel</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <packaging>jar</packaging>
+<name>hudsel</name>
+<properties>
+ <suiteXmlFile>src/test/resources/testng.xml</suiteXmlFile>
+ <skipTests>false</skipTests>
+ </properties>
+<dependencies>
+ <!-- Adding TestNG for unit test support -->
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>6.8</version>
+ </dependency>
+ <!-- Adding Selenium dependency -->
+ <dependency>
+ <groupId>org.seleniumhq.selenium</groupId>
+ <artifactId>selenium-server</artifactId>
+ <version>2.41.0</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <!-- using the compiler plug-in to specify that this project is to be compiled with JDK 1.6 -->
+ <!-- This is needed so that we get the JDK annotation support that was introduced recently -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.6</version>
+ <configuration>
+ <!-- You can specify a specific testng.xml file here <suiteXmlFiles>
+ <suiteXmlFile>src/test/resources/testng-sample.xml</suiteXmlFile> </suiteXmlFiles> -->
+ <!-- Or dynamically with something like '-DsuiteXmlFile=src/test/resources/testng-sample.xml' -->
+ <suiteXmlFiles>
+ <suiteXmlFile>${suiteXmlFile}</suiteXmlFile>
+ </suiteXmlFiles>
+ <!-- Build with '-DskipTests=true' to bypass test execution @ build time Default: false -->
+ <skipTests>${skipTests}</skipTests>
+ <testFailureIgnore>true</testFailureIgnore>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </project>

test/test_blob.rb

@@ -202,6 +202,7 @@ class TestBlob < Minitest::Test
     assert sample_blob_memory("JavaScript/gen-js-linguist-thrift.js").generated?
     assert sample_blob_memory("Ruby/gen-rb-linguist-thrift.rb").generated?
     assert sample_blob_memory("Objective-C/gen-cocoa-linguist-thrift.m").generated?
+    assert sample_blob_memory("PHP/ThriftGenerated.php").generated?
 
     # Generated JNI
     assert sample_blob_memory("C/jni_layer.h").generated?

test/test_file_blob.rb

@@ -294,6 +294,8 @@ class TestFileBlob < Minitest::Test
     assert !sample_blob("Godeps/Godeps.json").vendored?
     assert sample_blob("Godeps/_workspace/src/github.com/kr/s3/sign.go").vendored?
 
+    assert sample_blob(".indent.pro").vendored?
+
     # Rails vendor/
     assert sample_blob("vendor/plugins/will_paginate/lib/will_paginate.rb").vendored?
 

test/test_grammars.rb

@@ -6,7 +6,6 @@ class TestGrammars < Minitest::Test
   # List of projects that are allowed without licenses
   PROJECT_WHITELIST = [
     "vendor/grammars/Sublime-Lasso",
-    "vendor/grammars/ant.tmbundle",
     "vendor/grammars/sublime-spintools",
     "vendor/grammars/blitzmax"
   ].freeze
@@ -40,6 +39,7 @@ class TestGrammars < Minitest::Test
     "c9118c370411f2f049c746c0fd096554e877aea2", # perl6fe
     "8ccf886749c32fb7e65d4d1316a7ed0479c93dc9", # language-less
     "2f03492b52d7dd83b4e7472f01b87c6121e5b1a4", # monkey
+    "241e5ddbb4423d792216783e9f668bd670b026e4", # ant.tmbundle
     "bdab9fdc21e6790b479ccb5945b78bc0f6ce2493"  # language-blade
   ].freeze
 

test/test_heuristics.rb

@@ -73,6 +73,15 @@ class TestHeuristcs < Minitest::Test
     })
   end
 
+  def test_cls_by_heuristics
+    assert_heuristics({
+      "TeX" => all_fixtures("TeX", "*.cls"),
+      nil => all_fixtures("Apex", "*.cls"),
+      nil => all_fixtures("OpenEdge ABL", "*.cls"),
+      nil => all_fixtures("Visual Basic", "*.cls"),
+    })
+  end
+
   def test_cs_by_heuristics
     assert_heuristics({
       "C#" => all_fixtures("C#", "*.cs"),

test/test_language.rb

@@ -60,8 +60,8 @@ class TestLanguage < Minitest::Test
     assert_equal Language['SuperCollider'], Language.find_by_alias('supercollider')
     assert_equal Language['TeX'], Language.find_by_alias('tex')
     assert_equal Language['TypeScript'], Language.find_by_alias('ts')
-    assert_equal Language['VimL'], Language.find_by_alias('vim')
-    assert_equal Language['VimL'], Language.find_by_alias('viml')
+    assert_equal Language['Vim script'], Language.find_by_alias('vim')
+    assert_equal Language['Vim script'], Language.find_by_alias('viml')
     assert_equal Language['reStructuredText'], Language.find_by_alias('rst')
     assert_equal Language['YAML'], Language.find_by_alias('yml')
     assert_nil Language.find_by_alias(nil)
@@ -90,7 +90,7 @@ class TestLanguage < Minitest::Test
     assert_equal Language['Ruby'], Language['Ruby'].group
 
     # Test a few special groups
-    assert_equal Language['Assembly'], Language['GAS'].group
+    assert_equal Language['Assembly'], Language['Unix Assembly'].group
     assert_equal Language['C'], Language['OpenCL'].group
     assert_equal Language['Haskell'], Language['Literate Haskell'].group
     assert_equal Language['Java'], Language['Java Server Pages'].group
@@ -106,39 +106,6 @@ class TestLanguage < Minitest::Test
     end
   end
 
-  # Used for code search indexing. Changing any of these values may
-  # require reindexing repositories.
-  def test_search_term
-    assert_equal 'perl',        Language['Perl'].search_term
-    assert_equal 'python',      Language['Python'].search_term
-    assert_equal 'ruby',        Language['Ruby'].search_term
-    assert_equal 'common-lisp', Language['Common Lisp'].search_term
-    assert_equal 'html+erb',    Language['HTML+ERB'].search_term
-    assert_equal 'max/msp',     Language['Max'].search_term
-    assert_equal 'puppet',      Language['Puppet'].search_term
-    assert_equal 'pure-data',   Language['Pure Data'].search_term
-
-    assert_equal 'aspx-vb',       Language['ASP'].search_term
-    assert_equal 'as3',           Language['ActionScript'].search_term
-    assert_equal 'nasm',          Language['Assembly'].search_term
-    assert_equal 'bat',           Language['Batchfile'].search_term
-    assert_equal 'csharp',        Language['C#'].search_term
-    assert_equal 'cpp',           Language['C++'].search_term
-    assert_equal 'cfm',           Language['ColdFusion'].search_term
-    assert_equal 'dpatch',        Language['Darcs Patch'].search_term
-    assert_equal 'fsharp',        Language['F#'].search_term
-    assert_equal 'pot',           Language['Gettext Catalog'].search_term
-    assert_equal 'irc',           Language['IRC log'].search_term
-    assert_equal 'lhs',           Language['Literate Haskell'].search_term
-    assert_equal 'mirah',         Language['Mirah'].search_term
-    assert_equal 'raw',           Language['Raw token data'].search_term
-    assert_equal 'bash',          Language['Shell'].search_term
-    assert_equal 'vim',           Language['VimL'].search_term
-    assert_equal 'jsp',           Language['Java Server Pages'].search_term
-    assert_equal 'rst',           Language['reStructuredText'].search_term
-    assert_equal 'supercollider', Language['SuperCollider'].search_term
-  end
-
   def test_popular
     assert Language['Ruby'].popular?
     assert Language['Perl'].popular?
@@ -203,10 +170,16 @@ class TestLanguage < Minitest::Test
 
   def test_find_by_extension
     assert_equal [], Language.find_by_extension('.factor-rc')
-    assert_equal [], Language.find_by_extension('foo.rb')
-    assert_equal [Language['Ruby']], Language.find_by_extension('rb')
-    assert_equal [Language['Ruby']], Language.find_by_extension('.rb')
-    assert_equal [Language['Limbo'], Language['M'], Language['MUF'], Language['Mathematica'], Language['Matlab'], Language['Mercury'], Language['Objective-C']], Language.find_by_extension('.m')
+    assert_equal [Language['Limbo'], Language['M'], Language['MUF'], Language['Mathematica'], Language['Matlab'], Language['Mercury'], Language['Objective-C']], Language.find_by_extension('foo.m')
+    assert_equal [Language['Ruby']], Language.find_by_extension('foo.rb')
+    assert_equal [Language['Ruby']], Language.find_by_extension('foo/bar.rb')
+    assert_equal [Language['Ruby']], Language.find_by_extension('PKGBUILD.rb')
+    assert_equal ['C', 'C++', 'Objective-C'], Language.find_by_extension('foo.h').map(&:name).sort
+    assert_equal [], Language.find_by_extension('rb')
+    assert_equal [], Language.find_by_extension('.null')
+    assert_equal [Language['HTML+Django']], Language.find_by_extension('index.jinja')
+    assert_equal [Language['Chapel']], Language.find_by_extension('examples/hello.chpl')
+    assert_equal [], Language.find_by_filename('F.I.L.E.')
   end
 
   def test_find_all_by_extension
@@ -219,22 +192,17 @@ class TestLanguage < Minitest::Test
 
   def test_find_by_filename
     assert_equal [Language['Shell']], Language.find_by_filename('PKGBUILD')
-    assert_equal [Language['Ruby']], Language.find_by_filename('foo.rb')
-    assert_equal [Language['Ruby']], Language.find_by_filename('foo/bar.rb')
     assert_equal [Language['Ruby']], Language.find_by_filename('Rakefile')
-    assert_equal [Language['Ruby']], Language.find_by_filename('PKGBUILD.rb')
     assert_equal Language['ApacheConf'], Language.find_by_filename('httpd.conf').first
     assert_equal [Language['ApacheConf']], Language.find_by_filename('.htaccess')
     assert_equal Language['Nginx'], Language.find_by_filename('nginx.conf').first
-    assert_equal ['C', 'C++', 'Objective-C'], Language.find_by_filename('foo.h').map(&:name).sort
+    assert_equal [], Language.find_by_filename('foo.rb')
     assert_equal [], Language.find_by_filename('rb')
     assert_equal [], Language.find_by_filename('.null')
     assert_equal [Language['Shell']], Language.find_by_filename('.bashrc')
     assert_equal [Language['Shell']], Language.find_by_filename('bash_profile')
     assert_equal [Language['Shell']], Language.find_by_filename('.zshrc')
     assert_equal [Language['Clojure']], Language.find_by_filename('riemann.config')
-    assert_equal [Language['HTML+Django']], Language.find_by_filename('index.jinja')
-    assert_equal [Language['Chapel']], Language.find_by_filename('examples/hello.chpl')
   end
 
   def test_find_by_interpreter
@@ -296,9 +264,9 @@ class TestLanguage < Minitest::Test
   end
 
   def test_doesnt_blow_up_with_blank_lookup
-    assert_equal nil, Language.find_by_alias('')
-    assert_equal nil, Language.find_by_name(nil)
-    assert_equal nil, Language[""]
+    assert_nil Language.find_by_alias('')
+    assert_nil Language.find_by_name(nil)
+    assert_nil Language[""]
   end
 
   def test_name
@@ -344,13 +312,6 @@ class TestLanguage < Minitest::Test
     assert_equal 'text', Language['FORTRAN'].ace_mode
   end
 
-  def test_ace_modes
-    silence_warnings do
-      assert Language.ace_modes.include?(Language['Ruby'])
-      assert Language.ace_modes.include?(Language['FORTRAN'])
-    end
-  end
-
   def test_codemirror_mode
     assert_equal 'ruby', Language['Ruby'].codemirror_mode
     assert_equal 'javascript', Language['JavaScript'].codemirror_mode
@@ -378,17 +339,6 @@ class TestLanguage < Minitest::Test
     assert Language['SuperCollider'].extensions.include?('.scd')
   end
 
-  def test_primary_extension
-    assert_equal '.pl', Language['Perl'].primary_extension
-    assert_equal '.py', Language['Python'].primary_extension
-    assert_equal '.rb', Language['Ruby'].primary_extension
-    assert_equal '.js', Language['JavaScript'].primary_extension
-    assert_equal '.coffee', Language['CoffeeScript'].primary_extension
-    assert_equal '.t', Language['Turing'].primary_extension
-    assert_equal '.ts', Language['TypeScript'].primary_extension
-    assert_equal '.sc', Language['SuperCollider'].primary_extension
-  end
-
   def test_eql
     assert Language['Ruby'].eql?(Language['Ruby'])
     assert !Language['Ruby'].eql?(Language['Python'])
@@ -426,6 +376,14 @@ class TestLanguage < Minitest::Test
     assert missing.empty?, message
   end
 
+  def test_all_languages_have_a_valid_id
+    invalid = Language.all.select { |language| language.language_id < 0 || language.language_id >= (2**31 - 1) }
+
+    message = "The following languages do not have a valid language_id. Please use script/set-language-ids --update as per the contribution guidelines.\n"
+    invalid.each { |language| message << "#{language.name}\n" }
+    assert invalid.empty?, message
+  end
+
   def test_all_language_id_are_unique
     duplicates = Language.all.group_by{ |language| language.language_id }.select { |k, v| v.size > 1 }.map(&:first)
 

test/test_modelines.rb

@@ -4,7 +4,11 @@ class TestModelines < Minitest::Test
   include Linguist
 
   def assert_modeline(language, blob)
-    assert_equal language, Linguist::Strategy::Modeline.call(blob).first
+    if language.nil?
+      assert_nil Linguist::Strategy::Modeline.call(blob).first
+    else
+      assert_equal language, Linguist::Strategy::Modeline.call(blob).first
+    end
   end
 
   def test_modeline_strategy

test/test_shebang.rb

@@ -4,7 +4,11 @@ class TestShebang < Minitest::Test
   include Linguist
 
   def assert_interpreter(interpreter, body)
-    assert_equal interpreter, Shebang.interpreter(body)
+    if interpreter.nil?
+      assert_nil Shebang.interpreter(body)
+    else
+      assert_equal interpreter, Shebang.interpreter(body)
+    end
   end
 
   def test_shebangs

vendor/README.md

@@ -6,6 +6,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 <!-- Everything below this line is auto-generated by script/list-grammars. Manual edits will be lost -->
 - **1C Enterprise:** [xDrivenDevelopment/atom-language-1c-bsl](https://github.com/xDrivenDevelopment/atom-language-1c-bsl)
 - **ABAP:** [pvl/abap.tmbundle](https://github.com/pvl/abap.tmbundle)
+- **ABNF:** [sanssecours/ABNF.tmbundle](https://github.com/sanssecours/ABNF.tmbundle)
 - **ActionScript:** [simongregory/actionscript3-tmbundle](https://github.com/simongregory/actionscript3-tmbundle)
 - **Ada:** [textmate/ada.tmbundle](https://github.com/textmate/ada.tmbundle)
 - **Agda:** [mokus0/Agda.tmbundle](https://github.com/mokus0/Agda.tmbundle)
@@ -91,6 +92,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **DTrace:** [textmate/c.tmbundle](https://github.com/textmate/c.tmbundle)
 - **Dylan:** [textmate/dylan.tmbundle](https://github.com/textmate/dylan.tmbundle)
 - **Eagle:** [textmate/xml.tmbundle](https://github.com/textmate/xml.tmbundle)
+- **EBNF:** [sanssecours/EBNF.tmbundle](https://github.com/sanssecours/EBNF.tmbundle)
 - **eC:** [ecere/ec.tmbundle](https://github.com/ecere/ec.tmbundle)
 - **Ecere Projects:** [textmate/json.tmbundle](https://github.com/textmate/json.tmbundle)
 - **ECLiPSe:** [alnkpa/sublimeprolog](https://github.com/alnkpa/sublimeprolog)
@@ -123,6 +125,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **Gettext Catalog:** [textmate/gettext.tmbundle](https://github.com/textmate/gettext.tmbundle)
 - **GLSL:** [euler0/sublime-glsl](https://github.com/euler0/sublime-glsl)
 - **Glyph:** [textmate/tcl.tmbundle](https://github.com/textmate/tcl.tmbundle)
+- **GN:** [devoncarew/language-gn](https://github.com/devoncarew/language-gn)
 - **Gnuplot:** [mattfoster/gnuplot-tmbundle](https://github.com/mattfoster/gnuplot-tmbundle)
 - **Go:** [AlanQuatermain/go-tmbundle](https://github.com/AlanQuatermain/go-tmbundle)
 - **Golo:** [TypeUnsafe/sublime-golo](https://github.com/TypeUnsafe/sublime-golo)
@@ -136,7 +139,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **Groovy:** [textmate/groovy.tmbundle](https://github.com/textmate/groovy.tmbundle)
 - **Groovy Server Pages:** [textmate/java.tmbundle](https://github.com/textmate/java.tmbundle)
 - **Hack:** [textmate/php.tmbundle](https://github.com/textmate/php.tmbundle)
-- **Haml:** [textmate/ruby-haml.tmbundle](https://github.com/textmate/ruby-haml.tmbundle)
+- **Haml:** [ezekg/language-haml](https://github.com/ezekg/language-haml)
 - **Handlebars:** [daaain/Handlebars](https://github.com/daaain/Handlebars)
 - **Harbour:** [hernad/atom-language-harbour](https://github.com/hernad/atom-language-harbour)
 - **Haskell:** [atom-haskell/language-haskell](https://github.com/atom-haskell/language-haskell)
@@ -222,7 +225,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **NewLisp:** [textmate/lisp.tmbundle](https://github.com/textmate/lisp.tmbundle)
 - **Nginx:** [brandonwamboldt/sublime-nginx](https://github.com/brandonwamboldt/sublime-nginx)
 - **Nimrod:** [Varriount/NimLime](https://github.com/Varriount/NimLime)
-- **Ninja:** [textmate/ninja.tmbundle](https://github.com/textmate/ninja.tmbundle)
+- **Ninja:** [khyo/language-ninja](https://github.com/khyo/language-ninja)
 - **Nit:** [R4PaSs/Sublime-Nit](https://github.com/R4PaSs/Sublime-Nit)
 - **Nix:** [wmertens/sublime-nix](https://github.com/wmertens/sublime-nix)
 - **NSIS:** [github-linguist/NSIS](https://github.com/github-linguist/NSIS)
@@ -238,6 +241,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **OpenCL:** [textmate/c.tmbundle](https://github.com/textmate/c.tmbundle)
 - **OpenEdge ABL:** [jfairbank/Sublime-Text-2-OpenEdge-ABL](https://github.com/jfairbank/Sublime-Text-2-OpenEdge-ABL)
 - **OpenRC runscript:** [atom/language-shellscript](https://github.com/atom/language-shellscript)
+- **OpenType Feature File:** [Alhadis/language-fontforge](https://github.com/Alhadis/language-fontforge)
 - **Ox:** [andreashetland/sublime-text-ox](https://github.com/andreashetland/sublime-text-ox)
 - **Oz:** [eregon/oz-tmbundle](https://github.com/eregon/oz-tmbundle)
 - **Papyrus:** [Kapiainen/SublimePapyrus](https://github.com/Kapiainen/SublimePapyrus)
@@ -264,6 +268,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **Puppet:** [russCloak/SublimePuppet](https://github.com/russCloak/SublimePuppet)
 - **PureScript:** [purescript-contrib/atom-language-purescript](https://github.com/purescript-contrib/atom-language-purescript)
 - **Python:** [MagicStack/MagicPython](https://github.com/MagicStack/MagicPython)
+- **Python console:** [atom/language-python](https://github.com/atom/language-python)
 - **Python traceback:** [atom/language-python](https://github.com/atom/language-python)
 - **QMake:** [textmate/cpp-qt.tmbundle](https://github.com/textmate/cpp-qt.tmbundle)
 - **QML:** [skozlovf/Sublime-QML](https://github.com/skozlovf/Sublime-QML)
@@ -304,6 +309,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **SMT:** [SRI-CSL/SMT.tmbundle](https://github.com/SRI-CSL/SMT.tmbundle)
 - **SourcePawn:** [github-linguist/sublime-sourcepawn](https://github.com/github-linguist/sublime-sourcepawn)
 - **SPARQL:** [peta/turtle.tmbundle](https://github.com/peta/turtle.tmbundle)
+- **Spline Font Database:** [Alhadis/language-fontforge](https://github.com/Alhadis/language-fontforge)
 - **SQF:** [JonBons/Sublime-SQF-Language](https://github.com/JonBons/Sublime-SQF-Language)
 - **SQL:** [textmate/sql.tmbundle](https://github.com/textmate/sql.tmbundle)
 - **SQLPL:** [textmate/sql.tmbundle](https://github.com/textmate/sql.tmbundle)