Release v6.0.1 (#4016)

* Update grammar submodule refs

* Bump version to v.6.0.1

.gitmodules

@@ -301,9 +301,6 @@
 [submodule "vendor/grammars/make.tmbundle"]
 	path = vendor/grammars/make.tmbundle
 	url = https://github.com/textmate/make.tmbundle
-[submodule "vendor/grammars/matlab.tmbundle"]
-	path = vendor/grammars/matlab.tmbundle
-	url = https://github.com/textmate/matlab.tmbundle
 [submodule "vendor/grammars/maven.tmbundle"]
 	path = vendor/grammars/maven.tmbundle
 	url = https://github.com/textmate/maven.tmbundle
@@ -886,6 +883,9 @@
 [submodule "vendor/grammars/Syntax-highlighting-for-PostCSS"]
 	path = vendor/grammars/Syntax-highlighting-for-PostCSS
 	url = https://github.com/hudochenkov/Syntax-highlighting-for-PostCSS
+[submodule "vendor/grammars/MATLAB-Language-grammar"]
+	path = vendor/grammars/MATLAB-Language-grammar
+	url = https://github.com/mathworks/MATLAB-Language-grammar
 [submodule "vendor/grammars/javadoc.tmbundle"]
 	path = vendor/grammars/javadoc.tmbundle
 	url = https://github.com/textmate/javadoc.tmbundle

github-linguist.gemspec

@@ -2,7 +2,7 @@ require File.expand_path('../lib/linguist/version', __FILE__)
 
 Gem::Specification.new do |s|
   s.name    = 'github-linguist'
-  s.version = Linguist::VERSION
+  s.version = ENV['GEM_VERSION'] || Linguist::VERSION
   s.summary = "GitHub Language detection"
   s.description = 'We use this library at GitHub to detect blob languages, highlight code, ignore binary files, suppress generated files in diffs, and generate language breakdown graphs.'
 

grammars.yml

@@ -48,6 +48,8 @@ vendor/grammars/Lean.tmbundle:
 - source.lean
 vendor/grammars/LiveScript.tmbundle:
 - source.livescript
+vendor/grammars/MATLAB-Language-grammar:
+- source.matlab
 vendor/grammars/MQL5-sublime:
 - source.mql5
 vendor/grammars/MagicPython:
@@ -541,9 +543,6 @@ vendor/grammars/marko-tmbundle:
 - text.marko
 vendor/grammars/mathematica-tmbundle:
 - source.mathematica
-vendor/grammars/matlab.tmbundle:
-- source.matlab
-- source.octave
 vendor/grammars/maven.tmbundle:
 - text.xml.pom
 vendor/grammars/mediawiki.tmbundle:

lib/linguist/version.rb

@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "5.3.3"
+  VERSION = "6.0.1"
 end

script/grammar-compiler

@@ -6,6 +6,8 @@ cd "$(dirname "$0")/.."
 image="linguist/grammar-compiler:latest"
 mkdir -p grammars
 
+docker pull $image
+
 exec docker run --rm \
     -u $(id -u $USER):$(id -g $USER) \
     -v $PWD:/src/linguist \

tools/grammars/Dockerfile

@@ -1,16 +1,13 @@
 FROM golang:1.9.2
 
-RUN apt-get update
-RUN apt-get upgrade -y
-RUN apt-get install -y curl gnupg
+WORKDIR /go/src/github.com/github/linguist/tools/grammars
 
-RUN curl -sL https://deb.nodesource.com/setup_6.x | bash -
-RUN apt-get install -y nodejs
-RUN npm install -g season
-
-RUN apt-get install -y cmake
-RUN cd /tmp && git clone https://github.com/vmg/pcre
-RUN mkdir -p /tmp/pcre/build && cd /tmp/pcre/build && \
+RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - && \
+	apt-get update && \
+	apt-get install -y nodejs cmake && \
+	npm install -g season && \
+	cd /tmp && git clone https://github.com/vmg/pcre && \
+	mkdir -p /tmp/pcre/build && cd /tmp/pcre/build && \
 	cmake .. \
 		-DPCRE_SUPPORT_JIT=ON \
 		-DPCRE_SUPPORT_UTF=ON \
@@ -22,14 +19,12 @@ RUN mkdir -p /tmp/pcre/build && cd /tmp/pcre/build && \
 		-DPCRE_BUILD_PCREGREP=OFF \
 		-DPCRE_BUILD_TESTS=OFF \
 		-G "Unix Makefiles" && \
-    make && make install
-RUN rm -rf /tmp/pcre
+    make && make install && \
+	rm -rf /tmp/pcre && \
+	cd /go && go get -u github.com/golang/dep/cmd/dep && \
+	rm -rf /var/lib/apt/lists/*
 
-RUN go get -u github.com/golang/dep/cmd/dep
-WORKDIR /go/src/github.com/github/linguist/tools/grammars
 COPY . .
-
-RUN dep ensure
-RUN go install ./cmd/grammar-compiler
+RUN dep ensure && go install ./cmd/grammar-compiler
 
 ENTRYPOINT ["grammar-compiler"]

tools/grammars/compiler/converter.go

@@ -145,7 +145,10 @@ func (conv *Converter) ConvertGrammars(update bool) error {
 		repo.FixRules(knownScopes)
 
 		if update {
-			conv.grammars[source] = repo.Scopes()
+			scopes := repo.Scopes()
+			if len(scopes) > 0 {
+				conv.grammars[source] = scopes
+			}
 		} else {
 			expected := conv.grammars[source]
 			repo.CompareScopes(expected)

tools/grammars/compiler/loader.go

@@ -108,6 +108,11 @@ func isValidGrammar(path string, info os.FileInfo) bool {
 		return false
 	}
 
+	// Tree-Sitter grammars are not supported
+	if strings.HasPrefix(filepath.Base(path), "tree-sitter-") {
+		return false
+	}
+
 	dir := filepath.Dir(path)
 	ext := filepath.Ext(path)
 
@@ -117,7 +122,7 @@ func isValidGrammar(path string, info os.FileInfo) bool {
 	case ".tmlanguage", ".yaml-tmlanguage":
 		return true
 	case ".cson", ".json":
-		return strings.HasSuffix(dir, "/grammars")
+		return strings.HasSuffix(dir, "/grammars") || strings.HasSuffix(dir, "/syntaxes")
 	default:
 		return false
 	}

tools/grammars/compiler/loader_fs.go

@@ -6,6 +6,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"sort"
 	"strings"
 )
 
@@ -14,14 +15,43 @@ type fsLoader struct {
 	abspath string
 }
 
+var preferredGrammars = map[string]int{
+	".tmlanguage":      0,
+	".cson":            1,
+	".json":            1,
+	".plist":           2,
+	".yaml-tmlanguage": 3,
+}
+
+func findPreferredExtension(ext []string) string {
+	if len(ext) > 1 {
+		sort.Slice(ext, func(i, j int) bool {
+			a := strings.ToLower(ext[i])
+			b := strings.ToLower(ext[j])
+			return preferredGrammars[a] < preferredGrammars[b]
+		})
+	}
+	return ext[0]
+}
+
 func (l *fsLoader) findGrammars() (files []string, err error) {
+	grammars := make(map[string][]string)
+
 	err = filepath.Walk(l.abspath,
 		func(path string, info os.FileInfo, err error) error {
 			if err == nil && isValidGrammar(path, info) {
-				files = append(files, path)
+				ext := filepath.Ext(path)
+				base := path[0 : len(path)-len(ext)]
+				grammars[base] = append(grammars[base], ext)
 			}
 			return nil
 		})
+
+	for base, ext := range grammars {
+		pref := findPreferredExtension(ext)
+		files = append(files, base+pref)
+	}
+
 	return
 }
 

vendor/README.md

@@ -213,7 +213,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **Marko:** [marko-js/marko-tmbundle](https://github.com/marko-js/marko-tmbundle)
 - **Mask:** [tenbits/sublime-mask](https://github.com/tenbits/sublime-mask)
 - **Mathematica:** [shadanan/mathematica-tmbundle](https://github.com/shadanan/mathematica-tmbundle)
-- **Matlab:** [textmate/matlab.tmbundle](https://github.com/textmate/matlab.tmbundle)
+- **Matlab:** [mathworks/MATLAB-Language-grammar](https://github.com/mathworks/MATLAB-Language-grammar)
 - **Maven POM:** [textmate/maven.tmbundle](https://github.com/textmate/maven.tmbundle)
 - **Max:** [textmate/json.tmbundle](https://github.com/textmate/json.tmbundle)
 - **MAXScript:** [Alhadis/language-maxscript](https://github.com/Alhadis/language-maxscript)

vendor/licenses/grammar/MATLAB-Language-grammar.txt

@@ -0,0 +1,14 @@
+---
+type: grammar
+name: MATLAB-Language-grammar
+license: bsd-2-clause
+---
+Copyright 2018 The MathWorks, Inc.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.