compiler: Do not output empty grammars

.gitmodules

@@ -301,6 +301,9 @@
 [submodule "vendor/grammars/make.tmbundle"]
 	path = vendor/grammars/make.tmbundle
 	url = https://github.com/textmate/make.tmbundle
+[submodule "vendor/grammars/matlab.tmbundle"]
+	path = vendor/grammars/matlab.tmbundle
+	url = https://github.com/textmate/matlab.tmbundle
 [submodule "vendor/grammars/maven.tmbundle"]
 	path = vendor/grammars/maven.tmbundle
 	url = https://github.com/textmate/maven.tmbundle
@@ -739,6 +742,9 @@
 [submodule "vendor/grammars/language-emacs-lisp"]
 	path = vendor/grammars/language-emacs-lisp
 	url = https://github.com/Alhadis/language-emacs-lisp
+[submodule "vendor/grammars/language-babel"]
+	path = vendor/grammars/language-babel
+	url = https://github.com/github-linguist/language-babel
 [submodule "vendor/CodeMirror"]
 	path = vendor/CodeMirror
 	url = https://github.com/codemirror/CodeMirror
@@ -880,9 +886,6 @@
 [submodule "vendor/grammars/Syntax-highlighting-for-PostCSS"]
 	path = vendor/grammars/Syntax-highlighting-for-PostCSS
 	url = https://github.com/hudochenkov/Syntax-highlighting-for-PostCSS
-[submodule "vendor/grammars/MATLAB-Language-grammar"]
-	path = vendor/grammars/MATLAB-Language-grammar
-	url = https://github.com/mathworks/MATLAB-Language-grammar
 [submodule "vendor/grammars/javadoc.tmbundle"]
 	path = vendor/grammars/javadoc.tmbundle
 	url = https://github.com/textmate/javadoc.tmbundle
@@ -895,7 +898,3 @@
 [submodule "vendor/grammars/atom-language-nextflow"]
 	path = vendor/grammars/atom-language-nextflow
 	url = https://github.com/nextflow-io/atom-language-nextflow
-[submodule "vendor/grammars/language-babel"]
-	path = vendor/grammars/language-babel
-	url = https://github.com/lildude/language-babel
-	branch = make-pcre-friendly

README.md

@@ -97,7 +97,7 @@ If the language stats bar is reporting a language that you don't expect:
 1. If the files are misclassified, search for [open issues][issues] to see if anyone else has already reported the issue. Any information you can add, especially links to public repositories, is helpful. You can also use the [manual overrides](#overrides) feature to correctly classify them in your repository.
 1. If there are no reported issues of this misclassification, [open an issue][new-issue] and include a link to the repository or a sample of the code that is being misclassified.
 
-Keep in mind that the repository language stats are only [updated when you push changes](#how-linguist-works-on-githubcom), and the results are cached for the lifetime of your repository. If you have not made any changes to your repository in a while, you may find pushing another change will correct the stats.
+Keep in mind that the repository language stats are only [updated when you push changes](#how-linguist-works-on-github-com), and the results are cached for the lifetime of your repository. If you have not made any changes to your repository in a while, you may find pushing another change will correct the stats.
 
 ### My repository isn't showing my language
 

github-linguist.gemspec

@@ -2,7 +2,7 @@ require File.expand_path('../lib/linguist/version', __FILE__)
 
 Gem::Specification.new do |s|
   s.name    = 'github-linguist'
-  s.version = ENV['GEM_VERSION'] || Linguist::VERSION
+  s.version = Linguist::VERSION
   s.summary = "GitHub Language detection"
   s.description = 'We use this library at GitHub to detect blob languages, highlight code, ignore binary files, suppress generated files in diffs, and generate language breakdown graphs.'
 

grammars.yml

@@ -48,8 +48,6 @@ vendor/grammars/Lean.tmbundle:
 - source.lean
 vendor/grammars/LiveScript.tmbundle:
 - source.livescript
-vendor/grammars/MATLAB-Language-grammar:
-- source.matlab
 vendor/grammars/MQL5-sublime:
 - source.mql5
 vendor/grammars/MagicPython:
@@ -543,6 +541,9 @@ vendor/grammars/marko-tmbundle:
 - text.marko
 vendor/grammars/mathematica-tmbundle:
 - source.mathematica
+vendor/grammars/matlab.tmbundle:
+- source.matlab
+- source.octave
 vendor/grammars/maven.tmbundle:
 - text.xml.pom
 vendor/grammars/mediawiki.tmbundle:

lib/linguist/languages.yml

@@ -301,7 +301,6 @@ Assembly:
   type: programming
   color: "#6E4C13"
   aliases:
-  - asm
   - nasm
   extensions:
   - ".asm"
@@ -3632,7 +3631,6 @@ Python:
   - python3
   aliases:
   - rusthon
-  - python3
   language_id: 303
 Python console:
   type: programming

lib/linguist/version.rb

@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "6.0.1"
+  VERSION = "5.3.3"
 end

script/grammar-compiler

@@ -6,8 +6,6 @@ cd "$(dirname "$0")/.."
 image="linguist/grammar-compiler:latest"
 mkdir -p grammars
 
-docker pull $image
-
 exec docker run --rm \
     -u $(id -u $USER):$(id -g $USER) \
     -v $PWD:/src/linguist \

tools/grammars/Dockerfile

@@ -1,13 +1,16 @@
 FROM golang:1.9.2
 
-WORKDIR /go/src/github.com/github/linguist/tools/grammars
+RUN apt-get update
+RUN apt-get upgrade -y
+RUN apt-get install -y curl gnupg
 
-RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - && \
-	apt-get update && \
-	apt-get install -y nodejs cmake && \
-	npm install -g season && \
-	cd /tmp && git clone https://github.com/vmg/pcre && \
-	mkdir -p /tmp/pcre/build && cd /tmp/pcre/build && \
+RUN curl -sL https://deb.nodesource.com/setup_6.x | bash -
+RUN apt-get install -y nodejs
+RUN npm install -g season
+
+RUN apt-get install -y cmake
+RUN cd /tmp && git clone https://github.com/vmg/pcre
+RUN mkdir -p /tmp/pcre/build && cd /tmp/pcre/build && \
 	cmake .. \
 		-DPCRE_SUPPORT_JIT=ON \
 		-DPCRE_SUPPORT_UTF=ON \
@@ -19,12 +22,14 @@ RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - && \
 		-DPCRE_BUILD_PCREGREP=OFF \
 		-DPCRE_BUILD_TESTS=OFF \
 		-G "Unix Makefiles" && \
-    make && make install && \
-	rm -rf /tmp/pcre && \
-	cd /go && go get -u github.com/golang/dep/cmd/dep && \
-	rm -rf /var/lib/apt/lists/*
+    make && make install
+RUN rm -rf /tmp/pcre
 
+RUN go get -u github.com/golang/dep/cmd/dep
+WORKDIR /go/src/github.com/github/linguist/tools/grammars
 COPY . .
-RUN dep ensure && go install ./cmd/grammar-compiler
+
+RUN dep ensure
+RUN go install ./cmd/grammar-compiler
 
 ENTRYPOINT ["grammar-compiler"]

tools/grammars/compiler/loader.go

@@ -108,11 +108,6 @@ func isValidGrammar(path string, info os.FileInfo) bool {
 		return false
 	}
 
-	// Tree-Sitter grammars are not supported
-	if strings.HasPrefix(filepath.Base(path), "tree-sitter-") {
-		return false
-	}
-
 	dir := filepath.Dir(path)
 	ext := filepath.Ext(path)
 
@@ -122,7 +117,7 @@ func isValidGrammar(path string, info os.FileInfo) bool {
 	case ".tmlanguage", ".yaml-tmlanguage":
 		return true
 	case ".cson", ".json":
-		return strings.HasSuffix(dir, "/grammars") || strings.HasSuffix(dir, "/syntaxes")
+		return strings.HasSuffix(dir, "/grammars")
 	default:
 		return false
 	}

tools/grammars/compiler/loader_fs.go

@@ -6,7 +6,6 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
-	"sort"
 	"strings"
 )
 
@@ -15,43 +14,14 @@ type fsLoader struct {
 	abspath string
 }
 
-var preferredGrammars = map[string]int{
-	".tmlanguage":      0,
-	".cson":            1,
-	".json":            1,
-	".plist":           2,
-	".yaml-tmlanguage": 3,
-}
-
-func findPreferredExtension(ext []string) string {
-	if len(ext) > 1 {
-		sort.Slice(ext, func(i, j int) bool {
-			a := strings.ToLower(ext[i])
-			b := strings.ToLower(ext[j])
-			return preferredGrammars[a] < preferredGrammars[b]
-		})
-	}
-	return ext[0]
-}
-
 func (l *fsLoader) findGrammars() (files []string, err error) {
-	grammars := make(map[string][]string)
-
 	err = filepath.Walk(l.abspath,
 		func(path string, info os.FileInfo, err error) error {
 			if err == nil && isValidGrammar(path, info) {
-				ext := filepath.Ext(path)
-				base := path[0 : len(path)-len(ext)]
-				grammars[base] = append(grammars[base], ext)
+				files = append(files, path)
 			}
 			return nil
 		})
-
-	for base, ext := range grammars {
-		pref := findPreferredExtension(ext)
-		files = append(files, base+pref)
-	}
-
 	return
 }
 

vendor/README.md

@@ -181,7 +181,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **JSON5:** [atom/language-javascript](https://github.com/atom/language-javascript)
 - **JSONiq:** [wcandillon/language-jsoniq](https://github.com/wcandillon/language-jsoniq)
 - **JSONLD:** [atom/language-javascript](https://github.com/atom/language-javascript)
-- **JSX:** [lildude/language-babel](https://github.com/lildude/language-babel)
+- **JSX:** [github-linguist/language-babel](https://github.com/github-linguist/language-babel)
 - **Julia:** [JuliaEditorSupport/atom-language-julia](https://github.com/JuliaEditorSupport/atom-language-julia)
 - **Jupyter Notebook:** [textmate/json.tmbundle](https://github.com/textmate/json.tmbundle)
 - **KiCad Layout:** [Alhadis/language-pcb](https://github.com/Alhadis/language-pcb)
@@ -213,7 +213,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **Marko:** [marko-js/marko-tmbundle](https://github.com/marko-js/marko-tmbundle)
 - **Mask:** [tenbits/sublime-mask](https://github.com/tenbits/sublime-mask)
 - **Mathematica:** [shadanan/mathematica-tmbundle](https://github.com/shadanan/mathematica-tmbundle)
-- **Matlab:** [mathworks/MATLAB-Language-grammar](https://github.com/mathworks/MATLAB-Language-grammar)
+- **Matlab:** [textmate/matlab.tmbundle](https://github.com/textmate/matlab.tmbundle)
 - **Maven POM:** [textmate/maven.tmbundle](https://github.com/textmate/maven.tmbundle)
 - **Max:** [textmate/json.tmbundle](https://github.com/textmate/json.tmbundle)
 - **MAXScript:** [Alhadis/language-maxscript](https://github.com/Alhadis/language-maxscript)

vendor/licenses/grammar/MATLAB-Language-grammar.txt

@@ -1,14 +0,0 @@
----
-type: grammar
-name: MATLAB-Language-grammar
-license: bsd-2-clause
----
-Copyright 2018 The MathWorks, Inc.
-
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.