Update submod sha

Pygments has separate highlighters for "python" (meaning python 2) and "python3" (meaning python 3). As a result, there are lots of files out there (especially ReSTructured text) that contain code blocks whose language is explicitly given as "python3" or "py3". Currently these are unrecognized by linguist. Instead, we should use our python highlighter for them (which works for both python 2 and python 3).

References:
  http://pygments.org/docs/lexers/#pygments.lexers.python.Python3Lexer
  https://github.com/github/markup/issues/1019
  https://github.com/python-trio/async_generator/pull/12

.gitmodules

@@ -739,9 +739,6 @@
 [submodule "vendor/grammars/language-emacs-lisp"]
 	path = vendor/grammars/language-emacs-lisp
 	url = https://github.com/Alhadis/language-emacs-lisp
-[submodule "vendor/grammars/language-babel"]
-	path = vendor/grammars/language-babel
-	url = https://github.com/github-linguist/language-babel
 [submodule "vendor/CodeMirror"]
 	path = vendor/CodeMirror
 	url = https://github.com/codemirror/CodeMirror
@@ -898,3 +895,7 @@
 [submodule "vendor/grammars/atom-language-nextflow"]
 	path = vendor/grammars/atom-language-nextflow
 	url = https://github.com/nextflow-io/atom-language-nextflow
+[submodule "vendor/grammars/language-babel"]
+	path = vendor/grammars/language-babel
+	url = https://github.com/lildude/language-babel
+	branch = make-pcre-friendly

README.md

@@ -97,7 +97,7 @@ If the language stats bar is reporting a language that you don't expect:
 1. If the files are misclassified, search for [open issues][issues] to see if anyone else has already reported the issue. Any information you can add, especially links to public repositories, is helpful. You can also use the [manual overrides](#overrides) feature to correctly classify them in your repository.
 1. If there are no reported issues of this misclassification, [open an issue][new-issue] and include a link to the repository or a sample of the code that is being misclassified.
 
-Keep in mind that the repository language stats are only [updated when you push changes](#how-linguist-works-on-github-com), and the results are cached for the lifetime of your repository. If you have not made any changes to your repository in a while, you may find pushing another change will correct the stats.
+Keep in mind that the repository language stats are only [updated when you push changes](#how-linguist-works-on-githubcom), and the results are cached for the lifetime of your repository. If you have not made any changes to your repository in a while, you may find pushing another change will correct the stats.
 
 ### My repository isn't showing my language
 

lib/linguist/languages.yml

@@ -301,6 +301,7 @@ Assembly:
   type: programming
   color: "#6E4C13"
   aliases:
+  - asm
   - nasm
   extensions:
   - ".asm"
@@ -3631,6 +3632,7 @@ Python:
   - python3
   aliases:
   - rusthon
+  - python3
   language_id: 303
 Python console:
   type: programming

lib/linguist/version.rb

@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "6.0.0"
+  VERSION = "6.0.1"
 end

tools/grammars/Dockerfile

@@ -1,16 +1,13 @@
 FROM golang:1.9.2
 
-RUN apt-get update
+WORKDIR /go/src/github.com/github/linguist/tools/grammars
-RUN apt-get upgrade -y
-RUN apt-get install -y curl gnupg
 
-RUN curl -sL https://deb.nodesource.com/setup_6.x | bash -
+RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - && \
-RUN apt-get install -y nodejs
+	apt-get update && \
-RUN npm install -g season
+	apt-get install -y nodejs cmake && \
-
+	npm install -g season && \
-RUN apt-get install -y cmake
+	cd /tmp && git clone https://github.com/vmg/pcre && \
-RUN cd /tmp && git clone https://github.com/vmg/pcre
+	mkdir -p /tmp/pcre/build && cd /tmp/pcre/build && \
-RUN mkdir -p /tmp/pcre/build && cd /tmp/pcre/build && \
 	cmake .. \
 		-DPCRE_SUPPORT_JIT=ON \
 		-DPCRE_SUPPORT_UTF=ON \
@@ -22,14 +19,12 @@ RUN mkdir -p /tmp/pcre/build && cd /tmp/pcre/build && \
 		-DPCRE_BUILD_PCREGREP=OFF \
 		-DPCRE_BUILD_TESTS=OFF \
 		-G "Unix Makefiles" && \
-    make && make install
+    make && make install && \
-RUN rm -rf /tmp/pcre
+	rm -rf /tmp/pcre && \
+	cd /go && go get -u github.com/golang/dep/cmd/dep && \
+	rm -rf /var/lib/apt/lists/*
 
-RUN go get -u github.com/golang/dep/cmd/dep
-WORKDIR /go/src/github.com/github/linguist/tools/grammars
 COPY . .
-
+RUN dep ensure && go install ./cmd/grammar-compiler
-RUN dep ensure
-RUN go install ./cmd/grammar-compiler
 
 ENTRYPOINT ["grammar-compiler"]

tools/grammars/compiler/loader.go

@@ -108,6 +108,11 @@ func isValidGrammar(path string, info os.FileInfo) bool {
 		return false
 	}
 
+	// Tree-Sitter grammars are not supported
+	if strings.HasPrefix(filepath.Base(path), "tree-sitter-") {
+		return false
+	}
+
 	dir := filepath.Dir(path)
 	ext := filepath.Ext(path)
 
@@ -117,7 +122,7 @@ func isValidGrammar(path string, info os.FileInfo) bool {
 	case ".tmlanguage", ".yaml-tmlanguage":
 		return true
 	case ".cson", ".json":
-		return strings.HasSuffix(dir, "/grammars")
+		return strings.HasSuffix(dir, "/grammars") || strings.HasSuffix(dir, "/syntaxes")
 	default:
 		return false
 	}

tools/grammars/compiler/loader_fs.go

@@ -6,6 +6,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"sort"
 	"strings"
 )
 
@@ -14,14 +15,43 @@ type fsLoader struct {
 	abspath string
 }
 
+var preferredGrammars = map[string]int{
+	".tmlanguage":      0,
+	".cson":            1,
+	".json":            1,
+	".plist":           2,
+	".yaml-tmlanguage": 3,
+}
+
+func findPreferredExtension(ext []string) string {
+	if len(ext) > 1 {
+		sort.Slice(ext, func(i, j int) bool {
+			a := strings.ToLower(ext[i])
+			b := strings.ToLower(ext[j])
+			return preferredGrammars[a] < preferredGrammars[b]
+		})
+	}
+	return ext[0]
+}
+
 func (l *fsLoader) findGrammars() (files []string, err error) {
+	grammars := make(map[string][]string)
+
 	err = filepath.Walk(l.abspath,
 		func(path string, info os.FileInfo, err error) error {
 			if err == nil && isValidGrammar(path, info) {
-				files = append(files, path)
+				ext := filepath.Ext(path)
+				base := path[0 : len(path)-len(ext)]
+				grammars[base] = append(grammars[base], ext)
 			}
 			return nil
 		})
+
+	for base, ext := range grammars {
+		pref := findPreferredExtension(ext)
+		files = append(files, base+pref)
+	}
+
 	return
 }
 

vendor/README.md

@@ -181,7 +181,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **JSON5:** [atom/language-javascript](https://github.com/atom/language-javascript)
 - **JSONiq:** [wcandillon/language-jsoniq](https://github.com/wcandillon/language-jsoniq)
 - **JSONLD:** [atom/language-javascript](https://github.com/atom/language-javascript)
-- **JSX:** [github-linguist/language-babel](https://github.com/github-linguist/language-babel)
+- **JSX:** [lildude/language-babel](https://github.com/lildude/language-babel)
 - **Julia:** [JuliaEditorSupport/atom-language-julia](https://github.com/JuliaEditorSupport/atom-language-julia)
 - **Jupyter Notebook:** [textmate/json.tmbundle](https://github.com/textmate/json.tmbundle)
 - **KiCad Layout:** [Alhadis/language-pcb](https://github.com/Alhadis/language-pcb)