New Grammars Compiler (#3915)

* grammars: Update several grammars with compat issues

* [WIP] Add new grammar conversion tools

* Wrap in a Docker script

* Proper Dockerfile support

* Add Javadoc grammar

* Remove NPM package.json

* Remove superfluous test

This is now always checked by the grammars compiler

* Update JSyntax grammar to new submodule

* Approve Javadoc license

* grammars: Remove checked-in dependencies

* grammars: Add regex checks to the compiler

* grammars: Point Oz to its actual submodule

* grammars: Refactor compiler to group errors by repo

* grammars: Cleanups to error reporting

tools/grammars/compiler/walker.go

@@ -0,0 +1,79 @@
+package compiler
+
+import (
+	"strings"
+
+	grammar "github.com/github/linguist/tools/grammars/proto"
+)
+
+func (w *walker) checkInclude(rule *grammar.Rule) {
+	include := rule.Include
+
+	if include == "" || include[0] == '#' || include[0] == '$' {
+		return
+	}
+
+	if alias, ok := GrammarAliases[include]; ok {
+		rule.Include = alias
+		return
+	}
+
+	include = strings.Split(include, "#")[0]
+	_, ok := w.Known[include]
+	if !ok {
+		if !w.Missing[include] {
+			w.Missing[include] = true
+			w.Errors = append(w.Errors, &MissingIncludeError{w.File, include})
+		}
+		rule.Include = ""
+	}
+}
+
+func (w *walker) checkRegexps(rule *grammar.Rule) {
+	check := func(re string) string {
+		re2, err := CheckPCRE(re)
+		if err != nil {
+			w.Errors = append(w.Errors, &InvalidRegexError{w.File, err})
+		}
+		return re2
+	}
+
+	rule.Match = check(rule.Match)
+	rule.Begin = check(rule.Begin)
+	rule.While = check(rule.While)
+	rule.End = check(rule.End)
+}
+
+func (w *walker) walk(rule *grammar.Rule) {
+	w.checkInclude(rule)
+	w.checkRegexps(rule)
+
+	for _, rule := range rule.Patterns {
+		w.walk(rule)
+	}
+	for _, rule := range rule.Captures {
+		w.walk(rule)
+	}
+	for _, rule := range rule.BeginCaptures {
+		w.walk(rule)
+	}
+	for _, rule := range rule.WhileCaptures {
+		w.walk(rule)
+	}
+	for _, rule := range rule.EndCaptures {
+		w.walk(rule)
+	}
+	for _, rule := range rule.Repository {
+		w.walk(rule)
+	}
+	for _, rule := range rule.Injections {
+		w.walk(rule)
+	}
+}
+
+type walker struct {
+	File    *LoadedFile
+	Known   map[string]*Repository
+	Missing map[string]bool
+	Errors  []error
+}