New Grammars Compiler (#3915)

* grammars: Update several grammars with compat issues

* [WIP] Add new grammar conversion tools

* Wrap in a Docker script

* Proper Dockerfile support

* Add Javadoc grammar

* Remove NPM package.json

* Remove superfluous test

This is now always checked by the grammars compiler

* Update JSyntax grammar to new submodule

* Approve Javadoc license

* grammars: Remove checked-in dependencies

* grammars: Add regex checks to the compiler

* grammars: Point Oz to its actual submodule

* grammars: Refactor compiler to group errors by repo

* grammars: Cleanups to error reporting

tools/grammars/compiler/loader_fs.go

@@ -0,0 +1,80 @@
+package compiler
+
+import (
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+type fsLoader struct {
+	*Repository
+	abspath string
+}
+
+func (l *fsLoader) findGrammars() (files []string, err error) {
+	err = filepath.Walk(l.abspath,
+		func(path string, info os.FileInfo, err error) error {
+			if err == nil && isValidGrammar(path, info) {
+				files = append(files, path)
+			}
+			return nil
+		})
+	return
+}
+
+func (l *fsLoader) load() {
+	grammars, err := l.findGrammars()
+	if err != nil {
+		l.Fail(err)
+		return
+	}
+
+	for _, path := range grammars {
+		data, err := ioutil.ReadFile(path)
+		if err != nil {
+			l.Fail(err)
+			continue
+		}
+
+		if rel, err := filepath.Rel(l.abspath, path); err == nil {
+			path = rel
+		}
+
+		rule, unknown, err := ConvertProto(filepath.Ext(path), data)
+		if err != nil {
+			l.Fail(&ConversionError{path, err})
+			continue
+		}
+
+		if _, ok := l.Files[rule.ScopeName]; ok {
+			continue
+		}
+
+		l.AddFile(path, rule, unknown)
+	}
+}
+
+func gitRemoteName(path string) (string, error) {
+	remote, err := exec.Command("git", "-C", path, "remote", "get-url", "origin").Output()
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(remote)), nil
+}
+
+func LoadFromFilesystem(root, src string) *Repository {
+	loader := fsLoader{
+		Repository: newRepository(src),
+		abspath:    path.Join(root, src),
+	}
+	loader.load()
+
+	if ups, err := gitRemoteName(loader.abspath); err == nil {
+		loader.Repository.Upstream = ups
+	}
+
+	return loader.Repository
+}