New Grammars Compiler (#3915)

* grammars: Update several grammars with compat issues

* [WIP] Add new grammar conversion tools

* Wrap in a Docker script

* Proper Dockerfile support

* Add Javadoc grammar

* Remove NPM package.json

* Remove superfluous test

This is now always checked by the grammars compiler

* Update JSyntax grammar to new submodule

* Approve Javadoc license

* grammars: Remove checked-in dependencies

* grammars: Add regex checks to the compiler

* grammars: Point Oz to its actual submodule

* grammars: Refactor compiler to group errors by repo

* grammars: Cleanups to error reporting

tools/grammars/compiler/loader_url.go

@@ -0,0 +1,93 @@
+package compiler
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
+	"strings"
+)
+
+type urlLoader struct {
+	*Repository
+}
+
+func (l *urlLoader) loadTarball(r io.Reader) {
+	gzf, err := gzip.NewReader(r)
+	if err != nil {
+		l.Fail(err)
+		return
+	}
+	defer gzf.Close()
+
+	tarReader := tar.NewReader(gzf)
+	for true {
+		header, err := tarReader.Next()
+
+		if err != nil {
+			if err != io.EOF {
+				l.Fail(err)
+			}
+			return
+		}
+
+		if isValidGrammar(header.Name, header.FileInfo()) {
+			data, err := ioutil.ReadAll(tarReader)
+			if err != nil {
+				l.Fail(err)
+				return
+			}
+
+			ext := filepath.Ext(header.Name)
+			rule, unknown, err := ConvertProto(ext, data)
+			if err != nil {
+				l.Fail(&ConversionError{header.Name, err})
+				continue
+			}
+
+			if _, ok := l.Files[rule.ScopeName]; ok {
+				continue
+			}
+
+			l.AddFile(header.Name, rule, unknown)
+		}
+	}
+}
+
+func (l *urlLoader) load() {
+	res, err := http.Get(l.Source)
+	if err != nil {
+		l.Fail(err)
+		return
+	}
+	defer res.Body.Close()
+
+	if strings.HasSuffix(l.Source, ".tar.gz") {
+		l.loadTarball(res.Body)
+		return
+	}
+
+	data, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		l.Fail(err)
+		return
+	}
+
+	ext := filepath.Ext(l.Source)
+	filename := filepath.Base(l.Source)
+	rule, unknown, err := ConvertProto(ext, data)
+	if err != nil {
+		l.Fail(&ConversionError{filename, err})
+		return
+	}
+
+	l.AddFile(filename, rule, unknown)
+}
+
+func LoadFromURL(src string) *Repository {
+	loader := urlLoader{newRepository(src)}
+	loader.load()
+	return loader.Repository
+}