New Grammars Compiler (#3915)

* grammars: Update several grammars with compat issues

* [WIP] Add new grammar conversion tools

* Wrap in a Docker script

* Proper Dockerfile support

* Add Javadoc grammar

* Remove NPM package.json

* Remove superfluous test

This is now always checked by the grammars compiler

* Update JSyntax grammar to new submodule

* Approve Javadoc license

* grammars: Remove checked-in dependencies

* grammars: Add regex checks to the compiler

* grammars: Point Oz to its actual submodule

* grammars: Refactor compiler to group errors by repo

* grammars: Cleanups to error reporting

tools/grammars/cmd/grammar-compiler/main.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/github/linguist/tools/grammars/compiler"
+)
+
+var linguistRoot = flag.String("linguist", "", "path to Linguist installation")
+var protoOut = flag.String("proto", "", "dump Protobuf library")
+var jsonOut = flag.String("json", "", "dump JSON output")
+var addGrammar = flag.String("add", "", "add a new grammar source")
+var updateList = flag.Bool("update", false, "update grammars.yml instead of verifying its contents")
+var report = flag.String("report", "", "write report to file")
+
+func fatal(err error) {
+	fmt.Fprintf(os.Stderr, "FATAL: %s\n", err)
+	os.Exit(1)
+}
+
+func main() {
+	flag.Parse()
+
+	if _, err := exec.LookPath("csonc"); err != nil {
+		fatal(err)
+	}
+
+	if *linguistRoot == "" {
+		cwd, err := os.Getwd()
+		if err != nil {
+			fatal(err)
+		}
+		*linguistRoot = cwd
+	}
+
+	conv, err := compiler.NewConverter(*linguistRoot)
+	if err != nil {
+		fatal(err)
+	}
+
+	if *addGrammar != "" {
+		if err := conv.AddGrammar(*addGrammar); err != nil {
+			fatal(err)
+		}
+	}
+
+	if err := conv.ConvertGrammars(*updateList); err != nil {
+		fatal(err)
+	}
+
+	if err := conv.WriteGrammarList(); err != nil {
+		fatal(err)
+	}
+
+	if *protoOut != "" {
+		if err := conv.WriteProto(*protoOut); err != nil {
+			fatal(err)
+		}
+	}
+
+	if *jsonOut != "" {
+		if err := conv.WriteJSON(*jsonOut); err != nil {
+			fatal(err)
+		}
+	}
+
+	if *report == "" {
+		conv.Report(os.Stderr)
+	} else {
+		f, err := os.Create(*report)
+		if err != nil {
+			fatal(err)
+		}
+		conv.Report(f)
+		f.Close()
+	}
+}