New Grammars Compiler (#3915)

* grammars: Update several grammars with compat issues

* [WIP] Add new grammar conversion tools

* Wrap in a Docker script

* Proper Dockerfile support

* Add Javadoc grammar

* Remove NPM package.json

* Remove superfluous test

This is now always checked by the grammars compiler

* Update JSyntax grammar to new submodule

* Approve Javadoc license

* grammars: Remove checked-in dependencies

* grammars: Add regex checks to the compiler

* grammars: Point Oz to its actual submodule

* grammars: Refactor compiler to group errors by repo

* grammars: Cleanups to error reporting

tools/grammars/compiler/pcre.go

@@ -0,0 +1,68 @@
+package compiler
+
+import (
+	"fmt"
+
+	"github.com/github/linguist/tools/grammars/pcre"
+)
+
+type replacement struct {
+	pos int
+	len int
+	val string
+}
+
+func fixRegex(re string) (string, bool) {
+	var (
+		replace     []replacement
+		escape      = false
+		hasBackRefs = false
+	)
+
+	for i, ch := range re {
+		if escape {
+			if ch == 'h' {
+				replace = append(replace, replacement{i - 1, 2, "[[:xdigit:]]"})
+			}
+			if '0' <= ch && ch <= '9' {
+				hasBackRefs = true
+			}
+		}
+		escape = !escape && ch == '\\'
+	}
+
+	if len(replace) > 0 {
+		reb := []byte(re)
+		offset := 0
+		for _, repl := range replace {
+			reb = append(
+				reb[:offset+repl.pos],
+				append([]byte(repl.val), reb[offset+repl.pos+repl.len:]...)...)
+			offset += len(repl.val) - repl.len
+		}
+		return string(reb), hasBackRefs
+	}
+
+	return re, hasBackRefs
+}
+
+func CheckPCRE(re string) (string, error) {
+	hasBackRefs := false
+
+	if re == "" {
+		return "", nil
+	}
+	if len(re) > 32*1024 {
+		return "", fmt.Errorf(
+			"regex %s: definition too long (%d bytes)",
+			pcre.RegexPP(re), len(re))
+	}
+
+	re, hasBackRefs = fixRegex(re)
+	if !hasBackRefs {
+		if err := pcre.CheckRegexp(re, pcre.DefaultFlags); err != nil {
+			return "", err
+		}
+	}
+	return re, nil
+}