New Grammars Compiler (#3915)

* grammars: Update several grammars with compat issues

* [WIP] Add new grammar conversion tools

* Wrap in a Docker script

* Proper Dockerfile support

* Add Javadoc grammar

* Remove NPM package.json

* Remove superfluous test

This is now always checked by the grammars compiler

* Update JSyntax grammar to new submodule

* Approve Javadoc license

* grammars: Remove checked-in dependencies

* grammars: Add regex checks to the compiler

* grammars: Point Oz to its actual submodule

* grammars: Refactor compiler to group errors by repo

* grammars: Cleanups to error reporting

tools/grammars/compiler/loader.go

@@ -0,0 +1,124 @@
+package compiler
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	grammar "github.com/github/linguist/tools/grammars/proto"
+)
+
+type LoadedFile struct {
+	Path string
+	Rule *grammar.Rule
+}
+
+func (f *LoadedFile) String() string {
+	return fmt.Sprintf("`%s` (in `%s`)", f.Rule.ScopeName, f.Path)
+}
+
+type Repository struct {
+	Source   string
+	Upstream string
+	Files    map[string]*LoadedFile
+	Errors   []error
+}
+
+func newRepository(src string) *Repository {
+	return &Repository{
+		Source: src,
+		Files:  make(map[string]*LoadedFile),
+	}
+}
+
+func (repo *Repository) String() string {
+	str := fmt.Sprintf("repository `%s`", repo.Source)
+	if repo.Upstream != "" {
+		str = str + fmt.Sprintf(" (from %s)", repo.Upstream)
+	}
+	return str
+}
+
+func (repo *Repository) Fail(err error) {
+	repo.Errors = append(repo.Errors, err)
+}
+
+func (repo *Repository) AddFile(path string, rule *grammar.Rule, uk []string) {
+	file := &LoadedFile{
+		Path: path,
+		Rule: rule,
+	}
+
+	repo.Files[rule.ScopeName] = file
+	if len(uk) > 0 {
+		repo.Fail(&UnknownKeysError{file, uk})
+	}
+}
+
+func toMap(slice []string) map[string]bool {
+	m := make(map[string]bool)
+	for _, s := range slice {
+		m[s] = true
+	}
+	return m
+}
+
+func (repo *Repository) CompareScopes(scopes []string) {
+	expected := toMap(scopes)
+
+	for scope, file := range repo.Files {
+		if !expected[scope] {
+			repo.Fail(&UnexpectedScopeError{file, scope})
+		}
+	}
+
+	for scope := range expected {
+		if _, ok := repo.Files[scope]; !ok {
+			repo.Fail(&MissingScopeError{scope})
+		}
+	}
+}
+
+func (repo *Repository) FixRules(knownScopes map[string]*Repository) {
+	for _, file := range repo.Files {
+		w := walker{
+			File:    file,
+			Known:   knownScopes,
+			Missing: make(map[string]bool),
+		}
+
+		w.walk(file.Rule)
+		repo.Errors = append(repo.Errors, w.Errors...)
+
+	}
+}
+
+func (repo *Repository) Scopes() (scopes []string) {
+	for s := range repo.Files {
+		scopes = append(scopes, s)
+	}
+	sort.Strings(scopes)
+	return
+}
+
+func isValidGrammar(path string, info os.FileInfo) bool {
+	if info.IsDir() {
+		return false
+	}
+
+	dir := filepath.Dir(path)
+	ext := filepath.Ext(path)
+
+	switch strings.ToLower(ext) {
+	case ".plist":
+		return strings.HasSuffix(dir, "/Syntaxes")
+	case ".tmlanguage", ".yaml-tmlanguage":
+		return true
+	case ".cson", ".json":
+		return strings.HasSuffix(dir, "/grammars")
+	default:
+		return false
+	}
+}