Merge pull request #1954 from github/test-grammar-licenses

Test that all grammar submodules have an appropriate license

.gitmodules

@@ -503,7 +503,6 @@
 [submodule "vendor/grammars/sublime-mask"]
 	path = vendor/grammars/sublime-mask
 	url = https://github.com/tenbits/sublime-mask
-	branch = release
 [submodule "vendor/grammars/sublime_cobol"]
 	path = vendor/grammars/sublime_cobol
 	url = https://bitbucket.org/bitlang/sublime_cobol

test/test_grammars.rb

@@ -3,6 +3,14 @@ require_relative "./helper"
 class TestGrammars < Minitest::Test
   ROOT = File.expand_path("../..", __FILE__)
 
+  # These grammars have no license but have been grandfathered in. New grammars
+  # must have a license that allows redistribution.
+  UNLICENSED_GRAMMARS_WHITELIST = %w[
+    vendor/grammars/Sublime-Lasso
+    vendor/grammars/Sublime-REBOL
+    vendor/grammars/x86-assembly-textmate-bundle
+  ].freeze
+
   def setup
     @grammars = YAML.load(File.read(File.join(ROOT, "grammars.yml")))
   end
@@ -14,12 +22,11 @@ class TestGrammars < Minitest::Test
   end
 
   def test_submodules_are_in_sync
-    submodules = `git config --list --file "#{File.join(ROOT, ".gitmodules")}"`.lines.grep(/\.path=/).map { |line| line.chomp.split("=", 2).last }
     # Strip off paths inside the submodule so that just the submodule path remains.
     listed_submodules = @grammars.keys.grep(/vendor\/grammars/).map { |source| source[%r{vendor/grammars/[^/]+}] }
 
-    nonexistent_submodules = listed_submodules - submodules
+    nonexistent_submodules = listed_submodules - submodule_paths
-    unlisted_submodules = submodules - listed_submodules
+    unlisted_submodules = submodule_paths - listed_submodules
 
     message = ""
     unless nonexistent_submodules.empty?
@@ -49,4 +56,79 @@ class TestGrammars < Minitest::Test
       assert_equal v, actual[k], "The scopes listed for #{k} in grammars.yml don't match the scopes found in that repository"
     end
   end
+
+  def test_submodules_have_licenses
+    categories = submodule_paths.group_by do |submodule|
+      files = Dir[File.join(ROOT, submodule, "*")]
+      license = files.find { |path| File.basename(path) =~ /\blicense\b/i } || files.find { |path| File.basename(path) =~ /\bcopying\b/i }
+      if license.nil?
+        if readme = files.find { |path| File.basename(path) =~ /\Areadme\b/i }
+          license = readme if File.read(readme) =~ /\blicense\b/i
+        end
+      end
+      if license.nil?
+        :unlicensed
+      elsif classify_license(license)
+        :licensed
+      else
+        :unrecognized
+      end
+    end
+
+    unlicensed = categories[:unlicensed] || []
+    unrecognized = categories[:unrecognized] || []
+    disallowed_unlicensed = unlicensed - UNLICENSED_GRAMMARS_WHITELIST
+    disallowed_unrecognized = unrecognized - UNLICENSED_GRAMMARS_WHITELIST
+    extra_whitelist_entries = UNLICENSED_GRAMMARS_WHITELIST - (unlicensed | unrecognized)
+
+    message = ""
+    if disallowed_unlicensed.any?
+      message << "The following grammar submodules don't seem to have a license. All grammars must have a license that permits redistribution.\n"
+      message << disallowed_unlicensed.sort.join("\n")
+    end
+    if disallowed_unrecognized.any?
+      message << "\n\n" unless message.empty?
+      message << "The following grammar submodules have an unrecognized license. Please update #{__FILE__} to recognize the license.\n"
+      message << disallowed_unrecognized.sort.join("\n")
+    end
+    if extra_whitelist_entries.any?
+      message << "\n\n" unless message.empty?
+      message << "The following grammar submodules are listed in UNLICENSED_GRAMMARS_WHITELIST but either have a license (yay!)\n"
+      message << "or have been removed from the repository. Please remove them from the whitelist.\n"
+      message << extra_whitelist_entries.sort.join("\n")
+    end
+
+    assert disallowed_unlicensed.empty? && disallowed_unrecognized.empty? && extra_whitelist_entries.empty?, message
+  end
+
+  private
+
+  def submodule_paths
+    @submodule_paths ||= `git config --list --file "#{File.join(ROOT, ".gitmodules")}"`.lines.grep(/\.path=/).map { |line| line.chomp.split("=", 2).last }
+  end
+
+  def classify_license(path)
+    content = File.read(path)
+    if content.include?("Apache License") && content.include?("2.0")
+      "Apache 2.0"
+    elsif content.include?("GNU") && content =~ /general/i && content =~ /public/i
+      if content =~ /version 2/i
+        "GPLv2"
+      elsif content =~ /version 3/i
+        "GPLv3"
+      end
+    elsif content.include?("GPL") && content.include?("http://www.gnu.org/licenses/gpl.html")
+      "GPLv3"
+    elsif content.include?("Creative Commons")
+      "CC"
+    elsif content.include?("tidy-license.txt") || content.include?("If not otherwise specified (see below)")
+      "textmate"
+    elsif content =~ /^\s*[*-]\s+Redistribution/ || content.include?("Redistributions of source code")
+      "BSD"
+    elsif content.include?("Permission is hereby granted") || content =~ /\bMIT\b/
+      "MIT"
+    elsif content.include?("unlicense.org")
+      "unlicense"
+    end
+  end
 end