Merge branch 'master' into newlisp

test/helper.rb

@@ -1,4 +1,4 @@
 require "bundler/setup"
-require "test/unit"
+require "minitest/autorun"
 require "mocha/setup"
 require "linguist"

test/test_blob.rb

@@ -1,6 +1,6 @@
 require_relative "./helper"
 
-class TestBlob < Test::Unit::TestCase
+class TestBlob < Minitest::Test
   include Linguist
 
   def setup

test/test_classifier.rb

@@ -1,6 +1,6 @@
 require_relative "./helper"
 
-class TestClassifier < Test::Unit::TestCase
+class TestClassifier < Minitest::Test
   include Linguist
 
   def samples_path

test/test_file_blob.rb

@@ -1,7 +1,6 @@
-require 'linguist/file_blob'
-require 'test/unit'
+require_relative "./helper"
 
-class TestFileBlob < Test::Unit::TestCase
+class TestFileBlob < Minitest::Test
   def test_extensions
     assert_equal [".gitignore"], Linguist::FileBlob.new(".gitignore").extensions
     assert_equal [".xml"],  Linguist::FileBlob.new("build.xml").extensions

test/test_generated.rb

@@ -0,0 +1,55 @@
+require_relative "./helper"
+
+class TestGenerated < Minitest::Test
+  include Linguist
+
+  def samples_path
+    File.expand_path("../../samples", __FILE__)
+  end
+
+  class DataLoadedError < StandardError; end
+
+  def generated_without_loading_data(name)
+    blob = File.join(samples_path, name)
+    begin
+      assert Generated.generated?(blob, lambda { raise DataLoadedError.new }), "#{name} was not recognized as a generated file"
+    rescue DataLoadedError
+      assert false, "Data was loaded when calling generated? on #{name}"
+    end
+  end
+
+  def generated_loading_data(name)
+    blob = File.join(samples_path, name)
+    assert_raises(DataLoadedError, "Data wasn't loaded when calling generated? on #{name}") do
+      Generated.generated?(blob, lambda { raise DataLoadedError.new })
+    end
+  end
+
+  def test_check_generated_without_loading_data
+    # Xcode project files
+    generated_without_loading_data("Binary/MainMenu.nib")
+    generated_without_loading_data("Dummy/foo.xcworkspacedata")
+    generated_without_loading_data("Dummy/foo.xcuserstate")
+
+    # .NET designer file
+    generated_without_loading_data("Dummu/foo.designer.cs")
+
+    # Composer generated composer.lock file
+    generated_without_loading_data("JSON/composer.lock")
+
+    # Node modules
+    generated_without_loading_data("Dummy/node_modules/foo.js")
+
+    # Godep saved dependencies
+    generated_without_loading_data("Godeps/Godeps.json")
+    generated_without_loading_data("Godeps/_workspace/src/github.com/kr/s3/sign.go")
+
+    # Generated by Zephir
+    generated_without_loading_data("C/exception.zep.c")
+    generated_without_loading_data("C/exception.zep.h")
+    generated_without_loading_data("PHP/exception.zep.php")
+
+    # Minified files
+    generated_loading_data("JavaScript/jquery-1.6.1.min.js")
+  end
+end

test/test_grammars.rb

@@ -0,0 +1,136 @@
+require_relative "./helper"
+
+class TestGrammars < Minitest::Test
+  ROOT = File.expand_path("../..", __FILE__)
+
+  # These grammars have no license but have been grandfathered in. New grammars
+  # must have a license that allows redistribution.
+  UNLICENSED_GRAMMARS_WHITELIST = %w[
+    vendor/grammars/Sublime-Lasso
+    vendor/grammars/Sublime-REBOL
+    vendor/grammars/x86-assembly-textmate-bundle
+  ].freeze
+
+  def setup
+    @grammars = YAML.load(File.read(File.join(ROOT, "grammars.yml")))
+  end
+
+  def test_no_duplicate_scopes
+    scopes = @grammars.values.flatten
+    duplicates = scopes.group_by { |s| s }.select { |k, v| v.length > 1 }.map(&:first)
+    assert duplicates.empty?, "The following scopes appear in grammars.yml more than once:\n#{duplicates.sort.join("\n")}"
+  end
+
+  def test_submodules_are_in_sync
+    # Strip off paths inside the submodule so that just the submodule path remains.
+    listed_submodules = @grammars.keys.grep(/vendor\/grammars/).map { |source| source[%r{vendor/grammars/[^/]+}] }
+
+    nonexistent_submodules = listed_submodules - submodule_paths
+    unlisted_submodules = submodule_paths - listed_submodules
+
+    message = ""
+    unless nonexistent_submodules.empty?
+      message << "The following submodules are listed in grammars.yml but don't seem to exist in the repository.\n"
+      message << "Either add them using `git submodule add` or remove them from grammars.yml.\n"
+      message << nonexistent_submodules.sort.join("\n")
+    end
+    unless unlisted_submodules.empty?
+      message << "\n" unless message.empty?
+      message << "The following submodules exist in the repository but aren't listed in grammars.yml.\n"
+      message << "Either add them to grammars.yml or remove them from the repository using `git rm`.\n"
+      message << unlisted_submodules.sort.join("\n")
+    end
+
+    assert nonexistent_submodules.empty? && unlisted_submodules.empty?, message
+  end
+
+  def test_local_scopes_are_in_sync
+    actual = YAML.load(`"#{File.join(ROOT, "script", "convert-grammars")}" --output - --no-install --no-remote`)
+    assert $?.success?, "script/convert-grammars failed"
+
+    # We're not checking remote grammars. That can take a long time and make CI
+    # flaky if network conditions are poor.
+    @grammars.delete_if { |k, v| k.start_with?("http:", "https:") }
+
+    @grammars.each do |k, v|
+      assert_equal v, actual[k], "The scopes listed for #{k} in grammars.yml don't match the scopes found in that repository"
+    end
+  end
+
+  def test_submodules_have_licenses
+    categories = submodule_paths.group_by do |submodule|
+      files = Dir[File.join(ROOT, submodule, "*")]
+      license = files.find { |path| File.basename(path) =~ /\b(un)?licen[cs]e\b/i } || files.find { |path| File.basename(path) =~ /\bcopying\b/i }
+      if license.nil?
+        if readme = files.find { |path| File.basename(path) =~ /\Areadme\b/i }
+          license = readme if File.read(readme) =~ /\blicen[cs]e\b/i
+        end
+      end
+      if license.nil?
+        :unlicensed
+      elsif classify_license(license)
+        :licensed
+      else
+        :unrecognized
+      end
+    end
+
+    unlicensed = categories[:unlicensed] || []
+    unrecognized = categories[:unrecognized] || []
+    disallowed_unlicensed = unlicensed - UNLICENSED_GRAMMARS_WHITELIST
+    disallowed_unrecognized = unrecognized - UNLICENSED_GRAMMARS_WHITELIST
+    extra_whitelist_entries = UNLICENSED_GRAMMARS_WHITELIST - (unlicensed | unrecognized)
+
+    message = ""
+    if disallowed_unlicensed.any?
+      message << "The following grammar submodules don't seem to have a license. All grammars must have a license that permits redistribution.\n"
+      message << disallowed_unlicensed.sort.join("\n")
+    end
+    if disallowed_unrecognized.any?
+      message << "\n\n" unless message.empty?
+      message << "The following grammar submodules have an unrecognized license. Please update #{__FILE__} to recognize the license.\n"
+      message << disallowed_unrecognized.sort.join("\n")
+    end
+    if extra_whitelist_entries.any?
+      message << "\n\n" unless message.empty?
+      message << "The following grammar submodules are listed in UNLICENSED_GRAMMARS_WHITELIST but either have a license (yay!)\n"
+      message << "or have been removed from the repository. Please remove them from the whitelist.\n"
+      message << extra_whitelist_entries.sort.join("\n")
+    end
+
+    assert disallowed_unlicensed.empty? && disallowed_unrecognized.empty? && extra_whitelist_entries.empty?, message
+  end
+
+  private
+
+  def submodule_paths
+    @submodule_paths ||= `git config --list --file "#{File.join(ROOT, ".gitmodules")}"`.lines.grep(/\.path=/).map { |line| line.chomp.split("=", 2).last }
+  end
+
+  def classify_license(path)
+    content = File.read(path)
+    if content.include?("Apache License") && content.include?("2.0")
+      "Apache 2.0"
+    elsif content.include?("GNU") && content =~ /general/i && content =~ /public/i
+      if content =~ /version 2/i
+        "GPLv2"
+      elsif content =~ /version 3/i
+        "GPLv3"
+      end
+    elsif content.include?("GPL") && content.include?("http://www.gnu.org/licenses/gpl.html")
+      "GPLv3"
+    elsif content.include?("Creative Commons")
+      "CC"
+    elsif content.include?("tidy-license.txt") || content.include?("If not otherwise specified (see below)")
+      "textmate"
+    elsif content =~ /^\s*[*-]\s+Redistribution/ || content.include?("Redistributions of source code")
+      "BSD"
+    elsif content.include?("Permission is hereby granted") || content =~ /\bMIT\b/
+      "MIT"
+    elsif content.include?("unlicense.org")
+      "unlicense"
+    elsif content.include?("http://www.wtfpl.net/txt/copying/")
+      "WTFPL"
+    end
+  end
+end

test/test_heuristics.rb

@@ -1,6 +1,6 @@
 require_relative "./helper"
 
-class TestHeuristcs < Test::Unit::TestCase
+class TestHeuristcs < Minitest::Test
   include Linguist
 
   def samples_path
@@ -140,6 +140,13 @@ class TestHeuristcs < Test::Unit::TestCase
     })
   end
 
+  def test_cs_by_heuristics
+    assert_heuristics({
+      "C#" => all_fixtures("C#", "*.cs"),
+      "Smalltalk" => all_fixtures("Smalltalk", "*.cs")
+    })
+  end
+
   def assert_heuristics(hash)
     candidates = hash.keys.map { |l| Language[l] }
 

test/test_language.rb

@@ -1,6 +1,6 @@
 require_relative "./helper"
 
-class TestLanguage < Test::Unit::TestCase
+class TestLanguage < Minitest::Test
   include Linguist
 
   def test_find_by_alias
@@ -198,7 +198,7 @@ class TestLanguage < Test::Unit::TestCase
   def test_find_all_by_extension
     Language.all.each do |language|
       language.extensions.each do |extension|
-        assert_include Language.find_by_extension(extension), language
+        assert_includes Language.find_by_extension(extension), language
       end
     end
   end
@@ -283,7 +283,7 @@ class TestLanguage < Test::Unit::TestCase
   end
 
   def test_error_without_name
-    assert_raise ArgumentError do
+    assert_raises ArgumentError do
       Language.new :name => nil
     end
   end

test/test_md5.rb

@@ -1,6 +1,6 @@
 require_relative "./helper"
 
-class TestMD5 < Test::Unit::TestCase
+class TestMD5 < Minitest::Test
   include Linguist
 
   def test_hexdigest_string
@@ -12,28 +12,28 @@ class TestMD5 < Test::Unit::TestCase
     assert_equal "450c1ae043459546517b3dd2f98250f0", MD5.hexdigest(:foo)
     assert_equal "f06967526af9d7a512594b0a81b31ede", MD5.hexdigest(:bar)
 
-    assert_not_equal MD5.hexdigest("foo"), MD5.hexdigest(:foo)
+    refute_equal MD5.hexdigest("foo"), MD5.hexdigest(:foo)
   end
 
   def test_hexdigest_integer
     assert_equal "7605ec17fd7fd213fdcd23cac302cbb4", MD5.hexdigest(1)
     assert_equal "097c311a46d330e4e119ba2b1dc0f9a5", MD5.hexdigest(2)
 
-    assert_not_equal MD5.hexdigest("1"), MD5.hexdigest(1)
+    refute_equal MD5.hexdigest("1"), MD5.hexdigest(1)
   end
 
   def test_hexdigest_boolean
     assert_equal "a690a0615820e2e5c53901d8b8958509", MD5.hexdigest(true)
     assert_equal "fca6a9b459e702fa93513c6a8b8c5dfe", MD5.hexdigest(false)
 
-    assert_not_equal MD5.hexdigest("true"), MD5.hexdigest(true)
-    assert_not_equal MD5.hexdigest("false"), MD5.hexdigest(false)
+    refute_equal MD5.hexdigest("true"), MD5.hexdigest(true)
+    refute_equal MD5.hexdigest("false"), MD5.hexdigest(false)
   end
 
   def test_hexdigest_nil
     assert_equal "35589a1cc0b3ca90fc52d0e711c0c434", MD5.hexdigest(nil)
 
-    assert_not_equal MD5.hexdigest("nil"), MD5.hexdigest(nil)
+    refute_equal MD5.hexdigest("nil"), MD5.hexdigest(nil)
   end
 
   def test_hexdigest_array
@@ -49,7 +49,7 @@ class TestMD5 < Test::Unit::TestCase
     assert_equal "868ee214faf277829a85667cf332749f", MD5.hexdigest({:a => 1})
     assert_equal "fa9df957c2b26de6fcca9d062ea8701e", MD5.hexdigest({:b => 2})
 
-    assert_not_equal MD5.hexdigest([:b, 2]), MD5.hexdigest({:b => 2})
+    refute_equal MD5.hexdigest([:b, 2]), MD5.hexdigest({:b => 2})
 
     assert_equal MD5.hexdigest({:b => 2, :a => 1}), MD5.hexdigest({:a => 1, :b => 2})
     assert_equal MD5.hexdigest({:c => 3, :b => 2, :a => 1}), MD5.hexdigest({:a => 1, :b => 2, :c => 3})

test/test_pedantic.rb

@@ -1,8 +1,9 @@
 require_relative "./helper"
 
-class TestPedantic < Test::Unit::TestCase
+class TestPedantic < Minitest::Test
   filename = File.expand_path("../../lib/linguist/languages.yml", __FILE__)
   LANGUAGES = YAML.load(File.read(filename))
+  GRAMMARS = YAML.load(File.read(File.expand_path("../../grammars.yml", __FILE__)))
 
   def test_language_names_are_sorted
     assert_sorted LANGUAGES.keys
@@ -21,6 +22,16 @@ class TestPedantic < Test::Unit::TestCase
     end
   end
 
+  def test_grammars_are_sorted
+    assert_sorted GRAMMARS.keys
+  end
+
+  def test_scopes_are_sorted
+    GRAMMARS.values.each do |scopes|
+      assert_sorted scopes
+    end
+  end
+
   def assert_sorted(list)
     list.each_cons(2) do |previous, item|
       flunk "#{previous} should come after #{item}" if previous > item

test/test_repository.rb

@@ -1,6 +1,6 @@
 require_relative "./helper"
 
-class TestRepository < Test::Unit::TestCase
+class TestRepository < Minitest::Test
   def rugged_repository
     @rugged ||= Rugged::Repository.new(File.expand_path("../../.git", __FILE__))
   end

test/test_samples.rb

@@ -1,7 +1,7 @@
 require_relative "./helper"
 require "tempfile"
 
-class TestSamples < Test::Unit::TestCase
+class TestSamples < Minitest::Test
   include Linguist
 
   def test_up_to_date
@@ -43,7 +43,7 @@ class TestSamples < Test::Unit::TestCase
       if extnames = Samples.cache['extnames'][name]
         extnames.each do |extname|
           next if extname == '.script!'
-          assert options['extensions'].include?(extname), "#{name} has a sample with extension (#{extname}) that isn't explicitly defined in languages.yml"
+          assert options['extensions'].index { |x| x.end_with? extname }, "#{name} has a sample with extension (#{extname}) that isn't explicitly defined in languages.yml"
         end
       end
 

test/test_shebang.rb

@@ -1,6 +1,6 @@
 require_relative "./helper"
 
-class TestShebang < Test::Unit::TestCase
+class TestShebang < Minitest::Test
   include Linguist
 
   def assert_interpreter(interpreter, body)

test/test_tokenizer.rb

@@ -1,6 +1,6 @@
 require_relative "./helper"
 
-class TestTokenizer < Test::Unit::TestCase
+class TestTokenizer < Minitest::Test
   include Linguist
 
   def samples_path