Merge branch 'master' into go-vendor

2025-10-29 17:50:22 +00:00 · 2016-10-26 18:34:02 -04:00
parent 789607d9bc 346aa99fcf
commit 4efc6f8c95
719 changed files with 83506 additions and 9105 deletions
--- a/test/fixtures/Data/Modelines/iamjs.pl
+++ b/test/fixtures/Data/Modelines/iamjs.pl
@@ -0,0 +1,3 @@
+# vim: noexpandtab: ft=javascript
+
+"It's JavaScript, baby";
--- a/test/fixtures/Data/Modelines/iamjs2.pl
+++ b/test/fixtures/Data/Modelines/iamjs2.pl
@@ -0,0 +1,4 @@
+# vim:noexpandtab titlestring=hi\|there\\\ ft=perl ts=4
+# vim:noexpandtab titlestring=hi|there\\ ft=javascript ts=4
+
+"Still JavaScript, bruh";
--- a/test/fixtures/Data/Modelines/ruby10
+++ b/test/fixtures/Data/Modelines/ruby10
@@ -0,0 +1,3 @@
+ ex: noexpandtab: ft=ruby
+
+# Still Ruby
--- a/test/fixtures/Data/Modelines/ruby11
+++ b/test/fixtures/Data/Modelines/ruby11
@@ -0,0 +1,3 @@
+# vim600: ft=ruby
+
+# Targets Vim 6.0 or later
--- a/test/fixtures/Data/Modelines/ruby12
+++ b/test/fixtures/Data/Modelines/ruby12
@@ -0,0 +1,3 @@
+vim<520: ft=ruby
+
+# Targets Vim 5.20 and earlier
--- a/test/fixtures/Data/Modelines/seeplusplusEmacs10
+++ b/test/fixtures/Data/Modelines/seeplusplusEmacs10
@@ -0,0 +1,3 @@
+// -*- foo-bar mode: c++ -*-
+
+"Malformed modeline, but still understood by Emacs to be C++."
--- a/test/fixtures/Data/Modelines/seeplusplusEmacs11
+++ b/test/fixtures/Data/Modelines/seeplusplusEmacs11
@@ -0,0 +1 @@
+/* -*- mode: c++ -------*- */
--- a/test/fixtures/Data/Modelines/seeplusplusEmacs12
+++ b/test/fixtures/Data/Modelines/seeplusplusEmacs12
@@ -0,0 +1 @@
+-*--------- foo:bar mode: c++ -*-
--- a/test/fixtures/Data/app.config
+++ b/test/fixtures/Data/app.config
@@ -0,0 +1,6 @@
+<?xml version="1.0"?>
+<configuration>
+  <startup>
+    <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.0" />
+  </startup>
+</configuration>
--- a/test/fixtures/Data/cars.csv
+++ b/test/fixtures/Data/cars.csv
@@ -1,3 +0,0 @@
-Year,Make,Model,Length
-1997,Ford,E350,2.34
-2000,Mercury,Cougar,2.38
--- a/test/helper.rb
+++ b/test/helper.rb
@@ -35,3 +35,11 @@ def sample_blob_memory(name)
  content = File.read(filepath)
  Linguist::Blob.new(name, content)
 end
+
+def silence_warnings
+  original_verbosity = $VERBOSE
+  $VERBOSE = nil
+  yield
+ensure
+  $VERBOSE = original_verbosity
+end
--- a/test/test_blob.rb
+++ b/test/test_blob.rb
@@ -4,14 +4,18 @@ class TestBlob < Minitest::Test
  include Linguist

  def setup
-    # git blobs are normally loaded as ASCII-8BIT since they may contain data
-    # with arbitrary encoding not known ahead of time
-    @original_external = Encoding.default_external
-    Encoding.default_external = Encoding.find("ASCII-8BIT")
+    silence_warnings do
+      # git blobs are normally loaded as ASCII-8BIT since they may contain data
+      # with arbitrary encoding not known ahead of time
+      @original_external = Encoding.default_external
+      Encoding.default_external = Encoding.find("ASCII-8BIT")
+    end
  end

  def teardown
-    Encoding.default_external = @original_external
+    silence_warnings do
+      Encoding.default_external = @original_external
+    end
  end

  def script_blob(name)
@@ -132,7 +136,7 @@ class TestBlob < Minitest::Test
  end

  def test_csv
-    assert fixture_blob_memory("Data/cars.csv").csv?
+    assert sample_blob_memory("CSV/cars.csv").csv?
  end

  def test_pdf
@@ -166,7 +170,7 @@ class TestBlob < Minitest::Test
    assert sample_blob_memory("JavaScript/jquery-1.4.2.min.js").generated?

    # Composer generated composer.lock file
-    assert sample_blob_memory("JSON/composer.lock").generated?
+    assert sample_blob_memory("JSON/filenames/composer.lock").generated?

    # PEG.js-generated parsers
    assert sample_blob_memory("JavaScript/parser.js").generated?
--- a/test/test_file_blob.rb
+++ b/test/test_file_blob.rb
@@ -1,17 +1,29 @@
 require_relative "./helper"

-class TestBlob < Minitest::Test
+class TestFileBlob < Minitest::Test
  include Linguist

+  def silence_warnings
+    original_verbosity = $VERBOSE
+    $VERBOSE = nil
+    yield
+  ensure
+    $VERBOSE = original_verbosity
+  end
+
  def setup
-    # git blobs are normally loaded as ASCII-8BIT since they may contain data
-    # with arbitrary encoding not known ahead of time
-    @original_external = Encoding.default_external
-    Encoding.default_external = Encoding.find("ASCII-8BIT")
+    silence_warnings do
+      # git blobs are normally loaded as ASCII-8BIT since they may contain data
+      # with arbitrary encoding not known ahead of time
+      @original_external = Encoding.default_external
+      Encoding.default_external = Encoding.find("ASCII-8BIT")
+    end
  end

  def teardown
-    Encoding.default_external = @original_external
+    silence_warnings do
+      Encoding.default_external = @original_external
+    end
  end

  def script_blob(name)
@@ -19,7 +31,7 @@ class TestBlob < Minitest::Test
    blob.instance_variable_set(:@name, 'script')
    blob
  end
-  
+
  def test_extensions
    assert_equal [".gitignore"], Linguist::FileBlob.new(".gitignore").extensions
    assert_equal [".xml"],  Linguist::FileBlob.new("build.xml").extensions
@@ -152,7 +164,7 @@ class TestBlob < Minitest::Test
  end

  def test_csv
-    assert fixture_blob("Data/cars.csv").csv?
+    assert sample_blob("CSV/cars.csv").csv?
  end

  def test_pdf
@@ -305,18 +317,20 @@ class TestBlob < Minitest::Test
    assert sample_blob("some/vendored/path/Chart.js").vendored?
    assert !sample_blob("some/vendored/path/chart.js").vendored?

-    # Codemirror deps
+    # CodeMirror deps
    assert sample_blob("codemirror/mode/blah.js").vendored?
    assert sample_blob("codemirror/5.0/mode/blah.js").vendored?

    # Debian packaging
    assert sample_blob("debian/cron.d").vendored?

+    # Django env
+    assert sample_blob("env/foo.py").vendored?
+
    # Erlang
    assert sample_blob("rebar").vendored?

    # git config files
-
    assert_predicate fixture_blob("some/path/.gitattributes"), :vendored?
    assert_predicate fixture_blob(".gitignore"), :vendored?
    assert_predicate fixture_blob("special/path/.gitmodules"), :vendored?
@@ -409,6 +423,22 @@ class TestBlob < Minitest::Test
    assert sample_blob("public/javascripts/tiny_mce_popup.js").vendored?
    assert sample_blob("public/javascripts/tiny_mce_src.js").vendored?

+    # Ace Editor
+    assert sample_blob("ace-builds/src/ace.js").vendored?
+    assert sample_blob("static/project/ace-builds/src/ace.js").vendored?
+
+    # Fontello CSS files
+    assert sample_blob("fontello.css").vendored?
+    assert sample_blob("fontello-ie7.css").vendored?
+    assert sample_blob("fontello-codes.css").vendored?
+    assert sample_blob("fontello-codes-ie7.css").vendored?
+    assert sample_blob("fontello-embedded.css").vendored?
+    assert sample_blob("assets/css/fontello.css").vendored?
+    assert sample_blob("assets/css/fontello-ie7.css").vendored?
+    assert sample_blob("assets/css/fontello-codes.css").vendored?
+    assert sample_blob("assets/css/fontello-codes-ie7.css").vendored?
+    assert sample_blob("assets/css/fontello-embedded.css").vendored?
+
    # AngularJS
    assert sample_blob("public/javascripts/angular.js").vendored?
    assert sample_blob("public/javascripts/angular.min.js").vendored?
@@ -522,6 +552,9 @@ class TestBlob < Minitest::Test
    assert sample_blob("myapp/My Template.xctemplate/___FILEBASENAME___.h").vendored?
    assert sample_blob("myapp/My Images.xcassets/some/stuff.imageset/Contents.json").vendored?
    assert !sample_blob("myapp/MyData.json").vendored?
+
+    # Jenkins
+    assert sample_blob("Jenkinsfile").vendored?
  end

  def test_documentation
--- a/test/test_generated.rb
+++ b/test/test_generated.rb
@@ -17,6 +17,7 @@ class TestGenerated < Minitest::Test
    assert_raises(DataLoadedError, "Data wasn't loaded when calling generated? on #{blob}") do
      Generated.generated?(blob, lambda { raise DataLoadedError.new })
    end
+    assert Generated.generated?(blob, lambda { IO.read(blob) }), "#{blob} was not recognized as a generated file"
  end

  def generated_fixture_without_loading_data(name)
@@ -50,6 +51,9 @@ class TestGenerated < Minitest::Test
    # Node modules
    generated_sample_without_loading_data("Dummy/node_modules/foo.js")

+    # npm shrinkwrap file
+    generated_sample_without_loading_data("Dummy/npm-shrinkwrap.json")
+
    # Godep saved dependencies
    generated_sample_without_loading_data("Godeps/Godeps.json")
    generated_sample_without_loading_data("Godeps/_workspace/src/github.com/kr/s3/sign.go")
@@ -62,6 +66,9 @@ class TestGenerated < Minitest::Test
    # Minified files
    generated_sample_loading_data("JavaScript/jquery-1.6.1.min.js")

+    # JS files with source map reference
+    generated_sample_loading_data("JavaScript/namespace.js")
+
    # Source Map
    generated_fixture_without_loading_data("Data/bootstrap.css.map")
    generated_fixture_loading_data("Data/sourcemap.v3.map")
--- a/test/test_grammars.rb
+++ b/test/test_grammars.rb
@@ -3,17 +3,44 @@ require_relative "./helper"
 class TestGrammars < Minitest::Test
  ROOT = File.expand_path("../..", __FILE__)

+  # List of projects that are allowed without licenses
  PROJECT_WHITELIST = [
-    # This grammar's MIT license is inside a subdirectory.
-    "vendor/grammars/SublimePapyrus",
-
-    # This grammar has a nonstandard but acceptable license.
-    "vendor/grammars/gap-tmbundle",
-
-    # These grammars have no license but have been grandfathered in. New grammars
-    # must have a license that allows redistribution.
    "vendor/grammars/Sublime-Lasso",
-    "vendor/grammars/x86-assembly-textmate-bundle"
+    "vendor/grammars/ant.tmbundle",
+    "vendor/grammars/sublime-spintools",
+    "vendor/grammars/blitzmax"
+  ].freeze
+
+  HASH_WHITELIST = [
+    "bc12b3b4917eab9aedb87ec1305c2a4376e34fd1", # TextMate bundles
+    "16c4748566b3dd996594af0410a1875b22d3a2b3", # language-yaml and atom-salt
+    "ebae2d87e06d3acef075d049fcfc8958c0364863", # go-tmbundle
+    "ff21db2554d69d78b2220db5615b16bbba0788d3", # factor
+    "b9a7428fd036eed8503995e06e989180c276b17d", # jflex.tmbundle
+    "da39a3ee5e6b4b0d3255bfef95601890afd80709", # SCSS.tmbundle
+    "5f772ff20ddf3dbac1ec9b6a98c5aa50ace555b2", # gradle.tmbundle
+    "b5432a1e1055de7eeede2dddf91e009480651fd6", # jasmin-sublime
+    "74143c4d2a5649eb179105afcb37f466558c22ce", # language-clojure
+    "760471435f5ab0b9dc99a628203cd8f9156d28ce", # language-coffee-script
+    "330e6d465e26bdd232aafcd3f5dba6a1d098a20e", # language-csharp
+    "70fb557a431891c2d634c33fa7367feab5066fd6", # language-javascript
+    "e0528c23cd967f999e058f1408ccb5b7237daaba", # language-python
+    "8653305b358375d0fced85dc24793b99919b11ef", # language-shellscript
+    "9f0c0b0926a18f5038e455e8df60221125fc3111", # elixir-tmbundle
+    "90af581219debd4e90ef041b46c294e8b4ae6d14", # mako-tmbundle
+    "b9b24778619dce325b651f0d77cbc72e7ae0b0a3", # Julia.tmbundle
+    "e06722add999e7428048abcc067cd85f1f7ca71c", # r.tmbundle
+    "50b14a0e3f03d7ca754dac42ffb33302b5882b78", # smalltalk-tmbundle
+    "eafbc4a2f283752858e6908907f3c0c90188785b", # gap-tmbundle
+    "1faa3a44cac6070f22384332434af37dfaaf2f70", # Stylus
+    "c87e7e574fca543941650e5b0a144b44c02c55d8", # language-crystal
+    "c78ec142ac3126cf639cfd67bd646ed8226d8b74", # atom-language-purescript
+    "341d7f66806fc41d081133d6e51ade856352e056", # FreeMarker.tmbundle
+    "15a394f6bc43400946570b299aee8ae264a1e3ff", # language-renpy
+    "c9118c370411f2f049c746c0fd096554e877aea2", # perl6fe
+    "8ccf886749c32fb7e65d4d1316a7ed0479c93dc9", # language-less
+    "2f03492b52d7dd83b4e7472f01b87c6121e5b1a4", # monkey
+    "bdab9fdc21e6790b479ccb5945b78bc0f6ce2493"  # language-blade
  ].freeze

  # List of allowed SPDX license names
@@ -21,12 +48,10 @@ class TestGrammars < Minitest::Test
    apache-2.0
    bsd-2-clause
    bsd-3-clause
-    cc-by-sa-3.0
-    gpl-2.0
-    gpl-3.0
-    lgpl-3.0
+    isc
    mit
    mpl-2.0
+    public
    textmate
    unlicense
    wtfpl
@@ -95,15 +120,34 @@ class TestGrammars < Minitest::Test
  end

  def test_submodules_have_approved_licenses
-    unapproved = submodule_licenses.reject { |k,v| LICENSE_WHITELIST.include?(v) || PROJECT_WHITELIST.include?(k) }.map { |k,v| "#{k}: #{v}"}
+    unapproved = submodule_licenses.reject { |k,v| LICENSE_WHITELIST.include?(v) ||
+                                                   PROJECT_WHITELIST.include?(k) ||
+                                                   HASH_WHITELIST.include?(v) }
+                                   .map { |k,v| "#{k}: #{v}"}
    message = "The following submodules have unapproved licenses:\n* #{unapproved.join("\n* ")}\n"
    message << "The license must be added to the LICENSE_WHITELIST in /test/test_grammars.rb once approved."
    assert_equal [], unapproved, message
  end

+  def test_whitelisted_submodules_dont_have_licenses
+    licensed = submodule_licenses.reject { |k,v| v.nil? }.select { |k,v| PROJECT_WHITELIST.include?(k) }
+    message = "The following whitelisted submodules have a license:\n* #{licensed.keys.join("\n* ")}\n"
+    message << "Please remove them from the project whitelist."
+    assert_equal Hash.new, licensed, message
+  end
+
+  def test_whitelisted_hashes_dont_have_licenses
+    used_hashes = submodule_licenses.values.reject { |v| v.nil? || LICENSE_WHITELIST.include?(v) }
+    unused_hashes = HASH_WHITELIST - used_hashes
+    message = "The following whitelisted license hashes are unused:\n* #{unused_hashes.join("\n* ")}\n"
+    message << "Please remove them from the hash whitelist."
+    assert_equal Array.new, unused_hashes, message
+  end
+
  def test_submodules_whitelist_has_no_extra_entries
+    skip("Need to work out how to handle dual-licensed entities")
    extra_whitelist_entries = PROJECT_WHITELIST - submodule_licenses.select { |k,v| v.nil? }.keys
-    not_present = extra_whitelist_entries.reject { |k,v| Dir.exists?(k) }
+    not_present = extra_whitelist_entries.reject { |k,v| Dir.exist?(k) }
    licensed = extra_whitelist_entries.select { |k,v| submodule_licenses[k] }

    msg = "The following whitelisted submodules don't appear to be part of the project:\n* #{not_present.join("\n* ")}"
@@ -130,7 +174,7 @@ class TestGrammars < Minitest::Test
  private

  def submodule_paths
-    @submodule_paths ||= `git config --list --file "#{File.join(ROOT, ".gitmodules")}"`.lines.grep(/\.path=/).map { |line| line.chomp.split("=", 2).last }
+    @submodule_paths ||= `git config --list --file "#{File.join(ROOT, ".gitmodules")}"`.lines.grep(/\.path=/).map { |line| line.chomp.split("=", 2).last }.reject { |path| path =~ /CodeMirror/ }
  end

  # Returns a hash of submodules in the form of submodule_path => license
@@ -143,49 +187,18 @@ class TestGrammars < Minitest::Test
  end

  # Given the path to a submodule, return its SPDX-compliant license key
+  # If the license is unrecognized, return its hash
  def submodule_license(submodule)
    # Prefer Licensee to detect a submodule's license
-    project = Licensee::FSProject.new(submodule)
+    project = Licensee::FSProject.new(submodule, detect_readme: true)
    return project.license.key if project.license

-    # We know a license file exists, but Licensee wasn't able to detect the license,
-    # Let's try our own more permissive regex method
+    # We know a license exists, but no method was able to recognize it.
+    # We return the license hash in this case, to uniquely identify it.
    if project.license_file
-      path = File.expand_path project.license_file.path, submodule
-      license = classify_license(path)
-      return license if license
-    end
-
-    # Neither Licensee nor our own regex was able to detect the license, let's check the readme
-    files = Dir[File.join(ROOT, submodule, "*")]
-    if readme = files.find { |path| File.basename(path) =~ /\Areadme\b/i }
-      classify_license(readme)
-    end
-  end
-
-  def classify_license(path)
-    content = File.read(path)
-    return unless content =~ /\blicen[cs]e\b/i
-    if content.include?("Apache License") && content.include?("2.0")
-      "apache-2.0"
-    elsif content.include?("GNU") && content =~ /general/i && content =~ /public/i
-      if content =~ /version 2/i
-        "gpl-2.0"
-      elsif content =~ /version 3/i
-        "gpl-3.0"
-      end
-    elsif content.include?("GPL") && content.include?("http://www.gnu.org/licenses/gpl.html")
-      "gpl-3.0"
-    elsif content.include?("Creative Commons Attribution-Share Alike 3.0")
-      "cc-by-sa-3.0"
-    elsif content.include?("tidy-license.txt") || content.include?("If not otherwise specified (see below)")
-      "textmate"
-    elsif content.include?("Permission is hereby granted") || content =~ /\bMIT\b/
-      "mit"
-    elsif content.include?("http://www.wtfpl.net/txt/copying/")
-      "wtfpl"
-    elsif content.include?("zlib") && content.include?("license") && content.include?("2. Altered source versions must be plainly marked as such")
-      "zlib"
+      return project.license_file.hash
+    elsif project.readme
+      return project.readme.hash
    end
  end
 end
--- a/test/test_heuristics.rb
+++ b/test/test_heuristics.rb
@@ -13,7 +13,8 @@ class TestHeuristcs < Minitest::Test
  end

  def all_fixtures(language_name, file="*")
-    Dir.glob("#{samples_path}/#{language_name}/#{file}")
+    Dir.glob("#{samples_path}/#{language_name}/#{file}") -
+      ["#{samples_path}/#{language_name}/filenames"]
  end

  def test_no_match
@@ -28,14 +29,18 @@ class TestHeuristcs < Minitest::Test
    hash.each do |language, blobs|
      Array(blobs).each do |blob|
        result = Heuristics.call(file_blob(blob), candidates)
-        assert_equal [Language[language]], result, "Failed for #{blob}"
+        if language.nil?
+          assert_equal [], result, "Failed for #{blob}"
+        else
+          assert_equal [Language[language]], result, "Failed for #{blob}"
+        end
      end
    end
  end

  def test_detect_still_works_if_nothing_matches
    blob = Linguist::FileBlob.new(File.join(samples_path, "Objective-C/hello.m"))
-    match = Language.detect(blob)
+    match = Linguist.detect(blob)
    assert_equal Language["Objective-C"], match
  end

@@ -82,6 +87,13 @@ class TestHeuristcs < Minitest::Test
      "ECLiPSe" => all_fixtures("ECLiPSe", "*.ecl")
    })
  end
+  
+  def test_es_by_heuristics
+    assert_heuristics({
+      "Erlang" => all_fixtures("Erlang", "*.es"),
+      "JavaScript" => all_fixtures("JavaScript", "*.es")
+    })
+  end

  def test_f_by_heuristics
    assert_heuristics({
@@ -114,6 +126,15 @@ class TestHeuristcs < Minitest::Test
    })
  end

+  # Candidate languages = ["Assembly", "C++", "HTML", "PAWN", "PHP",
+  #                        "POV-Ray SDL", "Pascal", "SQL", "SourcePawn"]
+  def test_inc_by_heuristics
+    assert_heuristics({
+      "PHP" => all_fixtures("PHP", "*.inc"),
+      "POV-Ray SDL" => all_fixtures("POV-Ray SDL", "*.inc")
+    })
+  end
+
  def test_ls_by_heuristics
    assert_heuristics({
      "LiveScript" => all_fixtures("LiveScript", "*.ls"),
@@ -128,12 +149,32 @@ class TestHeuristcs < Minitest::Test
    })
  end

+  def test_m_by_heuristics
+    assert_heuristics({
+      "Objective-C" => all_fixtures("Objective-C", "*.m") - all_fixtures("Objective-C", "cocoa_monitor.m"),
+      "Mercury" => all_fixtures("Mercury", "*.m"),
+      "MUF" => all_fixtures("MUF", "*.m"),
+      "M" => all_fixtures("M", "MDB.m"),
+      "Mathematica" => all_fixtures("Mathematica", "*.m") - all_fixtures("Mathematica", "Problem12.m"),
+      "Matlab" => all_fixtures("Matlab", "create_ieee_paper_plots.m"),
+      "Limbo" => all_fixtures("Limbo", "*.m"),
+      nil => ["Objective-C/cocoa_monitor.m"]
+    })
+  end
+
+  def test_md_by_heuristics
+    assert_heuristics({
+      "Markdown" => all_fixtures("Markdown", "*.md"),
+      "GCC machine description" => all_fixtures("GCC machine description", "*.md")
+    })
+  end
+
  # Candidate languages = ["C++", "Objective-C"]
  def test_obj_c_by_heuristics
    # Only calling out '.h' filenames as these are the ones causing issues
    assert_heuristics({
      "Objective-C" => all_fixtures("Objective-C", "*.h"),
-      "C++" => ["C++/scanner.h", "C++/qscicommand.h", "C++/v8.h", "C++/gdsdbreader.h"],
+      "C++" => ["C++/scanner.h", "C++/protocol-buffer.pb.h", "C++/v8.h", "C++/gdsdbreader.h"],
      "C" => nil
    })
  end
@@ -188,6 +229,16 @@ class TestHeuristcs < Minitest::Test
    })
  end

+  # Candidate languages = ["SQL", "PLpgSQL", "SQLPL", "PLSQL"]
+  def test_sql_by_heuristics
+    assert_heuristics({
+      "SQL" => ["SQL/create_stuff.sql", "SQL/db.sql", "SQL/dual.sql"],
+      "PLpgSQL" => all_fixtures("PLpgSQL", "*.sql"),
+      "SQLPL" => ["SQLPL/trigger.sql"],
+      "PLSQL" => all_fixtures("PLSQL", "*.sql")
+    })
+  end
+
  # Candidate languages = ["Perl", "Perl6"]
  def test_t_perl_by_heuristics
    assert_heuristics({
--- a/test/test_instrumentation.rb
+++ b/test/test_instrumentation.rb
@@ -28,7 +28,7 @@ class TestInstrumentation < Minitest::Test

  def test_detection_instrumentation_with_binary_blob
    binary_blob = fixture_blob("Binary/octocat.ai")
-    Language.detect(binary_blob)
+    Linguist.detect(binary_blob)

    # Shouldn't instrument this (as it's binary)
    assert_equal 0, Linguist.instrumenter.events.size
@@ -36,7 +36,7 @@ class TestInstrumentation < Minitest::Test

  def test_modeline_instrumentation
    blob = fixture_blob("Data/Modelines/ruby")
-    Language.detect(blob)
+    Linguist.detect(blob)

    detect_event = Linguist.instrumenter.events.last
    detect_event_payload = detect_event[:args].first
--- a/test/test_language.rb
+++ b/test/test_language.rb
@@ -67,6 +67,22 @@ class TestLanguage < Minitest::Test
    assert_nil Language.find_by_alias(nil)
  end

+  # Note these are set by script/set-language-ids. If these tests fail then someone
+  # has changed the language_id fields set in languages.yml which is almost certainly
+  # not what you want to happen (these fields are used in GitHub's search indexes)
+  def test_language_ids
+    assert_equal 4, Language['ANTLR'].language_id
+    assert_equal 54, Language['Ceylon'].language_id
+    assert_equal 326, Language['Ruby'].language_id
+    assert_equal 421, Language['xBase'].language_id
+  end
+
+  def test_find_by_id
+    assert_equal Language['Elixir'], Language.find_by_id(100)
+    assert_equal Language['Ruby'], Language.find_by_id(326)
+    assert_equal Language['xBase'], Language.find_by_id(421)
+  end
+
  def test_groups
    # Test a couple identity cases
    assert_equal Language['Perl'], Language['Perl'].group
@@ -329,8 +345,24 @@ class TestLanguage < Minitest::Test
  end

  def test_ace_modes
-    assert Language.ace_modes.include?(Language['Ruby'])
-    assert Language.ace_modes.include?(Language['FORTRAN'])
+    silence_warnings do
+      assert Language.ace_modes.include?(Language['Ruby'])
+      assert Language.ace_modes.include?(Language['FORTRAN'])
+    end
+  end
+
+  def test_codemirror_mode
+    assert_equal 'ruby', Language['Ruby'].codemirror_mode
+    assert_equal 'javascript', Language['JavaScript'].codemirror_mode
+    assert_equal 'clike', Language['C'].codemirror_mode
+    assert_equal 'clike', Language['C++'].codemirror_mode
+  end
+
+  def test_codemirror_mime_type
+    assert_equal 'text/x-ruby', Language['Ruby'].codemirror_mime_type
+    assert_equal 'text/javascript', Language['JavaScript'].codemirror_mime_type
+    assert_equal 'text/x-csrc', Language['C'].codemirror_mime_type
+    assert_equal 'text/x-c++src', Language['C++'].codemirror_mime_type
  end

  def test_wrap
@@ -386,6 +418,22 @@ class TestLanguage < Minitest::Test
    assert missing.empty?, message
  end

+  def test_all_languages_have_a_language_id_set
+    missing = Language.all.select { |language| language.language_id.nil? }
+
+    message = "The following languages do not have a language_id listed in languages.yml. Please add language_id fields for all new languages.\n"
+    missing.each { |language| message << "#{language.name}\n" }
+    assert missing.empty?, message
+  end
+
+  def test_all_language_id_are_unique
+    duplicates = Language.all.group_by{ |language| language.language_id }.select { |k, v| v.size > 1 }.map(&:first)
+
+    message = "The following language_id are used several times in languages.yml. Please use script/set-language-ids --update as per the contribution guidelines.\n"
+    duplicates.each { |language_id| message << "#{language_id}\n" }
+    assert duplicates.empty?, message
+  end
+
  def test_all_languages_have_a_valid_ace_mode
    ace_fixture_path = File.join('test', 'fixtures', 'ace_modes.json')
    skip("No ace_modes.json file") unless File.exist?(ace_fixture_path)
@@ -404,6 +452,40 @@ class TestLanguage < Minitest::Test
    assert missing.empty?, message
  end

+  def test_codemirror_modes_present
+    Language.all.each do |language|
+      if language.codemirror_mode || language.codemirror_mime_type
+        assert language.codemirror_mode, "#{language.inspect} missing CodeMirror mode"
+        assert language.codemirror_mime_type, "#{language.inspect} missing CodeMirror MIME mode"
+      end
+    end
+  end
+
+  def test_valid_codemirror_mode
+    Language.all.each do |language|
+      if mode = language.codemirror_mode
+        assert File.exist?(File.expand_path("../../vendor/CodeMirror/mode/#{mode}", __FILE__)), "#{mode} isn't a valid CodeMirror mode"
+      end
+    end
+  end
+
+  def test_codemirror_mode_and_mime_defined_by_meta_mapping
+    meta = File.read(File.expand_path("../../vendor/CodeMirror/mode/meta.js", __FILE__))
+    Language.all.each do |language|
+      next unless language.codemirror_mode && language.codemirror_mime_type
+      assert meta.match(/^.+#{Regexp.escape(language.codemirror_mime_type)}.+#{Regexp.escape(language.codemirror_mode)}.+$/), "#{language.inspect}: #{language.codemirror_mime_type} not defined under #{language.codemirror_mode}"
+    end
+  end
+
+  def test_codemirror_mime_declared_in_mode_file
+    Language.all.each do |language|
+      next unless language.codemirror_mode && language.codemirror_mime_type
+      filename = File.expand_path("../../vendor/CodeMirror/mode/#{language.codemirror_mode}/#{language.codemirror_mode}.js", __FILE__)
+      assert File.exist?(filename), "#{filename} does not exist"
+      assert File.read(filename).match(language.codemirror_mime_type), "#{language.inspect}: #{language.codemirror_mime_type} not defined in #{filename}"
+    end
+  end
+
  def test_all_popular_languages_exist
    popular = YAML.load(File.read(File.expand_path("../../lib/linguist/popular.yml", __FILE__)))

@@ -412,4 +494,12 @@ class TestLanguage < Minitest::Test
    message << missing.sort.join("\n")
    assert missing.empty?, message
  end
+
+  def test_no_unused_colours
+    Language.all.each do |language|
+      next unless language.type == :data || language.type == :prose ||
+        language.group.to_s != language.name
+      assert !language.color, "Unused colour assigned to #{language.name}"
+    end
+  end
 end
--- a/test/test_modelines.rb
+++ b/test/test_modelines.rb
@@ -17,6 +17,9 @@ class TestModelines < Minitest::Test
    assert_modeline Language["Ruby"], fixture_blob("Data/Modelines/ruby7")
    assert_modeline Language["Ruby"], fixture_blob("Data/Modelines/ruby8")
    assert_modeline Language["Ruby"], fixture_blob("Data/Modelines/ruby9")
+    assert_modeline Language["Ruby"], fixture_blob("Data/Modelines/ruby10")
+    assert_modeline Language["Ruby"], fixture_blob("Data/Modelines/ruby11")
+    assert_modeline Language["Ruby"], fixture_blob("Data/Modelines/ruby12")
    assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplus")
    assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs1")
    assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs2")
@@ -27,10 +30,16 @@ class TestModelines < Minitest::Test
    assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs7")
    assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs8")
    assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs9")
+    assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs10")
+    assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs11")
+    assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs12")
    assert_modeline Language["Text"], fixture_blob("Data/Modelines/fundamentalEmacs.c")
    assert_modeline Language["Prolog"], fixture_blob("Data/Modelines/not_perl.pl")
    assert_modeline Language["Smalltalk"], fixture_blob("Data/Modelines/example_smalltalk.md")
+    assert_modeline Language["JavaScript"], fixture_blob("Data/Modelines/iamjs.pl")
+    assert_modeline Language["JavaScript"], fixture_blob("Data/Modelines/iamjs2.pl")
    assert_modeline Language["PHP"], fixture_blob("Data/Modelines/iamphp.inc")
+    assert_modeline nil, sample_blob("C/main.c")
  end

  def test_modeline_languages
@@ -47,9 +56,14 @@ class TestModelines < Minitest::Test
    assert_equal Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs7").language
    assert_equal Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs8").language
    assert_equal Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs9").language
+    assert_equal Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs10").language
+    assert_equal Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs11").language
+    assert_equal Language["C++"], fixture_blob("Data/Modelines/seeplusplusEmacs12").language
    assert_equal Language["Text"], fixture_blob("Data/Modelines/fundamentalEmacs.c").language
    assert_equal Language["Prolog"], fixture_blob("Data/Modelines/not_perl.pl").language
    assert_equal Language["Smalltalk"], fixture_blob("Data/Modelines/example_smalltalk.md").language
+    assert_equal Language["JavaScript"], fixture_blob("Data/Modelines/iamjs.pl").language
+    assert_equal Language["JavaScript"], fixture_blob("Data/Modelines/iamjs2.pl").language
    assert_equal Language["PHP"], fixture_blob("Data/Modelines/iamphp.inc").language
  end
 end
--- a/test/test_repository.rb
+++ b/test/test_repository.rb
@@ -81,7 +81,7 @@ class TestRepository < Minitest::Test

  def test_linguist_override_vendored?
    attr_commit = '351c1cc8fd57340839bdb400d7812332af80e9bd'
-    repo = linguist_repo(attr_commit).read_index
+    linguist_repo(attr_commit).read_index

    override_vendored = Linguist::LazyBlob.new(rugged_repository, attr_commit, 'Gemfile')

@@ -91,7 +91,7 @@ class TestRepository < Minitest::Test

  def test_linguist_override_unvendored?
    attr_commit = '351c1cc8fd57340839bdb400d7812332af80e9bd'
-    repo = linguist_repo(attr_commit).read_index
+    linguist_repo(attr_commit).read_index

    # lib/linguist/vendor.yml defines this as vendored.
    override_unvendored = Linguist::LazyBlob.new(rugged_repository, attr_commit, 'test/fixtures/foo.rb')
@@ -102,7 +102,7 @@ class TestRepository < Minitest::Test

  def test_linguist_override_documentation?
    attr_commit = "d4c8fb8a28e91f97a7e53428a365c0abbac36d3d"
-    repo = linguist_repo(attr_commit).read_index
+    linguist_repo(attr_commit).read_index

    readme = Linguist::LazyBlob.new(rugged_repository, attr_commit, "README.md")
    arduino = Linguist::LazyBlob.new(rugged_repository, attr_commit, "samples/Arduino/hello.ino")
@@ -114,7 +114,7 @@ class TestRepository < Minitest::Test

  def test_linguist_override_generated?
    attr_commit = "351c1cc8fd57340839bdb400d7812332af80e9bd"
-    repo = linguist_repo(attr_commit).read_index
+    linguist_repo(attr_commit).read_index

    rakefile = Linguist::LazyBlob.new(rugged_repository, attr_commit, "Rakefile")

--- a/test/test_samples.rb
+++ b/test/test_samples.rb
@@ -83,7 +83,7 @@ class TestSamples < Minitest::Test
        # Check for samples if more than one language matches the given filename
        if Language.find_by_filename(filename).size > 1
          sample = "samples/#{language.name}/filenames/#{filename}"
-          assert File.exists?(sample),
+          assert File.exist?(sample),
            "Missing sample in #{sample.inspect}. See https://github.com/github/linguist/blob/master/CONTRIBUTING.md"
        end
      end