Merge branch 'master' into 1036-local

Conflicts: lib/linguist/heuristics.rb lib/linguist/languages.yml test/test_heuristics.rb
2025-10-29 17:50:22 +00:00 · 2014-11-25 13:06:11 -06:00
parent 0c05a6c3ac 7cbc4bc144
commit 8de2cd15ed
116 changed files with 6266 additions and 574 deletions
--- a/test/test_blob.rb
+++ b/test/test_blob.rb
@@ -4,13 +4,10 @@ require 'linguist/samples'
 require 'test/unit'
 require 'mocha/setup'
 require 'mime/types'
-require 'pygments'

 class TestBlob < Test::Unit::TestCase
  include Linguist

-  Lexer = Pygments::Lexer
-
  def setup
    # git blobs are normally loaded as ASCII-8BIT since they may contain data
    # with arbitrary encoding not known ahead of time
@@ -196,8 +193,8 @@ class TestBlob < Test::Unit::TestCase
    assert blob("Binary/MainMenu.nib").generated?
    assert !blob("XML/project.pbxproj").generated?

-    # Gemfile.locks
-    assert blob("Gemfile.lock").generated?
+    # Gemfile.lock is NOT generated
+    assert !blob("Gemfile.lock").generated?

    # Generated .NET Docfiles
    assert blob("XML/net_docfile.xml").generated?
@@ -229,7 +226,6 @@ class TestBlob < Test::Unit::TestCase
    assert !blob("PostScript/sierpinski.ps").generated?

    # These examples are too basic to tell
-    assert !blob("JavaScript/empty.js").generated?
    assert !blob("JavaScript/hello.js").generated?

    assert blob("JavaScript/intro-old.js").generated?
@@ -301,9 +297,24 @@ class TestBlob < Test::Unit::TestCase
    assert blob("deps/http_parser/http_parser.c").vendored?
    assert blob("deps/v8/src/v8.h").vendored?

+    # Chart.js
+    assert blob("some/vendored/path/Chart.js").vendored?
+    assert !blob("some/vendored/path/chart.js").vendored?
+
+    # Codemirror deps
+    assert blob("codemirror/mode/blah.js").vendored?
+
    # Debian packaging
    assert blob("debian/cron.d").vendored?

+    # Minified JavaScript and CSS
+    assert blob("foo.min.js").vendored?
+    assert blob("foo.min.css").vendored?
+    assert blob("foo-min.js").vendored?
+    assert blob("foo-min.css").vendored?
+    assert !blob("foomin.css").vendored?
+    assert !blob("foo.min.txt").vendored?
+
    # Prototype
    assert !blob("public/javascripts/application.js").vendored?
    assert blob("public/javascripts/prototype.js").vendored?
@@ -461,24 +472,16 @@ class TestBlob < Test::Unit::TestCase
    end
  end

-  def test_lexer
-    assert_equal Lexer['Ruby'], blob("Ruby/foo.rb").lexer
+  def test_minified_files_not_safe_to_highlight
+    assert !blob("JavaScript/jquery-1.6.1.min.js").safe_to_colorize?
  end

-  def test_colorize
-    assert_equal <<-HTML.chomp, blob("Ruby/foo.rb").colorize
-<div class="highlight"><pre><span class="k">module</span> <span class="nn">Foo</span>
-<span class="k">end</span>
-</pre></div>
-    HTML
-  end
+  def test_empty
+    blob = Struct.new(:data) { include Linguist::BlobHelper }

-  def test_colorize_does_skip_minified_files
-    assert_nil blob("JavaScript/jquery-1.6.1.min.js").colorize
-  end
-
-  # Pygments.rb was taking exceeding long on this particular file
-  def test_colorize_doesnt_blow_up_with_files_with_high_ratio_of_long_lines
-    assert_nil blob("JavaScript/steelseries-min.js").colorize
+    assert blob.new("").empty?
+    assert blob.new(nil).empty?
+    refute blob.new(" ").empty?
+    refute blob.new("nope").empty?
  end
 end
--- a/test/test_heuristics.rb
+++ b/test/test_heuristics.rb
@@ -20,27 +20,26 @@ class TestHeuristcs < Test::Unit::TestCase
    Dir.glob("#{samples_path}/#{language_name}/#{file}")
  end

+  # Candidate languages = ["C++", "Objective-C"]
  def test_obj_c_by_heuristics
-    languages = ["C++", "Objective-C"]
    # Only calling out '.h' filenames as these are the ones causing issues
    all_fixtures("Objective-C", "*.h").each do |fixture|
-      results = Heuristics.disambiguate_c(fixture("Objective-C/#{File.basename(fixture)}"), languages)
+      results = Heuristics.disambiguate_c(fixture("Objective-C/#{File.basename(fixture)}"))
      assert_equal Language["Objective-C"], results.first, "Failed for #{File.basename(fixture)}"
    end
  end

+  # Candidate languages = ["C++", "Objective-C"]
  def test_cpp_by_heuristics
-    languages = ["C++", "Objective-C"]
-    results = Heuristics.disambiguate_c(fixture("C++/render_adapter.cpp"), languages)
+    results = Heuristics.disambiguate_c(fixture("C++/render_adapter.cpp"))
    assert_equal Language["C++"], results.first
-    languages = ["C++", "Objective-C", "C"]
-    results = Heuristics.disambiguate_c(fixture("C++/ThreadedQueue.h"), languages)
+    results = Heuristics.disambiguate_c(fixture("C++/ThreadedQueue.h"))
    assert_equal Language["C++"], results.first
  end

  def test_c_by_heuristics
    languages = ["C++", "Objective-C", "C"]
-    results = Heuristics.disambiguate_c(fixture("C/ArrowLeft.h"), languages)
+    results = Heuristics.disambiguate_c(fixture("C/ArrowLeft.h"))
    assert_equal nil, results.first
  end

@@ -50,51 +49,57 @@ class TestHeuristcs < Test::Unit::TestCase
    assert_equal Language["Objective-C"], match
  end

+  # Candidate languages = ["Perl", "Prolog"]
  def test_pl_prolog_by_heuristics
-    languages = ["Perl", "Prolog"]
-    results = Heuristics.disambiguate_pl(fixture("Prolog/turing.pl"), languages)
+    results = Heuristics.disambiguate_pl(fixture("Prolog/turing.pl"))
    assert_equal Language["Prolog"], results.first
  end

+  # Candidate languages = ["Perl", "Prolog"]
  def test_pl_perl_by_heuristics
-    languages = ["Perl", "Prolog"]
-    results = Heuristics.disambiguate_pl(fixture("Perl/perl-test.t"), languages)
+    results = Heuristics.disambiguate_pl(fixture("Perl/perl-test.t"))
    assert_equal Language["Perl"], results.first
  end

+  # Candidate languages = ["ECL", "Prolog"]
  def test_ecl_prolog_by_heuristics
-    languages = ["ECL", "Prolog"]
-    results = Heuristics.disambiguate_ecl(fixture("Prolog/or-constraint.ecl"), languages)
+    results = Heuristics.disambiguate_ecl(fixture("Prolog/or-constraint.ecl"))
    assert_equal Language["Prolog"], results.first
  end

+  # Candidate languages = ["ECL", "Prolog"]
  def test_ecl_ecl_by_heuristics
-    languages = ["ECL", "Prolog"]
-    results = Heuristics.disambiguate_ecl(fixture("ECL/sample.ecl"), languages)
+    results = Heuristics.disambiguate_ecl(fixture("ECL/sample.ecl"))
    assert_equal Language["ECL"], results.first
  end

+  # Candidate languages = ["IDL", "Prolog"]
  def test_pro_prolog_by_heuristics
-    languages = ["IDL", "Prolog"]
-    results = Heuristics.disambiguate_pro(fixture("Prolog/logic-problem.pro"), languages)
+    results = Heuristics.disambiguate_pro(fixture("Prolog/logic-problem.pro"))
    assert_equal Language["Prolog"], results.first
  end

+  # Candidate languages = ["IDL", "Prolog"]
  def test_pro_idl_by_heuristics
-    languages = ["IDL", "Prolog"]
-    results = Heuristics.disambiguate_pro(fixture("IDL/mg_acosh.pro"), languages)
+    results = Heuristics.disambiguate_pro(fixture("IDL/mg_acosh.pro"))
    assert_equal Language["IDL"], results.first
  end

+  # Candidate languages = ["AGS Script", "AsciiDoc"]
+  def test_asc_asciidoc_by_heuristics
+    results = Heuristics.disambiguate_asc(fixture("AsciiDoc/list.asc"))
+    assert_equal Language["AsciiDoc"], results.first
+  end
+
+  # Candidate languages = ["TypeScript", "XML"]
  def test_ts_typescript_by_heuristics
-    languages = ["TypeScript", "XML"]
-    results = Heuristics.disambiguate_ts(fixture("TypeScript/classes.ts"), languages)
+    results = Heuristics.disambiguate_ts(fixture("TypeScript/classes.ts"))
    assert_equal Language["TypeScript"], results.first
  end

+  # Candidate languages = ["TypeScript", "XML"]
  def test_ts_xml_by_heuristics
-    languages = ["TypeScript", "XML"]
-    results = Heuristics.disambiguate_ts(fixture("XML/pt_BR.xml"), languages)
+    results = Heuristics.disambiguate_ts(fixture("XML/pt_BR.xml"))
    assert_equal Language["XML"], results.first
  end

@@ -102,9 +107,37 @@ class TestHeuristcs < Test::Unit::TestCase
    languages = ["Common Lisp", "OpenCL"]
    languages.each do |language|
      all_fixtures(language).each do |fixture|
-        results = Heuristics.disambiguate_cl(fixture("#{language}/#{File.basename(fixture)}"), languages)
+        results = Heuristics.disambiguate_cl(fixture("#{language}/#{File.basename(fixture)}"))
        assert_equal Language[language], results.first
      end
    end
  end
+
+  def test_f_by_heuristics
+    languages = ["FORTRAN", "Forth"]
+    languages.each do |language|
+      all_fixtures(language).each do |fixture|
+        results = Heuristics.disambiguate_f(fixture("#{language}/#{File.basename(fixture)}"))
+        assert_equal Language[language], results.first
+      end
+    end
+  end
+
+  # Candidate languages = ["Hack", "PHP"]
+  def test_hack_by_heuristics
+    results = Heuristics.disambiguate_hack(fixture("Hack/funs.php"))
+    assert_equal Language["Hack"], results.first
+  end
+
+  # Candidate languages = ["Scala", "SuperCollider"]
+  def test_sc_supercollider_by_heuristics
+    results = Heuristics.disambiguate_sc(fixture("SuperCollider/WarpPreset.sc"))
+    assert_equal Language["SuperCollider"], results.first
+  end
+
+  # Candidate languages = ["Scala", "SuperCollider"]
+  def test_sc_scala_by_heuristics
+    results = Heuristics.disambiguate_sc(fixture("Scala/node11.sc"))
+    assert_equal Language["Scala"], results.first
+  end
 end
--- a/test/test_language.rb
+++ b/test/test_language.rb
@@ -1,64 +1,10 @@
 require 'linguist/language'
-
 require 'test/unit'
-require 'pygments'
+require 'yaml'

 class TestLanguage < Test::Unit::TestCase
  include Linguist

-  Lexer = Pygments::Lexer
-
-  def test_lexer
-    assert_equal Lexer['ActionScript 3'], Language['ActionScript'].lexer
-    assert_equal Lexer['AspectJ'], Language['AspectJ'].lexer
-    assert_equal Lexer['Bash'], Language['Gentoo Ebuild'].lexer
-    assert_equal Lexer['Bash'], Language['Gentoo Eclass'].lexer
-    assert_equal Lexer['Bash'], Language['Shell'].lexer
-    assert_equal Lexer['C'], Language['OpenCL'].lexer
-    assert_equal Lexer['C'], Language['XS'].lexer
-    assert_equal Lexer['C++'], Language['C++'].lexer
-    assert_equal Lexer['Chapel'], Language['Chapel'].lexer
-    assert_equal Lexer['Coldfusion HTML'], Language['ColdFusion'].lexer
-    assert_equal Lexer['Coq'], Language['Coq'].lexer
-    assert_equal Lexer['FSharp'], Language['F#'].lexer
-    assert_equal Lexer['FSharp'], Language['F#'].lexer
-    assert_equal Lexer['Fortran'], Language['FORTRAN'].lexer
-    assert_equal Lexer['Gherkin'], Language['Cucumber'].lexer
-    assert_equal Lexer['Groovy'], Language['Groovy'].lexer
-    assert_equal Lexer['HTML'], Language['HTML'].lexer
-    assert_equal Lexer['HTML+Django/Jinja'], Language['HTML+Django'].lexer
-    assert_equal Lexer['HTML+PHP'], Language['HTML+PHP'].lexer
-    assert_equal Lexer['HTTP'], Language['HTTP'].lexer
-    assert_equal Lexer['JSON'], Language['JSON'].lexer
-    assert_equal Lexer['Java'], Language['ChucK'].lexer
-    assert_equal Lexer['Java'], Language['Java'].lexer
-    assert_equal Lexer['JavaScript'], Language['JavaScript'].lexer
-    assert_equal Lexer['LSL'], Language['LSL'].lexer
-    assert_equal Lexer['MOOCode'], Language['Moocode'].lexer
-    assert_equal Lexer['MuPAD'], Language['mupad'].lexer
-    assert_equal Lexer['NASM'], Language['Assembly'].lexer
-    assert_equal Lexer['OCaml'], Language['OCaml'].lexer
-    assert_equal Lexer['Ooc'], Language['ooc'].lexer
-    assert_equal Lexer['OpenEdge ABL'], Language['OpenEdge ABL'].lexer
-    assert_equal Lexer['REBOL'], Language['Rebol'].lexer
-    assert_equal Lexer['RHTML'], Language['HTML+ERB'].lexer
-    assert_equal Lexer['RHTML'], Language['RHTML'].lexer
-    assert_equal Lexer['Ruby'], Language['Crystal'].lexer
-    assert_equal Lexer['Ruby'], Language['Mirah'].lexer
-    assert_equal Lexer['Ruby'], Language['Ruby'].lexer
-    assert_equal Lexer['S'], Language['R'].lexer
-    assert_equal Lexer['Scheme'], Language['Nu'].lexer
-    assert_equal Lexer['Racket'], Language['Racket'].lexer
-    assert_equal Lexer['Scheme'], Language['Scheme'].lexer
-    assert_equal Lexer['Standard ML'], Language['Standard ML'].lexer
-    assert_equal Lexer['TeX'], Language['TeX'].lexer
-    assert_equal Lexer['Verilog'], Language['Verilog'].lexer
-    assert_equal Lexer['XSLT'], Language['XSLT'].lexer
-    assert_equal Lexer['aspx-vb'], Language['ASP'].lexer
-    assert_equal Lexer['haXe'], Language['Haxe'].lexer
-    assert_equal Lexer['reStructuredText'], Language['reStructuredText'].lexer
-  end
-
  def test_find_by_alias
    assert_equal Language['ASP'], Language.find_by_alias('asp')
    assert_equal Language['ASP'], Language.find_by_alias('aspx')
@@ -119,6 +65,7 @@ class TestLanguage < Test::Unit::TestCase
    assert_equal Language['VimL'], Language.find_by_alias('viml')
    assert_equal Language['reStructuredText'], Language.find_by_alias('rst')
    assert_equal Language['YAML'], Language.find_by_alias('yml')
+    assert_nil Language.find_by_alias(nil)
  end

  def test_groups
@@ -193,6 +140,7 @@ class TestLanguage < Test::Unit::TestCase
    assert_equal :programming, Language['Python'].type
    assert_equal :programming, Language['Ruby'].type
    assert_equal :programming, Language['TypeScript'].type
+    assert_equal :programming, Language['Makefile'].type
  end

  def test_markup
@@ -211,7 +159,6 @@ class TestLanguage < Test::Unit::TestCase

  def test_other
    assert_nil Language['Brainfuck'].type
-    assert_nil Language['Makefile'].type
  end

  def test_searchable
@@ -221,6 +168,7 @@ class TestLanguage < Test::Unit::TestCase
  end

  def test_find_by_name
+    assert_nil Language.find_by_name(nil)
    ruby = Language['Ruby']
    assert_equal ruby, Language.find_by_name('Ruby')
  end
@@ -241,6 +189,22 @@ class TestLanguage < Test::Unit::TestCase
    end
  end

+  def test_find_by_extension
+    assert_equal [], Language.find_by_extension('.factor-rc')
+    assert_equal [], Language.find_by_extension('foo.rb')
+    assert_equal [Language['Ruby']], Language.find_by_extension('rb')
+    assert_equal [Language['Ruby']], Language.find_by_extension('.rb')
+    assert_equal [Language['M'], Language['Mathematica'], Language['Matlab'], Language['Mercury'], Language['Objective-C']], Language.find_by_extension('.m')
+  end
+
+  def test_find_all_by_extension
+    Language.all.each do |language|
+      language.extensions.each do |extension|
+        assert_include Language.find_by_extension(extension), language
+      end
+    end
+  end
+
  def test_find_by_filename
    assert_equal [Language['Shell']], Language.find_by_filename('PKGBUILD')
    assert_equal [Language['Ruby']], Language.find_by_filename('foo.rb')
@@ -301,6 +265,22 @@ class TestLanguage < Test::Unit::TestCase
    assert_equal 'C#', Language['c#'].name
    assert_equal 'C#', Language['csharp'].name
    assert_nil Language['defunkt']
+    assert_nil Language[nil]
+  end
+
+  def test_find_ignores_case
+    assert_equal 'AGS Script', Language['ags script'].name
+    assert_equal 'AGS Script', Language['ags sCRIPT'].name
+  end
+
+  def test_find_by_name_ignores_case
+    assert_equal 'AGS Script', Language.find_by_name('ags script').name
+    assert_equal 'AGS Script', Language.find_by_name('ags sCRIPT').name
+  end
+
+  def test_find_by_alias_ignores_case
+    refute_includes Language['AGS Script'].aliases, 'AGS'
+    assert_equal 'AGS Script', Language.find_by_alias('AGS').name
  end

  def test_name
@@ -370,12 +350,6 @@ class TestLanguage < Test::Unit::TestCase
    assert_equal '.coffee', Language['CoffeeScript'].primary_extension
    assert_equal '.t', Language['Turing'].primary_extension
    assert_equal '.ts', Language['TypeScript'].primary_extension
-
-    # This is a nasty requirement, but there's some code in GitHub that
-    # expects this. Really want to drop this.
-    Language.all.each do |language|
-      assert language.primary_extension, "#{language} has no primary extension"
-    end
  end

  def test_eql
@@ -387,21 +361,14 @@ class TestLanguage < Test::Unit::TestCase
    assert !Language.by_type(:prose).nil?
  end

-  def test_colorize
-    assert_equal <<-HTML.chomp, Language['Ruby'].colorize("def foo\n  'foo'\nend\n")
-<div class="highlight"><pre><span class="k">def</span> <span class="nf">foo</span>
-  <span class="s1">&#39;foo&#39;</span>
-<span class="k">end</span>
-</pre></div>
-    HTML
-  end
+  def test_all_languages_have_grammars
+    scopes = YAML.load(File.read(File.expand_path("../../grammars.yml", __FILE__))).values.flatten
+    missing = Language.all.reject { |language| language.tm_scope == "none" || scopes.include?(language.tm_scope) }
+    message = "The following languages' scopes are not listed in grammars.yml. Please add grammars for all new languages.\n"
+    message << "If no grammar exists for a language, mark the language with `tm_scope: none` in lib/linguist/languages.yml.\n"

-  def test_colorize_with_options
-    assert_equal <<-HTML.chomp, Language['Ruby'].colorize("def foo\n  'foo'\nend\n", :options => { :cssclass => "highlight highlight-ruby" })
-<div class="highlight highlight-ruby"><pre><span class="k">def</span> <span class="nf">foo</span>
-  <span class="s1">&#39;foo&#39;</span>
-<span class="k">end</span>
-</pre></div>
-    HTML
+    width = missing.map { |language| language.name.length }.max
+    message << missing.map { |language| sprintf("%-#{width}s %s", language.name, language.tm_scope) }.sort.join("\n")
+    assert missing.empty?, message
  end
 end
--- a/test/test_pedantic.rb
+++ b/test/test_pedantic.rb
@@ -1,57 +1,30 @@
 require 'test/unit'
+require 'yaml'

 class TestPedantic < Test::Unit::TestCase
-  Lib = File.expand_path("../../lib/linguist", __FILE__)
-
-  def file(name)
-    File.read(File.join(Lib, name))
-  end
+  filename = File.expand_path("../../lib/linguist/languages.yml", __FILE__)
+  LANGUAGES = YAML.load(File.read(filename))

  def test_language_names_are_sorted
-    languages = []
-    file("languages.yml").lines.each do |line|
-      if line =~ /^(\w+):$/
-        languages << $1
-      end
-    end
-    assert_sorted languages
+    assert_sorted LANGUAGES.keys
  end

  def test_extensions_are_sorted
-    extensions = nil
-    file("languages.yml").lines.each do |line|
-      if line =~ /^  extensions:$/
-        extensions = []
-      elsif extensions && line =~ /^  - \.([\w-]+)( *#.*)?$/
-        extensions << $1
-      else
-        assert_sorted extensions[1..-1] if extensions
-        extensions = nil
-      end
+    LANGUAGES.each do |name, language|
+      extensions = language['extensions']
+      assert_sorted extensions[1..-1] if extensions && extensions.size > 1
    end
  end

  def test_filenames_are_sorted
-    filenames = nil
-    file("languages.yml").lines.each do |line|
-      if line =~ /^  filenames:$/
-        filenames = []
-      elsif filenames && line =~ /^  - \.(\w+)$/
-        filenames << $1
-      else
-        assert_sorted filenames if filenames
-        filenames = nil
-      end
+    LANGUAGES.each do |name, language|
+      assert_sorted language['filenames'] if language['filenames']
    end
  end

  def assert_sorted(list)
-    previous = nil
-    list.each do |item|
-      if previous && previous > item
-        flunk "#{previous} should come after #{item}"
-      end
-      previous = item
+    list.each_cons(2) do |previous, item|
+      flunk "#{previous} should come after #{item}" if previous > item
    end
  end
 end
--- a/test/test_samples.rb
+++ b/test/test_samples.rb
@@ -16,11 +16,11 @@ class TestSamples < Test::Unit::TestCase
      warn "Samples database is out of date. Run `bundle exec rake samples`."

      expected = Tempfile.new('expected.json')
-      expected.write Yajl::Encoder.encode(serialized, :pretty => true)
+      expected.write Yajl.dump(serialized, :pretty => true)
      expected.close

      actual = Tempfile.new('actual.json')
-      actual.write Yajl::Encoder.encode(latest, :pretty => true)
+      actual.write Yajl.dump(latest, :pretty => true)
      actual.close

      expected.unlink
@@ -55,20 +55,28 @@ class TestSamples < Test::Unit::TestCase
  end

  # If a language extension isn't globally unique then make sure there are samples
-  def test_presence
-    Linguist::Language.all.each do |language|
-      language.all_extensions.each do |extension|
-        language_matches = Language.find_by_filename("foo#{extension}")
+  Linguist::Language.all.each do |language|
+    define_method "test_#{language.name}_has_samples" do
+      language.extensions.each do |extension|
+        language_matches = Language.find_by_extension(extension)

-        # If there is more than one language match for a given extension
-        # then check that there are examples for that language with the extension
+        # Check for samples if more than one language matches the given extension.
        if language_matches.length > 1
-          language_matches.each do |language|
-            assert File.directory?("samples/#{language.name}"), "#{language.name} is missing a samples directory"
-            assert Dir.glob("samples/#{language.name}/*#{extension}").any?, "#{language.name} is missing samples for extension #{extension}"
+          language_matches.each do |match|
+            samples = "samples/#{match.name}/*#{extension}"
+            assert Dir.glob(samples).any?, "Missing samples in #{samples.inspect}. See https://github.com/github/linguist/blob/master/CONTRIBUTING.md"
          end
        end
      end
+
+      language.filenames.each do |filename|
+        # Check for samples if more than one language matches the given filename
+        if Language.find_by_filename(filename).size > 1
+          sample = "samples/#{language.name}/filenames/#{filename}"
+          assert File.exists?(sample),
+            "Missing sample in #{sample.inspect}. See https://github.com/github/linguist/blob/master/CONTRIBUTING.md"
+        end
+      end
    end
  end
 end