Merge branch 'master' into pro

2025-10-29 17:50:22 +00:00 · 2014-09-25 10:23:21 -04:00
parent d94bffb198 5152bd7124
commit f59cf24a82
409 changed files with 132166 additions and 49243 deletions
--- a/test/test_blob.rb
+++ b/test/test_blob.rb
@@ -11,6 +11,17 @@ class TestBlob < Test::Unit::TestCase

  Lexer = Pygments::Lexer

+  def setup
+    # git blobs are normally loaded as ASCII-8BIT since they may contain data
+    # with arbitrary encoding not known ahead of time
+    @original_external = Encoding.default_external
+    Encoding.default_external = Encoding.find("ASCII-8BIT")
+  end
+
+  def teardown
+    Encoding.default_external = @original_external
+  end
+
  def samples_path
    File.expand_path("../../samples", __FILE__)
  end
@@ -67,6 +78,14 @@ class TestBlob < Test::Unit::TestCase
    assert_equal 475, blob("Emacs Lisp/ess-julia.el").lines.length
  end

+  def test_lines_maintains_original_encoding
+    # Even if the file's encoding is detected as something like UTF-16LE,
+    # earlier versions of the gem made implicit guarantees that the encoding of
+    # each `line` is in the same encoding as the file was originally read (in
+    # practice, UTF-8 or ASCII-8BIT)
+    assert_equal Encoding.default_external, blob("Text/utf16le.txt").lines.first.encoding
+  end
+
  def test_size
    assert_equal 15, blob("Ruby/foo.rb").size
  end
@@ -77,13 +96,27 @@ class TestBlob < Test::Unit::TestCase

  def test_sloc
    assert_equal 2, blob("Ruby/foo.rb").sloc
+    assert_equal 3, blob("Text/utf16le-windows.txt").sloc
+    assert_equal 1, blob("Text/iso8859-8-i.txt").sloc
  end

  def test_encoding
    assert_equal "ISO-8859-2", blob("Text/README").encoding
+    assert_equal "ISO-8859-2", blob("Text/README").ruby_encoding
    assert_equal "ISO-8859-1", blob("Text/dump.sql").encoding
+    assert_equal "ISO-8859-1", blob("Text/dump.sql").ruby_encoding
    assert_equal "UTF-8", blob("Text/foo.txt").encoding
+    assert_equal "UTF-8", blob("Text/foo.txt").ruby_encoding
+    assert_equal "UTF-16LE", blob("Text/utf16le.txt").encoding
+    assert_equal "UTF-16LE", blob("Text/utf16le.txt").ruby_encoding
+    assert_equal "UTF-16LE", blob("Text/utf16le-windows.txt").encoding
+    assert_equal "UTF-16LE", blob("Text/utf16le-windows.txt").ruby_encoding
+    assert_equal "ISO-2022-KR", blob("Text/ISO-2022-KR.txt").encoding
+    assert_equal "binary", blob("Text/ISO-2022-KR.txt").ruby_encoding
    assert_nil blob("Binary/dog.o").encoding
+
+    assert_equal "windows-1252", blob("Text/Visual_Battlers.rb").encoding
+    assert_equal "Windows-1252", blob("Text/Visual_Battlers.rb").ruby_encoding
  end

  def test_binary
@@ -107,6 +140,13 @@ class TestBlob < Test::Unit::TestCase
    assert !blob("Perl/script.pl").binary?
  end

+  def test_all_binary
+    Samples.each do |sample|
+      blob = blob(sample[:path])
+      assert ! (blob.likely_binary? || blob.binary?), "#{sample[:path]} is a binary file"
+    end
+  end
+
  def test_text
    assert blob("Text/README").text?
    assert blob("Text/dump.sql").text?
@@ -152,9 +192,9 @@ class TestBlob < Test::Unit::TestCase
    assert !blob("Text/README").generated?

    # Xcode project files
-    assert blob("XML/MainMenu.xib").generated?
+    assert !blob("XML/MainMenu.xib").generated?
    assert blob("Binary/MainMenu.nib").generated?
-    assert blob("XML/project.pbxproj").generated?
+    assert !blob("XML/project.pbxproj").generated?

    # Gemfile.locks
    assert blob("Gemfile.lock").generated?
@@ -211,7 +251,21 @@ class TestBlob < Test::Unit::TestCase
    assert !blob("CSS/bootstrap.css").generated?
    assert blob("CSS/bootstrap.min.css").generated?

+    # Generated VCR
+    assert blob("YAML/vcr_cassette.yml").generated?
+
+    # Generated by Zephir
+    assert blob("Zephir/filenames/exception.zep.c").generated?
+    assert blob("Zephir/filenames/exception.zep.h").generated?
+    assert blob("Zephir/filenames/exception.zep.php").generated?
+    assert !blob("Zephir/Router.zep").generated?
+
+
    assert Linguist::Generated.generated?("node_modules/grunt/lib/grunt.js", nil)
+
+    # Godep saved dependencies
+    assert blob("Godeps/Godeps.json").generated?
+    assert blob("Godeps/_workspace/src/github.com/kr/s3/sign.go").generated?
  end

  def test_vendored
@@ -229,12 +283,20 @@ class TestBlob < Test::Unit::TestCase
    assert blob("app/bower_components/custom/custom.js").vendored?
    assert blob("vendor/assets/bower_components/custom/custom.js").vendored?

+    # Go dependencies
+    assert !blob("Godeps/Godeps.json").vendored?
+    assert blob("Godeps/_workspace/src/github.com/kr/s3/sign.go").vendored?
+
    # Rails vendor/
    assert blob("vendor/plugins/will_paginate/lib/will_paginate.rb").vendored?

    # 'thirdparty' directory
    assert blob("thirdparty/lib/main.c").vendored?

+    # 'extern(al)' directory
+    assert blob("extern/util/__init__.py").vendored?
+    assert blob("external/jquery.min.js").vendored?
+
    # C deps
    assert blob("deps/http_parser/http_parser.c").vendored?
    assert blob("deps/v8/src/v8.h").vendored?
@@ -303,6 +365,14 @@ class TestBlob < Test::Unit::TestCase
    assert blob("public/javascripts/angular.js").vendored?
    assert blob("public/javascripts/angular.min.js").vendored?

+    # D3.js
+    assert blob("public/javascripts/d3.v3.js").vendored?
+    assert blob("public/javascripts/d3.v3.min.js").vendored?
+
+    # Modernizr
+    assert blob("public/javascripts/modernizr-2.7.1.js").vendored?
+    assert blob("public/javascripts/modernizr.custom.01009.js").vendored?
+
    # Fabric
    assert blob("fabfile.py").vendored?

@@ -331,6 +401,20 @@ class TestBlob < Test::Unit::TestCase
    # NuGet Packages
    assert blob("packages/Modernizr.2.0.6/Content/Scripts/modernizr-2.0.6-development-only.js").vendored?

+    # Font Awesome
+    assert blob("some/asset/path/font-awesome.min.css").vendored?
+    assert blob("some/asset/path/font-awesome.css").vendored?
+
+    # Normalize
+    assert blob("some/asset/path/normalize.css").vendored?
+
+    # Cocoapods
+    assert blob('Pods/blah').vendored?
+
+    # Html5shiv
+    assert blob("Scripts/html5shiv.js").vendored?
+    assert blob("Scripts/html5shiv.min.js").vendored?
+
    # Test fixtures
    assert blob("test/fixtures/random.rkt").vendored?
    assert blob("Test/fixtures/random.rkt").vendored?
@@ -341,8 +425,32 @@ class TestBlob < Test::Unit::TestCase
    assert blob("cordova-2.1.0.js").vendored?
    assert blob("cordova-2.1.0.min.js").vendored?

+    # Foundation js
+    assert blob("foundation.js").vendored?
+    assert blob("foundation.min.js").vendored?
+    assert blob("foundation.abide.js").vendored?
+
    # Vagrant
    assert blob("Vagrantfile").vendored?
+
+    # Gradle
+    assert blob("gradlew").vendored?
+    assert blob("gradlew.bat").vendored?
+    assert blob("gradle/wrapper/gradle-wrapper.properties").vendored?
+    assert blob("subproject/gradlew").vendored?
+    assert blob("subproject/gradlew.bat").vendored?
+    assert blob("subproject/gradle/wrapper/gradle-wrapper.properties").vendored?
+
+    # Octicons
+    assert blob("octicons.css").vendored?
+    assert blob("public/octicons.min.css").vendored?
+    assert blob("public/octicons/sprockets-octicons.scss").vendored?
+
+    # Typesafe Activator
+    assert blob("activator").vendored?
+    assert blob("activator.bat").vendored?
+    assert blob("subproject/activator").vendored?
+    assert blob("subproject/activator.bat").vendored?
  end

  def test_language
--- a/test/test_classifier.rb
+++ b/test/test_classifier.rb
@@ -44,12 +44,12 @@ class TestClassifier < Test::Unit::TestCase
  end

  def test_instance_classify_empty
-    results = Classifier.classify(Samples::DATA, "")
+    results = Classifier.classify(Samples.cache, "")
    assert results.first[1] < 0.5, results.first.inspect
  end

  def test_instance_classify_nil
-    assert_equal [], Classifier.classify(Samples::DATA, nil)
+    assert_equal [], Classifier.classify(Samples.cache, nil)
  end

  def test_classify_ambiguous_languages
@@ -58,7 +58,7 @@ class TestClassifier < Test::Unit::TestCase
      languages = Language.find_by_filename(sample[:path]).map(&:name)
      next unless languages.length > 1

-      results = Classifier.classify(Samples::DATA, File.read(sample[:path]), languages)
+      results = Classifier.classify(Samples.cache, File.read(sample[:path]), languages)
      assert_equal language.name, results.first[0], "#{sample[:path]}\n#{results.inspect}"
    end
  end
--- a/test/test_heuristics.rb
+++ b/test/test_heuristics.rb
@@ -1,6 +1,7 @@
 require 'linguist/heuristics'
 require 'linguist/language'
 require 'linguist/samples'
+require 'linguist/file_blob'

 require 'test/unit'

@@ -35,7 +36,8 @@ class TestHeuristcs < Test::Unit::TestCase
  end

  def test_detect_still_works_if_nothing_matches
-    match = Language.detect("Hello.m", fixture("Objective-C/hello.m"))
+    blob = Linguist::FileBlob.new(File.join(samples_path, "Objective-C/hello.m"))
+    match = Language.detect(blob)
    assert_equal Language["Objective-C"], match
  end
  
--- a/test/test_language.rb
+++ b/test/test_language.rb
@@ -10,12 +10,14 @@ class TestLanguage < Test::Unit::TestCase

  def test_lexer
    assert_equal Lexer['ActionScript 3'], Language['ActionScript'].lexer
+    assert_equal Lexer['AspectJ'], Language['AspectJ'].lexer
    assert_equal Lexer['Bash'], Language['Gentoo Ebuild'].lexer
    assert_equal Lexer['Bash'], Language['Gentoo Eclass'].lexer
    assert_equal Lexer['Bash'], Language['Shell'].lexer
    assert_equal Lexer['C'], Language['OpenCL'].lexer
    assert_equal Lexer['C'], Language['XS'].lexer
    assert_equal Lexer['C++'], Language['C++'].lexer
+    assert_equal Lexer['Chapel'], Language['Chapel'].lexer
    assert_equal Lexer['Coldfusion HTML'], Language['ColdFusion'].lexer
    assert_equal Lexer['Coq'], Language['Coq'].lexer
    assert_equal Lexer['FSharp'], Language['F#'].lexer
@@ -31,6 +33,7 @@ class TestLanguage < Test::Unit::TestCase
    assert_equal Lexer['Java'], Language['ChucK'].lexer
    assert_equal Lexer['Java'], Language['Java'].lexer
    assert_equal Lexer['JavaScript'], Language['JavaScript'].lexer
+    assert_equal Lexer['LSL'], Language['LSL'].lexer
    assert_equal Lexer['MOOCode'], Language['Moocode'].lexer
    assert_equal Lexer['MuPAD'], Language['mupad'].lexer
    assert_equal Lexer['NASM'], Language['Assembly'].lexer
@@ -44,7 +47,6 @@ class TestLanguage < Test::Unit::TestCase
    assert_equal Lexer['Ruby'], Language['Mirah'].lexer
    assert_equal Lexer['Ruby'], Language['Ruby'].lexer
    assert_equal Lexer['S'], Language['R'].lexer
-    assert_equal Lexer['Scheme'], Language['Emacs Lisp'].lexer
    assert_equal Lexer['Scheme'], Language['Nu'].lexer
    assert_equal Lexer['Racket'], Language['Racket'].lexer
    assert_equal Lexer['Scheme'], Language['Scheme'].lexer
@@ -70,6 +72,7 @@ class TestLanguage < Test::Unit::TestCase
    assert_equal Language['C'], Language.find_by_alias('c')
    assert_equal Language['C++'], Language.find_by_alias('c++')
    assert_equal Language['C++'], Language.find_by_alias('cpp')
+    assert_equal Language['Chapel'], Language.find_by_alias('chpl')
    assert_equal Language['CoffeeScript'], Language.find_by_alias('coffee')
    assert_equal Language['CoffeeScript'], Language.find_by_alias('coffee-script')
    assert_equal Language['ColdFusion'], Language.find_by_alias('cfm')
@@ -166,7 +169,7 @@ class TestLanguage < Test::Unit::TestCase
    assert_equal 'pot',           Language['Gettext Catalog'].search_term
    assert_equal 'irc',           Language['IRC log'].search_term
    assert_equal 'lhs',           Language['Literate Haskell'].search_term
-    assert_equal 'ruby',          Language['Mirah'].search_term
+    assert_equal 'mirah',         Language['Mirah'].search_term
    assert_equal 'raw',           Language['Raw token data'].search_term
    assert_equal 'bash',          Language['Shell'].search_term
    assert_equal 'vim',           Language['VimL'].search_term
@@ -184,6 +187,7 @@ class TestLanguage < Test::Unit::TestCase

  def test_programming
    assert_equal :programming, Language['JavaScript'].type
+    assert_equal :programming, Language['LSL'].type
    assert_equal :programming, Language['Perl'].type
    assert_equal :programming, Language['PowerShell'].type
    assert_equal :programming, Language['Python'].type
@@ -213,7 +217,7 @@ class TestLanguage < Test::Unit::TestCase
  def test_searchable
    assert Language['Ruby'].searchable?
    assert !Language['Gettext Catalog'].searchable?
-    assert !Language['SQL'].searchable?
+    assert Language['SQL'].searchable?
  end

  def test_find_by_name
@@ -248,13 +252,13 @@ class TestLanguage < Test::Unit::TestCase
    assert_equal Language['Nginx'], Language.find_by_filename('nginx.conf').first
    assert_equal ['C', 'C++', 'Objective-C'], Language.find_by_filename('foo.h').map(&:name).sort
    assert_equal [], Language.find_by_filename('rb')
-    assert_equal [], Language.find_by_filename('.rb')
-    assert_equal [], Language.find_by_filename('.nkt')
+    assert_equal [], Language.find_by_filename('.null')
    assert_equal [Language['Shell']], Language.find_by_filename('.bashrc')
    assert_equal [Language['Shell']], Language.find_by_filename('bash_profile')
    assert_equal [Language['Shell']], Language.find_by_filename('.zshrc')
    assert_equal [Language['Clojure']], Language.find_by_filename('riemann.config')
    assert_equal [Language['HTML+Django']], Language.find_by_filename('index.jinja')
+    assert_equal [Language['Chapel']], Language.find_by_filename('examples/hello.chpl')
  end

  def test_find_by_shebang
@@ -322,8 +326,9 @@ class TestLanguage < Test::Unit::TestCase
  def test_color
    assert_equal '#701516', Language['Ruby'].color
    assert_equal '#3581ba', Language['Python'].color
-    assert_equal '#f15501', Language['JavaScript'].color
+    assert_equal '#f1e05a', Language['JavaScript'].color
    assert_equal '#31859c', Language['TypeScript'].color
+    assert_equal '#3d9970', Language['LSL'].color
  end

  def test_colors
@@ -336,6 +341,7 @@ class TestLanguage < Test::Unit::TestCase
    assert_equal 'coffee', Language['CoffeeScript'].ace_mode
    assert_equal 'csharp', Language['C#'].ace_mode
    assert_equal 'css', Language['CSS'].ace_mode
+    assert_equal 'lsl', Language['LSL'].ace_mode
    assert_equal 'javascript', Language['JavaScript'].ace_mode
  end

@@ -350,6 +356,7 @@ class TestLanguage < Test::Unit::TestCase
  end

  def test_extensions
+    assert Language['LSL'].extensions.include?('.lsl')
    assert Language['Perl'].extensions.include?('.pl')
    assert Language['Python'].extensions.include?('.py')
    assert Language['Ruby'].extensions.include?('.rb')
@@ -385,6 +392,15 @@ class TestLanguage < Test::Unit::TestCase
 <div class="highlight"><pre><span class="k">def</span> <span class="nf">foo</span>
  <span class="s1">&#39;foo&#39;</span>
 <span class="k">end</span>
+</pre></div>
+    HTML
+  end
+
+  def test_colorize_with_options
+    assert_equal <<-HTML.chomp, Language['Ruby'].colorize("def foo\n  'foo'\nend\n", :options => { :cssclass => "highlight highlight-ruby" })
+<div class="highlight highlight-ruby"><pre><span class="k">def</span> <span class="nf">foo</span>
+  <span class="s1">&#39;foo&#39;</span>
+<span class="k">end</span>
 </pre></div>
    HTML
  end
--- a/test/test_pedantic.rb
+++ b/test/test_pedantic.rb
@@ -22,10 +22,10 @@ class TestPedantic < Test::Unit::TestCase
    file("languages.yml").lines.each do |line|
      if line =~ /^  extensions:$/
        extensions = []
-      elsif extensions && line =~ /^  - \.(\w+)$/
+      elsif extensions && line =~ /^  - \.([\w-]+)( *#.*)?$/
        extensions << $1
      else
-        assert_sorted extensions if extensions
+        assert_sorted extensions[1..-1] if extensions
        extensions = nil
      end
    end
--- a/test/test_repository.rb
+++ b/test/test_repository.rb
@@ -3,22 +3,24 @@ require 'linguist/repository'
 require 'test/unit'

 class TestRepository < Test::Unit::TestCase
-  include Linguist
-
-  def repo(base_path)
-    Repository.from_directory(base_path)
+  def rugged_repository
+    @rugged ||= Rugged::Repository.new(File.expand_path("../../.git", __FILE__))
  end

-  def linguist_repo
-    repo(File.expand_path("../..", __FILE__))
+  def master_oid
+    'd40b4a33deba710e2f494db357c654fbe5d4b419'
+  end
+
+  def linguist_repo(oid = master_oid)
+    Linguist::Repository.new(rugged_repository, oid)
  end

  def test_linguist_language
-    # assert_equal Language['Ruby'], linguist_repo.language
+    assert_equal 'Ruby', linguist_repo.language
  end

  def test_linguist_languages
-    # assert linguist_repo.languages[Language['Ruby']] > 10_000
+    assert linguist_repo.languages['Ruby'] > 10_000
  end

  def test_linguist_size
@@ -31,7 +33,18 @@ class TestRepository < Test::Unit::TestCase
    assert linguist_repo.breakdown_by_file["Ruby"].include?("lib/linguist/language.rb")
  end

-  def test_binary_override
-    assert_equal repo(File.expand_path("../../samples/Nimrod", __FILE__)).language, Language["Nimrod"]
+  def test_incremental_stats
+    old_commit = '3d7364877d6794f6cc2a86b493e893968a597332'
+    old_repo = linguist_repo(old_commit)
+
+    assert old_repo.languages['Ruby'] > 10_000
+    assert old_repo.size > 30_000
+
+    new_repo = Linguist::Repository.incremental(rugged_repository, master_oid, old_commit, old_repo.cache)
+
+    assert new_repo.languages['Ruby'] > old_repo.languages['Ruby']
+    assert new_repo.size > old_repo.size
+
+    assert_equal linguist_repo.cache, new_repo.cache
  end
 end
--- a/test/test_samples.rb
+++ b/test/test_samples.rb
@@ -1,14 +1,14 @@
 require 'linguist/samples'
+require 'linguist/language'
 require 'tempfile'
 require 'yajl'
-
 require 'test/unit'

 class TestSamples < Test::Unit::TestCase
  include Linguist

  def test_up_to_date
-    assert serialized = Samples::DATA
+    assert serialized = Samples.cache
    assert latest = Samples.data

    # Just warn, it shouldn't scare people off by breaking the build.
@@ -29,10 +29,46 @@ class TestSamples < Test::Unit::TestCase
  end

  def test_verify
-    assert data = Samples::DATA
+    assert data = Samples.cache

    assert_equal data['languages_total'], data['languages'].inject(0) { |n, (_, c)| n += c }
    assert_equal data['tokens_total'], data['language_tokens'].inject(0) { |n, (_, c)| n += c }
    assert_equal data['tokens_total'], data['tokens'].inject(0) { |n, (_, ts)| n += ts.inject(0) { |m, (_, c)| m += c } }
  end
+
+  # Check that there aren't samples with extensions that aren't explicitly defined in languages.yml
+  def test_parity
+    extensions = Samples.cache['extnames']
+    languages_yml = File.expand_path("../../lib/linguist/languages.yml", __FILE__)
+    languages = YAML.load_file(languages_yml)
+
+    languages.each do |name, options|
+      options['extensions'] ||= []
+
+      if extnames = extensions[name]
+        extnames.each do |extname|
+          next if extname == '.script!'
+          assert options['extensions'].include?(extname), "#{name} has a sample with extension (#{extname}) that isn't explicitly defined in languages.yml"
+        end
+      end
+    end
+  end
+
+  # If a language extension isn't globally unique then make sure there are samples
+  def test_presence
+    Linguist::Language.all.each do |language|
+      language.all_extensions.each do |extension|
+        language_matches = Language.find_by_filename("foo#{extension}")
+
+        # If there is more than one language match for a given extension
+        # then check that there are examples for that language with the extension
+        if language_matches.length > 1
+          language_matches.each do |language|
+            assert File.directory?("samples/#{language.name}"), "#{language.name} is missing a samples directory"
+            assert Dir.glob("samples/#{language.name}/*#{extension}").any?, "#{language.name} is missing samples for extension #{extension}"
+          end
+        end
+      end
+    end
+  end
 end