Merge branch 'master' into revert-2014-revert-1976-path-for-fileblob

Conflicts:
	lib/linguist/version.rb

test/fixtures/Data/Modelines/example_smalltalk.md

@@ -0,0 +1 @@
+; -*-mode:Smalltalk-*-

test/fixtures/Data/Modelines/iamphp.inc

@@ -0,0 +1 @@
+; -*- MoDe: PhP;-*-

test/fixtures/Data/Modelines/not_perl.pl

@@ -0,0 +1,3 @@
+/* vim: set filEtype=pRoloG: */
+
+# I am Prolog

test/fixtures/Data/Modelines/ruby

@@ -0,0 +1,3 @@
+/* vim: set filetype=ruby: */
+
+# I am Ruby

test/fixtures/Data/Modelines/seeplusplus

@@ -0,0 +1,3 @@
+/* vim: set ft=cpp: */
+
+I would like to be C++ please.

test/fixtures/Shell/crossbuild_liblua5.1

@@ -0,0 +1,57 @@
+#! /bin/sh
+#
+# Builds and installs liblua5.1 for the cross toolchain.
+# Executed by build-uqm-dependencies.chroot
+
+# Include our common functions
+. /usr/lib/crossbuild/crossbuild.subr
+
+# envvar LIBLUA51_URL
+#
+# Specifies the URL of the liblua5.1 source tarball you want to use.
+export LIBLUA51_URL="http://www.lua.org/ftp/lua-5.1.5.tar.gz"
+
+# envvar INSTALL_TOP
+#
+# This determines where lua's makefiles install everything (we don't want to use
+# /usr/local!).
+export INSTALL_TOP="/usr/${HOST_TRIPLET}"
+
+# envvar TO_BIN
+#
+# Names of the binary files to install (that's right, lua's makefiles don't
+# determine this automatically, and since we end up with files named according
+# to Windows conventions the install chokes without these)
+export TO_BIN="lua.exe luac.exe"
+
+# envvar TO_LIB
+#
+# Names of the libraries to install, see TO_BIN
+export TO_LIB="liblua.a lua51.dll"
+
+
+# liblua5.1 uses custom makefiles and does not natively support cross-building.
+# However, with our cross toolchain in its PATH it successfully builds the mingw
+# target.
+export PATH=/usr/${HOST_TRIPLET}/bin:${PATH}
+
+echo "*************************************************************************"
+echo "--- BEGIN: crossbuild_liblua5.1 ---"
+
+get_tarball "liblua5.1" "${LIBLUA51_URL}" gz
+
+cd ${SRC_ROOT_DIR}/liblua5.1/*
+
+if [ -f Makefile ]; then
+  make clean
+  make --environment-overrides mingw install
+
+
+else
+  echo "crossbuild_liblua5.1 failed: Could not find Makefile"
+  echo "(is the liblua5.1 source tarball sane?)"
+  exit 1
+fi
+
+echo "--- END: crossbuild_liblua5.1 ---"
+echo "*************************************************************************"

test/fixtures/Shell/graylog2-server.init.d

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+### BEGIN INIT INFO
+# Provides:        graylog2-server
+# Required-Start:  $network
+# Required-Stop:   $network
+# Default-Start:   2 3 4 5
+# Default-Stop:    1
+# Short-Description: Start Graylog2 server
+### END INIT INFO
+
+# Written by Lital Natan <litaln@gmail.com>
+
+PREFIX=/usr
+SHAREDIR=$PREFIX/share/graylog2-server
+SERVER_JAR=$SHAREDIR/graylog2-server.jar
+SYSLOG4J_JAR=$SHAREDIR/syslog4j-0.9.46-bin.jar
+SVCNAME="graylog2-server"
+
+CONFIG="/etc/graylog2.conf"
+LOGFILE="/var/log/graylog2.log"
+PIDFILE="/var/run/graylog2.pid"
+
+start() {
+    if [ ! -e $CONFIG ]; then
+        echo "Config file $CONFIG does not exist"
+        return 1
+    fi
+
+    echo "Starting ${SVCNAME}"
+        nohup `which java` -cp $SERVER_JAR:$SYSLOG4J_JAR org.graylog2.Main \
+      -p ${PIDFILE} -f ${CONFIG} > $LOGFILE 2>&1 &
+
+    # Sleep before testing the service
+    sleep 2
+
+    graylog2_test || return 1
+}
+
+stop() {
+    pid=`< $PIDFILE`
+    kill $pid
+    rm -f ${PIDFILE} # just in case
+}
+
+graylog2_test() {
+    # Graylog2 only deletes its PID file if it hits a config error
+    if [ ! -e ${PIDFILE} ]; then
+        echo "Configuration error, check ${CONFIG}"
+        return 1
+    fi
+
+    local pid=`cat ${PIDFILE}`
+
+    # Graylog2 isn't running, so that means there was a problem
+    if [ ! -e /proc/$pid ]; then
+        echo "Something went wrong, check ${LOGFILE}"
+        rm -f ${PIDFILE}
+        return 1
+    else
+        return 0
+    fi
+}
+
+
+status() {
+    graylog2_test > /dev/null 2>&1
+    if [ "$?" == "0" ]; then
+       echo "Graylog2 server is up"
+       return 0
+    else
+       echo "Graylog2 server is down"
+       return 1
+    fi
+}
+
+
+restart() {
+    stop
+    start
+}
+
+case "$1" in
+    start)
+  start
+  ;;
+    stop)
+  stop
+  ;;
+    status)
+        status
+        ;;
+    restart)
+  restart
+  ;;
+    *)
+  echo "Usage $0 {start|stop|restart|status}"
+  RETVAL=1
+esac

test/helper.rb

@@ -2,3 +2,21 @@ require "bundler/setup"
 require "minitest/autorun"
 require "mocha/setup"
 require "linguist"
+
+def fixtures_path
+  File.expand_path("../fixtures", __FILE__)
+end
+
+def fixture_blob(name)
+  name = File.join(fixtures_path, name) unless name =~ /^\//
+  Linguist::FileBlob.new(name, fixtures_path)
+end
+
+def samples_path
+  File.expand_path("../../samples", __FILE__)
+end
+
+def sample_blob(name)
+  name = File.join(samples_path, name) unless name =~ /^\//
+  Linguist::FileBlob.new(name, samples_path)
+end

test/test_blob.rb

@@ -14,24 +14,6 @@ class TestBlob < Minitest::Test
     Encoding.default_external = @original_external
   end
 
-  def samples_path
-    File.expand_path("../../samples", __FILE__)
-  end
-
-  def fixtures_path
-    File.expand_path("../fixtures", __FILE__)
-  end
-
-  def sample_blob(name)
-    name = File.join(samples_path, name) unless name =~ /^\//
-    FileBlob.new(name, samples_path)
-  end
-
-  def fixture_blob(name)
-    name = File.join(fixtures_path, name) unless name =~ /^\//
-    FileBlob.new(name, fixtures_path)
-  end
-
   def script_blob(name)
     blob = sample_blob(name)
     blob.instance_variable_set(:@name, 'script')
@@ -234,6 +216,7 @@ class TestBlob < Minitest::Test
     assert sample_blob("C++/protocol-buffer.pb.cc").generated?
     assert sample_blob("Java/ProtocolBuffer.java").generated?
     assert sample_blob("Python/protocol_buffer_pb2.py").generated?
+    assert sample_blob("Go/api.pb.go").generated?
 
     # Generated JNI
     assert sample_blob("C/jni_layer.h").generated?
@@ -467,6 +450,54 @@ class TestBlob < Minitest::Test
     assert sample_blob("activator.bat").vendored?
     assert sample_blob("subproject/activator").vendored?
     assert sample_blob("subproject/activator.bat").vendored?
+
+    assert_predicate fixture_blob(".google_apis/bar.jar"), :vendored?
+    assert_predicate fixture_blob("foo/.google_apis/bar.jar"), :vendored?
+  end
+
+  def test_documentation
+    assert_predicate fixture_blob("doc/foo.html"), :documentation?
+    assert_predicate fixture_blob("docs/foo.html"), :documentation?
+    refute_predicate fixture_blob("project/doc/foo.html"), :documentation?
+    refute_predicate fixture_blob("project/docs/foo.html"), :documentation?
+
+    assert_predicate fixture_blob("Documentation/foo.md"), :documentation?
+    assert_predicate fixture_blob("documentation/foo.md"), :documentation?
+    assert_predicate fixture_blob("project/Documentation/foo.md"), :documentation?
+    assert_predicate fixture_blob("project/documentation/foo.md"), :documentation?
+
+    assert_predicate fixture_blob("javadoc/foo.html"), :documentation?
+    assert_predicate fixture_blob("project/javadoc/foo.html"), :documentation?
+
+    assert_predicate fixture_blob("man/foo.html"), :documentation?
+    refute_predicate fixture_blob("project/man/foo.html"), :documentation?
+
+    assert_predicate fixture_blob("README"), :documentation?
+    assert_predicate fixture_blob("README.md"), :documentation?
+    assert_predicate fixture_blob("README.txt"), :documentation?
+    assert_predicate fixture_blob("foo/README"), :documentation?
+
+    assert_predicate fixture_blob("CONTRIBUTING"), :documentation?
+    assert_predicate fixture_blob("CONTRIBUTING.md"), :documentation?
+    assert_predicate fixture_blob("CONTRIBUTING.txt"), :documentation?
+    assert_predicate fixture_blob("foo/CONTRIBUTING"), :documentation?
+
+    assert_predicate fixture_blob("LICENSE"), :documentation?
+    assert_predicate fixture_blob("LICENCE.md"), :documentation?
+    assert_predicate fixture_blob("LICENSE.txt"), :documentation?
+    assert_predicate fixture_blob("foo/LICENSE"), :documentation?
+
+    assert_predicate fixture_blob("COPYING"), :documentation?
+    assert_predicate fixture_blob("COPYING.md"), :documentation?
+    assert_predicate fixture_blob("COPYING.txt"), :documentation?
+    assert_predicate fixture_blob("foo/COPYING"), :documentation?
+
+    assert_predicate fixture_blob("INSTALL"), :documentation?
+    assert_predicate fixture_blob("INSTALL.md"), :documentation?
+    assert_predicate fixture_blob("INSTALL.txt"), :documentation?
+    assert_predicate fixture_blob("foo/INSTALL"), :documentation?
+
+    refute_predicate fixture_blob("foo.md"), :documentation?
   end
 
   def test_language
@@ -513,4 +544,29 @@ class TestBlob < Minitest::Test
     refute blob.new(" ").empty?
     refute blob.new("nope").empty?
   end
+
+  def test_include_in_language_stats
+    vendored = sample_blob("bower_components/custom/custom.js")
+    assert_predicate vendored, :vendored?
+    refute_predicate vendored, :include_in_language_stats?
+
+    documentation = fixture_blob("README")
+    assert_predicate documentation, :documentation?
+    refute_predicate documentation, :include_in_language_stats?
+
+    generated = sample_blob("CSS/bootstrap.min.css")
+    assert_predicate generated, :generated?
+    refute_predicate generated, :include_in_language_stats?
+
+    data = sample_blob("Ant Build System/filenames/ant.xml")
+    assert_equal :data, data.language.type
+    refute_predicate data, :include_in_language_stats?
+
+    prose = sample_blob("Markdown/tender.md")
+    assert_equal :prose, prose.language.type
+    refute_predicate prose, :include_in_language_stats?
+
+    included = sample_blob("HTML/pages.html")
+    assert_predicate included, :include_in_language_stats?
+  end
 end

test/test_classifier.rb

@@ -3,10 +3,6 @@ require_relative "./helper"
 class TestClassifier < Minitest::Test
   include Linguist
 
-  def samples_path
-    File.expand_path("../../samples", __FILE__)
-  end
-
   def fixture(name)
     File.read(File.join(samples_path, name))
   end

test/test_generated.rb

@@ -3,10 +3,6 @@ require_relative "./helper"
 class TestGenerated < Minitest::Test
   include Linguist
 
-  def samples_path
-    File.expand_path("../../samples", __FILE__)
-  end
-
   class DataLoadedError < StandardError; end
 
   def generated_without_loading_data(name)

test/test_grammars.rb

@@ -3,12 +3,18 @@ require_relative "./helper"
 class TestGrammars < Minitest::Test
   ROOT = File.expand_path("../..", __FILE__)
 
-  # These grammars have no license but have been grandfathered in. New grammars
-  # must have a license that allows redistribution.
-  UNLICENSED_GRAMMARS_WHITELIST = %w[
-    vendor/grammars/Sublime-Lasso
-    vendor/grammars/Sublime-REBOL
-    vendor/grammars/x86-assembly-textmate-bundle
+  LICENSE_WHITELIST = [
+    # This grammar's MIT license is inside a subdirectory.
+    "vendor/grammars/SublimePapyrus",
+
+    # This grammar has a nonstandard but acceptable license.
+    "vendor/grammars/gap-tmbundle",
+
+    # These grammars have no license but have been grandfathered in. New grammars
+    # must have a license that allows redistribution.
+    "vendor/grammars/Sublime-Lasso",
+    "vendor/grammars/Sublime-REBOL",
+    "vendor/grammars/x86-assembly-textmate-bundle",
   ].freeze
 
   def setup
@@ -77,9 +83,9 @@ class TestGrammars < Minitest::Test
 
     unlicensed = categories[:unlicensed] || []
     unrecognized = categories[:unrecognized] || []
-    disallowed_unlicensed = unlicensed - UNLICENSED_GRAMMARS_WHITELIST
-    disallowed_unrecognized = unrecognized - UNLICENSED_GRAMMARS_WHITELIST
-    extra_whitelist_entries = UNLICENSED_GRAMMARS_WHITELIST - (unlicensed | unrecognized)
+    disallowed_unlicensed = unlicensed - LICENSE_WHITELIST
+    disallowed_unrecognized = unrecognized - LICENSE_WHITELIST
+    extra_whitelist_entries = LICENSE_WHITELIST - (unlicensed | unrecognized)
 
     message = ""
     if disallowed_unlicensed.any?
@@ -93,7 +99,7 @@ class TestGrammars < Minitest::Test
     end
     if extra_whitelist_entries.any?
       message << "\n\n" unless message.empty?
-      message << "The following grammar submodules are listed in UNLICENSED_GRAMMARS_WHITELIST but either have a license (yay!)\n"
+      message << "The following grammar submodules are listed in LICENSE_WHITELIST but either have a license (yay!)\n"
       message << "or have been removed from the repository. Please remove them from the whitelist.\n"
       message << extra_whitelist_entries.sort.join("\n")
     end
@@ -131,6 +137,8 @@ class TestGrammars < Minitest::Test
       "unlicense"
     elsif content.include?("http://www.wtfpl.net/txt/copying/")
       "WTFPL"
+    elsif content.include?("zlib") && content.include?("license") && content.include?("2. Altered source versions must be plainly marked as such")
+      "zlib"
     end
   end
 end

test/test_heuristics.rb

@@ -3,10 +3,6 @@ require_relative "./helper"
 class TestHeuristcs < Minitest::Test
   include Linguist
 
-  def samples_path
-    File.expand_path("../../samples", __FILE__)
-  end
-
   def fixture(name)
     File.read(File.join(samples_path, name))
   end
@@ -52,7 +48,7 @@ class TestHeuristcs < Minitest::Test
   def test_pl_prolog_perl_by_heuristics
     assert_heuristics({
       "Prolog" => "Prolog/turing.pl",
-      "Perl" => "Perl/perl-test.t",
+      "Perl" => ["Perl/perl-test.t", "Perl/use5.pl"]
     })
   end
 
@@ -72,11 +68,12 @@ class TestHeuristcs < Minitest::Test
     })
   end
 
-  # Candidate languages = ["AGS Script", "AsciiDoc"]
-  def test_asc_asciidoc_by_heuristics
+  # Candidate languages = ["AGS Script", "AsciiDoc", "Public Key"]
+  def test_asc_by_heuristics
     assert_heuristics({
       "AsciiDoc" => "AsciiDoc/list.asc",
-      "AGS Script" => nil
+      "AGS Script" => "AGS Script/GlobalScript.asc",
+      "Public Key" => "Public Key/sunCert.asc"
     })
   end
 
@@ -133,6 +130,13 @@ class TestHeuristcs < Minitest::Test
     })
   end
 
+  def test_lsp_by_heuristics
+    assert_heuristics({
+      "Common Lisp" => all_fixtures("Common Lisp"),
+      "NewLisp" => all_fixtures("NewLisp")
+    })
+  end
+
   def test_cs_by_heuristics
     assert_heuristics({
       "C#" => all_fixtures("C#", "*.cs"),

test/test_modelines.rb

@@ -0,0 +1,25 @@
+require_relative "./helper"
+
+class TestModelines < Minitest::Test
+  include Linguist
+
+  def assert_modeline(language, blob)
+    assert_equal language, Linguist::Strategy::Modeline.call(blob).first
+  end
+
+  def test_modeline_strategy
+    assert_modeline Language["Ruby"], fixture_blob("Data/Modelines/ruby")
+    assert_modeline Language["C++"], fixture_blob("Data/Modelines/seeplusplus")
+    assert_modeline Language["Prolog"], fixture_blob("Data/Modelines/not_perl.pl")
+    assert_modeline Language["Smalltalk"], fixture_blob("Data/Modelines/example_smalltalk.md")
+    assert_modeline Language["PHP"], fixture_blob("Data/Modelines/iamphp.inc")
+  end
+
+  def test_modeline_languages
+    assert_equal Language["Ruby"], fixture_blob("Data/Modelines/ruby").language
+    assert_equal Language["C++"], fixture_blob("Data/Modelines/seeplusplus").language
+    assert_equal Language["Prolog"], fixture_blob("Data/Modelines/not_perl.pl").language
+    assert_equal Language["Smalltalk"], fixture_blob("Data/Modelines/example_smalltalk.md").language
+    assert_equal Language["PHP"], fixture_blob("Data/Modelines/iamphp.inc").language
+  end
+end

test/test_repository.rb

@@ -99,4 +99,16 @@ class TestRepository < Minitest::Test
     # overridden .gitattributes
     assert !override_unvendored.vendored?
   end
+
+  def test_linguist_override_documentation?
+    attr_commit = "d4c8fb8a28e91f97a7e53428a365c0abbac36d3d"
+    repo = linguist_repo(attr_commit).read_index
+
+    readme = Linguist::LazyBlob.new(rugged_repository, attr_commit, "README.md")
+    arduino = Linguist::LazyBlob.new(rugged_repository, attr_commit, "samples/Arduino/hello.ino")
+
+    # overridden by .gitattributes
+    refute_predicate readme, :documentation?
+    assert_predicate arduino, :documentation?
+  end
 end

test/test_tokenizer.rb

@@ -3,10 +3,6 @@ require_relative "./helper"
 class TestTokenizer < Minitest::Test
   include Linguist
 
-  def samples_path
-    File.expand_path("../../samples", __FILE__)
-  end
-
   def tokenize(data)
     data = File.read(File.join(samples_path, data.to_s)) if data.is_a?(Symbol)
     Tokenizer.tokenize(data)
@@ -41,6 +37,8 @@ class TestTokenizer < Minitest::Test
     assert_equal %w(foo), tokenize("foo {- Comment -}")
     assert_equal %w(foo), tokenize("foo (* Comment *)")
     assert_equal %w(%), tokenize("2 % 10\n% Comment")
+    assert_equal %w(foo bar), tokenize("foo\n\"\"\"\nComment\n\"\"\"\nbar")
+    assert_equal %w(foo bar), tokenize("foo\n'''\nComment\n'''\nbar")
   end
 
   def test_sgml_tags