Merge branch 'master' into sqlllll

test/fixtures/Shell/crossbuild_liblua5.1

@@ -0,0 +1,57 @@
+#! /bin/sh
+#
+# Builds and installs liblua5.1 for the cross toolchain.
+# Executed by build-uqm-dependencies.chroot
+
+# Include our common functions
+. /usr/lib/crossbuild/crossbuild.subr
+
+# envvar LIBLUA51_URL
+#
+# Specifies the URL of the liblua5.1 source tarball you want to use.
+export LIBLUA51_URL="http://www.lua.org/ftp/lua-5.1.5.tar.gz"
+
+# envvar INSTALL_TOP
+#
+# This determines where lua's makefiles install everything (we don't want to use
+# /usr/local!).
+export INSTALL_TOP="/usr/${HOST_TRIPLET}"
+
+# envvar TO_BIN
+#
+# Names of the binary files to install (that's right, lua's makefiles don't
+# determine this automatically, and since we end up with files named according
+# to Windows conventions the install chokes without these)
+export TO_BIN="lua.exe luac.exe"
+
+# envvar TO_LIB
+#
+# Names of the libraries to install, see TO_BIN
+export TO_LIB="liblua.a lua51.dll"
+
+
+# liblua5.1 uses custom makefiles and does not natively support cross-building.
+# However, with our cross toolchain in its PATH it successfully builds the mingw
+# target.
+export PATH=/usr/${HOST_TRIPLET}/bin:${PATH}
+
+echo "*************************************************************************"
+echo "--- BEGIN: crossbuild_liblua5.1 ---"
+
+get_tarball "liblua5.1" "${LIBLUA51_URL}" gz
+
+cd ${SRC_ROOT_DIR}/liblua5.1/*
+
+if [ -f Makefile ]; then
+  make clean
+  make --environment-overrides mingw install
+
+
+else
+  echo "crossbuild_liblua5.1 failed: Could not find Makefile"
+  echo "(is the liblua5.1 source tarball sane?)"
+  exit 1
+fi
+
+echo "--- END: crossbuild_liblua5.1 ---"
+echo "*************************************************************************"

test/fixtures/Shell/graylog2-server.init.d

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+### BEGIN INIT INFO
+# Provides:        graylog2-server
+# Required-Start:  $network
+# Required-Stop:   $network
+# Default-Start:   2 3 4 5
+# Default-Stop:    1
+# Short-Description: Start Graylog2 server
+### END INIT INFO
+
+# Written by Lital Natan <litaln@gmail.com>
+
+PREFIX=/usr
+SHAREDIR=$PREFIX/share/graylog2-server
+SERVER_JAR=$SHAREDIR/graylog2-server.jar
+SYSLOG4J_JAR=$SHAREDIR/syslog4j-0.9.46-bin.jar
+SVCNAME="graylog2-server"
+
+CONFIG="/etc/graylog2.conf"
+LOGFILE="/var/log/graylog2.log"
+PIDFILE="/var/run/graylog2.pid"
+
+start() {
+    if [ ! -e $CONFIG ]; then
+        echo "Config file $CONFIG does not exist"
+        return 1
+    fi
+
+    echo "Starting ${SVCNAME}"
+        nohup `which java` -cp $SERVER_JAR:$SYSLOG4J_JAR org.graylog2.Main \
+      -p ${PIDFILE} -f ${CONFIG} > $LOGFILE 2>&1 &
+
+    # Sleep before testing the service
+    sleep 2
+
+    graylog2_test || return 1
+}
+
+stop() {
+    pid=`< $PIDFILE`
+    kill $pid
+    rm -f ${PIDFILE} # just in case
+}
+
+graylog2_test() {
+    # Graylog2 only deletes its PID file if it hits a config error
+    if [ ! -e ${PIDFILE} ]; then
+        echo "Configuration error, check ${CONFIG}"
+        return 1
+    fi
+
+    local pid=`cat ${PIDFILE}`
+
+    # Graylog2 isn't running, so that means there was a problem
+    if [ ! -e /proc/$pid ]; then
+        echo "Something went wrong, check ${LOGFILE}"
+        rm -f ${PIDFILE}
+        return 1
+    else
+        return 0
+    fi
+}
+
+
+status() {
+    graylog2_test > /dev/null 2>&1
+    if [ "$?" == "0" ]; then
+       echo "Graylog2 server is up"
+       return 0
+    else
+       echo "Graylog2 server is down"
+       return 1
+    fi
+}
+
+
+restart() {
+    stop
+    start
+}
+
+case "$1" in
+    start)
+  start
+  ;;
+    stop)
+  stop
+  ;;
+    status)
+        status
+        ;;
+    restart)
+  restart
+  ;;
+    *)
+  echo "Usage $0 {start|stop|restart|status}"
+  RETVAL=1
+esac

test/test_blob.rb

@@ -216,6 +216,7 @@ class TestBlob < Minitest::Test
     assert sample_blob("C++/protocol-buffer.pb.cc").generated?
     assert sample_blob("Java/ProtocolBuffer.java").generated?
     assert sample_blob("Python/protocol_buffer_pb2.py").generated?
+    assert sample_blob("Go/api.pb.go").generated?
 
     # Generated JNI
     assert sample_blob("C/jni_layer.h").generated?
@@ -439,6 +440,54 @@ class TestBlob < Minitest::Test
     assert sample_blob("activator.bat").vendored?
     assert sample_blob("subproject/activator").vendored?
     assert sample_blob("subproject/activator.bat").vendored?
+
+    assert_predicate fixture_blob(".google_apis/bar.jar"), :vendored?
+    assert_predicate fixture_blob("foo/.google_apis/bar.jar"), :vendored?
+  end
+
+  def test_documentation
+    assert_predicate fixture_blob("doc/foo.html"), :documentation?
+    assert_predicate fixture_blob("docs/foo.html"), :documentation?
+    refute_predicate fixture_blob("project/doc/foo.html"), :documentation?
+    refute_predicate fixture_blob("project/docs/foo.html"), :documentation?
+
+    assert_predicate fixture_blob("Documentation/foo.md"), :documentation?
+    assert_predicate fixture_blob("documentation/foo.md"), :documentation?
+    assert_predicate fixture_blob("project/Documentation/foo.md"), :documentation?
+    assert_predicate fixture_blob("project/documentation/foo.md"), :documentation?
+
+    assert_predicate fixture_blob("javadoc/foo.html"), :documentation?
+    assert_predicate fixture_blob("project/javadoc/foo.html"), :documentation?
+
+    assert_predicate fixture_blob("man/foo.html"), :documentation?
+    refute_predicate fixture_blob("project/man/foo.html"), :documentation?
+
+    assert_predicate fixture_blob("README"), :documentation?
+    assert_predicate fixture_blob("README.md"), :documentation?
+    assert_predicate fixture_blob("README.txt"), :documentation?
+    assert_predicate fixture_blob("foo/README"), :documentation?
+
+    assert_predicate fixture_blob("CONTRIBUTING"), :documentation?
+    assert_predicate fixture_blob("CONTRIBUTING.md"), :documentation?
+    assert_predicate fixture_blob("CONTRIBUTING.txt"), :documentation?
+    assert_predicate fixture_blob("foo/CONTRIBUTING"), :documentation?
+
+    assert_predicate fixture_blob("LICENSE"), :documentation?
+    assert_predicate fixture_blob("LICENCE.md"), :documentation?
+    assert_predicate fixture_blob("LICENSE.txt"), :documentation?
+    assert_predicate fixture_blob("foo/LICENSE"), :documentation?
+
+    assert_predicate fixture_blob("COPYING"), :documentation?
+    assert_predicate fixture_blob("COPYING.md"), :documentation?
+    assert_predicate fixture_blob("COPYING.txt"), :documentation?
+    assert_predicate fixture_blob("foo/COPYING"), :documentation?
+
+    assert_predicate fixture_blob("INSTALL"), :documentation?
+    assert_predicate fixture_blob("INSTALL.md"), :documentation?
+    assert_predicate fixture_blob("INSTALL.txt"), :documentation?
+    assert_predicate fixture_blob("foo/INSTALL"), :documentation?
+
+    refute_predicate fixture_blob("foo.md"), :documentation?
   end
 
   def test_language
@@ -485,4 +534,29 @@ class TestBlob < Minitest::Test
     refute blob.new(" ").empty?
     refute blob.new("nope").empty?
   end
+
+  def test_include_in_language_stats
+    vendored = sample_blob("bower_components/custom/custom.js")
+    assert_predicate vendored, :vendored?
+    refute_predicate vendored, :include_in_language_stats?
+
+    documentation = fixture_blob("README")
+    assert_predicate documentation, :documentation?
+    refute_predicate documentation, :include_in_language_stats?
+
+    generated = sample_blob("CSS/bootstrap.min.css")
+    assert_predicate generated, :generated?
+    refute_predicate generated, :include_in_language_stats?
+
+    data = sample_blob("Ant Build System/filenames/ant.xml")
+    assert_equal :data, data.language.type
+    refute_predicate data, :include_in_language_stats?
+
+    prose = sample_blob("Markdown/tender.md")
+    assert_equal :prose, prose.language.type
+    refute_predicate prose, :include_in_language_stats?
+
+    included = sample_blob("HTML/pages.html")
+    assert_predicate included, :include_in_language_stats?
+  end
 end

test/test_grammars.rb

@@ -3,13 +3,19 @@ require_relative "./helper"
 class TestGrammars < Minitest::Test
   ROOT = File.expand_path("../..", __FILE__)
 
-  # These grammars have no license but have been grandfathered in. New grammars
-  # must have a license that allows redistribution.
-  UNLICENSED_GRAMMARS_WHITELIST = %w[
-    vendor/grammars/Sublime-Lasso
-    vendor/grammars/Sublime-REBOL
-    vendor/grammars/x86-assembly-textmate-bundle
-    vendor/grammars/oracle.tmbundle
+  LICENSE_WHITELIST = [
+    # This grammar's MIT license is inside a subdirectory.
+    "vendor/grammars/SublimePapyrus",
+
+    # This grammar has a nonstandard but acceptable license.
+    "vendor/grammars/gap-tmbundle",
+
+    # These grammars have no license but have been grandfathered in. New grammars
+    # must have a license that allows redistribution.
+    "vendor/grammars/Sublime-Lasso",
+    "vendor/grammars/Sublime-REBOL",
+    "vendor/grammars/x86-assembly-textmate-bundle",
+    "vendor/grammars/oracle.tmbundle"
   ].freeze
 
   def setup
@@ -78,9 +84,9 @@ class TestGrammars < Minitest::Test
 
     unlicensed = categories[:unlicensed] || []
     unrecognized = categories[:unrecognized] || []
-    disallowed_unlicensed = unlicensed - UNLICENSED_GRAMMARS_WHITELIST
-    disallowed_unrecognized = unrecognized - UNLICENSED_GRAMMARS_WHITELIST
-    extra_whitelist_entries = UNLICENSED_GRAMMARS_WHITELIST - (unlicensed | unrecognized)
+    disallowed_unlicensed = unlicensed - LICENSE_WHITELIST
+    disallowed_unrecognized = unrecognized - LICENSE_WHITELIST
+    extra_whitelist_entries = LICENSE_WHITELIST - (unlicensed | unrecognized)
 
     message = ""
     if disallowed_unlicensed.any?
@@ -94,7 +100,7 @@ class TestGrammars < Minitest::Test
     end
     if extra_whitelist_entries.any?
       message << "\n\n" unless message.empty?
-      message << "The following grammar submodules are listed in UNLICENSED_GRAMMARS_WHITELIST but either have a license (yay!)\n"
+      message << "The following grammar submodules are listed in LICENSE_WHITELIST but either have a license (yay!)\n"
       message << "or have been removed from the repository. Please remove them from the whitelist.\n"
       message << extra_whitelist_entries.sort.join("\n")
     end
@@ -132,6 +138,8 @@ class TestGrammars < Minitest::Test
       "unlicense"
     elsif content.include?("http://www.wtfpl.net/txt/copying/")
       "WTFPL"
+    elsif content.include?("zlib") && content.include?("license") && content.include?("2. Altered source versions must be plainly marked as such")
+      "zlib"
     end
   end
 end

test/test_heuristics.rb

@@ -48,7 +48,7 @@ class TestHeuristcs < Minitest::Test
   def test_pl_prolog_perl_by_heuristics
     assert_heuristics({
       "Prolog" => "Prolog/turing.pl",
-      "Perl" => "Perl/perl-test.t",
+      "Perl" => ["Perl/perl-test.t", "Perl/use5.pl"]
     })
   end
 
@@ -60,19 +60,22 @@ class TestHeuristcs < Minitest::Test
     })
   end
 
-  # Candidate languages = ["IDL", "Prolog"]
-  def test_pro_prolog_idl_by_heuristics
+  # Candidate languages = ["IDL", "Prolog", "QMake", "INI"]
+  def test_pro_by_heuristics
     assert_heuristics({
-      "Prolog" => "Prolog/logic-problem.pro",
-      "IDL" => "IDL/mg_acosh.pro"
+      "Prolog" => all_fixtures("Prolog", "*.pro"),
+      "IDL" => all_fixtures("IDL", "*.pro"),
+      "INI" => all_fixtures("INI", "*.pro"),
+      "QMake" => all_fixtures("QMake", "*.pro")
     })
   end
 
-  # Candidate languages = ["AGS Script", "AsciiDoc"]
-  def test_asc_asciidoc_by_heuristics
+  # Candidate languages = ["AGS Script", "AsciiDoc", "Public Key"]
+  def test_asc_by_heuristics
     assert_heuristics({
       "AsciiDoc" => "AsciiDoc/list.asc",
-      "AGS Script" => nil
+      "AGS Script" => "AGS Script/GlobalScript.asc",
+      "Public Key" => "Public Key/sunCert.asc"
     })
   end
 

test/test_language.rb

@@ -155,10 +155,6 @@ class TestLanguage < Minitest::Test
     assert_equal :prose, Language['Org'].type
   end
 
-  def test_other
-    assert_nil Language['Brainfuck'].type
-  end
-
   def test_searchable
     assert Language['Ruby'].searchable?
     assert !Language['Gettext Catalog'].searchable?
@@ -192,7 +188,7 @@ class TestLanguage < Minitest::Test
     assert_equal [], Language.find_by_extension('foo.rb')
     assert_equal [Language['Ruby']], Language.find_by_extension('rb')
     assert_equal [Language['Ruby']], Language.find_by_extension('.rb')
-    assert_equal [Language['M'], Language['Mathematica'], Language['Matlab'], Language['Mercury'], Language['Objective-C']], Language.find_by_extension('.m')
+    assert_equal [Language['M'], Language['MUF'], Language['Mathematica'], Language['Matlab'], Language['Mercury'], Language['Objective-C']], Language.find_by_extension('.m')
   end
 
   def test_find_all_by_extension
@@ -358,6 +354,15 @@ class TestLanguage < Minitest::Test
     assert missing.empty?, message
   end
 
+  def test_all_languages_have_type
+    missing = Language.all.select { |language| language.type.nil? }
+    message = "The following languages do not have a type listed in grammars.yml. Please add types for all new languages.\n"
+
+    width = missing.map { |language| language.name.length }.max
+    message << missing.map { |language| sprintf("%-#{width}s", language.name) }.sort.join("\n")
+    assert missing.empty?, message
+  end
+
   def test_all_languages_have_a_valid_ace_mode
     ace_fixture_path = File.join('test', 'fixtures', 'ace_modes.json')
     skip("No ace_modes.json file") unless File.exist?(ace_fixture_path)

test/test_repository.rb

@@ -99,4 +99,16 @@ class TestRepository < Minitest::Test
     # overridden .gitattributes
     assert !override_unvendored.vendored?
   end
+
+  def test_linguist_override_documentation?
+    attr_commit = "d4c8fb8a28e91f97a7e53428a365c0abbac36d3d"
+    repo = linguist_repo(attr_commit).read_index
+
+    readme = Linguist::LazyBlob.new(rugged_repository, attr_commit, "README.md")
+    arduino = Linguist::LazyBlob.new(rugged_repository, attr_commit, "samples/Arduino/hello.ino")
+
+    # overridden by .gitattributes
+    refute_predicate readme, :documentation?
+    assert_predicate arduino, :documentation?
+  end
 end