Add some linguist-detectable attributes (#3806)

This allows to test a use-case where markdown should be reported
in the language stats (e.g because the main purpose of the repo is to
hold prose in markdown format) while HTML should be excluded from the
language stats (e.g because it's example output).

.gitattributes

@@ -0,0 +1,9 @@
+Gemfile linguist-vendored=true
+lib/linguist.rb linguist-language=Java
+test/*.rb linguist-language=Java
+Rakefile linguist-generated
+test/fixtures/* linguist-vendored=false
+README.md linguist-documentation=false
+samples/Arduino/* linguist-documentation
+samples/Markdown/*.md linguist-detectable=true
+samples/HTML/*.html linguist-detectable=false

lib/linguist/heuristics.rb

@@ -1,7 +1,7 @@
 module Linguist
   # A collection of simple heuristics that can be used to better analyze languages.
   class Heuristics
-    ACTIVE = false
+    ACTIVE = true
 
     # Public: Given an array of String language names,
     # apply heuristics against the given data and return an array
@@ -13,24 +13,13 @@ module Linguist
     # Returns an array of Languages or []
     def self.find_by_heuristics(data, languages)
       if active?
-        if languages.all? { |l| ["Objective-C", "C++"].include?(l) }
-          disambiguate_c(data, languages)
-        end
         if languages.all? { |l| ["Perl", "Prolog"].include?(l) }
-          disambiguate_pl(data, languages)
+          result = disambiguate_pl(data, languages)
         end
         if languages.all? { |l| ["ECL", "Prolog"].include?(l) }
-          disambiguate_ecl(data, languages)
-        end
-        if languages.all? { |l| ["TypeScript", "XML"].include?(l) }
-          disambiguate_ts(data, languages)
-        end
-        if languages.all? { |l| ["Common Lisp", "OpenCL"].include?(l) }
-          disambiguate_cl(data, languages)
-        end
-        if languages.all? { |l| ["Rebol", "R"].include?(l) }
-          disambiguate_r(data, languages)
+          result = disambiguate_ecl(data, languages)
         end
+        return result
       end
     end
 

lib/linguist/language.rb

@@ -136,7 +136,7 @@ module Linguist
         elsif (determined = Heuristics.find_by_heuristics(data, possible_language_names)) && !determined.empty?
           determined.first
         # Lastly, fall back to the probablistic classifier.
-        elsif classified = Classifier.classify(Samples::DATA, data, possible_language_names ).first
+        elsif classified = Classifier.classify(Samples::DATA, data, possible_language_names).first
           # Return the actual Language object based of the string language name (i.e., first element of `#classify`)
           Language[classified[0]]
         end

lib/linguist/languages.yml

@@ -647,7 +647,7 @@ Elm:
 
 Emacs Lisp:
   type: programming
-  lexer: Scheme
+  lexer: Common Lisp
   color: "#c065db"
   aliases:
   - elisp
@@ -1001,6 +1001,12 @@ IDL:
   - .pro
   - .dlm
 
+IGOR Pro:
+  type: programming
+  lexer: Igor
+  extensions:
+  - .ipf
+
 INI:
   type: data
   extensions:
@@ -1196,7 +1202,17 @@ LFE:
 LLVM:
   extensions:
   - .ll
-  
+
+LSL:
+  type: programming
+  lexer: LSL
+  ace_mode: lsl
+  extensions:
+  - .lsl
+  interpreters:
+  - lsl
+  color: '#3d9970'
+
 LabVIEW:
   type: programming
   lexer: Text only
@@ -1778,9 +1794,9 @@ Prolog:
   type: programming
   color: "#74283c"
   extensions:
-  - .prolog
-  - .ecl
   - .pl
+  - .ecl
+  - .prolog
 
 Propeller Spin:
   type: programming
@@ -1825,6 +1841,7 @@ Python:
   color: "#3581ba"
   extensions:
   - .py
+  - .cgi
   - .gyp
   - .lmi
   - .pyde
@@ -2134,6 +2151,7 @@ Shell:
   - .sh
   - .bash
   - .bats
+  - .cgi
   - .tmux
   - .zsh
   interpreters:

lib/linguist/version.rb

@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "3.1.2"
+  VERSION = "3.1.5"
 end

samples/IGOR Pro/functions.ipf

@@ -0,0 +1,38 @@
+#pragma rtGlobals=3
+
+Function FooBar()
+	return 0
+End
+
+Function FooBarSubType() : ButtonControl
+	return 0
+End
+
+Function/D FooBarVar()
+	return 0
+End
+
+static Function FooBarStatic()
+	return 0
+End
+
+threadsafe static Function FooBarStaticThreadsafe()
+	return 0
+End
+
+threadsafe Function FooBarThread()
+	return 0
+End
+
+Function CallOperationsAndBuiltInFuncs(string var)
+
+	string someDQString = "abcd"
+
+	Make/N=(1,2,3,4) myWave
+	Redimension/N=(-1,-1,-1,5) myWave
+
+	print strlen(someDQString)
+
+	return 0
+End
+

samples/IGOR Pro/generic.ipf

@@ -0,0 +1,21 @@
+#pragma rtGlobals=3
+
+StrConstant myConstString="abcd"
+// some comment
+constant myConst=123
+
+Structure struct1
+	string str
+	variable var
+EndStructure
+
+static Structure struct2
+	string str
+	variable var
+EndStructure
+
+#include "someFile"
+
+#ifdef NOT_DEFINED
+	// conditional compilation
+#endif

samples/LSL/LSL.lsl

@@ -0,0 +1,74 @@
+/*
+    Testing syntax highlighting
+    for the Linden Scripting Language
+*/
+
+integer someIntNormal       = 3672;
+integer someIntHex          = 0x00000000;
+integer someIntMath         = PI_BY_TWO;
+
+integer event               = 5673;// 'event' is invalid.illegal
+
+key someKeyTexture          = TEXTURE_DEFAULT;
+string someStringSpecial    = EOF;
+
+some_user_defined_function_without_return_type(string inputAsString)
+{
+    llSay(PUBLIC_CHANNEL, inputAsString);
+}
+
+string user_defined_function_returning_a_string(key inputAsKey)
+{
+    return (string)inputAsKey;
+}
+
+default
+{
+    state_entry()
+    {
+        key someKey = NULL_KEY;
+        someKey = llGetOwner();
+
+        string someString = user_defined_function_returning_a_string(someKey);
+
+        some_user_defined_function_without_return_type(someString);
+    }
+
+    touch_start(integer num_detected)
+    {
+        list agentsInRegion = llGetAgentList(AGENT_LIST_REGION, []);
+        integer numOfAgents = llGetListLength(agentsInRegion);
+
+        integer index;                                                          // defaults to 0
+        for (; index <= numOfAgents - 1; index++)                               // for each agent in region
+        {
+            llRegionSayTo(llList2Key(agentsInRegion, index), PUBLIC_CHANNEL, "Hello, Avatar!");
+        }
+    }
+
+    touch_end(integer num_detected)
+    {
+        someIntNormal       = 3672;
+        someIntHex          = 0x00000000;
+        someIntMath         = PI_BY_TWO;
+
+        event               = 5673;// 'event' is invalid.illegal
+
+        someKeyTexture      = TEXTURE_DEFAULT;
+        someStringSpecial   = EOF;
+
+        llSetInventoryPermMask("some item", MASK_NEXT, PERM_ALL);// 'llSetInventoryPermMask' is reserved.godmode
+
+        llWhisper(PUBLIC_CHANNEL, "Leaving \"default\" now...");
+        state other;
+    }
+}
+
+state other
+{
+    state_entry()
+    {
+        llWhisper(PUBLIC_CHANNEL, "Entered \"state other\", returning to \"default\" again...");
+        state default;
+    }
+}

samples/Prolog/dleak-report.script!

@@ -0,0 +1,11 @@
+#!/usr/bin/env swipl
+
+:- set_prolog_flag(verbose, silent).
+:- use_module(dleak).
+
+:- initialization
+	main, halt.
+
+main :-
+	current_prolog_flag(argv, [File]),
+	dleak(File).

samples/Prolog/ex6.pl

@@ -0,0 +1,5 @@
+%6.8
+subset(Set, Subset) :-
+	append(L1, Subset, Set).
+powerset(Set, Subset) :-
+	bagof(Subset, subset(Set, Subset), Subset).

samples/Python/action.cgi

@@ -0,0 +1,82 @@
+#!/usr/bin/python
+
+from model import Feed
+import session
+import datetime
+import sys
+
+argv = session.argv()
+
+feed = Feed.get(guid=argv[1])
+action = argv[2]
+
+if action == 'done':
+    when = feed.notify_interval * feed.notify_unit
+elif action == 'snooze':
+    if len(argv) > 3:
+        when = int(argv[3])
+    else:
+        when = 3600
+else:
+    print '''Status: 400 Bad request
+Content-type: text/html
+
+Unknown action %s''' % action
+    sys.exit(1)
+
+feed.notify_next = datetime.datetime.utcnow() + datetime.timedelta(seconds=when)
+feed.save()
+
+response = '''Content-type: text/html
+
+<html><head><title>Alarm reset</title>
+<link rel="stylesheet" href="{base_url}/style.css">
+</head>
+<body>
+
+<div class="container">
+<h1>Alarm reset</h1>
+<div>
+<p id="reset">Alarm "<span class="name">{name}</span>" has been reset. You won't be notified for another <span class="duration">{duration}</span>.</p>
+
+<p>Actions:</p>
+<ul>
+<li><a href="{edit_url}?feed={guid}">Edit this reminder</a></li>
+<li><a href="{edit_url}">Create another reminder</a></li>
+<li><a href="{base_url}">Visit the Reminder Me site</a></li>
+</ul>
+</div>
+</div>
+
+<p class="back"><a href=".">Reminder Me</a></p>
+
+</body></html>'''
+
+when_left = when
+duration_list = []
+for (label,period) in [('month',86400*365/12),
+                       ('week',86400*7),
+                       ('day',86400),
+                       ('hour',3600),
+                       ('minute',60),
+                       ('second',1)]:
+    if when == period:
+        duration_list = [label]
+        break
+    
+    val = when_left/period
+    if val:
+        duration_list.append("%d %s%s" % (
+            val,
+            label,
+            val > 1 and 's' or ''))
+        when_left -= val*period
+
+basedir=session.request_script_dir()
+
+print response.format(guid=feed.guid,
+                      name=feed.name,
+                      edit_url="%s/edit.cgi" % basedir,
+                      base_url=basedir,
+                      duration=', '.join(duration_list))
+                      

samples/Shell/settime.cgi

@@ -0,0 +1,27 @@
+#!/bin/bash
+echo "Content-type: text/html"
+day=`echo "$QUERY_STRING" | sed -n 's/^.*day=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"`
+month=`echo "$QUERY_STRING" | sed -n 's/^.*month=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"`
+year=`echo "$QUERY_STRING" | sed -n 's/^.*year=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"`
+hour=`echo "$QUERY_STRING" | sed -n 's/^.*hour=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"`
+minute=`echo "$QUERY_STRING" | sed -n 's/^.*minute=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"`
+second=`echo "$QUERY_STRING" | sed -n 's/^.*second=\([^&]*\).*$/\1/p' | sed "s/%20/ /g"`
+echo ""
+echo "<html><body>"
+
+echo "<pre> $(killall ems) </pre>"
+
+
+
+echo "<pre> $(date $month$day$hour$minute$year.$second) </pre>"
+
+echo "<pre> $(/sbin/hwclock -w>/dev/null & /sbin/reboot) </pre>"
+
+echo "<pre> $(/sbin/reboot) </pre>"
+
+
+
+
+
+
+echo "</body></html>"

test/fixtures/foo.rb

@@ -0,0 +1,3 @@
+def foo
+  return "BAR"
+end

test/test_language.rb

@@ -33,6 +33,7 @@ class TestLanguage < Test::Unit::TestCase
     assert_equal Lexer['Java'], Language['ChucK'].lexer
     assert_equal Lexer['Java'], Language['Java'].lexer
     assert_equal Lexer['JavaScript'], Language['JavaScript'].lexer
+    assert_equal Lexer['LSL'], Language['LSL'].lexer
     assert_equal Lexer['MOOCode'], Language['Moocode'].lexer
     assert_equal Lexer['MuPAD'], Language['mupad'].lexer
     assert_equal Lexer['NASM'], Language['Assembly'].lexer
@@ -46,7 +47,6 @@ class TestLanguage < Test::Unit::TestCase
     assert_equal Lexer['Ruby'], Language['Mirah'].lexer
     assert_equal Lexer['Ruby'], Language['Ruby'].lexer
     assert_equal Lexer['S'], Language['R'].lexer
-    assert_equal Lexer['Scheme'], Language['Emacs Lisp'].lexer
     assert_equal Lexer['Scheme'], Language['Nu'].lexer
     assert_equal Lexer['Racket'], Language['Racket'].lexer
     assert_equal Lexer['Scheme'], Language['Scheme'].lexer
@@ -187,6 +187,7 @@ class TestLanguage < Test::Unit::TestCase
 
   def test_programming
     assert_equal :programming, Language['JavaScript'].type
+    assert_equal :programming, Language['LSL'].type
     assert_equal :programming, Language['Perl'].type
     assert_equal :programming, Language['PowerShell'].type
     assert_equal :programming, Language['Python'].type
@@ -327,6 +328,7 @@ class TestLanguage < Test::Unit::TestCase
     assert_equal '#3581ba', Language['Python'].color
     assert_equal '#f1e05a', Language['JavaScript'].color
     assert_equal '#31859c', Language['TypeScript'].color
+    assert_equal '#3d9970', Language['LSL'].color
   end
 
   def test_colors
@@ -339,6 +341,7 @@ class TestLanguage < Test::Unit::TestCase
     assert_equal 'coffee', Language['CoffeeScript'].ace_mode
     assert_equal 'csharp', Language['C#'].ace_mode
     assert_equal 'css', Language['CSS'].ace_mode
+    assert_equal 'lsl', Language['LSL'].ace_mode
     assert_equal 'javascript', Language['JavaScript'].ace_mode
   end
 
@@ -353,6 +356,7 @@ class TestLanguage < Test::Unit::TestCase
   end
 
   def test_extensions
+    assert Language['LSL'].extensions.include?('.lsl')
     assert Language['Perl'].extensions.include?('.pl')
     assert Language['Python'].extensions.include?('.py')
     assert Language['Ruby'].extensions.include?('.rb')