Merge branch 'master' into clarion

lib/linguist.rb

@@ -6,3 +6,15 @@ require 'linguist/repository'
 require 'linguist/samples'
 require 'linguist/shebang'
 require 'linguist/version'
+
+class << Linguist
+  attr_accessor :instrumenter
+
+  def instrument(*args, &bk)
+    if instrumenter
+      instrumenter.instrument(*args, &bk)
+    else
+      yield if block_given?
+    end
+  end
+end

lib/linguist/blob_helper.rb

@@ -99,7 +99,7 @@ module Linguist
       elsif name.nil?
         "attachment"
       else
-        "attachment; filename=#{EscapeUtils.escape_url(File.basename(name))}"
+        "attachment; filename=#{EscapeUtils.escape_url(name)}"
       end
     end
 
@@ -233,7 +233,7 @@ module Linguist
     #
     # Return true or false
     def vendored?
-      name =~ VendoredRegexp ? true : false
+      path =~ VendoredRegexp ? true : false
     end
 
     documentation_paths = YAML.load_file(File.expand_path("../documentation.yml", __FILE__))
@@ -248,7 +248,7 @@ module Linguist
     #
     # Return true or false
     def documentation?
-      name =~ DocumentationRegexp ? true : false
+      path =~ DocumentationRegexp ? true : false
     end
 
     # Public: Get each line of data
@@ -316,7 +316,7 @@ module Linguist
     #
     # Return true or false
     def generated?
-      @_generated ||= Generated.generated?(name, lambda { data })
+      @_generated ||= Generated.generated?(path, lambda { data })
     end
 
     # Public: Detects the Language of the blob.

lib/linguist/documentation.yml

@@ -7,10 +7,14 @@
 # Please add additional test coverage to
 # `test/test_blob.rb#test_documentation` if you make any changes.
 
-## Documentation Conventions ##
+## Documentation directories ##
 
 - ^docs?/
-- ^Documentation/
+- (^|/)[Dd]ocumentation/
+- (^|/)javadoc/
+- ^man/
+
+## Documentation files ##
 
 - (^|/)CONTRIBUTING(\.|$)
 - (^|/)COPYING(\.|$)

lib/linguist/file_blob.rb

@@ -3,7 +3,7 @@ require 'linguist/blob_helper'
 module Linguist
   # A FileBlob is a wrapper around a File object to make it quack
   # like a Grit::Blob. It provides the basic interface: `name`,
-  # `data`, and `size`.
+  # `data`, `path` and `size`.
   class FileBlob
     include BlobHelper
 
@@ -14,43 +14,50 @@ module Linguist
     #
     # Returns a FileBlob.
     def initialize(path, base_path = nil)
-      @path = path
-      @name = base_path ? path.sub("#{base_path}/", '') : path
+      @fullpath = path
+      @path = base_path ? path.sub("#{base_path}/", '') : path
     end
 
     # Public: Filename
     #
     # Examples
     #
-    #   FileBlob.new("/path/to/linguist/lib/linguist.rb").name
+    #   FileBlob.new("/path/to/linguist/lib/linguist.rb").path
     #   # =>  "/path/to/linguist/lib/linguist.rb"
     #
     #   FileBlob.new("/path/to/linguist/lib/linguist.rb",
-    #                "/path/to/linguist").name
+    #                "/path/to/linguist").path
     #   # =>  "lib/linguist.rb"
     #
     # Returns a String
-    attr_reader :name
+    attr_reader :path
 
     # Public: Read file permissions
     #
     # Returns a String like '100644'
     def mode
-      File.stat(@path).mode.to_s(8)
+      File.stat(@fullpath).mode.to_s(8)
+    end
+
+    # Public: File name
+    #
+    # Returns a String
+    def name
+      File.basename(@fullpath)
     end
 
     # Public: Read file contents.
     #
     # Returns a String.
     def data
-      File.read(@path)
+      File.read(@fullpath)
     end
 
     # Public: Get byte size
     #
     # Returns an Integer.
     def size
-      File.size(@path)
+      File.size(@fullpath)
     end
 
     # Public: Get file extension.
@@ -67,7 +74,7 @@ module Linguist
     #
     # Returns an Array
     def extensions
-      basename, *segments = File.basename(name).split(".")
+      basename, *segments = name.downcase.split(".")
 
       segments.map.with_index do |segment, index|
         "." + segments[index..-1].join(".")

lib/linguist/generated.rb

@@ -58,10 +58,13 @@ module Linguist
       godeps? ||
       generated_by_zephir? ||
       minified_files? ||
+      source_map? ||
       compiled_coffeescript? ||
       generated_parser? ||
       generated_net_docfile? ||
       generated_postscript? ||
+      compiled_cython_file? ||
+      generated_protocol_buffer_go? ||
       generated_protocol_buffer? ||
       generated_jni_header? ||
       vcr_cassette?
@@ -94,6 +97,20 @@ module Linguist
       end
     end
 
+    # Internal: Is the blob a generated source map?
+    #
+    # Source Maps usually have .css.map or .js.map extensions. In case they
+    # are not following the name convention, detect them based on the content.
+    #
+    # Returns true or false.
+    def source_map?
+      return false unless extname.downcase == '.map'
+
+      name =~ /(\.css|\.js)\.map$/i ||                 # Name convention
+      lines[0] =~ /^{"version":\d+,/ ||                # Revision 2 and later begin with the version number
+      lines[0] =~ /^\/\*\* Begin line maps\. \*\*\/{/  # Revision 1 begins with a magic comment
+    end
+
     # Internal: Is the blob of JS generated by CoffeeScript?
     #
     # CoffeeScript is meant to output JS that would be difficult to
@@ -202,6 +219,13 @@ module Linguist
         creator.include?("ImageMagick")
     end
 
+    def generated_protocol_buffer_go?
+      return false unless extname == '.go'
+      return false unless lines.count > 1
+
+      return lines[0].include?("Code generated by protoc-gen-go")
+    end
+
     # Internal: Is the blob a C++, Java or Python source file generated by the
     # Protocol Buffer compiler?
     #
@@ -262,5 +286,18 @@ module Linguist
       # VCR Cassettes have "recorded_with: VCR" in the second last line.
       return lines[-2].include?("recorded_with: VCR")
     end
+
+    # Internal: Is this a compiled C/C++ file from Cython?
+    #
+    # Cython-compiled C/C++ files typically contain:
+    # /* Generated by Cython x.x.x on ... */
+    # on the first line.
+    #
+    # Return true or false
+    def compiled_cython_file?
+      return false unless ['.c', '.cpp'].include? extname
+      return false unless lines.count > 1
+      return lines[0].include?("Generated by Cython")
+    end
   end
 end

lib/linguist/heuristics.rb

@@ -33,7 +33,7 @@ module Linguist
     #     disambiguate "Perl", "Prolog" do |data|
     #       if data.include?("use strict")
     #         Language["Perl"]
-    #       elsif data.include?(":-")
+    #       elsif /^[^#]+:-/.match(data)
     #         Language["Prolog"]
     #       end
     #     end
@@ -94,23 +94,27 @@ module Linguist
         Language["Perl6"]
       elsif data.match(/use strict|use\s+v?5\./)
         Language["Perl"]
-      elsif data.include?(":-")
+      elsif /^[^#]+:-/.match(data)
         Language["Prolog"]
       end
     end
 
     disambiguate "ECL", "Prolog" do |data|
-      if data.include?(":-")
+      if /^[^#]+:-/.match(data)
         Language["Prolog"]
       elsif data.include?(":=")
         Language["ECL"]
       end
     end
 
-    disambiguate "IDL", "Prolog" do |data|
-      if data.include?(":-")
+    disambiguate "IDL", "Prolog", "INI", "QMake" do |data|
+      if /^[^#]+:-/.match(data)
         Language["Prolog"]
-      else
+      elsif data.include?("last_client=")
+        Language["INI"]
+      elsif data.include?("HEADERS") && data.include?("SOURCES")
+        Language["QMake"]
+      elsif /^\s*function[ \w,]+$/.match(data)
         Language["IDL"]
       end
     end
@@ -160,7 +164,7 @@ module Linguist
       end
     end
 
-    disambiguate "FORTRAN", "Forth" do |data|
+    disambiguate "FORTRAN", "Forth", "Formatted" do |data|
       if /^: /.match(data)
         Language["Forth"]
       elsif /^([c*][^a-z]|      (subroutine|program)\s|\s*!)/i.match(data)
@@ -168,27 +172,33 @@ module Linguist
       end
     end
 
-    disambiguate "F#", "Forth", "GLSL" do |data|
+    disambiguate "F#", "Forth", "GLSL", "Filterscript" do |data|
       if /^(: |new-device)/.match(data)
         Language["Forth"]
       elsif /^\s*(#light|import|let|module|namespace|open|type)/.match(data)
         Language["F#"]
-      elsif /^\s*(#include|#pragma|precision|uniform|varying|void)/.match(data)
+      elsif /^\s*(#version|precision|uniform|varying|vec[234])/.match(data)
         Language["GLSL"]
+      elsif /#include|#pragma\s+(rs|version)|__attribute__/.match(data)
+        Language["Filterscript"]
       end
     end
 
-    disambiguate "M", "Mathematica", "Matlab", "Mercury", "Objective-C" do |data|
+    disambiguate "Limbo", "M", "MUF", "Mathematica", "Matlab", "Mercury", "Objective-C" do |data|
       if ObjectiveCRegex.match(data)
         Language["Objective-C"]
       elsif data.include?(":- module")
         Language["Mercury"]
+      elsif /^: /.match(data)
+        Language["MUF"]
       elsif /^\s*;/.match(data)
         Language["M"]
       elsif /^\s*\(\*/.match(data)
         Language["Mathematica"]
       elsif /^\s*%/.match(data)
         Language["Matlab"]
+      elsif /^\w+\s*:\s*module\s*{/.match(data)
+        Language["Limbo"]
       end
     end
 
@@ -229,5 +239,55 @@ module Linguist
         Language["Text"]
       end
     end
+
+    disambiguate "PLSQL", "SQLPL", "PLpgSQL", "SQL" do |data|
+      if /^\\i\b|AS \$\$|LANGUAGE '+plpgsql'+/i.match(data) || /SECURITY (DEFINER|INVOKER)/i.match(data) || /BEGIN( WORK| TRANSACTION)?;/i.match(data)
+        #Postgres
+        Language["PLpgSQL"]
+      elsif /(alter module)|(language sql)|(begin( NOT)+ atomic)/i.match(data)  || /signal SQLSTATE '[0-9]+'/i.match(data)
+        #IBM db2
+        Language["SQLPL"]
+      elsif /pragma|\$\$PLSQL_|XMLTYPE|sysdate|systimestamp|\.nextval|connect by|AUTHID (DEFINER|CURRENT_USER)/i.match(data) || /constructor\W+function/i.match(data)
+        #Oracle
+        Language["PLSQL"]
+      elsif ! /begin|boolean|package|exception/i.match(data)
+        #Generic SQL
+        Language["SQL"]
+      end
+    end
+
+    disambiguate "D", "DTrace", "Makefile" do |data|
+      if /^module /.match(data)
+        Language["D"]
+      elsif /^((dtrace:::)?BEGIN|provider |#pragma (D (option|attributes)|ident)\s)/.match(data)
+        Language["DTrace"]
+      elsif /(\/.*:( .* \\)$| : \\$|^ : |: \\$)/.match(data)
+        Language["Makefile"]
+      end
+    end
+
+    disambiguate "OCaml", "Standard ML" do |data|
+      if /(^\s*module)|let rec |match\s+(\S+\s)+with/.match(data)
+        Language["OCaml"]
+      elsif /=> |case\s+(\S+\s)+of/.match(data)
+        Language["Standard ML"]
+      end
+    end
+
+    disambiguate "NL", "NewLisp" do |data|
+      if /^g3 /.match(data)
+        Language["NL"]
+      else
+        Language["NewLisp"]
+      end
+    end
+
+    disambiguate "Rust", "RenderScript" do |data|
+      if data.include?("^(use |fn |mod |pub |macro_rules|impl|#!?\[)")
+        Language["Rust"]
+      elsif /#include|#pragma\s+(rs|version)|__attribute__/.match(data)
+        Language["RenderScript"]
+      end
+    end
   end
 end

lib/linguist/language.rb

@@ -73,7 +73,7 @@ module Linguist
           raise ArgumentError, "Extension is missing a '.': #{extension.inspect}"
         end
 
-        @extension_index[extension] << language
+        @extension_index[extension.downcase] << language
       end
 
       language.interpreters.each do |interpreter|
@@ -105,19 +105,31 @@ module Linguist
       # Bail early if the blob is binary or empty.
       return nil if blob.likely_binary? || blob.binary? || blob.empty?
 
-      # Call each strategy until one candidate is returned.
-      STRATEGIES.reduce([]) do |languages, strategy|
-        candidates = strategy.call(blob, languages)
-        if candidates.size == 1
-          return candidates.first
-        elsif candidates.size > 1
-          # More than one candidate was found, pass them to the next strategy.
-          candidates
-        else
-          # No candiates were found, pass on languages from the previous strategy.
-          languages
+      Linguist.instrument("linguist.detection", :blob => blob) do
+        # Call each strategy until one candidate is returned.
+        languages = []
+        returning_strategy = nil
+
+        STRATEGIES.each do |strategy|
+          returning_strategy = strategy
+          candidates = Linguist.instrument("linguist.strategy", :blob => blob, :strategy => strategy, :candidates => languages) do
+            strategy.call(blob, languages)
+          end
+          if candidates.size == 1
+            languages = candidates
+            break
+          elsif candidates.size > 1
+            # More than one candidate was found, pass them to the next strategy.
+            languages = candidates
+          else
+            # No candidates, try the next strategy
+          end
         end
-      end.first
+
+        Linguist.instrument("linguist.detected", :blob => blob, :strategy => returning_strategy, :language => languages.first)
+
+        languages.first
+      end
     end
 
     # Public: Get all Languages
@@ -191,7 +203,7 @@ module Linguist
     # Returns all matching Languages or [] if none were found.
     def self.find_by_extension(extname)
       extname = ".#{extname}" unless extname.start_with?(".")
-      @extension_index[extname]
+      @extension_index[extname.downcase]
     end
 
     # DEPRECATED
@@ -528,8 +540,8 @@ module Linguist
 
     if extnames = extensions[name]
       extnames.each do |extname|
-        if !options['extensions'].index { |x| x.end_with? extname }
-          warn "#{name} has a sample with extension (#{extname}) that isn't explicitly defined in languages.yml" unless extname == '.script!'
+        if !options['extensions'].index { |x| x.downcase.end_with? extname.downcase }
+          warn "#{name} has a sample with extension (#{extname.downcase}) that isn't explicitly defined in languages.yml" unless extname == '.script!'
           options['extensions'] << extname
         end
       end

lib/linguist/lazy_blob.rb

@@ -14,13 +14,15 @@ module Linguist
 
     attr_reader :repository
     attr_reader :oid
-    attr_reader :name
+    attr_reader :path
     attr_reader :mode
 
-    def initialize(repo, oid, name, mode = nil)
+    alias :name :path
+
+    def initialize(repo, oid, path, mode = nil)
       @repository = repo
       @oid = oid
-      @name = name
+      @path = path
       @mode = mode
     end
 
@@ -49,7 +51,7 @@ module Linguist
       return @language if defined?(@language)
 
       @language = if lang = git_attributes['linguist-language']
-        Language.find_by_name(lang)
+        Language.find_by_alias(lang)
       else
         super
       end

lib/linguist/popular.yml

@@ -9,21 +9,21 @@
 - CSS
 - Clojure
 - CoffeeScript
-- Common Lisp
-- Diff
-- Emacs Lisp
-- Erlang
+- Go
 - HTML
 - Haskell
 - Java
 - JavaScript
 - Lua
+- Matlab
 - Objective-C
 - PHP
 - Perl
 - Python
+- R
 - Ruby
-- SQL
 - Scala
-- Scheme
 - Shell
+- Swift
+- TeX
+- VimL

lib/linguist/shebang.rb

@@ -23,17 +23,20 @@ module Linguist
       # First line must start with #!
       return unless shebang && shebang.start_with?("#!")
 
-      # Get the parts of the shebang without the #!
-      tokens = shebang.sub(/^#!\s*/, '').strip.split(' ')
+      s = StringScanner.new(shebang)
 
       # There was nothing after the #!
-      return if tokens.empty?
+      return unless path = s.scan(/^#!\s*\S+/)
 
-      # Get the name of the interpreter
-      script = File.basename(tokens.first)
+      # Keep going
+      script = path.split('/').last
 
-      # Get next argument if interpreter was /usr/bin/env
-      script = tokens[1] if script == 'env'
+      # if /usr/bin/env type shebang then walk the string
+      if script == 'env'
+        s.scan(/\s+/)
+        s.scan(/.*=[^\s]+\s+/) # skip over variable arguments e.g. foo=bar
+        script = s.scan(/\S+/)
+      end
 
       # Interpreter was /usr/bin/env with no arguments
       return unless script
@@ -41,6 +44,9 @@ module Linguist
       # "python2.6" -> "python2"
       script.sub! /(\.\d+)$/, ''
 
+      # #! perl -> perl
+      script.sub! /^#!\s*/, ''
+
       # Check for multiline shebang hacks that call `exec`
       if script == 'sh' &&
         data.lines.first(5).any? { |l| l.match(/exec (\w+).+\$0.+\$@/) }

lib/linguist/strategy/modeline.rb

@@ -1,7 +1,7 @@
 module Linguist
   module Strategy
     class Modeline
-      EmacsModeline = /-\*-\s*mode:\s*(\w+);?\s*-\*-/i
+      EmacsModeline = /-\*-\s*(?:(?!mode)[\w-]+\s*:\s*(?:[\w+-]+)\s*;?\s*)*(?:mode\s*:)?\s*([\w+-]+)\s*(?:;\s*(?!mode)[\w-]+\s*:\s*[\w+-]+\s*)*;?\s*-\*-/i
       VimModeline = /\/\*\s*vim:\s*set\s*(?:ft|filetype)=(\w+):\s*\*\//i
 
       # Public: Detects language based on Vim and Emacs modelines

lib/linguist/tokenizer.rb

@@ -22,8 +22,10 @@ module Linguist
     # Start state on token, ignore anything till the next newline
     SINGLE_LINE_COMMENTS = [
       '//', # C
+      '--', # Ada, Haskell, AppleScript
       '#',  # Ruby
       '%',  # Tex
+      '"',  # Vim
     ]
 
     # Start state on opening token, ignore anything until the closing
@@ -130,6 +132,9 @@ module Linguist
     #   extract_shebang("#!/usr/bin/env node")
     #   # => "node"
     #
+    #   extract_shebang("#!/usr/bin/env A=B foo=bar awk -f")
+    #   # => "awk"
+    #
     # Returns String token or nil it couldn't be parsed.
     def extract_shebang(data)
       s = StringScanner.new(data)
@@ -138,6 +143,7 @@ module Linguist
         script = path.split('/').last
         if script == 'env'
           s.scan(/\s+/)
+          s.scan(/.*=[^\s]+\s+/)
           script = s.scan(/\S+/)
         end
         script = script[/[^\d]+/, 0] if script

lib/linguist/vendor.yml

@@ -24,6 +24,9 @@
 - (^|/)config.guess$
 - (^|/)config.sub$
 
+# Linters
+- cpplint.py
+
 # Node dependencies
 - node_modules/
 
@@ -143,6 +146,9 @@
 
 ## Python ##
 
+# Sphinx
+- (^|/)docs?/_?(build|themes?|templates?|static)/
+
 # django
 - (^|/)admin_media/
 
@@ -221,7 +227,7 @@
 - ^readme$
 
 # Test fixtures
-- ^[Tt]est/fixtures/
+- ^[Tt]ests?/fixtures/
 
 # PhoneGap/Cordova
 - (^|/)cordova([^.]*)\.js$
@@ -233,7 +239,7 @@
 # Vagrant
 - ^Vagrantfile$
 
-# .DS_Store's
+# .DS_Stores
 - .[Dd][Ss]_[Ss]tore$
 
 # R packages
@@ -251,3 +257,9 @@
 # ProGuard
 - proguard.pro
 - proguard-rules.pro
+
+# PuPHPet
+- ^puphpet/
+
+# Android Google APIs
+- (^|/)\.google_apis/

lib/linguist/version.rb

@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "4.4.0"
+  VERSION = "4.5.4"
 end