Merge branch 'master' into revert-2014-revert-1976-path-for-fileblob

Conflicts: lib/linguist/version.rb
2025-10-29 17:50:22 +00:00 · 2015-02-24 14:54:14 -06:00
parent 496b3e5a78 c7d8f39ad8
commit 8561f2a6b0
124 changed files with 15649 additions and 615 deletions
--- a/lib/linguist/blob_helper.rb
+++ b/lib/linguist/blob_helper.rb
@@ -236,6 +236,21 @@ module Linguist
      path =~ VendoredRegexp ? true : false
    end

+    documentation_paths = YAML.load_file(File.expand_path("../documentation.yml", __FILE__))
+    DocumentationRegexp = Regexp.new(documentation_paths.join('|'))
+
+    # Public: Is the blob in a documentation directory?
+    #
+    # Documentation files are ignored by language statistics.
+    #
+    # See "documentation.yml" for a list of documentation conventions that match
+    # this pattern.
+    #
+    # Return true or false
+    def documentation?
+      name =~ DocumentationRegexp ? true : false
+    end
+
    # Public: Get each line of data
    #
    # Requires Blob#data
@@ -317,5 +332,15 @@ module Linguist
    def tm_scope
      language && language.tm_scope
    end
+
+    DETECTABLE_TYPES = [:programming, :markup].freeze
+
+    # Internal: Should this blob be included in repository language statistics?
+    def include_in_language_stats?
+      !vendored? &&
+      !documentation? &&
+      !generated? &&
+      language && DETECTABLE_TYPES.include?(language.type)
+    end
  end
 end
--- a/lib/linguist/documentation.yml
+++ b/lib/linguist/documentation.yml
@@ -0,0 +1,23 @@
+# Documentation files and directories are excluded from language
+# statistics.
+#
+# Lines in this file are Regexps that are matched against the file
+# pathname.
+#
+# Please add additional test coverage to
+# `test/test_blob.rb#test_documentation` if you make any changes.
+
+## Documentation directories ##
+
+- ^docs?/
+- (^|/)[Dd]ocumentation/
+- (^|/)javadoc/
+- ^man/
+
+## Documentation files ##
+
+- (^|/)CONTRIBUTING(\.|$)
+- (^|/)COPYING(\.|$)
+- (^|/)INSTALL(\.|$)
+- (^|/)LICEN[CS]E(\.|$)
+- (^|/)README(\.|$)
--- a/lib/linguist/generated.rb
+++ b/lib/linguist/generated.rb
@@ -62,6 +62,7 @@ module Linguist
      generated_parser? ||
      generated_net_docfile? ||
      generated_postscript? ||
+      generated_protocol_buffer_go? ||
      generated_protocol_buffer? ||
      generated_jni_header? ||
      vcr_cassette?
@@ -202,6 +203,13 @@ module Linguist
        creator.include?("ImageMagick")
    end

+    def generated_protocol_buffer_go?
+      return false unless extname == '.go'
+      return false unless lines.count > 1
+
+      return lines[0].include?("Code generated by protoc-gen-go")
+    end
+
    # Internal: Is the blob a C++, Java or Python source file generated by the
    # Protocol Buffer compiler?
    #
--- a/lib/linguist/heuristics.rb
+++ b/lib/linguist/heuristics.rb
@@ -61,6 +61,9 @@ module Linguist
      @heuristic.call(data)
    end

+    # Common heuristics
+    ObjectiveCRegex = /^[ \t]*@(interface|class|protocol|property|end|synchronised|selector|implementation)\b/
+
    disambiguate "BitBake", "BlitzBasic" do |data|
      if /^\s*; /.match(data) || data.include?("End Function")
        Language["BlitzBasic"]
@@ -78,7 +81,7 @@ module Linguist
    end

    disambiguate "Objective-C", "C++", "C" do |data|
-      if (/^[ \t]*@(interface|class|protocol|property|end|synchronised|selector|implementation)\b/.match(data))
+      if ObjectiveCRegex.match(data)
        Language["Objective-C"]
      elsif (/^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>/.match(data) ||
        /^\s*template\s*</.match(data) || /^[ \t]*try/.match(data) || /^[ \t]*catch\s*\(/.match(data) || /^[ \t]*(class|(using[ \t]+)?namespace)\s+\w+/.match(data) || /^[ \t]*(private|public|protected):$/.match(data) || /std::\w+/.match(data))
@@ -89,7 +92,7 @@ module Linguist
    disambiguate "Perl", "Perl6", "Prolog" do |data|
      if data.include?("use v6")
        Language["Perl6"]
-      elsif data.include?("use strict")
+      elsif data.match(/use strict|use\s+v?5\./)
        Language["Perl"]
      elsif data.include?(":-")
        Language["Prolog"]
@@ -112,6 +115,15 @@ module Linguist
      end
    end

+    disambiguate "GAP", "Scilab" do |data|
+      if (data.include?("gap> "))
+        Language["GAP"]
+      # Heads up - we don't usually write heuristics like this (with no regex match)
+      else
+        Language["Scilab"]
+      end
+    end
+
    disambiguate "Common Lisp", "OpenCL", "Cool" do |data|
      if data.include?("(defun ")
        Language["Common Lisp"]
@@ -138,14 +150,20 @@ module Linguist
      end
    end

-    disambiguate "AsciiDoc", "AGS Script" do |data|
-      Language["AsciiDoc"] if /^=+(\s|\n)/.match(data)
+    disambiguate "AsciiDoc", "AGS Script", "Public Key" do |data|
+      if /^[=-]+(\s|\n)|{{[A-Za-z]/.match(data)
+        Language["AsciiDoc"]
+      elsif /^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])/.match(data)
+        Language["AGS Script"]
+      elsif /^-----BEGIN/.match(data)
+        Language["Public Key"]
+      end
    end

    disambiguate "FORTRAN", "Forth" do |data|
      if /^: /.match(data)
        Language["Forth"]
-      elsif /^([c*][^a-z]|      (subroutine|program)\s|!)/i.match(data)
+      elsif /^([c*][^a-z]|      (subroutine|program)\s|\s*!)/i.match(data)
        Language["FORTRAN"]
      end
    end
@@ -160,6 +178,20 @@ module Linguist
      end
    end

+    disambiguate "M", "Mathematica", "Matlab", "Mercury", "Objective-C" do |data|
+      if ObjectiveCRegex.match(data)
+        Language["Objective-C"]
+      elsif data.include?(":- module")
+        Language["Mercury"]
+      elsif /^\s*;/.match(data)
+        Language["M"]
+      elsif /^\s*\(\*/.match(data)
+        Language["Mathematica"]
+      elsif /^\s*%/.match(data)
+        Language["Matlab"]
+      end
+    end
+
    disambiguate "Gosu", "JavaScript" do |data|
      Language["Gosu"] if /^uses java\./.match(data)
    end
@@ -172,6 +204,14 @@ module Linguist
      end
    end

+    disambiguate "Common Lisp", "NewLisp" do |data|
+      if /^\s*\((defun|in-package|defpackage) /.match(data)
+        Language["Common Lisp"]
+      elsif /^\s*\(define /.match(data)
+        Language["NewLisp"]
+      end
+    end
+
    disambiguate "TypeScript", "XML" do |data|
      if data.include?("<TS ")
        Language["XML"]
--- a/lib/linguist/language.rb
+++ b/lib/linguist/language.rb
@@ -11,6 +11,7 @@ require 'linguist/samples'
 require 'linguist/file_blob'
 require 'linguist/blob_helper'
 require 'linguist/strategy/filename'
+require 'linguist/strategy/modeline'
 require 'linguist/shebang'

 module Linguist
@@ -31,13 +32,6 @@ module Linguist
    # Valid Languages types
    TYPES = [:data, :markup, :programming, :prose]

-    # Names of non-programming languages that we will still detect
-    #
-    # Returns an array
-    def self.detectable_markup
-      ["CSS", "Less", "Sass", "SCSS", "Stylus", "TeX"]
-    end
-
    # Detect languages by a specific type
    #
    # type - A symbol that exists within TYPES
@@ -94,8 +88,9 @@ module Linguist
    end

    STRATEGIES = [
-      Linguist::Strategy::Filename,
+      Linguist::Strategy::Modeline,
      Linguist::Shebang,
+      Linguist::Strategy::Filename,
      Linguist::Heuristics,
      Linguist::Classifier
    ]
@@ -155,7 +150,7 @@ module Linguist
    #   Language.find_by_alias('cpp')
    #   # => #<Language name="C++">
    #
-    # Returns the Lexer or nil if none was found.
+    # Returns the Language or nil if none was found.
    def self.find_by_alias(name)
      name && @alias_index[name.downcase]
    end
@@ -219,7 +214,7 @@ module Linguist
    end


-    # Public: Look up Language by its name or lexer.
+    # Public: Look up Language by its name.
    #
    # name - The String name of the Language
    #
@@ -243,7 +238,7 @@ module Linguist
    #
    # This list is configured in "popular.yml".
    #
-    # Returns an Array of Lexers.
+    # Returns an Array of Languages.
    def self.popular
      @popular ||= all.select(&:popular?).sort_by { |lang| lang.name.downcase }
    end
@@ -255,7 +250,7 @@ module Linguist
    #
    # This list is created from all the languages not listed in "popular.yml".
    #
-    # Returns an Array of Lexers.
+    # Returns an Array of Languages.
    def self.unpopular
      @unpopular ||= all.select(&:unpopular?).sort_by { |lang| lang.name.downcase }
    end
@@ -375,11 +370,6 @@ module Linguist
    # Returns the name String
    attr_reader :search_term

-    # Public: Get Lexer
-    #
-    # Returns the Lexer
-    attr_reader :lexer
-
    # Public: Get the name of a TextMate-compatible scope
    #
    # Returns the scope
@@ -495,16 +485,6 @@ module Linguist
      @searchable
    end

-    # Public: Highlight syntax of text
-    #
-    # text    - String of code to be highlighted
-    # options - A Hash of options (defaults to {})
-    #
-    # Returns html String
-    def colorize(text, options = {})
-      lexer.highlight(text, options)
-    end
-
    # Public: Return name as String representation
    def to_s
      name
@@ -580,7 +560,6 @@ module Linguist
      :color             => options['color'],
      :type              => options['type'],
      :aliases           => options['aliases'],
-      :lexer             => options['lexer'],
      :tm_scope          => options['tm_scope'],
      :ace_mode          => options['ace_mode'],
      :wrap              => options['wrap'],
--- a/lib/linguist/languages.yml
+++ b/lib/linguist/languages.yml
@@ -54,13 +54,14 @@ APL:
  extensions:
  - .apl
  - .dyalog
-  tm_scope: none
+  tm_scope: source.apl
  ace_mode: text

 ASP:
  type: programming
  color: "#6a40fd"
  search_term: aspx-vb
+  tm_scope: text.html.asp
  aliases:
  - aspx
  - aspx-vb
@@ -81,10 +82,9 @@ ATS:
  - ats2
  extensions:
  - .dats
-  - .atxt
  - .hats
  - .sats
-  tm_scope: source.ocaml
+  tm_scope: source.ats
  ace_mode: ocaml

 ActionScript:
@@ -186,7 +186,7 @@ AsciiDoc:
  - .asciidoc
  - .adoc
  - .asc
-  tm_scope: none
+  tm_scope: text.html.asciidoc

 AspectJ:
  type: programming
@@ -198,7 +198,7 @@ AspectJ:

 Assembly:
  type: programming
-  color: "#a67219"
+  color: "#6E4C13"
  search_term: nasm
  aliases:
  - nasm
@@ -206,6 +206,7 @@ Assembly:
  - .asm
  - .ASM
  - .a51
+  - .nasm
  tm_scope: source.asm.x86
  ace_mode: assembly_x86

@@ -335,7 +336,7 @@ Brightscript:
  type: programming
  extensions:
  - .brs
-  tm_scope: none
+  tm_scope: source.brightscript
  ace_mode: text

 Bro:
@@ -355,6 +356,8 @@ C:
  - .h
  - .idc
  - .w
+  interpreters:
+  - tcc
  ace_mode: c_cpp

 C#:
@@ -381,6 +384,7 @@ C++:
  - .cpp
  - .c++
  - .cc
+  - .cp
  - .cxx
  - .h
  - .h++
@@ -413,7 +417,7 @@ CLIPS:
  type: programming
  extensions:
  - .clp
-  tm_scope: none
+  tm_scope: source.clips
  ace_mode: text

 CMake:
@@ -437,6 +441,8 @@ COBOL:
  ace_mode: cobol

 CSS:
+  type: markup
+  tm_scope: source.css
  ace_mode: css
  color: "#563d7c"
  extensions:
@@ -622,7 +628,7 @@ Creole:
  wrap: true
  extensions:
  - .creole
-  tm_scope: none
+  tm_scope: text.html.creole
  ace_mode: text

 Crystal:
@@ -811,9 +817,11 @@ Emacs Lisp:
  - emacs
  filenames:
  - .emacs
+  - .emacs.desktop
  extensions:
  - .el
  - .emacs
+  - .emacs.desktop
  ace_mode: lisp

 EmberScript:
@@ -915,6 +923,7 @@ Forth:
  color: "#341708"
  extensions:
  - .fth
+  - .4TH
  - .4th
  - .F
  - .f
@@ -939,7 +948,7 @@ G-code:
  - .g
  - .gco
  - .gcode
-  tm_scope: none
+  tm_scope: source.gcode
  ace_mode: gcode

 GAMS:
@@ -956,7 +965,8 @@ GAP:
  - .gap
  - .gd
  - .gi
-  tm_scope: none
+  - .tst
+  tm_scope: source.gap
  ace_mode: text

 GAS:
@@ -972,7 +982,7 @@ GDScript:
  type: programming
  extensions:
  - .gd
-  tm_scope: none
+  tm_scope: source.gdscript
  ace_mode: text

 GLSL:
@@ -1070,7 +1080,7 @@ Golo:
  color: "#f6a51f"
  extensions:
  - .golo
-  tm_scope: none
+  tm_scope: source.golo
  ace_mode: text

 Gosu:
@@ -1088,7 +1098,7 @@ Grace:
  type: programming
  extensions:
  - .grace
-  tm_scope: none
+  tm_scope: source.grace
  ace_mode: text

 Gradle:
@@ -1167,6 +1177,7 @@ HTML:
  type: markup
  tm_scope: text.html.basic
  ace_mode: html
+  color: "#e44b23"
  aliases:
  - xhtml
  extensions:
@@ -1236,6 +1247,7 @@ Handlebars:
  type: markup
  aliases:
  - hbs
+  - htmlbars
  extensions:
  - .handlebars
  - .hbs
@@ -1340,7 +1352,7 @@ Inform 7:
 Inno Setup:
  extensions:
  - .iss
-  tm_scope: none
+  tm_scope: source.inno
  ace_mode: text

 Io:
@@ -1369,9 +1381,10 @@ Isabelle:

 J:
  type: programming
+  color: "#2d8abd"
  extensions:
  - .ijs
-  tm_scope: none
+  tm_scope: source.j
  ace_mode: text

 JSON:
@@ -1596,7 +1609,7 @@ Liquid:
  type: markup
  extensions:
  - .liquid
-  tm_scope: none
+  tm_scope: text.html.liquid
  ace_mode: liquid

 Literate Agda:
@@ -1801,7 +1814,7 @@ MediaWiki:
  wrap: true
  extensions:
  - .mediawiki
-  tm_scope: none
+  tm_scope: text.html.mediawiki
  ace_mode: text

 Mercury:
@@ -1835,6 +1848,13 @@ Mirah:
  tm_scope: source.ruby
  ace_mode: ruby

+Modelica:
+  type: programming
+  extensions:
+  - .mo
+  tm_scope: source.modelica
+  ace_mode: text
+
 Monkey:
  type: programming
  extensions:
@@ -1883,6 +1903,19 @@ NetLogo:
  tm_scope: source.lisp
  ace_mode: lisp

+NewLisp:
+  type: programming
+  lexer: NewLisp
+  color: "#eedd66"
+  extensions:
+  - .nl
+  - .lisp
+  - .lsp
+  interpreters:
+  - newlisp
+  tm_scope: source.lisp
+  ace_mode: lisp
+
 Nginx:
  type: markup
  extensions:
@@ -2026,7 +2059,7 @@ Opal:
  color: "#f7ede0"
  extensions:
  - .opal
-  tm_scope: none
+  tm_scope: source.opal
  ace_mode: text

 OpenCL:
@@ -2055,7 +2088,7 @@ OpenSCAD:
  extensions:
  - .scad
  tm_scope: none
-  ace_mode: text
+  ace_mode: scad

 Org:
  type: prose
@@ -2071,7 +2104,7 @@ Ox:
  - .ox
  - .oxh
  - .oxo
-  tm_scope: none
+  tm_scope: source.ox
  ace_mode: text

 Oxygene:
@@ -2132,7 +2165,7 @@ Papyrus:
  color: "#6600cc"
  extensions:
  - .psc
-  tm_scope: none
+  tm_scope: source.papyrus
  ace_mode: text

 Parrot:
@@ -2224,7 +2257,7 @@ PigLatin:
  color: "#fcd7de"
  extensions:
  - .pig
-  tm_scope: none
+  tm_scope: source.pig_latin
  ace_mode: text

 Pike:
@@ -2250,7 +2283,7 @@ PogoScript:
  color: "#d80074"
  extensions:
  - .pogo
-  tm_scope: none
+  tm_scope: source.pogoscript
  ace_mode: text

 PostScript:
@@ -2297,7 +2330,7 @@ Propeller Spin:
  color: "#2b446d"
  extensions:
  - .spin
-  tm_scope: none
+  tm_scope: source.spin
  ace_mode: text

 Protocol Buffer:
@@ -2379,6 +2412,8 @@ Python:
  - python
  - python2
  - python3
+  aliases:
+  - rusthon

 Python traceback:
  type: data
@@ -2426,7 +2461,6 @@ R:

 RAML:
  type: data
-  lexer: YAML
  ace_mode: yaml
  tm_scope: source.yaml
  color: "#77d9fb"
@@ -2656,6 +2690,13 @@ STON:
  tm_scope: source.smalltalk
  ace_mode: text

+SVG:
+  type: data
+  extensions:
+  - .svg
+  tm_scope: text.xml
+  ace_mode: xml
+
 Sage:
  type: programming
  group: Python
@@ -2856,7 +2897,7 @@ Stylus:
  group: CSS
  extensions:
  - .styl
-  tm_scope: none
+  tm_scope: source.stylus
  ace_mode: stylus

 SuperCollider:
@@ -2895,7 +2936,7 @@ TXL:
  type: programming
  extensions:
  - .txl
-  tm_scope: none
+  tm_scope: source.txl
  ace_mode: text

 Tcl:
@@ -3127,7 +3168,7 @@ XC:
  ace_mode: c_cpp

 XML:
-  type: markup
+  type: data
  ace_mode: xml
  aliases:
  - rss
@@ -3171,7 +3212,6 @@ XML:
  - .srdf
  - .stTheme
  - .sublime-snippet
-  - .svg
  - .targets
  - .tmCommand
  - .tmLanguage
@@ -3291,13 +3331,21 @@ Zimpl:
  tm_scope: none
  ace_mode: text

+desktop:
+  type: data
+  extensions:
+  - .desktop
+  - .desktop.in
+  tm_scope: source.desktop
+  ace_mode: text
+
 eC:
  type: programming
  search_term: ec
  extensions:
  - .ec
  - .eh
-  tm_scope: none
+  tm_scope: source.c.ec
  ace_mode: text

 edn:
--- a/lib/linguist/lazy_blob.rb
+++ b/lib/linguist/lazy_blob.rb
@@ -4,7 +4,7 @@ require 'rugged'

 module Linguist
  class LazyBlob
-    GIT_ATTR = ['linguist-language', 'linguist-vendored']
+    GIT_ATTR = ['linguist-documentation', 'linguist-language', 'linguist-vendored']
    GIT_ATTR_OPTS = { :priority => [:index], :skip_system => true }
    GIT_ATTR_FLAGS = Rugged::Repository::Attributes.parse_opts(GIT_ATTR_OPTS)

@@ -39,11 +39,19 @@ module Linguist
      end
    end

+    def documentation?
+      if attr = git_attributes['linguist-documentation']
+        boolean_attribute(attr)
+      else
+        super
+      end
+    end
+
    def language
      return @language if defined?(@language)

      @language = if lang = git_attributes['linguist-language']
-        Language.find_by_name(lang)
+        Language.find_by_alias(lang)
      else
        super
      end
--- a/lib/linguist/repository.rb
+++ b/lib/linguist/repository.rb
@@ -156,13 +156,8 @@ module Linguist

          blob = Linguist::LazyBlob.new(repository, delta.new_file[:oid], new, mode.to_s(8))

-          # Skip vendored or generated blobs
-          next if blob.vendored? || blob.generated? || blob.language.nil?
-
-          # Only include programming languages and acceptable markup languages
-          if blob.language.type == :programming || Language.detectable_markup.include?(blob.language.name)
-            file_map[new] = [blob.language.group.name, blob.size]
-          end
+          next unless blob.include_in_language_stats?
+          file_map[new] = [blob.language.group.name, blob.size]
        end
      end

--- a/lib/linguist/strategy/modeline.rb
+++ b/lib/linguist/strategy/modeline.rb
@@ -0,0 +1,30 @@
+module Linguist
+  module Strategy
+    class Modeline
+      EmacsModeline = /-\*-\s*mode:\s*(\w+);?\s*-\*-/i
+      VimModeline = /\/\*\s*vim:\s*set\s*(?:ft|filetype)=(\w+):\s*\*\//i
+
+      # Public: Detects language based on Vim and Emacs modelines
+      #
+      # blob               - An object that quacks like a blob.
+      #
+      # Examples
+      #
+      #   Modeline.call(FileBlob.new("path/to/file"))
+      #
+      # Returns an Array with one Language if the blob has a Vim or Emacs modeline
+      # that matches a Language name or alias. Returns an empty array if no match.
+      def self.call(blob, _ = nil)
+        Array(Language.find_by_alias(modeline(blob.data)))
+      end
+
+      # Public: Get the modeline from the first n-lines of the file
+      #
+      # Returns a String or nil
+      def self.modeline(data)
+        match = data.match(EmacsModeline) || data.match(VimModeline)
+        match[1] if match
+      end
+    end
+  end
+end
--- a/lib/linguist/tokenizer.rb
+++ b/lib/linguist/tokenizer.rb
@@ -33,7 +33,8 @@ module Linguist
      ['<!--', '-->'], # XML
      ['{-', '-}'],    # Haskell
      ['(*', '*)'],    # Coq
-      ['"""', '"""']   # Python
+      ['"""', '"""'],  # Python
+      ["'''", "'''"]   # Python
    ]

    START_SINGLE_LINE_COMMENT =  Regexp.compile(SINGLE_LINE_COMMENTS.map { |c|
--- a/lib/linguist/vendor.yml
+++ b/lib/linguist/vendor.yml
@@ -40,7 +40,7 @@
 # Minified JavaScript and CSS
 - (\.|-)min\.(js|css)$

-#Stylesheets imported from packages
+# Stylesheets imported from packages
 - ([^\s]*)import\.(css|less|scss|styl)$

 # Bootstrap css and js
@@ -251,3 +251,6 @@
 # ProGuard
 - proguard.pro
 - proguard-rules.pro
+
+# Android Google APIs
+- (^|/)\.google_apis/
--- a/lib/linguist/version.rb
+++ b/lib/linguist/version.rb
@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "4.3.0b1"
+  VERSION = "4.5.0b1"
 end