Merge branch 'master' into ascii

lib/linguist/blob_helper.rb

@@ -236,6 +236,21 @@ module Linguist
       name =~ VendoredRegexp ? true : false
     end
 
+    documentation_paths = YAML.load_file(File.expand_path("../documentation.yml", __FILE__))
+    DocumentationRegexp = Regexp.new(documentation_paths.join('|'))
+
+    # Public: Is the blob in a documentation directory?
+    #
+    # Documentation files are ignored by language statistics.
+    #
+    # See "documentation.yml" for a list of documentation conventions that match
+    # this pattern.
+    #
+    # Return true or false
+    def documentation?
+      name =~ DocumentationRegexp ? true : false
+    end
+
     # Public: Get each line of data
     #
     # Requires Blob#data
@@ -317,5 +332,15 @@ module Linguist
     def tm_scope
       language && language.tm_scope
     end
+
+    DETECTABLE_TYPES = [:programming, :markup].freeze
+
+    # Internal: Should this blob be included in repository language statistics?
+    def include_in_language_stats?
+      !vendored? &&
+      !documentation? &&
+      !generated? &&
+      language && DETECTABLE_TYPES.include?(language.type)
+    end
   end
 end

lib/linguist/documentation.yml

@@ -0,0 +1,19 @@
+# Documentation files and directories are excluded from language
+# statistics.
+#
+# Lines in this file are Regexps that are matched against the file
+# pathname.
+#
+# Please add additional test coverage to
+# `test/test_blob.rb#test_documentation` if you make any changes.
+
+## Documentation Conventions ##
+
+- ^docs?/
+- ^Documentation/
+
+- (^|/)CONTRIBUTING(\.|$)
+- (^|/)COPYING(\.|$)
+- (^|/)INSTALL(\.|$)
+- (^|/)LICEN[CS]E(\.|$)
+- (^|/)README(\.|$)

lib/linguist/heuristics.rb

@@ -92,7 +92,7 @@ module Linguist
     disambiguate "Perl", "Perl6", "Prolog" do |data|
       if data.include?("use v6")
         Language["Perl6"]
-      elsif data.include?("use strict")
+      elsif data.match(/use strict|use\s+v?5\./)
         Language["Perl"]
       elsif data.include?(":-")
         Language["Prolog"]
@@ -163,7 +163,7 @@ module Linguist
     disambiguate "FORTRAN", "Forth" do |data|
       if /^: /.match(data)
         Language["Forth"]
-      elsif /^([c*][^a-z]|      (subroutine|program)\s|!)/i.match(data)
+      elsif /^([c*][^a-z]|      (subroutine|program)\s|\s*!)/i.match(data)
         Language["FORTRAN"]
       end
     end

lib/linguist/language.rb

@@ -32,13 +32,6 @@ module Linguist
     # Valid Languages types
     TYPES = [:data, :markup, :programming, :prose]
 
-    # Names of non-programming languages that we will still detect
-    #
-    # Returns an array
-    def self.detectable_markup
-      ["CSS", "Less", "Sass", "SCSS", "Stylus", "TeX"]
-    end
-
     # Detect languages by a specific type
     #
     # type - A symbol that exists within TYPES
@@ -96,8 +89,8 @@ module Linguist
 
     STRATEGIES = [
       Linguist::Strategy::Modeline,
-      Linguist::Strategy::Filename,
       Linguist::Shebang,
+      Linguist::Strategy::Filename,
       Linguist::Heuristics,
       Linguist::Classifier
     ]

lib/linguist/languages.yml

@@ -206,6 +206,7 @@ Assembly:
   - .asm
   - .ASM
   - .a51
+  - .nasm
   tm_scope: source.asm.x86
   ace_mode: assembly_x86
 
@@ -438,6 +439,8 @@ COBOL:
   ace_mode: cobol
 
 CSS:
+  type: markup
+  tm_scope: source.css
   ace_mode: css
   color: "#563d7c"
   extensions:
@@ -918,6 +921,7 @@ Forth:
   color: "#341708"
   extensions:
   - .fth
+  - .4TH
   - .4th
   - .F
   - .f
@@ -942,7 +946,7 @@ G-code:
   - .g
   - .gco
   - .gcode
-  tm_scope: none
+  tm_scope: source.gcode
   ace_mode: gcode
 
 GAMS:
@@ -960,7 +964,7 @@ GAP:
   - .gd
   - .gi
   - .tst
-  tm_scope: none
+  tm_scope: source.gap
   ace_mode: text
 
 GAS:
@@ -1092,7 +1096,7 @@ Grace:
   type: programming
   extensions:
   - .grace
-  tm_scope: none
+  tm_scope: source.grace
   ace_mode: text
 
 Gradle:
@@ -1171,6 +1175,7 @@ HTML:
   type: markup
   tm_scope: text.html.basic
   ace_mode: html
+  color: "#e44b23"
   aliases:
   - xhtml
   extensions:
@@ -1345,7 +1350,7 @@ Inform 7:
 Inno Setup:
   extensions:
   - .iss
-  tm_scope: none
+  tm_scope: source.inno
   ace_mode: text
 
 Io:
@@ -1806,7 +1811,7 @@ MediaWiki:
   wrap: true
   extensions:
   - .mediawiki
-  tm_scope: none
+  tm_scope: text.html.mediawiki
   ace_mode: text
 
 Mercury:
@@ -2051,7 +2056,7 @@ Opal:
   color: "#f7ede0"
   extensions:
   - .opal
-  tm_scope: none
+  tm_scope: source.opal
   ace_mode: text
 
 OpenCL:
@@ -2096,7 +2101,7 @@ Ox:
   - .ox
   - .oxh
   - .oxo
-  tm_scope: none
+  tm_scope: source.ox
   ace_mode: text
 
 Oxygene:
@@ -2157,7 +2162,7 @@ Papyrus:
   color: "#6600cc"
   extensions:
   - .psc
-  tm_scope: none
+  tm_scope: source.papyrus
   ace_mode: text
 
 Parrot:
@@ -2275,7 +2280,7 @@ PogoScript:
   color: "#d80074"
   extensions:
   - .pogo
-  tm_scope: none
+  tm_scope: source.pogoscript
   ace_mode: text
 
 PostScript:
@@ -2322,7 +2327,7 @@ Propeller Spin:
   color: "#2b446d"
   extensions:
   - .spin
-  tm_scope: none
+  tm_scope: source.spin
   ace_mode: text
 
 Protocol Buffer:
@@ -2682,6 +2687,13 @@ STON:
   tm_scope: source.smalltalk
   ace_mode: text
 
+SVG:
+  type: data
+  extensions:
+  - .svg
+  tm_scope: text.xml
+  ace_mode: xml
+
 Sage:
   type: programming
   group: Python
@@ -2921,7 +2933,7 @@ TXL:
   type: programming
   extensions:
   - .txl
-  tm_scope: none
+  tm_scope: source.txl
   ace_mode: text
 
 Tcl:
@@ -3197,7 +3209,6 @@ XML:
   - .srdf
   - .stTheme
   - .sublime-snippet
-  - .svg
   - .targets
   - .tmCommand
   - .tmLanguage
@@ -3331,7 +3342,7 @@ eC:
   extensions:
   - .ec
   - .eh
-  tm_scope: none
+  tm_scope: source.c.ec
   ace_mode: text
 
 edn:

lib/linguist/lazy_blob.rb

@@ -4,7 +4,7 @@ require 'rugged'
 
 module Linguist
   class LazyBlob
-    GIT_ATTR = ['linguist-language', 'linguist-vendored']
+    GIT_ATTR = ['linguist-documentation', 'linguist-language', 'linguist-vendored']
     GIT_ATTR_OPTS = { :priority => [:index], :skip_system => true }
     GIT_ATTR_FLAGS = Rugged::Repository::Attributes.parse_opts(GIT_ATTR_OPTS)
 
@@ -37,6 +37,14 @@ module Linguist
       end
     end
 
+    def documentation?
+      if attr = git_attributes['linguist-documentation']
+        boolean_attribute(attr)
+      else
+        super
+      end
+    end
+
     def language
       return @language if defined?(@language)
 

lib/linguist/repository.rb

@@ -156,13 +156,8 @@ module Linguist
 
           blob = Linguist::LazyBlob.new(repository, delta.new_file[:oid], new, mode.to_s(8))
 
-          # Skip vendored or generated blobs
-          next if blob.vendored? || blob.generated? || blob.language.nil?
-
-          # Only include programming languages and acceptable markup languages
-          if blob.language.type == :programming || Language.detectable_markup.include?(blob.language.name)
-            file_map[new] = [blob.language.group.name, blob.size]
-          end
+          next unless blob.include_in_language_stats?
+          file_map[new] = [blob.language.group.name, blob.size]
         end
       end
 

lib/linguist/strategy/modeline.rb

@@ -1,8 +1,8 @@
 module Linguist
   module Strategy
     class Modeline
-      EmacsModeline = /-\*-\s*(?:mode:)?\s*(\w+);?\s*-\*-/
-      VimModeline = /\/\*\s*vim:\s*set\s*(?:ft|filetype)=(\w+):\s*\*\//
+      EmacsModeline = /-\*-\s*mode:\s*(\w+);?\s*-\*-/i
+      VimModeline = /\/\*\s*vim:\s*set\s*(?:ft|filetype)=(\w+):\s*\*\//i
 
       # Public: Detects language based on Vim and Emacs modelines
       #

lib/linguist/vendor.yml

@@ -40,7 +40,7 @@
 # Minified JavaScript and CSS
 - (\.|-)min\.(js|css)$
 
-#Stylesheets imported from packages
+# Stylesheets imported from packages
 - ([^\s]*)import\.(css|less|scss|styl)$
 
 # Bootstrap css and js

lib/linguist/version.rb

@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "4.3.0"
+  VERSION = "4.4.0"
 end