Merge branch 'master' into 815-update

Conflicts:
	lib/linguist/samples.json

lib/linguist.rb

@@ -4,3 +4,4 @@ require 'linguist/heuristics'
 require 'linguist/language'
 require 'linguist/repository'
 require 'linguist/samples'
+require 'linguist/version'

lib/linguist/blob_helper.rb

@@ -241,7 +241,25 @@ module Linguist
     def lines
       @lines ||=
         if viewable? && data
-          data.split(/\r\n|\r|\n/, -1)
+          # `data` is usually encoded as ASCII-8BIT even when the content has
+          # been detected as a different encoding. However, we are not allowed
+          # to change the encoding of `data` because we've made the implicit
+          # guarantee that each entry in `lines` is encoded the same way as
+          # `data`.
+          #
+          # Instead, we re-encode each possible newline sequence as the
+          # detected encoding, then force them back to the encoding of `data`
+          # (usually a binary encoding like ASCII-8BIT). This means that the
+          # byte sequence will match how newlines are likely encoded in the
+          # file, but we don't have to change the encoding of `data` as far as
+          # Ruby is concerned. This allows us to correctly parse out each line
+          # without changing the encoding of `data`, and
+          # also--importantly--without having to duplicate many (potentially
+          # large) strings.
+          encoded_newlines = ["\r\n", "\r", "\n"].
+            map { |nl| nl.encode(encoding).force_encoding(data.encoding) }
+
+          data.split(Regexp.union(encoded_newlines), -1)
         else
           []
         end

lib/linguist/heuristics.rb

@@ -28,6 +28,9 @@ module Linguist
         if languages.all? { |l| ["Common Lisp", "OpenCL"].include?(l) }
           disambiguate_cl(data, languages)
         end
+        if languages.all? { |l| ["Rebol", "R"].include?(l) }
+          disambiguate_r(data, languages)
+        end
       end
     end
 
@@ -73,6 +76,13 @@ module Linguist
       matches
     end
 
+    def self.disambiguate_r(data, languages)
+      matches = []
+      matches << Language["Rebol"] if /\bRebol\b/i.match(data)
+      matches << Language["R"] if data.include?("<-")
+      matches
+    end
+
     def self.active?
       !!ACTIVE
     end

lib/linguist/language.rb

@@ -24,7 +24,6 @@ module Linguist
     @extension_index          = Hash.new { |h,k| h[k] = [] }
     @interpreter_index        = Hash.new { |h,k| h[k] = [] }
     @filename_index           = Hash.new { |h,k| h[k] = [] }
-    @primary_extension_index  = {}
 
     # Valid Languages types
     TYPES = [:data, :markup, :programming, :prose]
@@ -80,12 +79,6 @@ module Linguist
         @extension_index[extension] << language
       end
 
-      if @primary_extension_index.key?(language.primary_extension)
-        raise ArgumentError, "Duplicate primary extension: #{language.primary_extension}"
-      end
-
-      @primary_extension_index[language.primary_extension] = language
-
       language.interpreters.each do |interpreter|
         @interpreter_index[interpreter] << language
       end
@@ -191,8 +184,7 @@ module Linguist
     # Returns all matching Languages or [] if none were found.
     def self.find_by_filename(filename)
       basename, extname = File.basename(filename), File.extname(filename)
-      langs = [@primary_extension_index[extname]] +
-              @filename_index[basename] +
+      langs = @filename_index[basename] +
               @extension_index[extname]
       langs.compact.uniq
     end
@@ -299,15 +291,6 @@ module Linguist
       @interpreters = attributes[:interpreters]   || []
       @filenames  = attributes[:filenames]  || []
 
-      unless @primary_extension = attributes[:primary_extension]
-        raise ArgumentError, "#{@name} is missing primary extension"
-      end
-
-      # Prepend primary extension unless its already included
-      if primary_extension && !extensions.include?(primary_extension)
-        @extensions = [primary_extension] + extensions
-      end
-
       # Set popular, and searchable flags
       @popular    = attributes.key?(:popular)    ? attributes[:popular]    : false
       @searchable = attributes.key?(:searchable) ? attributes[:searchable] : true
@@ -395,20 +378,6 @@ module Linguist
     # Returns the extensions Array
     attr_reader :extensions
 
-    # Deprecated: Get primary extension
-    #
-    # Defaults to the first extension but can be overridden
-    # in the languages.yml.
-    #
-    # The primary extension can not be nil. Tests should verify this.
-    #
-    # This attribute is only used by app/helpers/gists_helper.rb for
-    # creating the language dropdown. It really should be using `name`
-    # instead. Would like to drop primary extension.
-    #
-    # Returns the extension String.
-    attr_reader :primary_extension
-
     # Public: Get interpreters
     #
     # Examples
@@ -426,6 +395,27 @@ module Linguist
     #
     # Returns the extensions Array
     attr_reader :filenames
+    
+    # Public: Return all possible extensions for language
+    def all_extensions
+      (extensions + [primary_extension]).uniq
+    end
+
+    # Deprecated: Get primary extension
+    #
+    # Defaults to the first extension but can be overridden
+    # in the languages.yml.
+    #
+    # The primary extension can not be nil. Tests should verify this.
+    #
+    # This method is only used by app/helpers/gists_helper.rb for creating
+    # the language dropdown. It really should be using `name` instead.
+    # Would like to drop primary extension.
+    #
+    # Returns the extension String.
+    def primary_extension
+      extensions.first
+    end
 
     # Public: Get URL escaped name.
     #
@@ -568,9 +558,8 @@ module Linguist
       :group_name        => options['group'],
       :searchable        => options.key?('searchable') ? options['searchable'] : true,
       :search_term       => options['search_term'],
-      :extensions        => options['extensions'].sort,
+      :extensions        => [options['extensions'].first] + options['extensions'][1..-1].sort,
       :interpreters      => options['interpreters'].sort,
-      :primary_extension => options['primary_extension'],
       :filenames         => options['filenames'],
       :popular           => popular.include?(name)
     )

lib/linguist/vendor.yml

@@ -98,9 +98,16 @@
 # AngularJS
 - (^|/)angular([^.]*)(\.min)?\.js$
 
+# D3.js
+- (^|\/)d3(\.v\d+)?([^.]*)(\.min)?\.js$
+
 # React
 - (^|/)react(-[^.]*)?(\.min)?\.js$
 
+# Modernizr
+- (^|/)modernizr\-\d\.\d+(\.\d+)?(\.min)?\.js$
+- (^|/)modernizr\.custom\.\d+\.js$
+
 ## Python ##
 
 # django
@@ -131,6 +138,7 @@
 
 # Visual Studio IntelliSense
 - -vsdoc\.js$
+- \.intellisense\.js$
 
 # jQuery validation plugin (MS bundles this with asp.net mvc)
 - (^|/)jquery([^.]*)\.validate(\.unobtrusive)?(\.min)?\.js$
@@ -140,7 +148,7 @@
 - (^|/)[Mm]icrosoft([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?\.js$
 
 # NuGet
-- ^[Pp]ackages/
+- ^[Pp]ackages\/.+\.\d+\/
 
 # ExtJS
 - (^|/)extjs/.*?\.js$
@@ -185,3 +193,6 @@
 
 # .DS_Store's
 - .[Dd][Ss]_[Ss]tore$
+
+# Mercury --use-subdirs
+- Mercury/

lib/linguist/version.rb

@@ -0,0 +1,3 @@
+module Linguist
+  VERSION = "2.11.1"
+end