Merge branch 'master' into 1100-local

Conflicts:
	lib/linguist/languages.yml

lib/linguist/blob_helper.rb

@@ -112,6 +112,12 @@ module Linguist
       end
     end
 
+    def ruby_encoding
+      if hash = detect_encoding
+        hash[:ruby_encoding]
+      end
+    end
+
     # Try to guess the encoding
     #
     # Returns: a Hash, with :encoding, :confidence, :type
@@ -241,7 +247,31 @@ module Linguist
     def lines
       @lines ||=
         if viewable? && data
-          data.split(/\r\n|\r|\n/, -1)
+          # `data` is usually encoded as ASCII-8BIT even when the content has
+          # been detected as a different encoding. However, we are not allowed
+          # to change the encoding of `data` because we've made the implicit
+          # guarantee that each entry in `lines` is encoded the same way as
+          # `data`.
+          #
+          # Instead, we re-encode each possible newline sequence as the
+          # detected encoding, then force them back to the encoding of `data`
+          # (usually a binary encoding like ASCII-8BIT). This means that the
+          # byte sequence will match how newlines are likely encoded in the
+          # file, but we don't have to change the encoding of `data` as far as
+          # Ruby is concerned. This allows us to correctly parse out each line
+          # without changing the encoding of `data`, and
+          # also--importantly--without having to duplicate many (potentially
+          # large) strings.
+          begin
+            encoded_newlines = ["\r\n", "\r", "\n"].
+              map { |nl| nl.encode(ruby_encoding, "ASCII-8BIT").force_encoding(data.encoding) }
+
+            data.split(Regexp.union(encoded_newlines), -1)
+          rescue Encoding::ConverterNotFoundError
+            # The data is not splittable in the detected encoding.  Assume it's
+            # one big line.
+            [data]
+          end
         else
           []
         end

lib/linguist/generated.rb

@@ -63,7 +63,8 @@ module Linguist
         generated_jni_header? ||
         composer_lock? ||
         node_modules? ||
-        vcr_cassette?
+        vcr_cassette? ||
+        generated_by_zephir?
     end
 
     # Internal: Is the blob an XCode project file?
@@ -237,6 +238,13 @@ module Linguist
       !!name.match(/composer.lock/)
     end
 
+    # Internal: Is the blob a generated by Zephir
+    #
+    # Returns true or false.
+    def generated_by_zephir?
+      !!name.match(/.\.zep\.(?:c|h|php)$/)
+    end
+
     # Is the blob a VCR Cassette file?
     #
     # Returns true or false

lib/linguist/language.rb

@@ -24,7 +24,6 @@ module Linguist
     @extension_index          = Hash.new { |h,k| h[k] = [] }
     @interpreter_index        = Hash.new { |h,k| h[k] = [] }
     @filename_index           = Hash.new { |h,k| h[k] = [] }
-    @primary_extension_index  = {}
 
     # Valid Languages types
     TYPES = [:data, :markup, :programming, :prose]
@@ -80,12 +79,6 @@ module Linguist
         @extension_index[extension] << language
       end
 
-      if @primary_extension_index.key?(language.primary_extension)
-        raise ArgumentError, "Duplicate primary extension: #{language.primary_extension}"
-      end
-
-      @primary_extension_index[language.primary_extension] = language
-
       language.interpreters.each do |interpreter|
         @interpreter_index[interpreter] << language
       end
@@ -191,8 +184,7 @@ module Linguist
     # Returns all matching Languages or [] if none were found.
     def self.find_by_filename(filename)
       basename, extname = File.basename(filename), File.extname(filename)
-      langs = [@primary_extension_index[extname]] +
-              @filename_index[basename] +
+      langs = @filename_index[basename] +
               @extension_index[extname]
       langs.compact.uniq
     end
@@ -299,15 +291,6 @@ module Linguist
       @interpreters = attributes[:interpreters]   || []
       @filenames  = attributes[:filenames]  || []
 
-      unless @primary_extension = attributes[:primary_extension]
-        raise ArgumentError, "#{@name} is missing primary extension"
-      end
-
-      # Prepend primary extension unless its already included
-      if primary_extension && !extensions.include?(primary_extension)
-        @extensions = [primary_extension] + extensions
-      end
-
       # Set popular, and searchable flags
       @popular    = attributes.key?(:popular)    ? attributes[:popular]    : false
       @searchable = attributes.key?(:searchable) ? attributes[:searchable] : true
@@ -395,20 +378,6 @@ module Linguist
     # Returns the extensions Array
     attr_reader :extensions
 
-    # Deprecated: Get primary extension
-    #
-    # Defaults to the first extension but can be overridden
-    # in the languages.yml.
-    #
-    # The primary extension can not be nil. Tests should verify this.
-    #
-    # This attribute is only used by app/helpers/gists_helper.rb for
-    # creating the language dropdown. It really should be using `name`
-    # instead. Would like to drop primary extension.
-    #
-    # Returns the extension String.
-    attr_reader :primary_extension
-
     # Public: Get interpreters
     #
     # Examples
@@ -426,6 +395,27 @@ module Linguist
     #
     # Returns the extensions Array
     attr_reader :filenames
+    
+    # Public: Return all possible extensions for language
+    def all_extensions
+      (extensions + [primary_extension]).uniq
+    end
+
+    # Deprecated: Get primary extension
+    #
+    # Defaults to the first extension but can be overridden
+    # in the languages.yml.
+    #
+    # The primary extension can not be nil. Tests should verify this.
+    #
+    # This method is only used by app/helpers/gists_helper.rb for creating
+    # the language dropdown. It really should be using `name` instead.
+    # Would like to drop primary extension.
+    #
+    # Returns the extension String.
+    def primary_extension
+      extensions.first
+    end
 
     # Public: Get URL escaped name.
     #
@@ -568,9 +558,8 @@ module Linguist
       :group_name        => options['group'],
       :searchable        => options.key?('searchable') ? options['searchable'] : true,
       :search_term       => options['search_term'],
-      :extensions        => options['extensions'].sort,
+      :extensions        => [options['extensions'].first] + options['extensions'][1..-1].sort,
       :interpreters      => options['interpreters'].sort,
-      :primary_extension => options['primary_extension'],
       :filenames         => options['filenames'],
       :popular           => popular.include?(name)
     )

lib/linguist/vendor.yml

@@ -98,9 +98,16 @@
 # AngularJS
 - (^|/)angular([^.]*)(\.min)?\.js$
 
+# D3.js
+- (^|\/)d3(\.v\d+)?([^.]*)(\.min)?\.js$
+
 # React
 - (^|/)react(-[^.]*)?(\.min)?\.js$
 
+# Modernizr
+- (^|/)modernizr\-\d\.\d+(\.\d+)?(\.min)?\.js$
+- (^|/)modernizr\.custom\.\d+\.js$
+
 ## Python ##
 
 # django
@@ -141,7 +148,7 @@
 - (^|/)[Mm]icrosoft([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?\.js$
 
 # NuGet
-- ^[Pp]ackages/
+- ^[Pp]ackages\/.+\.\d+\/
 
 # ExtJS
 - (^|/)extjs/.*?\.js$
@@ -161,6 +168,9 @@
 - (^|/)extjs/src/
 - (^|/)extjs/welcome/
 
+# Html5shiv
+- (^|/)html5shiv(\.min)?\.js$
+
 # Samples folders
 - ^[Ss]amples/
 
@@ -189,3 +199,12 @@
 
 # Mercury --use-subdirs
 - Mercury/
+
+# R packages
+- ^vignettes/
+- ^inst/extdata/
+
+# Octicons
+- octicons.css
+- octicons.min.css
+- sprockets-octicons.scss

lib/linguist/version.rb

@@ -0,0 +1,3 @@
+module Linguist
+  VERSION = "2.12.0"
+end