Linguist 2.2.1

Fixes #203

github-linguist.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name    = 'github-linguist'
-  s.version = '2.1.2'
+  s.version = '2.2.1'
   s.summary = "GitHub Language detection"
 
   s.authors = "GitHub"

lib/linguist.rb

@@ -1,4 +1,5 @@
 require 'linguist/blob_helper'
+require 'linguist/generated'
 require 'linguist/language'
 require 'linguist/mime'
 require 'linguist/repository'

lib/linguist/blob_helper.rb

@@ -1,7 +1,6 @@
-require 'linguist/classifier'
+require 'linguist/generated'
 require 'linguist/language'
 require 'linguist/mime'
-require 'linguist/samples'
 
 require 'charlock_holmes'
 require 'escape_utils'
@@ -129,15 +128,6 @@ module Linguist
       ['.png', '.jpg', '.jpeg', '.gif'].include?(extname)
     end
 
-    # Public: Is the blob likely to have a shebang?
-    #
-    # Return true or false
-    def shebang_extname?
-      extname.empty? &&
-        mode &&
-        (mode.to_i(8) & 05) == 05
-    end
-
     MEGABYTE = 1024 * 1024
 
     # Public: Is the blob too big to load?
@@ -221,143 +211,16 @@ module Linguist
       lines.grep(/\S/).size
     end
 
-    # Internal: Compute average line length.
-    #
-    # Returns Integer.
-    def average_line_length
-      if lines.any?
-        lines.inject(0) { |n, l| n += l.length } / lines.length
-      else
-        0
-      end
-    end
-
     # Public: Is the blob a generated file?
     #
     # Generated source code is supressed in diffs and is ignored by
     # language statistics.
     #
-    # Requires Blob#data
-    #
-    # Includes:
-    # - XCode project XML files
-    # - Minified JavaScript
-    # - Compiled CoffeeScript
-    # - PEG.js-generated parsers
-    #
-    # Please add additional test coverage to
-    # `test/test_blob.rb#test_generated` if you make any changes.
+    # May load Blob#data
     #
     # Return true or false
     def generated?
-      if name == 'Gemfile.lock' || minified_javascript? || compiled_coffeescript? ||
-      xcode_project_file? || generated_net_docfile? || generated_parser?
-        true
-      else
-        false
-      end
-    end
-
-    # Internal: Is the blob an XCode project file?
-    #
-    # Generated if the file extension is an XCode project
-    # file extension.
-    #
-    # Returns true of false.
-    def xcode_project_file?
-      ['.xib', '.nib', '.storyboard', '.pbxproj', '.xcworkspacedata', '.xcuserstate'].include?(extname)
-    end
-
-    # Internal: Is the blob minified JS?
-    #
-    # Consider JS minified if the average line length is
-    # greater then 100c.
-    #
-    # Returns true or false.
-    def minified_javascript?
-      return unless extname == '.js'
-      average_line_length > 100
-    end
-
-    # Internal: Is the blob of JS a parser generated by PEG.js?
-    #
-    # Requires Blob#data
-    #
-    # PEG.js-generated parsers are not meant to be consumed by humans.
-    #
-    # Return true or false
-    def generated_parser?
-      return false unless extname == '.js'
-
-      # PEG.js-generated parsers include a comment near the top  of the file
-      # that marks them as such.
-      if lines[0..4].join('') =~ /^(?:[^\/]|\/[^\*])*\/\*(?:[^\*]|\*[^\/])*Generated by PEG.js/
-        return true
-      end
-
-      false
-    end
-
-    # Internal: Is the blob of JS generated by CoffeeScript?
-    #
-    # Requires Blob#data
-    #
-    # CoffeScript is meant to output JS that would be difficult to
-    # tell if it was generated or not. Look for a number of patterns
-    # output by the CS compiler.
-    #
-    # Return true or false
-    def compiled_coffeescript?
-      return false unless extname == '.js'
-
-      # CoffeeScript generated by > 1.2 include a comment on the first line
-      if lines[0] =~ /^\/\/ Generated by /
-        return true
-      end
-
-      if lines[0] == '(function() {' &&     # First line is module closure opening
-          lines[-2] == '}).call(this);' &&  # Second to last line closes module closure
-          lines[-1] == ''                   # Last line is blank
-
-        score = 0
-
-        lines.each do |line|
-          if line =~ /var /
-            # Underscored temp vars are likely to be Coffee
-            score += 1 * line.gsub(/(_fn|_i|_len|_ref|_results)/).count
-
-            # bind and extend functions are very Coffee specific
-            score += 3 * line.gsub(/(__bind|__extends|__hasProp|__indexOf|__slice)/).count
-          end
-        end
-
-        # Require a score of 3. This is fairly arbitrary. Consider
-        # tweaking later.
-        score >= 3
-      else
-        false
-      end
-    end
-
-    # Internal: Is this a generated documentation file for a .NET assembly?
-    #
-    # Requires Blob#data
-    #
-    # .NET developers often check in the XML Intellisense file along with an
-    # assembly - however, these don't have a special extension, so we have to
-    # dig into the contents to determine if it's a docfile. Luckily, these files
-    # are extremely structured, so recognizing them is easy.
-    #
-    # Returns true or false
-    def generated_net_docfile?
-      return false unless extname.downcase == ".xml"
-      return false unless lines.count > 3
-
-      # .NET Docfiles always open with <doc> and their first tag is an
-      # <assembly> tag
-      return lines[1].include?("<doc>") &&
-        lines[2].include?("<assembly>") &&
-        lines[-2].include?("</doc>")
+      @_generated ||= Generated.generated?(name, lambda { data })
     end
 
     # Public: Should the blob be indexed for searching?
@@ -375,6 +238,8 @@ module Linguist
     def indexable?
       if binary?
         false
+      elsif extname == '.txt'
+        true
       elsif language.nil?
         false
       elsif !language.searchable?
@@ -396,30 +261,11 @@ module Linguist
     def language
       if defined? @language
         @language
-      else
-        @language = guess_language
+      elsif !binary_mime_type?
+        @language = Language.detect(name.to_s, lambda { data }, mode)
       end
     end
 
-    # Internal: Guess language
-    #
-    # Please add additional test coverage to
-    # `test/test_blob.rb#test_language` if you make any changes.
-    #
-    # Returns a Language or nil
-    def guess_language
-      return if binary_mime_type?
-
-      # Disambiguate between multiple language extensions
-      disambiguate_extension_language ||
-
-        # See if there is a Language for the extension
-        Language.find_by_filename(name.to_s) ||
-
-        # Try to detect Language from shebang line
-        shebang_language
-    end
-
     # Internal: Get the lexer of the blob.
     #
     # Returns a Lexer.
@@ -427,86 +273,6 @@ module Linguist
       language ? language.lexer : Pygments::Lexer.find_by_name('Text only')
     end
 
-    # Internal: Disambiguates between multiple language extensions.
-    #
-    # Returns a Language or nil.
-    def disambiguate_extension_language
-      if Language.ambiguous?(extname)
-        possible_languages = Language.all.select { |l| l.extensions.include?(extname) }.map(&:name)
-        if possible_languages.any?
-          if result = Classifier.classify(Samples::DATA, data, possible_languages).first
-            Language[result[0]]
-          end
-        end
-      end
-    end
-
-    # Internal: Extract the script name from the shebang line
-    #
-    # Requires Blob#data
-    #
-    # Examples
-    #
-    #   '#!/usr/bin/ruby'
-    #   # => 'ruby'
-    #
-    #   '#!/usr/bin/env ruby'
-    #   # => 'ruby'
-    #
-    #   '#!/usr/bash/python2.4'
-    #   # => 'python'
-    #
-    # Please add additional test coverage to
-    # `test/test_blob.rb#test_shebang_script` if you make any changes.
-    #
-    # Returns a script name String or nil
-    def shebang_script
-      # Fail fast if blob isn't viewable?
-      return unless viewable?
-
-      if lines.any? && (match = lines[0].match(/(.+)\n?/)) && (bang = match[0]) =~ /^#!/
-        bang.sub!(/^#! /, '#!')
-        tokens = bang.split(' ')
-        pieces = tokens.first.split('/')
-        if pieces.size > 1
-          script = pieces.last
-        else
-          script = pieces.first.sub('#!', '')
-        end
-
-        script = script == 'env' ? tokens[1] : script
-
-        # python2.4 => python
-        if script =~ /((?:\d+\.?)+)/
-          script.sub! $1, ''
-        end
-
-        # Check for multiline shebang hacks that exec themselves
-        #
-        #   #!/bin/sh
-        #   exec foo "$0" "$@"
-        #
-        if script == 'sh' &&
-            lines[0...5].any? { |l| l.match(/exec (\w+).+\$0.+\$@/) }
-          script = $1
-        end
-
-        script
-      end
-    end
-
-    # Internal: Get Language for shebang script
-    #
-    # Returns the Language or nil
-    def shebang_language
-      # Skip file extensions unlikely to have shebangs
-      return unless shebang_extname?
-
-      if script = shebang_script
-        Language[script]
-      end
-    end
-
     # Public: Highlight syntax of blob
     #
     # options - A Hash of options (defaults to {})

lib/linguist/generated.rb

@@ -0,0 +1,162 @@
+module Linguist
+  class Generated
+    # Public: Is the blob a generated file?
+    #
+    # name - String filename
+    # data - String blob data. A block also maybe passed in for lazy
+    #        loading. This behavior is deprecated and you should always
+    #        pass in a String.
+    #
+    # Return true or false
+    def self.generated?(name, data)
+      new(name, data).generated?
+    end
+
+    # Internal: Initialize Generated instance
+    #
+    # name - String filename
+    # data - String blob data
+    def initialize(name, data)
+      @name = name
+      @extname = File.extname(name)
+      @_data = data
+    end
+
+    attr_reader :name, :extname
+
+    # Lazy load blob data if block was passed in.
+    #
+    # Awful, awful stuff happening here.
+    #
+    # Returns String data.
+    def data
+      @data ||= @_data.respond_to?(:call) ? @_data.call() : @_data
+    end
+
+    # Public: Get each line of data
+    #
+    # Returns an Array of lines
+    def lines
+      # TODO: data should be required to be a String, no nils
+      @lines ||= data ? data.split("\n", -1) : []
+    end
+
+    # Internal: Is the blob a generated file?
+    #
+    # Generated source code is supressed in diffs and is ignored by
+    # language statistics.
+    #
+    # Please add additional test coverage to
+    # `test/test_blob.rb#test_generated` if you make any changes.
+    #
+    # Return true or false
+    def generated?
+      name == 'Gemfile.lock' ||
+        minified_javascript? ||
+        compiled_coffeescript? ||
+        xcode_project_file? ||
+        generated_net_docfile? ||
+        generated_parser?
+    end
+
+    # Internal: Is the blob an XCode project file?
+    #
+    # Generated if the file extension is an XCode project
+    # file extension.
+    #
+    # Returns true of false.
+    def xcode_project_file?
+      ['.xib', '.nib', '.storyboard', '.pbxproj', '.xcworkspacedata', '.xcuserstate'].include?(extname)
+    end
+
+    # Internal: Is the blob minified JS?
+    #
+    # Consider JS minified if the average line length is
+    # greater then 100c.
+    #
+    # Returns true or false.
+    def minified_javascript?
+      return unless extname == '.js'
+      if lines.any?
+        (lines.inject(0) { |n, l| n += l.length } / lines.length) > 100
+      else
+        false
+      end
+    end
+
+    # Internal: Is the blob of JS generated by CoffeeScript?
+    #
+    # CoffeScript is meant to output JS that would be difficult to
+    # tell if it was generated or not. Look for a number of patterns
+    # output by the CS compiler.
+    #
+    # Return true or false
+    def compiled_coffeescript?
+      return false unless extname == '.js'
+
+      # CoffeeScript generated by > 1.2 include a comment on the first line
+      if lines[0] =~ /^\/\/ Generated by /
+        return true
+      end
+
+      if lines[0] == '(function() {' &&     # First line is module closure opening
+          lines[-2] == '}).call(this);' &&  # Second to last line closes module closure
+          lines[-1] == ''                   # Last line is blank
+
+        score = 0
+
+        lines.each do |line|
+          if line =~ /var /
+            # Underscored temp vars are likely to be Coffee
+            score += 1 * line.gsub(/(_fn|_i|_len|_ref|_results)/).count
+
+            # bind and extend functions are very Coffee specific
+            score += 3 * line.gsub(/(__bind|__extends|__hasProp|__indexOf|__slice)/).count
+          end
+        end
+
+        # Require a score of 3. This is fairly arbitrary. Consider
+        # tweaking later.
+        score >= 3
+      else
+        false
+      end
+    end
+
+    # Internal: Is this a generated documentation file for a .NET assembly?
+    #
+    # .NET developers often check in the XML Intellisense file along with an
+    # assembly - however, these don't have a special extension, so we have to
+    # dig into the contents to determine if it's a docfile. Luckily, these files
+    # are extremely structured, so recognizing them is easy.
+    #
+    # Returns true or false
+    def generated_net_docfile?
+      return false unless extname.downcase == ".xml"
+      return false unless lines.count > 3
+
+      # .NET Docfiles always open with <doc> and their first tag is an
+      # <assembly> tag
+      return lines[1].include?("<doc>") &&
+        lines[2].include?("<assembly>") &&
+        lines[-2].include?("</doc>")
+    end
+
+    # Internal: Is the blob of JS a parser generated by PEG.js?
+    #
+    # PEG.js-generated parsers are not meant to be consumed by humans.
+    #
+    # Return true or false
+    def generated_parser?
+      return false unless extname == '.js'
+
+      # PEG.js-generated parsers include a comment near the top  of the file
+      # that marks them as such.
+      if lines[0..4].join('') =~ /^(?:[^\/]|\/[^\*])*\/\*(?:[^\*]|\*[^\/])*Generated by PEG.js/
+        return true
+      end
+
+      false
+    end
+  end
+end

lib/linguist/language.rb

@@ -2,6 +2,7 @@ require 'escape_utils'
 require 'pygments'
 require 'yaml'
 
+require 'linguist/classifier'
 require 'linguist/samples'
 
 module Linguist
@@ -11,23 +12,15 @@ module Linguist
   # Languages are defined in `lib/linguist/languages.yml`.
   class Language
     @languages       = []
-    @overrides       = {}
     @index           = {}
     @name_index      = {}
     @alias_index     = {}
-    @extension_index = {}
-    @filename_index  = {}
+    @extension_index = Hash.new { |h,k| h[k] = [] }
+    @filename_index  = Hash.new { |h,k| h[k] = [] }
 
     # Valid Languages types
     TYPES = [:data, :markup, :programming]
 
-    # Internal: Test if extension maps to multiple Languages.
-    #
-    # Returns true or false.
-    def self.ambiguous?(extension)
-      @overrides.include?(extension)
-    end
-
     # Internal: Create a new Language object
     #
     # attributes - A hash of attributes
@@ -60,34 +53,45 @@ module Linguist
           raise ArgumentError, "Extension is missing a '.': #{extension.inspect}"
         end
 
-        unless ambiguous?(extension)
-          # Index the extension with a leading ".": ".rb"
-          @extension_index[extension] = language
-
-          # Index the extension without a leading ".": "rb"
-          @extension_index[extension.sub(/^\./, '')] = language
-        end
-      end
-
-      language.overrides.each do |extension|
-        if extension !~ /^\./
-          raise ArgumentError, "Extension is missing a '.': #{extension.inspect}"
-        end
-
-        if l = @overrides[extension]
-          raise ArgumentError, "#{extension} is already overridden by #{l.name}"
-        end
-
-        @overrides[extension] = language
+        @extension_index[extension] << language
       end
 
       language.filenames.each do |filename|
-        @filename_index[filename] = language
+        @filename_index[filename] << language
       end
 
       language
     end
 
+    # Public: Detects the Language of the blob.
+    #
+    # name - String filename
+    # data - String blob data. A block also maybe passed in for lazy
+    #        loading. This behavior is deprecated and you should always
+    #        pass in a String.
+    # mode - Optional String mode (defaults to nil)
+    #
+    # Returns Language or nil.
+    def self.detect(name, data, mode = nil)
+      # A bit of an elegant hack. If the file is exectable but extensionless,
+      # append a "magic" extension so it can be classified with other
+      # languages that have shebang scripts.
+      if File.extname(name).empty? && mode && (mode.to_i(8) & 05) == 05
+        name += ".script!"
+      end
+
+      possible_languages = find_by_filename(name)
+
+      if possible_languages.length > 1
+        data = data.call() if data.respond_to?(:call)
+        if result = Classifier.classify(Samples::DATA, data, possible_languages.map(&:name)).first
+          Language[result[0]]
+        end
+      else
+        possible_languages.first
+      end
+    end
+
     # Public: Get all Languages
     #
     # Returns an Array of Languages
@@ -123,33 +127,19 @@ module Linguist
       @alias_index[name]
     end
 
-    # Public: Look up Language by extension.
-    #
-    # extension - The extension String. May include leading "."
-    #
-    # Examples
-    #
-    #   Language.find_by_extension('.rb')
-    #   # => #<Language name="Ruby">
-    #
-    # Returns the Language or nil if none was found.
-    def self.find_by_extension(extension)
-      @extension_index[extension]
-    end
-
-    # Public: Look up Language by filename.
+    # Public: Look up Languages by filename.
     #
     # filename - The path String.
     #
     # Examples
     #
     #   Language.find_by_filename('foo.rb')
-    #   # => #<Language name="Ruby">
+    #   # => [#<Language name="Ruby">]
     #
-    # Returns the Language or nil if none was found.
+    # Returns all matching Languages or [] if none were found.
     def self.find_by_filename(filename)
       basename, extname = File.basename(filename), File.extname(filename)
-      @filename_index[basename] || @extension_index[extname]
+      @filename_index[basename] + @extension_index[extname]
     end
 
     # Public: Look up Language by its name or lexer.
@@ -236,7 +226,6 @@ module Linguist
 
       # Set extensions or default to [].
       @extensions = attributes[:extensions] || []
-      @overrides  = attributes[:overrides]  || []
       @filenames  = attributes[:filenames]  || []
 
       unless @primary_extension = attributes[:primary_extension]
@@ -344,11 +333,6 @@ module Linguist
     # Returns the extension String.
     attr_reader :primary_extension
 
-    # Internal: Get overridden extensions.
-    #
-    # Returns the extensions Array.
-    attr_reader :overrides
-
     # Public: Get filenames
     #
     # Examples
@@ -481,7 +465,6 @@ module Linguist
       :search_term       => options['search_term'],
       :extensions        => options['extensions'].sort,
       :primary_extension => options['primary_extension'],
-      :overrides         => options['overrides'],
       :filenames         => options['filenames'],
       :popular           => popular.include?(name)
     )

lib/linguist/languages.yml

@@ -15,7 +15,6 @@
 #                     the language. Must be unique. Used when a Language is picked
 #                     from a dropdown and we need to automatically choose an
 #                     extension.
-# overrides         - An Array of extensions that takes precedence over conflicts
 # searchable        - Boolean flag to enable searching (defaults to true)
 # search_term       - Deprecated: Some languages maybe indexed under a
 #                     different alias. Avoid defining new exceptions.
@@ -67,13 +66,12 @@ Apex:
   type: programming
   lexer: Text only
   primary_extension: .cls
-  overrides:
-  - .cls
 
 AppleScript:
+  type: programming
   aliases:
   - osascript
-  primary_extension: .scpt
+  primary_extension: .applescript
 
 Arc:
   type: programming
@@ -157,8 +155,6 @@ Bro:
 C:
   type: programming
   color: "#555"
-  overrides:
-  - .h
   primary_extension: .c
   extensions:
   - .w
@@ -533,8 +529,6 @@ Groovy:
 Groovy Server Pages:
   group: Groovy
   lexer: Java Server Page
-  overrides:
-  - .gsp
   aliases:
   - gsp
   primary_extension: .gsp
@@ -604,8 +598,6 @@ INI:
   - .prefs
   - .properties
   primary_extension: .ini
-  filenames:
-  - .gitconfig
 
 IRC log:
   lexer: IRC logs
@@ -634,8 +626,6 @@ JSON:
   ace_mode: json
   searchable: false
   primary_extension: .json
-  extensions:
-  - .json
 
 Java:
   type: programming
@@ -757,13 +747,15 @@ Matlab:
   extensions:
   - .matlab
 
-Max/MSP:
+Max:
   type: programming
   color: "#ce279c"
   lexer: Text only
+  aliases:
+  - max/msp
+  - maxmsp
+  search_term: max/msp
   primary_extension: .mxt
-  extensions:
-  - .mxt
 
 MiniD: # Legacy
   searchable: false
@@ -841,8 +833,6 @@ ObjDump:
 Objective-C:
   type: programming
   color: "#438eff"
-  overrides:
-  - .m
   primary_extension: .m
   extensions:
   - .mm
@@ -915,8 +905,6 @@ Perl:
   ace_mode: perl
   color: "#0298c3"
   primary_extension: .pl
-  overrides:
-  - .pl
   extensions:
   - .PL
   - .perl
@@ -983,8 +971,6 @@ R:
   type: programming
   color: "#198ce7"
   lexer: S
-  overrides:
-  - .r
   primary_extension: .r
   extensions:
   - .r
@@ -1129,12 +1115,7 @@ Shell:
   - zsh
   primary_extension: .sh
   filenames:
-  - .bash_profile
-  - .bashrc
-  - .profile
-  - .zlogin
   - .zsh
-  - .zshrc
   - bashrc
   - zshrc
 
@@ -1195,14 +1176,6 @@ Tea:
   type: markup
   primary_extension: .tea
 
-Text:
-  type: data
-  lexer: Text only
-  ace_mode: text
-  primary_extension: .txt
-  extensions:
-  - .txt
-
 Textile:
   type: markup
   lexer: Text only
@@ -1216,8 +1189,6 @@ Turing:
   color: "#45f715"
   lexer: Text only
   primary_extension: .t
-  overrides:
-  - .t
   extensions:
   - .tu
 
@@ -1249,8 +1220,6 @@ Verilog:
   type: programming
   lexer: verilog
   color: "#848bf3"
-  overrides:
-  - .v
   primary_extension: .v
 
 VimL:
@@ -1263,8 +1232,6 @@ VimL:
   extensions:
   - .vim
   filenames:
-  - .gvimrc
-  - .vimrc
   - vimrc
   - gvimrc
 
@@ -1332,8 +1299,6 @@ YAML:
   extensions:
   - .yaml
   - .yml
-  filenames:
-  - .gemrc
 
 eC:
   type: programming

lib/linguist/samples.rb

@@ -45,6 +45,10 @@ module Linguist
               })
             end
           else
+            if File.extname(filename) == ""
+              raise "#{File.join(dirname, filename)} is missing an extension, maybe it belongs in filenames/ subdir"
+            end
+
             yield({
               :path     => File.join(dirname, filename),
               :language => category,
@@ -68,18 +72,16 @@ module Linguist
       each do |sample|
         language_name = sample[:language]
 
-        # TODO: For now skip empty extnames
-        if sample[:extname] && sample[:extname] != ""
+        if sample[:extname]
           db['extnames'][language_name] ||= []
           if !db['extnames'][language_name].include?(sample[:extname])
             db['extnames'][language_name] << sample[:extname]
           end
         end
 
-        # TODO: For now skip empty extnames
-        if fn = sample[:filename]
+        if sample[:filename]
           db['filenames'][language_name] ||= []
-          db['filenames'][language_name] << fn
+          db['filenames'][language_name] << sample[:filename]
         end
 
         data = File.read(sample[:path])

lib/linguist/tokenizer.rb

@@ -1,3 +1,5 @@
+require 'strscan'
+
 module Linguist
   # Generic programming language tokenizer.
   #
@@ -50,8 +52,13 @@ module Linguist
 
       tokens = []
       until s.eos?
+        if token = s.scan(/^#!.+$/)
+          if name = extract_shebang(token)
+            tokens << "SHEBANG#!#{name}"
+          end
+
         # Single line comment
-        if token = s.scan(START_SINGLE_LINE_COMMENT)
+        elsif token = s.scan(START_SINGLE_LINE_COMMENT)
           tokens << token.strip
           s.skip_until(/\n|\Z/)
 
@@ -64,19 +71,27 @@ module Linguist
 
         # Skip single or double quoted strings
         elsif s.scan(/"/)
-          s.skip_until(/[^\\]"/)
+          if s.peek(1) == "\""
+            s.getch
+          else
+            s.skip_until(/[^\\]"/)
+          end
         elsif s.scan(/'/)
-          s.skip_until(/[^\\]'/)
+          if s.peek(1) == "'"
+            s.getch
+          else
+            s.skip_until(/[^\\]'/)
+          end
 
         # Skip number literals
-        elsif s.scan(/(0x)?\d+/)
+        elsif s.scan(/(0x)?\d(\d|\.)*/)
 
         # SGML style brackets
         elsif token = s.scan(/<[^\s<>][^<>]*>/)
           extract_sgml_tokens(token).each { |t| tokens << t }
 
         # Common programming punctuation
-        elsif token = s.scan(/;|\{|\}|\(|\)/)
+        elsif token = s.scan(/;|\{|\}|\(|\)|\[|\]/)
           tokens << token
 
         # Regular token
@@ -95,6 +110,33 @@ module Linguist
       tokens
     end
 
+    # Internal: Extract normalized shebang command token.
+    #
+    # Examples
+    #
+    #   extract_shebang("#!/usr/bin/ruby")
+    #   # => "ruby"
+    #
+    #   extract_shebang("#!/usr/bin/env node")
+    #   # => "node"
+    #
+    # Returns String token or nil it couldn't be parsed.
+    def extract_shebang(data)
+      s = StringScanner.new(data)
+
+      if path = s.scan(/^#!\s*\S+/)
+        script = path.split('/').last
+        if script == 'env'
+          s.scan(/\s+/)
+          script = s.scan(/\S+/)
+        end
+        script = script[/[^\d]+/, 0]
+        return script
+      end
+
+      nil
+    end
+
     # Internal: Extract tokens from inside SGML tag.
     #
     # data - SGML tag String.

samples/AppleScript/Convert To PDF.applescript

@@ -0,0 +1,87 @@
+(*
+Copyright 2003 Apple Computer, Inc.
+
+You may incorporate this Apple sample code into your program(s) without
+restriction.  This Apple sample code has been provided "AS IS" and the
+responsibility for its operation is yours.  You are not permitted to
+redistribute this Apple sample code as "Apple sample code" after having
+made changes.  If you're going to redistribute the code, we require
+that you make it clear that the code was descended from Apple sample
+code, but that you've made changes.
+*)
+
+property type_list : {"JPEG", "GIFf", "PICT", "TIFF", "PDF", "TEXT"}
+property extension_list : {"jpg", "gif", "pct", "tif", "pdf", "rtf"}
+--html is not currently handled
+
+on run {}
+	tell application "Finder" to set FinderSelection to the selection as alias list
+	
+	set FS to FinderSelection
+	--Ideally, this list could be passed to the open handler
+	
+	set SelectionCount to number of FS -- count	
+	if SelectionCount is 0 then
+		set FS to userPicksFolder()
+	else if the SelectionCount is 1 then
+		set MyPath to path to me
+		if MyPath is item 1 of FS then
+			--If I'm a droplet then I was double-clicked
+			set FS to userPicksFolder()
+		end if
+	else
+		--I'm not a double-clicked droplet
+	end if
+	open FS
+end run
+
+on userPicksFolder()
+	set these_items to {}
+	set these_items to (choose file with prompt "Select a file to convert to PDF:" of type {"JPEG", "GIFf", "PICT", "TIFF", "TEXT", "RTF"}) as list
+end userPicksFolder
+
+on open these_items
+	set thesefiles to {}
+	set the item_info to {}
+	repeat with i from 1 to the count of these_items
+		set this_item to (item i of these_items)
+		set the item_info to info for this_item
+		
+		if folder of the item_info is true then --if the item is a folder
+			processFolder(this_item)
+		else if ((folder of the item_info is false) and (alias of the item_info is false)) and (the file type of the item_info is in the type_list) or ((the name extension of the item_info) is in the extension_list) then
+			
+			set theFilePath to (item i of these_items as string)
+			set thePOSIXFilePath to POSIX path of theFilePath as string
+			processFile(thePOSIXFilePath)
+		end if
+	end repeat
+end open
+
+--process folders 
+on processFolder(theFolder)
+	set these_items to list folder theFolder without invisibles
+	repeat with i from 1 to the count of these_items
+		set this_item to alias ((theFolder as text) & (item i of these_items))
+		set the item_info to info for this_item
+		if folder of the item_info is true then
+			processFolder(this_item)
+		else if (alias of the item_info is false) and ((the file type of the item_info is in the type_list) or the name extension of the item_info is in the extension_list) then
+			set theFilePath to (this_item as string)
+			set thePOSIXFilePath to POSIX path of theFilePath as string
+			processFile(thePOSIXFilePath)
+		end if
+	end repeat
+end processFolder
+
+on processFile(thePOSIXFileName)
+	try
+		set terminalCommand to ""
+		set convertCommand to "/System/Library/Printers/Libraries/./convert "
+		set newFileName to thePOSIXFileName & ".pdf"
+		set terminalCommand to convertCommand & "-f " & "\"" & thePOSIXFileName & "\"" & " -o " & "\"" & newFileName & "\"" & " -j \"application/pdf\""
+		
+		do shell script terminalCommand
+	end try
+end processFile
+

samples/AppleScript/Convert To PostScript.applescript

@@ -0,0 +1,91 @@
+(*
+
+Copyright 2003 Apple Computer, Inc.
+
+You may incorporate this Apple sample code into your program(s) without
+restriction.  This Apple sample code has been provided "AS IS" and the
+responsibility for its operation is yours.  You are not permitted to
+redistribute this Apple sample code as "Apple sample code" after having
+made changes.  If you're going to redistribute the code, we require
+that you make it clear that the code was descended from Apple sample
+code, but that you've made changes.
+*)
+
+property type_list : {"JPEG", "GIFf", "PICT", "TIFF", "PDF", "TEXT"}
+property extension_list : {"jpg", "gif", "pct", "tif", "pdf", "rtf"}
+--html is not currently handled
+
+on run {}
+	tell application "Finder" to set FinderSelection to the selection as alias list
+	
+	set FS to FinderSelection
+	--Ideally, this list could be passed to the open handler
+	
+	set SelectionCount to number of FS -- count	
+	if SelectionCount is 0 then
+		set FS to userPicksFolder()
+	else if the SelectionCount is 1 then
+		set MyPath to path to me
+		if MyPath is item 1 of FS then
+			--If I'm a droplet then I was double-clicked
+			set FS to userPicksFolder()
+		end if
+	else
+		--I'm not a double-clicked droplet
+	end if
+	open FS
+end run
+
+on userPicksFolder()
+	set these_items to {}
+	set these_items to (choose file with prompt "Select a file to convert to PostScript:" of type {"JPEG", "GIFf", "PICT", "TIFF", "TEXT", "RTF"}) as list
+end userPicksFolder
+
+
+on open these_items
+	set thesefiles to {}
+	set the item_info to {}
+	repeat with i from 1 to the count of these_items
+		set this_item to (item i of these_items)
+		set the item_info to info for this_item
+		
+		if folder of the item_info is true then --if the item is a folder
+			processFolder(this_item)
+		else if ((folder of the item_info is false) and (alias of the item_info is false)) and (the file type of the item_info is in the type_list) or ((the name extension of the item_info) is in the extension_list) then
+			
+			set theFilePath to (item i of these_items as string)
+			set thePOSIXFilePath to POSIX path of theFilePath as string
+			processFile(thePOSIXFilePath)
+		end if
+	end repeat
+end open
+
+--process folders 
+on processFolder(theFolder)
+	set these_items to list folder theFolder without invisibles
+	repeat with i from 1 to the count of these_items
+		set this_item to alias ((theFolder as text) & (item i of these_items))
+		set the item_info to info for this_item
+		if folder of the item_info is true then
+			processFolder(this_item)
+		else if (alias of the item_info is false) and ((the file type of the item_info is in the type_list) or the name extension of the item_info is in the extension_list) then
+			set theFilePath to (this_item as string)
+			set thePOSIXFilePath to POSIX path of theFilePath as string
+			processFile(thePOSIXFilePath)
+		end if
+	end repeat
+end processFolder
+
+--need to pass the URL to Terminal
+on processFile(thePOSIXFileName)
+	try
+		set terminalCommand to ""
+		set convertCommand to "/System/Library/Printers/Libraries/./convert "
+		set newFileName to thePOSIXFileName & ".ps"
+		set terminalCommand to convertCommand & "-f " & "\"" & thePOSIXFileName & "\"" & " -o " & "\"" & newFileName & "\"" & " -j \"application/postscript\""
+		
+		do shell script terminalCommand
+	end try
+end processFile
+
+

samples/AppleScript/Count Messages in All Mailboxes.applescript

@@ -0,0 +1,80 @@
+(*
+Count Messages in All Mailboxes
+
+Copyright 2002-2012 Apple Inc. All rights reserved.
+
+You may incorporate this Apple sample code into your program(s) without
+restriction.  This Apple sample code has been provided "AS IS" and the
+responsibility for its operation is yours.  You are not permitted to
+redistribute this Apple sample code as "Apple sample code" after having
+made changes.  If you're going to redistribute the code, we require
+that you make it clear that the code was descended from Apple sample
+code, but that you've made changes.
+*)
+
+(*
+This script goes through each mailbox, gets the total message count and
+the unread count, then displays the final output in a new email message.
+*)
+
+tell application "Mail"
+	set localMailboxes to every mailbox
+	if (count of localMailboxes) is greater than 0 then
+		set messageCountDisplay to "Local mailboxes (On My Mac)" & return & my getMessageCountsForMailboxes(localMailboxes)
+	else
+		set messageCountDisplay to ""
+	end if
+	
+	set everyAccount to every account
+	repeat with eachAccount in everyAccount
+		set accountMailboxes to every mailbox of eachAccount
+		if (count of accountMailboxes) is greater than 0 then
+			set messageCountDisplay to messageCountDisplay & return & "Mailboxes for Account: " & name of eachAccount & return & my getMessageCountsForMailboxes(accountMailboxes)
+		end if
+	end repeat
+	
+	set outputMessage to make new outgoing message with properties {content:messageCountDisplay, subject:"Message counts for all my mailboxes", visible:true}
+	tell outputMessage
+		set font to "Courier"
+		set size to 12
+	end tell
+end tell
+
+on getMessageCountsForMailboxes(theMailboxes)
+	-- (list of mailboxes)
+	-- returns string
+	
+	set displayString to ""
+	
+	tell application "Mail"
+		repeat with eachMailbox in theMailboxes
+			set mailboxName to name of eachMailbox
+			set messageCount to (count of (messages of eachMailbox)) as string
+			set unreadCount to unread count of eachMailbox as string
+			
+			set displayString to displayString & "    " & my padString(mailboxName, 40) & "  " & messageCount & " (" & unreadCount & " unread)" & return
+		end repeat
+	end tell
+	
+	return displayString
+end getMessageCountsForMailboxes
+
+on padString(theString, fieldLength)
+	-- (string, integer)
+	-- returns string
+	
+	set stringLength to length of theString
+	
+	if stringLength is greater than fieldLength then
+		set paddedString to (text from character 1 to character (fieldLength - 3) of theString) & "..."
+	else -- stringLength is less than or equal to fieldLength
+		set paddedString to theString
+		
+		set paddingLength to fieldLength - stringLength
+		repeat paddingLength times
+			set paddedString to paddedString & space
+		end repeat
+	end if
+	
+	return paddedString
+end padString

samples/AppleScript/Crazy Message Text.applescript

@@ -0,0 +1,68 @@
+(*
+Crazy Message Text
+
+Copyright 2002-2012 Apple Inc. All rights reserved.
+
+You may incorporate this Apple sample code into your program(s) without
+restriction.  This Apple sample code has been provided "AS IS" and the
+responsibility for its operation is yours.  You are not permitted to
+redistribute this Apple sample code as "Apple sample code" after having
+made changes.  If you're going to redistribute the code, we require
+that you make it clear that the code was descended from Apple sample
+code, but that you've made changes.
+*)
+
+(*
+This script takes a string from the user and then makes a new message
+where each letter has a different font, size, and color.
+*)
+
+property lowFontSize : 36
+property highFontSize : 72
+property messageText : "Happy Birthday!"
+
+repeat
+	set userInput to display dialog "Enter some message text:" & return & return & "Minimum Character Size: " & (lowFontSize as string) & return & "Maximum Character Size: " & (highFontSize as string) default answer messageText buttons {"Cancel", "Set Prefs", "Continue"} default button 3
+	
+	if the button returned of userInput is "Set Prefs" then
+		set minimumFontSize to 9
+		
+		display dialog "Enter the minimum font size to use:" & return & return & "(Must be at least " & (minimumFontSize as string) & ")" default answer lowFontSize buttons {"OK"}
+		set newFontSize to text returned of the result as integer
+		if newFontSize is greater than or equal to minimumFontSize then
+			set lowFontSize to newFontSize
+		else
+			set lowFontSize to minimumFontSize
+		end if
+		
+		display dialog "Enter the maximum font size to use:" & return & return & "(Must be greater than " & (lowFontSize as string) & ")" default answer highFontSize buttons {"OK"}
+		set newFontSize to text returned of the result as integer
+		if newFontSize is greater than lowFontSize then
+			set highFontSize to newFontSize
+		else
+			set highFontSize to lowFontSize
+		end if
+		
+	else -- button returned of userInput is "Continue"
+		set theText to text returned of userInput
+		if theText is not "" then
+			set messageText to theText
+		end if
+		exit repeat
+	end if
+end repeat
+
+set fontList to {"American Typewriter", "American Typewriter Light", "American Typewriter Bold", "American Typewriter Condensed", "American Typewriter Condensed Light", "American Typewriter Condensed Bold", "Arial", "Arial Italic", "Arial Bold", "Arial Bold Italic", "Arial Black", "Baskerville", "Baskerville Italic", "Baskerville SemiBold", "Baskerville Bold", "Baskerville SemiBold Italic", "Baskerville Bold Italic", "Big Caslon Medium", "Comic Sans MS", "Comic Sans MS Bold", "Copperplate", "Copperplate Light", "Copperplate Bold", "Didot", "Didot Italic", "Didot Bold", "Futura Medium", "Futura Medium Italic", "Futura Condensed Medium", "Futura Condensed ExtraBold", "Geneva", "Gill Sans", "Gill Sans Italic", "Gill Sans Light", "Gill Sans Light Italic", "Gill Sans Bold", "Gill Sans Bold Italic", "Herculanum", "Lucida Grande", "Lucida Grande Bold", "Marker Felt Thin", "Marker Felt Wide", "Optima Regular", "Optima Italic", "Optima Bold", "Optima Bold Italic", "Optima ExtraBlack", "Papyrus", "Verdana", "Verdana Italic", "Verdana Bold", "Verdana Bold Italic", "Zapfino"}
+
+tell application "Mail"
+	activate
+	set crazyTextMessage to make new outgoing message with properties {content:messageText, visible:true}
+	
+	tell crazyTextMessage
+		repeat with eachCharacter in characters
+			set font of eachCharacter to (some item of fontList)
+			set size of eachCharacter to (random number from lowFontSize to highFontSize)
+			set color of eachCharacter to {random number from 0 to 65535, random number from 0 to 65535, random number from 0 to 65535}
+		end repeat
+	end tell
+end tell

samples/AppleScript/Get User Name.applescript

@@ -0,0 +1,41 @@
+(*
+Get User Name
+
+This script uses UI element scripting to get the name for the
+current user.
+
+If "Enable access for assistive devices" is not checked,
+this script will open the Universal Access System Preference and ask
+the user to check the checkbox.
+
+Copyright 2007 Apple Inc.
+
+You may incorporate this Apple sample code into your program(s) without
+restriction.  This Apple sample code has been provided "AS IS" and the
+responsibility for its operation is yours.  You are not permitted to
+redistribute this Apple sample code as "Apple sample code" after having
+made changes.  If you're going to redistribute the code, we require
+that you make it clear that the code was descended from Apple sample
+code, but that you've made changes.
+*)
+
+tell application "System Preferences"
+	activate
+	set current pane to pane "com.apple.preferences.users"
+end tell
+
+tell application "System Events"
+	if UI elements enabled then
+		tell tab group 1 of window "Accounts" of process "System Preferences"
+			click radio button 1
+			delay 2
+			get value of text field 1
+		end tell
+	else
+		tell application "System Preferences"
+			activate
+			set current pane to pane "com.apple.preference.universalaccess"
+			display dialog "UI element scripting is not enabled. Check \"Enable access for assistive devices\""
+		end tell
+	end if
+end tell

samples/AppleScript/Time Of Day.applescript

@@ -0,0 +1,75 @@
+(* 
+ Speaks the  date and time of day
+ 
+ Copyright 2008 Apple Inc. All rights reserved.
+ 
+ You may incorporate this Apple sample code into your program(s) without
+ restriction.  This Apple sample code has been provided "AS IS" and the
+ responsibility for its operation is yours.  You are not permitted to
+ redistribute this Apple sample code as "Apple sample code" after having
+ made changes.  If you're going to redistribute the code, we require
+ that you make it clear that the code was descended from Apple sample
+ code, but that you've made changes.
+ *)
+
+on isVoiceOverRunning()
+	set isRunning to false
+	tell application "System Events"
+		set isRunning to (name of processes) contains "VoiceOver"
+	end tell
+	return isRunning
+end isVoiceOverRunning
+
+on isVoiceOverRunningWithAppleScript()
+	if isVoiceOverRunning() then
+		set isRunningWithAppleScript to true
+		
+		-- is AppleScript enabled on VoiceOver --
+		tell application "VoiceOver"
+			try
+				set x to bounds of vo cursor
+			on error
+				set isRunningWithAppleScript to false
+			end try
+		end tell
+		return isRunningWithAppleScript
+	end if
+	return false
+end isVoiceOverRunningWithAppleScript
+
+set currentDate to current date
+set amPM to "AM"
+set currentHour to (currentDate's hours)
+set currentMinutes to currentDate's minutes
+
+if (currentHour > 12 and currentHour < 24) then
+	set amPM to "PM"
+else
+	set amPM to "AM"
+end if
+
+--  make minutes below 10 sound nice
+if currentMinutes < 10 then
+	set currentMinutes to ("0" & currentMinutes) as text
+end if
+
+--  ensure 0:nn gets set to 12:nn AM
+if currentHour is equal to 0 then
+	set currentHour to 12
+end if
+
+--  readjust for 12 hour time
+if (currentHour > 12) then
+	set currentHour to (currentHour - 12)
+end if
+
+set currentTime to ((currentDate's month) as text) & " " & ((currentDate's day) as text) & ", " & (currentHour as text) & ":" & ((currentMinutes) as text) & " " & amPM as text
+
+if isVoiceOverRunningWithAppleScript() then
+	tell application "VoiceOver"
+		output currentTime
+	end tell
+else
+	say currentTime
+	delay 2
+end if

samples/AppleScript/center.scpt

@@ -1,50 +0,0 @@
-set windowWidth to 800
-set windowHeight to 600
-delay 0.1
-
-set AppleScript's text item delimiters to "x"
-
-set res to text returned of (display dialog "Enter the width x height:" default answer ((windowWidth & windowHeight) as text))
-
-if res is "" then
-  display dialog "You need to enter a correct response"
-  return
-end if
-set {windowWidth, windowHeight} to text items of res
-
-set AppleScript's text item delimiters to ""
-
-tell application "Safari"
-  set screen_width to (do JavaScript "screen.availWidth" in document 1)
-  set screen_height to (do JavaScript "screen.availHeight" in document 1)
-end tell
-
-tell application "System Events"
-  set myFrontMost to name of first item of (processes whose frontmost is true)
-end tell
-
-tell application "Finder"
-  set {desktopTop, desktopLeft, desktopRight, desktopBottom} to bounds of desktop
-end tell
-
-try
-  tell application "System Events"
-    tell process myFrontMost
-      set {{w, h}} to size of drawer of window 1
-    end tell
-  end tell
-on error
-  set {w, h} to {0, 0}
-end try
-
-tell application "System Events"
-  tell process myFrontMost
-    try
-      set {{w, h}} to size of drawer of window 1
-    on error
-      set {w, h} to {0, 0}
-    end try
-    set position of window 1 to {((screen_width - windowWidth) / 2), ((screen_height - windowHeight) / 2.0) - desktopTop}
-    set size of window 1 to {windowWidth -w, windowHeight}
-  end tell
-end tell

samples/JSON/Hello.maxhelp

@@ -0,0 +1,367 @@
+{
+	"patcher" : 	{
+		"fileversion" : 1,
+		"appversion" : 		{
+			"major" : 5,
+			"minor" : 1,
+			"revision" : 9
+		}
+,
+		"rect" : [ 198.0, 92.0, 365.0, 407.0 ],
+		"bglocked" : 0,
+		"defrect" : [ 198.0, 92.0, 365.0, 407.0 ],
+		"openrect" : [ 0.0, 0.0, 0.0, 0.0 ],
+		"openinpresentation" : 0,
+		"default_fontsize" : 14.0,
+		"default_fontface" : 0,
+		"default_fontname" : "Arial",
+		"gridonopen" : 0,
+		"gridsize" : [ 20.0, 20.0 ],
+		"gridsnaponopen" : 0,
+		"toolbarvisible" : 1,
+		"boxanimatetime" : 200,
+		"imprint" : 0,
+		"enablehscroll" : 1,
+		"enablevscroll" : 1,
+		"devicewidth" : 0.0,
+		"boxes" : [ 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 260.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-22"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 240.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-20"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 220.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-18"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 200.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-16"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "route 0 1 2 3",
+					"patching_rect" : [ 200.0, 220.0, 99.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 5,
+					"outlettype" : [ "", "", "", "", "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-14"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "r jojo",
+					"patching_rect" : [ 200.0, 180.0, 41.0, 23.0 ],
+					"numinlets" : 0,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"color" : [ 0.827451, 0.737255, 0.835294, 1.0 ],
+					"id" : "obj-13"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "s jojo",
+					"patching_rect" : [ 20.0, 340.0, 43.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 0,
+					"fontsize" : 14.0,
+					"color" : [ 0.827451, 0.737255, 0.835294, 1.0 ],
+					"id" : "obj-12"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "append toto",
+					"patching_rect" : [ 20.0, 300.0, 84.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-11"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "% 4",
+					"patching_rect" : [ 20.0, 260.0, 35.0, 23.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-10"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "counter",
+					"patching_rect" : [ 20.0, 220.0, 73.0, 23.0 ],
+					"numinlets" : 5,
+					"fontname" : "Arial",
+					"numoutlets" : 4,
+					"outlettype" : [ "int", "", "", "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-9"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "metro 250",
+					"patching_rect" : [ 20.0, 180.0, 74.0, 23.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"fontsize" : 14.0,
+					"id" : "obj-8"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "toggle",
+					"patching_rect" : [ 20.0, 140.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"id" : "obj-7"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "t 0",
+					"patching_rect" : [ 140.0, 80.0, 26.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-5"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "t 1",
+					"patching_rect" : [ 20.0, 80.0, 26.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-4"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "message",
+					"text" : "Goodbye World !",
+					"patching_rect" : [ 140.0, 40.0, 115.0, 21.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-3"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "message",
+					"text" : "Hello World !",
+					"patching_rect" : [ 20.0, 40.0, 90.0, 21.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-2"
+				}
+
+			}
+ ],
+		"lines" : [ 			{
+				"patchline" : 				{
+					"source" : [ "obj-2", 0 ],
+					"destination" : [ "obj-4", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-3", 0 ],
+					"destination" : [ "obj-5", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-4", 0 ],
+					"destination" : [ "obj-7", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-5", 0 ],
+					"destination" : [ "obj-7", 0 ],
+					"hidden" : 0,
+					"midpoints" : [ 149.5, 121.0, 29.5, 121.0 ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-7", 0 ],
+					"destination" : [ "obj-8", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-8", 0 ],
+					"destination" : [ "obj-9", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-9", 0 ],
+					"destination" : [ "obj-10", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-10", 0 ],
+					"destination" : [ "obj-11", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-11", 0 ],
+					"destination" : [ "obj-12", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-13", 0 ],
+					"destination" : [ "obj-14", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 0 ],
+					"destination" : [ "obj-16", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 1 ],
+					"destination" : [ "obj-18", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 2 ],
+					"destination" : [ "obj-20", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 3 ],
+					"destination" : [ "obj-22", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+ ]
+	}
+
+}

samples/JSON/Hello.maxpat

@@ -0,0 +1,368 @@
+{
+	"patcher" : 	{
+		"fileversion" : 1,
+		"appversion" : 		{
+			"major" : 5,
+			"minor" : 1,
+			"revision" : 9
+		}
+,
+		"rect" : [ 198.0, 92.0, 365.0, 407.0 ],
+		"bglocked" : 0,
+		"defrect" : [ 198.0, 92.0, 365.0, 407.0 ],
+		"openrect" : [ 0.0, 0.0, 0.0, 0.0 ],
+		"openinpresentation" : 0,
+		"default_fontsize" : 14.0,
+		"default_fontface" : 0,
+		"default_fontname" : "Arial",
+		"gridonopen" : 0,
+		"gridsize" : [ 20.0, 20.0 ],
+		"gridsnaponopen" : 0,
+		"toolbarvisible" : 1,
+		"boxanimatetime" : 200,
+		"imprint" : 0,
+		"enablehscroll" : 1,
+		"enablevscroll" : 1,
+		"devicewidth" : 0.0,
+		"boxes" : [ 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 260.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-22"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 240.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-20"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 220.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-18"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 200.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-16"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "route 0 1 2 3",
+					"patching_rect" : [ 200.0, 220.0, 99.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 5,
+					"outlettype" : [ "", "", "", "", "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-14"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "r jojo",
+					"patching_rect" : [ 200.0, 180.0, 41.0, 23.0 ],
+					"numinlets" : 0,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"color" : [ 0.827451, 0.737255, 0.835294, 1.0 ],
+					"id" : "obj-13"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "s jojo",
+					"patching_rect" : [ 20.0, 340.0, 43.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 0,
+					"fontsize" : 14.0,
+					"color" : [ 0.827451, 0.737255, 0.835294, 1.0 ],
+					"id" : "obj-12"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "append toto",
+					"patching_rect" : [ 20.0, 300.0, 84.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-11"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "% 4",
+					"patching_rect" : [ 20.0, 260.0, 35.0, 23.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-10"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "counter",
+					"patching_rect" : [ 20.0, 220.0, 73.0, 23.0 ],
+					"numinlets" : 5,
+					"fontname" : "Arial",
+					"numoutlets" : 4,
+					"outlettype" : [ "int", "", "", "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-9"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "metro 250",
+					"patching_rect" : [ 20.0, 180.0, 74.0, 23.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"fontsize" : 14.0,
+					"id" : "obj-8"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "toggle",
+					"patching_rect" : [ 20.0, 140.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"id" : "obj-7"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "t 0",
+					"patching_rect" : [ 140.0, 80.0, 26.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-5"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "t 1",
+					"patching_rect" : [ 20.0, 80.0, 26.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-4"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "message",
+					"text" : "Goodbye World !",
+					"patching_rect" : [ 140.0, 40.0, 115.0, 21.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"presentation_rect" : [ 137.0, 42.0, 0.0, 0.0 ],
+					"id" : "obj-3"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "message",
+					"text" : "Hello World !",
+					"patching_rect" : [ 20.0, 40.0, 90.0, 21.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-2"
+				}
+
+			}
+ ],
+		"lines" : [ 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 3 ],
+					"destination" : [ "obj-22", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 2 ],
+					"destination" : [ "obj-20", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 1 ],
+					"destination" : [ "obj-18", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 0 ],
+					"destination" : [ "obj-16", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-13", 0 ],
+					"destination" : [ "obj-14", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-11", 0 ],
+					"destination" : [ "obj-12", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-10", 0 ],
+					"destination" : [ "obj-11", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-9", 0 ],
+					"destination" : [ "obj-10", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-8", 0 ],
+					"destination" : [ "obj-9", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-7", 0 ],
+					"destination" : [ "obj-8", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-5", 0 ],
+					"destination" : [ "obj-7", 0 ],
+					"hidden" : 0,
+					"midpoints" : [ 149.5, 121.0, 29.5, 121.0 ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-4", 0 ],
+					"destination" : [ "obj-7", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-3", 0 ],
+					"destination" : [ "obj-5", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-2", 0 ],
+					"destination" : [ "obj-4", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+ ]
+	}
+
+}

samples/JSON/person.json

@@ -0,0 +1,23 @@
+{
+     "firstName": "John",
+     "lastName" : "Smith",
+     "age"      : 25,
+     "address"  :
+     {
+         "streetAddress": "21 2nd Street",
+         "city"         : "New York",
+         "state"        : "NY",
+         "postalCode"   : "10021"
+     },
+     "phoneNumber":
+     [
+         {
+           "type"  : "home",
+           "number": "212 555-1234"
+         },
+         {
+           "type"  : "fax",
+           "number": "646 555-4567"
+         }
+     ]
+ }

samples/JSON/product.json

@@ -0,0 +1,7 @@
+{
+        "id": 1,
+        "name": "Foo",
+        "price": 123,
+        "tags": ["Bar","Eek"],
+        "stock": { "warehouse":300, "retail":20 }
+}

samples/JSON/schema.json

@@ -0,0 +1,47 @@
+{
+        "name":"Product",
+        "properties":
+        {
+                "id":
+                {
+                        "type":"number",
+                        "description":"Product identifier",
+                        "required":true
+                },
+                "name":
+                {
+                        "type":"string",
+                        "description":"Name of the product",
+                        "required":true
+                },
+                "price":
+                {
+                        "type":"number",
+                        "minimum":0,
+                        "required":true
+                },
+                "tags":
+                {
+                        "type":"array",
+                        "items":
+                        {
+                                "type":"string"
+                        }
+                },
+                "stock":
+                {
+                        "type":"object",
+                        "properties":
+                        {
+                                "warehouse":
+                                {
+                                        "type":"number"
+                                },
+                                "retail":
+                                {
+                                        "type":"number"
+                                }
+                        }
+                }
+        }
+}

samples/JavaScript/js2.script!

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+var http = require('http');
+http.createServer(function (req, res) {
+  res.writeHead(200, {'Content-Type': 'text/plain'});
+  res.end('Hello World\n');
+}).listen(1337, '127.0.0.1');
+console.log('Server running at http://127.0.0.1:1337/');

samples/Max/Hello.mxt

@@ -0,0 +1 @@
+max v2;

samples/Perl/perl.script!

@@ -0,0 +1,2 @@
+#!/usr/local/bin/perl
+print "Perl\n"

samples/Ruby/ruby2.script!

@@ -1,2 +1,2 @@
 #! /usr/bin/env ruby -w -Ilib:test
-echo "Ruby"
+puts "Ruby"

samples/Shell/bash.script!

@@ -0,0 +1,2 @@
+#!/bin/bash
+echo "bash"

samples/Shell/sh.script!

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "sh"

samples/Shell/zsh.script!

@@ -0,0 +1,2 @@
+#!/bin/zsh
+echo "zsh"

samples/Text/script.foo

@@ -1,2 +0,0 @@
-#!/bin/foo
-???

test/test_blob.rb

@@ -45,10 +45,6 @@ class TestBlob < Test::Unit::TestCase
     assert_equal "application/pdf", blob("Binary/foo.pdf").content_type
     assert_equal "image/png", blob("Binary/foo.png").content_type
     assert_equal "text/plain; charset=iso-8859-2", blob("Text/README").content_type
-    assert_equal "text/plain; charset=iso-8859-1", blob("Perl/script.pl").content_type
-    assert_equal "text/plain; charset=iso-8859-1", blob("Python/script.py").content_type
-    assert_equal "text/plain; charset=iso-8859-1", blob("Ruby/script.rb").content_type
-    assert_equal "text/plain; charset=iso-8859-1", blob("Shell/script.sh").content_type
   end
 
   def test_disposition
@@ -262,7 +258,6 @@ class TestBlob < Test::Unit::TestCase
   end
 
   def test_indexable
-    assert blob("Text/file.txt").indexable?
     assert blob("Ruby/foo.rb").indexable?
     assert !blob("Text/defu.nkt").indexable?
     assert !blob("Text/dump.sql").indexable?
@@ -281,25 +276,6 @@ class TestBlob < Test::Unit::TestCase
     assert_equal Lexer['Ruby'], blob("Ruby/foo.rb").lexer
   end
 
-  def test_shebang_script
-    assert_equal 'sh', script_blob("Shell/script.sh").shebang_script
-    assert_equal 'bash', script_blob("Shell/script.bash").shebang_script
-    assert_equal 'zsh', script_blob("Shell/script.zsh").shebang_script
-    assert_equal 'perl', script_blob("Perl/script.pl").shebang_script
-    assert_equal 'ruby', script_blob("Ruby/script.rb").shebang_script
-    assert_equal 'ruby', script_blob("Ruby/script2.rb").shebang_script
-    assert_equal 'python', script_blob("Python/script.py").shebang_script
-    assert_equal 'node', script_blob("JavaScript/script.js").shebang_script
-    assert_equal 'groovy', script_blob("Groovy/script.groovy").shebang_script
-    assert_equal 'macruby', script_blob("Ruby/macruby-script").shebang_script
-    assert_equal 'rake', script_blob("Ruby/script.rake").shebang_script
-    assert_equal 'foo', script_blob("Text/script.foo").shebang_script
-    assert_equal 'nush', script_blob("Nu/script.nu").shebang_script
-    assert_equal 'scala', script_blob("Scala/script.scala").shebang_script
-    assert_equal 'racket', script_blob("Racket/script.rkt").shebang_script
-    assert_equal nil, script_blob("Ruby/foo.rb").shebang_script
-  end
-
   def test_colorize
     assert_equal <<-HTML, blob("Ruby/foo.rb").colorize
 <div class="highlight"><pre><span class="k">module</span> <span class="nn">Foo</span>

test/test_classifier.rb

@@ -54,11 +54,8 @@ class TestClassifier < Test::Unit::TestCase
 
   def test_classify_ambiguous_languages
     Samples.each do |sample|
-      language = Linguist::Language.find_by_name(sample[:language])
-      next unless language.overrides.any?
-
-      extname   = File.extname(sample[:path])
-      languages = Language.all.select { |l| l.extensions.include?(extname) }.map(&:name)
+      language  = Linguist::Language.find_by_name(sample[:language])
+      languages = Language.find_by_filename(sample[:path]).map(&:name)
       next unless languages.length > 1
 
       results = Classifier.classify(Samples::DATA, File.read(sample[:path]), languages)

test/test_language.rb

@@ -8,29 +8,6 @@ class TestLanguage < Test::Unit::TestCase
 
   Lexer = Pygments::Lexer
 
-  def test_ambiguous_extensions
-    assert Language.ambiguous?('.cls')
-    assert_equal Language['Apex'], Language.find_by_extension('cls')
-
-    assert Language.ambiguous?('.h')
-    assert_equal Language['C'], Language.find_by_extension('h')
-
-    assert Language.ambiguous?('.m')
-    assert_equal Language['Objective-C'], Language.find_by_extension('m')
-
-    assert Language.ambiguous?('.pl')
-    assert_equal Language['Perl'], Language.find_by_extension('pl')
-
-    assert Language.ambiguous?('.r')
-    assert_equal Language['R'], Language.find_by_extension('r')
-
-    assert Language.ambiguous?('.t')
-    assert_equal Language['Turing'], Language.find_by_extension('t')
-
-    assert Language.ambiguous?('.v')
-    assert_equal Language['Verilog'], Language.find_by_extension('v')
-  end
-
   def test_lexer
     assert_equal Lexer['ActionScript 3'], Language['ActionScript'].lexer
     assert_equal Lexer['Bash'], Language['Gentoo Ebuild'].lexer
@@ -71,7 +48,6 @@ class TestLanguage < Test::Unit::TestCase
     assert_equal Lexer['Scheme'], Language['Scheme'].lexer
     assert_equal Lexer['Standard ML'], Language['Standard ML'].lexer
     assert_equal Lexer['TeX'], Language['TeX'].lexer
-    assert_equal Lexer['Text only'], Language['Text'].lexer
     assert_equal Lexer['Verilog'], Language['Verilog'].lexer
     assert_equal Lexer['XSLT'], Language['XSLT'].lexer
     assert_equal Lexer['aspx-vb'], Language['ASP'].lexer
@@ -165,7 +141,7 @@ class TestLanguage < Test::Unit::TestCase
     assert_equal 'ruby',        Language['Ruby'].search_term
     assert_equal 'common-lisp', Language['Common Lisp'].search_term
     assert_equal 'html+erb',    Language['HTML+ERB'].search_term
-    assert_equal 'max/msp',     Language['Max/MSP'].search_term
+    assert_equal 'max/msp',     Language['Max'].search_term
     assert_equal 'puppet',      Language['Puppet'].search_term
     assert_equal 'pure-data',   Language['Pure Data'].search_term
 
@@ -242,48 +218,16 @@ class TestLanguage < Test::Unit::TestCase
     end
   end
 
-  def test_find_by_extension
-    assert_equal Language['Ruby'], Language.find_by_extension('.rb')
-    assert_equal Language['Ruby'], Language.find_by_extension('rb')
-    assert_equal Language['Dart'], Language.find_by_extension('dart')
-    assert_equal Language['Groff'], Language.find_by_extension('man')
-    assert_equal Language['Groff'], Language.find_by_extension('1')
-    assert_equal Language['Groff'], Language.find_by_extension('2')
-    assert_equal Language['Groff'], Language.find_by_extension('3')
-    assert_equal Language['PHP'], Language.find_by_extension('php')
-    assert_equal Language['PHP'], Language.find_by_extension('php3')
-    assert_equal Language['PHP'], Language.find_by_extension('php4')
-    assert_equal Language['PHP'], Language.find_by_extension('php5')
-    assert_equal Language['PowerShell'], Language.find_by_extension('psm1')
-    assert_equal Language['PowerShell'], Language.find_by_extension('ps1')
-
-    # Aliases for Streamline.js ( https://github.com/Sage/streamlinejs )
-    assert_equal Language['JavaScript'], Language.find_by_extension('_js')
-    assert_equal Language['CoffeeScript'], Language.find_by_extension('_coffee')
-
-    assert_nil Language.find_by_extension('.nkt')
-  end
-
-  def test_find_all_by_extension
-    Language.all.each do |language|
-      assert_equal language, Language.find_by_extension(language.primary_extension)
-
-      language.extensions.each do |extension|
-        unless Language.ambiguous?(extension)
-          assert_equal language, Language.find_by_extension(extension)
-        end
-      end
-    end
-  end
-
   def test_find_by_filename
-    assert_equal Language['Shell'], Language.find_by_filename('PKGBUILD')
-    assert_equal Language['Ruby'], Language.find_by_filename('foo.rb')
-    assert_equal Language['Ruby'], Language.find_by_filename('foo/bar.rb')
-    assert_equal Language['Ruby'], Language.find_by_filename('Rakefile')
-    assert_nil Language.find_by_filename('rb')
-    assert_nil Language.find_by_filename('.rb')
-    assert_nil Language.find_by_filename('.nkt')
+    assert_equal [Language['Shell']], Language.find_by_filename('PKGBUILD')
+    assert_equal [Language['Ruby']], Language.find_by_filename('foo.rb')
+    assert_equal [Language['Ruby']], Language.find_by_filename('foo/bar.rb')
+    assert_equal [Language['Ruby']], Language.find_by_filename('Rakefile')
+    assert_equal [Language['Ruby']], Language.find_by_filename('PKGBUILD.rb')
+    assert_equal ['C', 'C++', 'Objective-C'], Language.find_by_filename('foo.h').map(&:name).sort
+    assert_equal [], Language.find_by_filename('rb')
+    assert_equal [], Language.find_by_filename('.rb')
+    assert_equal [], Language.find_by_filename('.nkt')
   end
 
   def test_find
@@ -310,7 +254,6 @@ class TestLanguage < Test::Unit::TestCase
     assert_equal 'C%2B%2B', Language['C++'].escaped_name
     assert_equal 'Objective-C', Language['Objective-C'].escaped_name
     assert_equal 'Common%20Lisp', Language['Common Lisp'].escaped_name
-    assert_equal 'Max%2FMSP', Language['Max/MSP'].escaped_name
   end
 
   def test_error_without_name
@@ -336,11 +279,9 @@ class TestLanguage < Test::Unit::TestCase
     assert_equal 'csharp', Language['C#'].ace_mode
     assert_equal 'css', Language['CSS'].ace_mode
     assert_equal 'javascript', Language['JavaScript'].ace_mode
-    assert_equal 'text', Language['Text'].ace_mode
   end
 
   def test_ace_modes
-    assert Language.ace_modes.include?(Language['Text'])
     assert Language.ace_modes.include?(Language['Ruby'])
     assert !Language.ace_modes.include?(Language['FORTRAN'])
   end
@@ -373,12 +314,6 @@ class TestLanguage < Test::Unit::TestCase
 
 
   def test_colorize
-    assert_equal <<-HTML, Language['Text'].colorize("Hello")
-<div class="highlight"><pre>Hello
-</pre>
-</div>
-    HTML
-
     assert_equal <<-HTML, Language['Ruby'].colorize("def foo\n  'foo'\nend\n")
 <div class="highlight"><pre><span class="k">def</span> <span class="nf">foo</span>
   <span class="s1">&#39;foo&#39;</span>

test/test_tokenizer.rb

@@ -20,12 +20,17 @@ class TestTokenizer < Test::Unit::TestCase
     assert_equal %w(print), tokenize("print 'Josh'")
     assert_equal %w(print), tokenize('print "Hello \"Josh\""')
     assert_equal %w(print), tokenize("print 'Hello \\'Josh\\''")
+    assert_equal %w(print), tokenize("print \"Hello\", \"Josh\"")
+    assert_equal %w(print), tokenize("print 'Hello', 'Josh'")
+    assert_equal %w(print), tokenize("print \"Hello\", \"\", \"Josh\"")
+    assert_equal %w(print), tokenize("print 'Hello', '', 'Josh'")
   end
 
   def test_skip_number_literals
     assert_equal %w(+), tokenize('1 + 1')
     assert_equal %w(add \( \)), tokenize('add(123, 456)')
     assert_equal %w(|), tokenize('0x01 | 0x10')
+    assert_equal %w(*), tokenize('500.42 * 1.0')
   end
 
   def test_skip_comments
@@ -77,16 +82,30 @@ class TestTokenizer < Test::Unit::TestCase
   def test_objective_c_tokens
     assert_equal %w(#import <Foundation/Foundation.h> @interface Foo NSObject { } @end), tokenize(:"Objective-C/Foo.h")
     assert_equal %w(#import @implementation Foo @end), tokenize(:"Objective-C/Foo.m")
-    assert_equal %w(#import <Cocoa/Cocoa.h> int main \( int argc char *argv \) { NSLog \( @ \) ; return ; }), tokenize(:"Objective-C/hello.m")
+    assert_equal %w(#import <Cocoa/Cocoa.h> int main \( int argc char *argv [ ] \) { NSLog \( @ \) ; return ; }), tokenize(:"Objective-C/hello.m")
+  end
+
+  def test_shebang
+    assert_equal "SHEBANG#!sh", tokenize(:"Shell/sh.script!")[0]
+    assert_equal "SHEBANG#!bash", tokenize(:"Shell/bash.script!")[0]
+    assert_equal "SHEBANG#!zsh", tokenize(:"Shell/zsh.script!")[0]
+    assert_equal "SHEBANG#!perl", tokenize(:"Perl/perl.script!")[0]
+    assert_equal "SHEBANG#!python", tokenize(:"Python/python.script!")[0]
+    assert_equal "SHEBANG#!ruby", tokenize(:"Ruby/ruby.script!")[0]
+    assert_equal "SHEBANG#!ruby", tokenize(:"Ruby/ruby2.script!")[0]
+    assert_equal "SHEBANG#!node", tokenize(:"JavaScript/js.script!")[0]
   end
 
   def test_javascript_tokens
     assert_equal %w( \( function \( \) { console.log \( \) ; } \) .call \( this \) ;), tokenize(:"JavaScript/hello.js")
   end
 
+  def test_json_tokens
+    assert_equal %w( { [ ] { } } ), tokenize(:"JSON/product.json")
+  end
+
   def test_ruby_tokens
     assert_equal %w(module Foo end), tokenize(:"Ruby/foo.rb")
-    assert_equal %w(# /usr/bin/env ruby puts), tokenize(:"Ruby/script.rb")
     assert_equal %w(task default do puts end), tokenize(:"Ruby/filenames/Rakefile")
   end
 end