Linguist 2.3.2

Similar to e415a13

bin/linguist

@@ -23,7 +23,6 @@ elsif File.file?(path)
 
   puts "#{blob.name}: #{blob.loc} lines (#{blob.sloc} sloc)"
   puts "  type:      #{type}"
-  puts "  extension: #{blob.pathname.extname}"
   puts "  mime type: #{blob.mime_type}"
   puts "  language:  #{blob.language}"
 

github-linguist.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name    = 'github-linguist'
-  s.version = '2.1.1'
+  s.version = '2.3.2'
   s.summary = "GitHub Language detection"
 
   s.authors = "GitHub"
@@ -10,8 +10,9 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'charlock_holmes', '~> 0.6.6'
   s.add_dependency 'escape_utils',    '~> 0.2.3'
-  s.add_dependency 'mime-types',      '~> 1.18'
+  s.add_dependency 'mime-types',      '~> 1.19'
   s.add_dependency 'pygments.rb',     '>= 0.2.13'
+  s.add_development_dependency 'mocha'
   s.add_development_dependency 'json'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'yajl-ruby'

lib/linguist.rb

@@ -1,5 +1,5 @@
 require 'linguist/blob_helper'
+require 'linguist/generated'
 require 'linguist/language'
-require 'linguist/mime'
 require 'linguist/repository'
 require 'linguist/samples'

lib/linguist/blob_helper.rb

@@ -1,10 +1,9 @@
-require 'linguist/classifier'
+require 'linguist/generated'
 require 'linguist/language'
-require 'linguist/mime'
-require 'linguist/samples'
 
 require 'charlock_holmes'
 require 'escape_utils'
+require 'mime/types'
 require 'pygments'
 require 'yaml'
 
@@ -21,7 +20,23 @@ module Linguist
     #
     # Returns a String
     def extname
-      File.extname(name)
+      File.extname(name.to_s)
+    end
+
+    # Internal: Lookup mime type for extension.
+    #
+    # Returns a MIME::Type
+    def _mime_type
+      if defined? @_mime_type
+        @_mime_type
+      else
+        guesses = ::MIME::Types.type_for(extname.to_s)
+
+        # Prefer text mime types over binary
+        @_mime_type = guesses.detect { |type| type.ascii? } ||
+          # Otherwise use the first guess
+          guesses.first
+      end
     end
 
     # Public: Get the actual blob mime type
@@ -33,7 +48,14 @@ module Linguist
     #
     # Returns a mime type String.
     def mime_type
-      @mime_type ||= Mime.mime_for(extname)
+      _mime_type ? _mime_type.to_s : 'text/plain'
+    end
+
+    # Internal: Is the blob binary according to its mime type
+    #
+    # Return true or false
+    def binary_mime_type?
+      _mime_type ? _mime_type.binary? : false
     end
 
     # Public: Get the Content-Type header value
@@ -84,15 +106,6 @@ module Linguist
       @detect_encoding ||= CharlockHolmes::EncodingDetector.new.detect(data) if data
     end
 
-    # Public: Is the blob binary according to its mime type
-    #
-    # Return true or false
-    def binary_mime_type?
-      if mime_type = Mime.lookup_mime_type_for(extname)
-        mime_type.binary?
-      end
-    end
-
     # Public: Is the blob binary?
     #
     # Return true or false
@@ -129,15 +142,6 @@ module Linguist
       ['.png', '.jpg', '.jpeg', '.gif'].include?(extname)
     end
 
-    # Public: Is the blob likely to have a shebang?
-    #
-    # Return true or false
-    def shebang_extname?
-      extname.empty? &&
-        mode &&
-        (mode.to_i(8) & 05) == 05
-    end
-
     MEGABYTE = 1024 * 1024
 
     # Public: Is the blob too big to load?
@@ -156,7 +160,7 @@ module Linguist
     #
     # Return true or false
     def safe_to_colorize?
-      text? && !large? && !high_ratio_of_long_lines?
+      !large? && text? && !high_ratio_of_long_lines?
     end
 
     # Internal: Does the blob have a ratio of long lines?
@@ -221,143 +225,16 @@ module Linguist
       lines.grep(/\S/).size
     end
 
-    # Internal: Compute average line length.
-    #
-    # Returns Integer.
-    def average_line_length
-      if lines.any?
-        lines.inject(0) { |n, l| n += l.length } / lines.length
-      else
-        0
-      end
-    end
-
     # Public: Is the blob a generated file?
     #
     # Generated source code is supressed in diffs and is ignored by
     # language statistics.
     #
-    # Requires Blob#data
-    #
-    # Includes:
-    # - XCode project XML files
-    # - Minified JavaScript
-    # - Compiled CoffeeScript
-    # - PEG.js-generated parsers
-    #
-    # Please add additional test coverage to
-    # `test/test_blob.rb#test_generated` if you make any changes.
+    # May load Blob#data
     #
     # Return true or false
     def generated?
-      if name == 'Gemfile.lock' || minified_javascript? || compiled_coffeescript? ||
-      xcode_project_file? || generated_net_docfile? || generated_parser?
-        true
-      else
-        false
-      end
-    end
-
-    # Internal: Is the blob an XCode project file?
-    #
-    # Generated if the file extension is an XCode project
-    # file extension.
-    #
-    # Returns true of false.
-    def xcode_project_file?
-      ['.xib', '.nib', '.storyboard', '.pbxproj', '.xcworkspacedata', '.xcuserstate'].include?(extname)
-    end
-
-    # Internal: Is the blob minified JS?
-    #
-    # Consider JS minified if the average line length is
-    # greater then 100c.
-    #
-    # Returns true or false.
-    def minified_javascript?
-      return unless extname == '.js'
-      average_line_length > 100
-    end
-
-    # Internal: Is the blob of JS a parser generated by PEG.js?
-    #
-    # Requires Blob#data
-    #
-    # PEG.js-generated parsers are not meant to be consumed by humans.
-    #
-    # Return true or false
-    def generated_parser?
-      return false unless extname == '.js'
-
-      # PEG.js-generated parsers include a comment near the top  of the file
-      # that marks them as such.
-      if lines[0..4].join('') =~ /^(?:[^\/]|\/[^\*])*\/\*(?:[^\*]|\*[^\/])*Generated by PEG.js/
-        return true
-      end
-
-      false
-    end
-
-    # Internal: Is the blob of JS generated by CoffeeScript?
-    #
-    # Requires Blob#data
-    #
-    # CoffeScript is meant to output JS that would be difficult to
-    # tell if it was generated or not. Look for a number of patterns
-    # output by the CS compiler.
-    #
-    # Return true or false
-    def compiled_coffeescript?
-      return false unless extname == '.js'
-
-      # CoffeeScript generated by > 1.2 include a comment on the first line
-      if lines[0] =~ /^\/\/ Generated by /
-        return true
-      end
-
-      if lines[0] == '(function() {' &&     # First line is module closure opening
-          lines[-2] == '}).call(this);' &&  # Second to last line closes module closure
-          lines[-1] == ''                   # Last line is blank
-
-        score = 0
-
-        lines.each do |line|
-          if line =~ /var /
-            # Underscored temp vars are likely to be Coffee
-            score += 1 * line.gsub(/(_fn|_i|_len|_ref|_results)/).count
-
-            # bind and extend functions are very Coffee specific
-            score += 3 * line.gsub(/(__bind|__extends|__hasProp|__indexOf|__slice)/).count
-          end
-        end
-
-        # Require a score of 3. This is fairly arbitrary. Consider
-        # tweaking later.
-        score >= 3
-      else
-        false
-      end
-    end
-
-    # Internal: Is this a generated documentation file for a .NET assembly?
-    #
-    # Requires Blob#data
-    #
-    # .NET developers often check in the XML Intellisense file along with an
-    # assembly - however, these don't have a special extension, so we have to
-    # dig into the contents to determine if it's a docfile. Luckily, these files
-    # are extremely structured, so recognizing them is easy.
-    #
-    # Returns true or false
-    def generated_net_docfile?
-      return false unless extname.downcase == ".xml"
-      return false unless lines.count > 3
-
-      # .NET Docfiles always open with <doc> and their first tag is an
-      # <assembly> tag
-      return lines[1].include?("<doc>") &&
-        lines[2].include?("<assembly>") &&
-        lines[-2].include?("</doc>")
+      @_generated ||= Generated.generated?(name, lambda { data })
     end
 
     # Public: Should the blob be indexed for searching?
@@ -373,16 +250,18 @@ module Linguist
     #
     # Return true or false
     def indexable?
-      if binary?
+      if size > 100 * 1024
         false
+      elsif binary?
+        false
+      elsif extname == '.txt'
+        true
       elsif language.nil?
         false
       elsif !language.searchable?
         false
       elsif generated?
         false
-      elsif size > 100 * 1024
-        false
       else
         true
       end
@@ -394,30 +273,15 @@ module Linguist
     #
     # Returns a Language or nil if none is detected
     def language
-      if defined? @language
-        @language
+      return @language if defined? @language
+
+      if defined?(@data) && @data.is_a?(String)
+        data = @data
       else
-        @language = guess_language
+        data = lambda { binary_mime_type? ? "" : self.data }
       end
-    end
 
-    # Internal: Guess language
-    #
-    # Please add additional test coverage to
-    # `test/test_blob.rb#test_language` if you make any changes.
-    #
-    # Returns a Language or nil
-    def guess_language
-      return if binary_mime_type?
-
-      # Disambiguate between multiple language extensions
-      disambiguate_extension_language ||
-
-        # See if there is a Language for the extension
-        Language.find_by_filename(name) ||
-
-        # Try to detect Language from shebang line
-        shebang_language
+      @language = Language.detect(name.to_s, data, mode)
     end
 
     # Internal: Get the lexer of the blob.
@@ -427,86 +291,6 @@ module Linguist
       language ? language.lexer : Pygments::Lexer.find_by_name('Text only')
     end
 
-    # Internal: Disambiguates between multiple language extensions.
-    #
-    # Returns a Language or nil.
-    def disambiguate_extension_language
-      if Language.ambiguous?(extname)
-        possible_languages = Language.all.select { |l| l.extensions.include?(extname) }.map(&:name)
-        if possible_languages.any?
-          if result = Classifier.classify(Samples::DATA, data, possible_languages).first
-            Language[result[0]]
-          end
-        end
-      end
-    end
-
-    # Internal: Extract the script name from the shebang line
-    #
-    # Requires Blob#data
-    #
-    # Examples
-    #
-    #   '#!/usr/bin/ruby'
-    #   # => 'ruby'
-    #
-    #   '#!/usr/bin/env ruby'
-    #   # => 'ruby'
-    #
-    #   '#!/usr/bash/python2.4'
-    #   # => 'python'
-    #
-    # Please add additional test coverage to
-    # `test/test_blob.rb#test_shebang_script` if you make any changes.
-    #
-    # Returns a script name String or nil
-    def shebang_script
-      # Fail fast if blob isn't viewable?
-      return unless viewable?
-
-      if lines.any? && (match = lines[0].match(/(.+)\n?/)) && (bang = match[0]) =~ /^#!/
-        bang.sub!(/^#! /, '#!')
-        tokens = bang.split(' ')
-        pieces = tokens.first.split('/')
-        if pieces.size > 1
-          script = pieces.last
-        else
-          script = pieces.first.sub('#!', '')
-        end
-
-        script = script == 'env' ? tokens[1] : script
-
-        # python2.4 => python
-        if script =~ /((?:\d+\.?)+)/
-          script.sub! $1, ''
-        end
-
-        # Check for multiline shebang hacks that exec themselves
-        #
-        #   #!/bin/sh
-        #   exec foo "$0" "$@"
-        #
-        if script == 'sh' &&
-            lines[0...5].any? { |l| l.match(/exec (\w+).+\$0.+\$@/) }
-          script = $1
-        end
-
-        script
-      end
-    end
-
-    # Internal: Get Language for shebang script
-    #
-    # Returns the Language or nil
-    def shebang_language
-      # Skip file extensions unlikely to have shebangs
-      return unless shebang_extname?
-
-      if script = shebang_script
-        Language[script]
-      end
-    end
-
     # Public: Highlight syntax of blob
     #
     # options - A Hash of options (defaults to {})

lib/linguist/generated.rb

@@ -0,0 +1,162 @@
+module Linguist
+  class Generated
+    # Public: Is the blob a generated file?
+    #
+    # name - String filename
+    # data - String blob data. A block also maybe passed in for lazy
+    #        loading. This behavior is deprecated and you should always
+    #        pass in a String.
+    #
+    # Return true or false
+    def self.generated?(name, data)
+      new(name, data).generated?
+    end
+
+    # Internal: Initialize Generated instance
+    #
+    # name - String filename
+    # data - String blob data
+    def initialize(name, data)
+      @name = name
+      @extname = File.extname(name)
+      @_data = data
+    end
+
+    attr_reader :name, :extname
+
+    # Lazy load blob data if block was passed in.
+    #
+    # Awful, awful stuff happening here.
+    #
+    # Returns String data.
+    def data
+      @data ||= @_data.respond_to?(:call) ? @_data.call() : @_data
+    end
+
+    # Public: Get each line of data
+    #
+    # Returns an Array of lines
+    def lines
+      # TODO: data should be required to be a String, no nils
+      @lines ||= data ? data.split("\n", -1) : []
+    end
+
+    # Internal: Is the blob a generated file?
+    #
+    # Generated source code is supressed in diffs and is ignored by
+    # language statistics.
+    #
+    # Please add additional test coverage to
+    # `test/test_blob.rb#test_generated` if you make any changes.
+    #
+    # Return true or false
+    def generated?
+      name == 'Gemfile.lock' ||
+        minified_javascript? ||
+        compiled_coffeescript? ||
+        xcode_project_file? ||
+        generated_net_docfile? ||
+        generated_parser?
+    end
+
+    # Internal: Is the blob an XCode project file?
+    #
+    # Generated if the file extension is an XCode project
+    # file extension.
+    #
+    # Returns true of false.
+    def xcode_project_file?
+      ['.xib', '.nib', '.storyboard', '.pbxproj', '.xcworkspacedata', '.xcuserstate'].include?(extname)
+    end
+
+    # Internal: Is the blob minified JS?
+    #
+    # Consider JS minified if the average line length is
+    # greater then 100c.
+    #
+    # Returns true or false.
+    def minified_javascript?
+      return unless extname == '.js'
+      if lines.any?
+        (lines.inject(0) { |n, l| n += l.length } / lines.length) > 100
+      else
+        false
+      end
+    end
+
+    # Internal: Is the blob of JS generated by CoffeeScript?
+    #
+    # CoffeScript is meant to output JS that would be difficult to
+    # tell if it was generated or not. Look for a number of patterns
+    # output by the CS compiler.
+    #
+    # Return true or false
+    def compiled_coffeescript?
+      return false unless extname == '.js'
+
+      # CoffeeScript generated by > 1.2 include a comment on the first line
+      if lines[0] =~ /^\/\/ Generated by /
+        return true
+      end
+
+      if lines[0] == '(function() {' &&     # First line is module closure opening
+          lines[-2] == '}).call(this);' &&  # Second to last line closes module closure
+          lines[-1] == ''                   # Last line is blank
+
+        score = 0
+
+        lines.each do |line|
+          if line =~ /var /
+            # Underscored temp vars are likely to be Coffee
+            score += 1 * line.gsub(/(_fn|_i|_len|_ref|_results)/).count
+
+            # bind and extend functions are very Coffee specific
+            score += 3 * line.gsub(/(__bind|__extends|__hasProp|__indexOf|__slice)/).count
+          end
+        end
+
+        # Require a score of 3. This is fairly arbitrary. Consider
+        # tweaking later.
+        score >= 3
+      else
+        false
+      end
+    end
+
+    # Internal: Is this a generated documentation file for a .NET assembly?
+    #
+    # .NET developers often check in the XML Intellisense file along with an
+    # assembly - however, these don't have a special extension, so we have to
+    # dig into the contents to determine if it's a docfile. Luckily, these files
+    # are extremely structured, so recognizing them is easy.
+    #
+    # Returns true or false
+    def generated_net_docfile?
+      return false unless extname.downcase == ".xml"
+      return false unless lines.count > 3
+
+      # .NET Docfiles always open with <doc> and their first tag is an
+      # <assembly> tag
+      return lines[1].include?("<doc>") &&
+        lines[2].include?("<assembly>") &&
+        lines[-2].include?("</doc>")
+    end
+
+    # Internal: Is the blob of JS a parser generated by PEG.js?
+    #
+    # PEG.js-generated parsers are not meant to be consumed by humans.
+    #
+    # Return true or false
+    def generated_parser?
+      return false unless extname == '.js'
+
+      # PEG.js-generated parsers include a comment near the top  of the file
+      # that marks them as such.
+      if lines[0..4].join('') =~ /^(?:[^\/]|\/[^\*])*\/\*(?:[^\*]|\*[^\/])*Generated by PEG.js/
+        return true
+      end
+
+      false
+    end
+  end
+end

lib/linguist/language.rb

@@ -2,6 +2,7 @@ require 'escape_utils'
 require 'pygments'
 require 'yaml'
 
+require 'linguist/classifier'
 require 'linguist/samples'
 
 module Linguist
@@ -11,23 +12,15 @@ module Linguist
   # Languages are defined in `lib/linguist/languages.yml`.
   class Language
     @languages       = []
-    @overrides       = {}
     @index           = {}
     @name_index      = {}
     @alias_index     = {}
-    @extension_index = {}
-    @filename_index  = {}
+    @extension_index = Hash.new { |h,k| h[k] = [] }
+    @filename_index  = Hash.new { |h,k| h[k] = [] }
 
     # Valid Languages types
     TYPES = [:data, :markup, :programming]
 
-    # Internal: Test if extension maps to multiple Languages.
-    #
-    # Returns true or false.
-    def self.ambiguous?(extension)
-      @overrides.include?(extension)
-    end
-
     # Internal: Create a new Language object
     #
     # attributes - A hash of attributes
@@ -60,34 +53,45 @@ module Linguist
           raise ArgumentError, "Extension is missing a '.': #{extension.inspect}"
         end
 
-        unless ambiguous?(extension)
-          # Index the extension with a leading ".": ".rb"
-          @extension_index[extension] = language
-
-          # Index the extension without a leading ".": "rb"
-          @extension_index[extension.sub(/^\./, '')] = language
-        end
-      end
-
-      language.overrides.each do |extension|
-        if extension !~ /^\./
-          raise ArgumentError, "Extension is missing a '.': #{extension.inspect}"
-        end
-
-        if l = @overrides[extension]
-          raise ArgumentError, "#{extension} is already overridden by #{l.name}"
-        end
-
-        @overrides[extension] = language
+        @extension_index[extension] << language
       end
 
       language.filenames.each do |filename|
-        @filename_index[filename] = language
+        @filename_index[filename] << language
       end
 
       language
     end
 
+    # Public: Detects the Language of the blob.
+    #
+    # name - String filename
+    # data - String blob data. A block also maybe passed in for lazy
+    #        loading. This behavior is deprecated and you should always
+    #        pass in a String.
+    # mode - Optional String mode (defaults to nil)
+    #
+    # Returns Language or nil.
+    def self.detect(name, data, mode = nil)
+      # A bit of an elegant hack. If the file is exectable but extensionless,
+      # append a "magic" extension so it can be classified with other
+      # languages that have shebang scripts.
+      if File.extname(name).empty? && mode && (mode.to_i(8) & 05) == 05
+        name += ".script!"
+      end
+
+      possible_languages = find_by_filename(name)
+
+      if possible_languages.length > 1
+        data = data.call() if data.respond_to?(:call)
+        if result = Classifier.classify(Samples::DATA, data, possible_languages.map(&:name)).first
+          Language[result[0]]
+        end
+      else
+        possible_languages.first
+      end
+    end
+
     # Public: Get all Languages
     #
     # Returns an Array of Languages
@@ -123,33 +127,19 @@ module Linguist
       @alias_index[name]
     end
 
-    # Public: Look up Language by extension.
-    #
-    # extension - The extension String. May include leading "."
-    #
-    # Examples
-    #
-    #   Language.find_by_extension('.rb')
-    #   # => #<Language name="Ruby">
-    #
-    # Returns the Language or nil if none was found.
-    def self.find_by_extension(extension)
-      @extension_index[extension]
-    end
-
-    # Public: Look up Language by filename.
+    # Public: Look up Languages by filename.
     #
     # filename - The path String.
     #
     # Examples
     #
     #   Language.find_by_filename('foo.rb')
-    #   # => #<Language name="Ruby">
+    #   # => [#<Language name="Ruby">]
     #
-    # Returns the Language or nil if none was found.
+    # Returns all matching Languages or [] if none were found.
     def self.find_by_filename(filename)
       basename, extname = File.basename(filename), File.extname(filename)
-      @filename_index[basename] || @extension_index[extname]
+      @filename_index[basename] + @extension_index[extname]
     end
 
     # Public: Look up Language by its name or lexer.
@@ -236,7 +226,6 @@ module Linguist
 
       # Set extensions or default to [].
       @extensions = attributes[:extensions] || []
-      @overrides  = attributes[:overrides]  || []
       @filenames  = attributes[:filenames]  || []
 
       unless @primary_extension = attributes[:primary_extension]
@@ -344,11 +333,6 @@ module Linguist
     # Returns the extension String.
     attr_reader :primary_extension
 
-    # Internal: Get overridden extensions.
-    #
-    # Returns the extensions Array.
-    attr_reader :overrides
-
     # Public: Get filenames
     #
     # Examples
@@ -481,7 +465,6 @@ module Linguist
       :search_term       => options['search_term'],
       :extensions        => options['extensions'].sort,
       :primary_extension => options['primary_extension'],
-      :overrides         => options['overrides'],
       :filenames         => options['filenames'],
       :popular           => popular.include?(name)
     )

lib/linguist/languages.yml

@@ -15,7 +15,6 @@
 #                     the language. Must be unique. Used when a Language is picked
 #                     from a dropdown and we need to automatically choose an
 #                     extension.
-# overrides         - An Array of extensions that takes precedence over conflicts
 # searchable        - Boolean flag to enable searching (defaults to true)
 # search_term       - Deprecated: Some languages maybe indexed under a
 #                     different alias. Avoid defining new exceptions.
@@ -67,13 +66,12 @@ Apex:
   type: programming
   lexer: Text only
   primary_extension: .cls
-  overrides:
-  - .cls
 
 AppleScript:
+  type: programming
   aliases:
   - osascript
-  primary_extension: .scpt
+  primary_extension: .applescript
 
 Arc:
   type: programming
@@ -157,8 +155,6 @@ Bro:
 C:
   type: programming
   color: "#555"
-  overrides:
-  - .h
   primary_extension: .c
   extensions:
   - .w
@@ -533,8 +529,6 @@ Groovy:
 Groovy Server Pages:
   group: Groovy
   lexer: Java Server Page
-  overrides:
-  - .gsp
   aliases:
   - gsp
   primary_extension: .gsp
@@ -604,8 +598,6 @@ INI:
   - .prefs
   - .properties
   primary_extension: .ini
-  filenames:
-  - .gitconfig
 
 IRC log:
   lexer: IRC logs
@@ -634,8 +626,6 @@ JSON:
   ace_mode: json
   searchable: false
   primary_extension: .json
-  extensions:
-  - .json
 
 Java:
   type: programming
@@ -757,13 +747,15 @@ Matlab:
   extensions:
   - .matlab
 
-Max/MSP:
+Max:
   type: programming
   color: "#ce279c"
   lexer: Text only
+  aliases:
+  - max/msp
+  - maxmsp
+  search_term: max/msp
   primary_extension: .mxt
-  extensions:
-  - .mxt
 
 MiniD: # Legacy
   searchable: false
@@ -841,8 +833,6 @@ ObjDump:
 Objective-C:
   type: programming
   color: "#438eff"
-  overrides:
-  - .m
   primary_extension: .m
   extensions:
   - .mm
@@ -915,8 +905,6 @@ Perl:
   ace_mode: perl
   color: "#0298c3"
   primary_extension: .pl
-  overrides:
-  - .pl
   extensions:
   - .PL
   - .perl
@@ -983,8 +971,6 @@ R:
   type: programming
   color: "#198ce7"
   lexer: S
-  overrides:
-  - .r
   primary_extension: .r
   extensions:
   - .r
@@ -1129,12 +1115,7 @@ Shell:
   - zsh
   primary_extension: .sh
   filenames:
-  - .bash_profile
-  - .bashrc
-  - .profile
-  - .zlogin
   - .zsh
-  - .zshrc
   - bashrc
   - zshrc
 
@@ -1195,14 +1176,6 @@ Tea:
   type: markup
   primary_extension: .tea
 
-Text:
-  type: data
-  lexer: Text only
-  ace_mode: text
-  primary_extension: .txt
-  extensions:
-  - .txt
-
 Textile:
   type: markup
   lexer: Text only
@@ -1216,8 +1189,6 @@ Turing:
   color: "#45f715"
   lexer: Text only
   primary_extension: .t
-  overrides:
-  - .t
   extensions:
   - .tu
 
@@ -1249,8 +1220,6 @@ Verilog:
   type: programming
   lexer: verilog
   color: "#848bf3"
-  overrides:
-  - .v
   primary_extension: .v
 
 VimL:
@@ -1263,8 +1232,6 @@ VimL:
   extensions:
   - .vim
   filenames:
-  - .gvimrc
-  - .vimrc
   - vimrc
   - gvimrc
 
@@ -1332,8 +1299,6 @@ YAML:
   extensions:
   - .yaml
   - .yml
-  filenames:
-  - .gemrc
 
 eC:
   type: programming

lib/linguist/mime.rb

@@ -1,91 +0,0 @@
-require 'mime/types'
-require 'yaml'
-
-class MIME::Type
-  attr_accessor :override
-end
-
-# Register additional mime type extensions
-#
-# Follows same format as mime-types data file
-#   https://github.com/halostatue/mime-types/blob/master/lib/mime/types.rb.data
-File.read(File.expand_path("../mimes.yml", __FILE__)).lines.each do |line|
-  # Regexp was cargo culted from mime-types lib
-  next unless line =~ %r{^
-    #{MIME::Type::MEDIA_TYPE_RE}
-    (?:\s@([^\s]+))?
-    (?:\s:(#{MIME::Type::ENCODING_RE}))?
-  }x
-
-  mediatype  = $1
-  subtype    = $2
-  extensions = $3
-  encoding   = $4
-
-  # Lookup existing mime type
-  mime_type = MIME::Types["#{mediatype}/#{subtype}"].first ||
-    # Or create a new instance
-    MIME::Type.new("#{mediatype}/#{subtype}")
-
-  if extensions
-    extensions.split(/,/).each do |extension|
-      mime_type.extensions << extension
-    end
-  end
-
-  if encoding
-    mime_type.encoding = encoding
-  end
-
-  mime_type.override = true
-
-  # Kind of hacky, but we need to reindex the mime type after making changes
-  MIME::Types.add_type_variant(mime_type)
-  MIME::Types.index_extensions(mime_type)
-end
-
-module Linguist
-  module Mime
-    # Internal: Look up mime type for extension.
-    #
-    # ext - The extension String. May include leading "."
-    #
-    # Examples
-    #
-    #   Mime.mime_for('.html')
-    #   # => 'text/html'
-    #
-    #   Mime.mime_for('txt')
-    #   # => 'text/plain'
-    #
-    # Return mime type String otherwise falls back to 'text/plain'.
-    def self.mime_for(ext)
-      mime_type = lookup_mime_type_for(ext)
-      mime_type ? mime_type.to_s : 'text/plain'
-    end
-
-    # Internal: Lookup mime type for extension or mime type
-    #
-    # ext_or_mime_type - A file extension ".txt" or mime type "text/plain".
-    #
-    # Returns a MIME::Type
-    def self.lookup_mime_type_for(ext_or_mime_type)
-      ext_or_mime_type ||= ''
-
-      if ext_or_mime_type =~ /\w+\/\w+/
-        guesses = ::MIME::Types[ext_or_mime_type]
-      else
-        guesses = ::MIME::Types.type_for(ext_or_mime_type)
-      end
-
-      # Use custom override first
-      guesses.detect { |type| type.override } ||
-
-        # Prefer text mime types over binary
-        guesses.detect { |type| type.ascii? } ||
-
-        # Otherwise use the first guess
-        guesses.first
-    end
-  end
-end

lib/linguist/mimes.yml

@@ -1,62 +0,0 @@
-# Additional types to add to MIME::Types
-#
-# MIME types are used to set the Content-Type of raw binary blobs. All text
-# blobs are served as text/plain regardless of their type to ensure they
-# open in the browser rather than downloading.
-#
-# The encoding helps determine whether a file should be treated as plain
-# text or binary. By default, a mime type's encoding is base64 (binary).
-# These types will show a "View Raw" link. To force a type to render as
-# plain text, set it to 8bit for UTF-8. text/* types will be treated as
-# text by default.
-#
-#   <type> @<extensions> :<encoding>
-#
-# type       - mediatype/subtype
-# extensions - comma seperated extension list
-# encoding   - base64 (binary), 7bit (ASCII), 8bit (UTF-8), or
-#              quoted-printable (Printable ASCII).
-#
-# Follows same format as mime-types data file
-#   https://github.com/halostatue/mime-types/blob/master/lib/mime/types.rb.data
-#
-# Any additions or modifications (even trivial) should have corresponding
-# test change in `test/test_mime.rb`.
-
-# TODO: Lookup actual types
-application/octet-stream @a,blend,gem,graffle,ipa,lib,mcz,nib,o,ogv,otf,pfx,pigx,plgx,psd,sib,spl,sqlite3,swc,ucode,xpi
-
-# Please keep this list alphabetized
-application/java-archive @ear,war
-application/netcdf :8bit
-application/ogg @ogg
-application/postscript :base64
-application/vnd.adobe.air-application-installer-package+zip @air
-application/vnd.mozilla.xul+xml :8bit
-application/vnd.oasis.opendocument.presentation @odp
-application/vnd.oasis.opendocument.spreadsheet @ods
-application/vnd.oasis.opendocument.text @odt
-application/vnd.openofficeorg.extension @oxt
-application/vnd.openxmlformats-officedocument.presentationml.presentation @pptx
-application/x-chrome-extension @crx
-application/x-iwork-keynote-sffkey @key
-application/x-iwork-numbers-sffnumbers @numbers
-application/x-iwork-pages-sffpages @pages
-application/x-ms-xbap @xbap :8bit
-application/x-parrot-bytecode @pbc
-application/x-shockwave-flash @swf
-application/x-silverlight-app @xap
-application/x-supercollider @sc :8bit
-application/x-troff-ms :8bit
-application/x-wais-source :8bit
-application/xaml+xml @xaml :8bit
-application/xslt+xml @xslt :8bit
-image/x-icns @icns
-text/cache-manifest @manifest
-text/plain @cu,cxx
-text/x-logtalk @lgt
-text/x-nemerle @n
-text/x-nimrod @nim
-text/x-ocaml @ml,mli,mll,mly,sig,sml
-text/x-rust @rs,rc
-text/x-scheme @rkt,scm,sls,sps,ss

lib/linguist/samples.rb

@@ -45,6 +45,10 @@ module Linguist
               })
             end
           else
+            if File.extname(filename) == ""
+              raise "#{File.join(dirname, filename)} is missing an extension, maybe it belongs in filenames/ subdir"
+            end
+
             yield({
               :path     => File.join(dirname, filename),
               :language => category,
@@ -68,18 +72,16 @@ module Linguist
       each do |sample|
         language_name = sample[:language]
 
-        # TODO: For now skip empty extnames
-        if sample[:extname] && sample[:extname] != ""
+        if sample[:extname]
           db['extnames'][language_name] ||= []
           if !db['extnames'][language_name].include?(sample[:extname])
             db['extnames'][language_name] << sample[:extname]
           end
         end
 
-        # TODO: For now skip empty extnames
-        if fn = sample[:filename]
+        if sample[:filename]
           db['filenames'][language_name] ||= []
-          db['filenames'][language_name] << fn
+          db['filenames'][language_name] << sample[:filename]
         end
 
         data = File.read(sample[:path])

lib/linguist/tokenizer.rb

@@ -1,3 +1,5 @@
+require 'strscan'
+
 module Linguist
   # Generic programming language tokenizer.
   #
@@ -14,12 +16,18 @@ module Linguist
       new.extract_tokens(data)
     end
 
+    # Read up to 100KB
+    BYTE_LIMIT = 100_000
+
+    # Start state on token, ignore anything till the next newline
     SINGLE_LINE_COMMENTS = [
       '//', # C
       '#',  # Ruby
       '%',  # Tex
     ]
 
+    # Start state on opening token, ignore anything until the closing
+    # token is reached.
     MULTI_LINE_COMMENTS = [
       ['/*', '*/'],    # C
       ['<!--', '-->'], # XML
@@ -28,7 +36,7 @@ module Linguist
     ]
 
     START_SINGLE_LINE_COMMENT =  Regexp.compile(SINGLE_LINE_COMMENTS.map { |c|
-      "^\s*#{Regexp.escape(c)} "
+      "\s*#{Regexp.escape(c)} "
     }.join("|"))
 
     START_MULTI_LINE_COMMENT =  Regexp.compile(MULTI_LINE_COMMENTS.map { |c|
@@ -50,33 +58,48 @@ module Linguist
 
       tokens = []
       until s.eos?
+        break if s.pos >= BYTE_LIMIT
+
+        if token = s.scan(/^#!.+$/)
+          if name = extract_shebang(token)
+            tokens << "SHEBANG#!#{name}"
+          end
+
         # Single line comment
-        if token = s.scan(START_SINGLE_LINE_COMMENT)
-          tokens << token.strip
+        elsif s.beginning_of_line? && token = s.scan(START_SINGLE_LINE_COMMENT)
+          # tokens << token.strip
           s.skip_until(/\n|\Z/)
 
         # Multiline comments
         elsif token = s.scan(START_MULTI_LINE_COMMENT)
-          tokens << token
+          # tokens << token
           close_token = MULTI_LINE_COMMENTS.assoc(token)[1]
           s.skip_until(Regexp.compile(Regexp.escape(close_token)))
-          tokens << close_token
+          # tokens << close_token
 
         # Skip single or double quoted strings
         elsif s.scan(/"/)
-          s.skip_until(/[^\\]"/)
+          if s.peek(1) == "\""
+            s.getch
+          else
+            s.skip_until(/[^\\]"/)
+          end
         elsif s.scan(/'/)
-          s.skip_until(/[^\\]'/)
+          if s.peek(1) == "'"
+            s.getch
+          else
+            s.skip_until(/[^\\]'/)
+          end
 
         # Skip number literals
-        elsif s.scan(/(0x)?\d+/)
+        elsif s.scan(/(0x)?\d(\d|\.)*/)
 
         # SGML style brackets
         elsif token = s.scan(/<[^\s<>][^<>]*>/)
           extract_sgml_tokens(token).each { |t| tokens << t }
 
         # Common programming punctuation
-        elsif token = s.scan(/;|\{|\}|\(|\)/)
+        elsif token = s.scan(/;|\{|\}|\(|\)|\[|\]/)
           tokens << token
 
         # Regular token
@@ -95,6 +118,33 @@ module Linguist
       tokens
     end
 
+    # Internal: Extract normalized shebang command token.
+    #
+    # Examples
+    #
+    #   extract_shebang("#!/usr/bin/ruby")
+    #   # => "ruby"
+    #
+    #   extract_shebang("#!/usr/bin/env node")
+    #   # => "node"
+    #
+    # Returns String token or nil it couldn't be parsed.
+    def extract_shebang(data)
+      s = StringScanner.new(data)
+
+      if path = s.scan(/^#!\s*\S+/)
+        script = path.split('/').last
+        if script == 'env'
+          s.scan(/\s+/)
+          script = s.scan(/\S+/)
+        end
+        script = script[/[^\d]+/, 0]
+        return script
+      end
+
+      nil
+    end
+
     # Internal: Extract tokens from inside SGML tag.
     #
     # data - SGML tag String.

samples/AppleScript/Convert To PDF.applescript

@@ -0,0 +1,87 @@
+(*
+Copyright 2003 Apple Computer, Inc.
+
+You may incorporate this Apple sample code into your program(s) without
+restriction.  This Apple sample code has been provided "AS IS" and the
+responsibility for its operation is yours.  You are not permitted to
+redistribute this Apple sample code as "Apple sample code" after having
+made changes.  If you're going to redistribute the code, we require
+that you make it clear that the code was descended from Apple sample
+code, but that you've made changes.
+*)
+
+property type_list : {"JPEG", "GIFf", "PICT", "TIFF", "PDF", "TEXT"}
+property extension_list : {"jpg", "gif", "pct", "tif", "pdf", "rtf"}
+--html is not currently handled
+
+on run {}
+	tell application "Finder" to set FinderSelection to the selection as alias list
+	
+	set FS to FinderSelection
+	--Ideally, this list could be passed to the open handler
+	
+	set SelectionCount to number of FS -- count	
+	if SelectionCount is 0 then
+		set FS to userPicksFolder()
+	else if the SelectionCount is 1 then
+		set MyPath to path to me
+		if MyPath is item 1 of FS then
+			--If I'm a droplet then I was double-clicked
+			set FS to userPicksFolder()
+		end if
+	else
+		--I'm not a double-clicked droplet
+	end if
+	open FS
+end run
+
+on userPicksFolder()
+	set these_items to {}
+	set these_items to (choose file with prompt "Select a file to convert to PDF:" of type {"JPEG", "GIFf", "PICT", "TIFF", "TEXT", "RTF"}) as list
+end userPicksFolder
+
+on open these_items
+	set thesefiles to {}
+	set the item_info to {}
+	repeat with i from 1 to the count of these_items
+		set this_item to (item i of these_items)
+		set the item_info to info for this_item
+		
+		if folder of the item_info is true then --if the item is a folder
+			processFolder(this_item)
+		else if ((folder of the item_info is false) and (alias of the item_info is false)) and (the file type of the item_info is in the type_list) or ((the name extension of the item_info) is in the extension_list) then
+			
+			set theFilePath to (item i of these_items as string)
+			set thePOSIXFilePath to POSIX path of theFilePath as string
+			processFile(thePOSIXFilePath)
+		end if
+	end repeat
+end open
+
+--process folders 
+on processFolder(theFolder)
+	set these_items to list folder theFolder without invisibles
+	repeat with i from 1 to the count of these_items
+		set this_item to alias ((theFolder as text) & (item i of these_items))
+		set the item_info to info for this_item
+		if folder of the item_info is true then
+			processFolder(this_item)
+		else if (alias of the item_info is false) and ((the file type of the item_info is in the type_list) or the name extension of the item_info is in the extension_list) then
+			set theFilePath to (this_item as string)
+			set thePOSIXFilePath to POSIX path of theFilePath as string
+			processFile(thePOSIXFilePath)
+		end if
+	end repeat
+end processFolder
+
+on processFile(thePOSIXFileName)
+	try
+		set terminalCommand to ""
+		set convertCommand to "/System/Library/Printers/Libraries/./convert "
+		set newFileName to thePOSIXFileName & ".pdf"
+		set terminalCommand to convertCommand & "-f " & "\"" & thePOSIXFileName & "\"" & " -o " & "\"" & newFileName & "\"" & " -j \"application/pdf\""
+		
+		do shell script terminalCommand
+	end try
+end processFile
+

samples/AppleScript/Convert To PostScript.applescript

@@ -0,0 +1,91 @@
+(*
+
+Copyright 2003 Apple Computer, Inc.
+
+You may incorporate this Apple sample code into your program(s) without
+restriction.  This Apple sample code has been provided "AS IS" and the
+responsibility for its operation is yours.  You are not permitted to
+redistribute this Apple sample code as "Apple sample code" after having
+made changes.  If you're going to redistribute the code, we require
+that you make it clear that the code was descended from Apple sample
+code, but that you've made changes.
+*)
+
+property type_list : {"JPEG", "GIFf", "PICT", "TIFF", "PDF", "TEXT"}
+property extension_list : {"jpg", "gif", "pct", "tif", "pdf", "rtf"}
+--html is not currently handled
+
+on run {}
+	tell application "Finder" to set FinderSelection to the selection as alias list
+	
+	set FS to FinderSelection
+	--Ideally, this list could be passed to the open handler
+	
+	set SelectionCount to number of FS -- count	
+	if SelectionCount is 0 then
+		set FS to userPicksFolder()
+	else if the SelectionCount is 1 then
+		set MyPath to path to me
+		if MyPath is item 1 of FS then
+			--If I'm a droplet then I was double-clicked
+			set FS to userPicksFolder()
+		end if
+	else
+		--I'm not a double-clicked droplet
+	end if
+	open FS
+end run
+
+on userPicksFolder()
+	set these_items to {}
+	set these_items to (choose file with prompt "Select a file to convert to PostScript:" of type {"JPEG", "GIFf", "PICT", "TIFF", "TEXT", "RTF"}) as list
+end userPicksFolder
+
+
+on open these_items
+	set thesefiles to {}
+	set the item_info to {}
+	repeat with i from 1 to the count of these_items
+		set this_item to (item i of these_items)
+		set the item_info to info for this_item
+		
+		if folder of the item_info is true then --if the item is a folder
+			processFolder(this_item)
+		else if ((folder of the item_info is false) and (alias of the item_info is false)) and (the file type of the item_info is in the type_list) or ((the name extension of the item_info) is in the extension_list) then
+			
+			set theFilePath to (item i of these_items as string)
+			set thePOSIXFilePath to POSIX path of theFilePath as string
+			processFile(thePOSIXFilePath)
+		end if
+	end repeat
+end open
+
+--process folders 
+on processFolder(theFolder)
+	set these_items to list folder theFolder without invisibles
+	repeat with i from 1 to the count of these_items
+		set this_item to alias ((theFolder as text) & (item i of these_items))
+		set the item_info to info for this_item
+		if folder of the item_info is true then
+			processFolder(this_item)
+		else if (alias of the item_info is false) and ((the file type of the item_info is in the type_list) or the name extension of the item_info is in the extension_list) then
+			set theFilePath to (this_item as string)
+			set thePOSIXFilePath to POSIX path of theFilePath as string
+			processFile(thePOSIXFilePath)
+		end if
+	end repeat
+end processFolder
+
+--need to pass the URL to Terminal
+on processFile(thePOSIXFileName)
+	try
+		set terminalCommand to ""
+		set convertCommand to "/System/Library/Printers/Libraries/./convert "
+		set newFileName to thePOSIXFileName & ".ps"
+		set terminalCommand to convertCommand & "-f " & "\"" & thePOSIXFileName & "\"" & " -o " & "\"" & newFileName & "\"" & " -j \"application/postscript\""
+		
+		do shell script terminalCommand
+	end try
+end processFile
+
+

samples/AppleScript/Count Messages in All Mailboxes.applescript

@@ -0,0 +1,80 @@
+(*
+Count Messages in All Mailboxes
+
+Copyright 2002-2012 Apple Inc. All rights reserved.
+
+You may incorporate this Apple sample code into your program(s) without
+restriction.  This Apple sample code has been provided "AS IS" and the
+responsibility for its operation is yours.  You are not permitted to
+redistribute this Apple sample code as "Apple sample code" after having
+made changes.  If you're going to redistribute the code, we require
+that you make it clear that the code was descended from Apple sample
+code, but that you've made changes.
+*)
+
+(*
+This script goes through each mailbox, gets the total message count and
+the unread count, then displays the final output in a new email message.
+*)
+
+tell application "Mail"
+	set localMailboxes to every mailbox
+	if (count of localMailboxes) is greater than 0 then
+		set messageCountDisplay to "Local mailboxes (On My Mac)" & return & my getMessageCountsForMailboxes(localMailboxes)
+	else
+		set messageCountDisplay to ""
+	end if
+	
+	set everyAccount to every account
+	repeat with eachAccount in everyAccount
+		set accountMailboxes to every mailbox of eachAccount
+		if (count of accountMailboxes) is greater than 0 then
+			set messageCountDisplay to messageCountDisplay & return & "Mailboxes for Account: " & name of eachAccount & return & my getMessageCountsForMailboxes(accountMailboxes)
+		end if
+	end repeat
+	
+	set outputMessage to make new outgoing message with properties {content:messageCountDisplay, subject:"Message counts for all my mailboxes", visible:true}
+	tell outputMessage
+		set font to "Courier"
+		set size to 12
+	end tell
+end tell
+
+on getMessageCountsForMailboxes(theMailboxes)
+	-- (list of mailboxes)
+	-- returns string
+	
+	set displayString to ""
+	
+	tell application "Mail"
+		repeat with eachMailbox in theMailboxes
+			set mailboxName to name of eachMailbox
+			set messageCount to (count of (messages of eachMailbox)) as string
+			set unreadCount to unread count of eachMailbox as string
+			
+			set displayString to displayString & "    " & my padString(mailboxName, 40) & "  " & messageCount & " (" & unreadCount & " unread)" & return
+		end repeat
+	end tell
+	
+	return displayString
+end getMessageCountsForMailboxes
+
+on padString(theString, fieldLength)
+	-- (string, integer)
+	-- returns string
+	
+	set stringLength to length of theString
+	
+	if stringLength is greater than fieldLength then
+		set paddedString to (text from character 1 to character (fieldLength - 3) of theString) & "..."
+	else -- stringLength is less than or equal to fieldLength
+		set paddedString to theString
+		
+		set paddingLength to fieldLength - stringLength
+		repeat paddingLength times
+			set paddedString to paddedString & space
+		end repeat
+	end if
+	
+	return paddedString
+end padString

samples/AppleScript/Crazy Message Text.applescript

@@ -0,0 +1,68 @@
+(*
+Crazy Message Text
+
+Copyright 2002-2012 Apple Inc. All rights reserved.
+
+You may incorporate this Apple sample code into your program(s) without
+restriction.  This Apple sample code has been provided "AS IS" and the
+responsibility for its operation is yours.  You are not permitted to
+redistribute this Apple sample code as "Apple sample code" after having
+made changes.  If you're going to redistribute the code, we require
+that you make it clear that the code was descended from Apple sample
+code, but that you've made changes.
+*)
+
+(*
+This script takes a string from the user and then makes a new message
+where each letter has a different font, size, and color.
+*)
+
+property lowFontSize : 36
+property highFontSize : 72
+property messageText : "Happy Birthday!"
+
+repeat
+	set userInput to display dialog "Enter some message text:" & return & return & "Minimum Character Size: " & (lowFontSize as string) & return & "Maximum Character Size: " & (highFontSize as string) default answer messageText buttons {"Cancel", "Set Prefs", "Continue"} default button 3
+	
+	if the button returned of userInput is "Set Prefs" then
+		set minimumFontSize to 9
+		
+		display dialog "Enter the minimum font size to use:" & return & return & "(Must be at least " & (minimumFontSize as string) & ")" default answer lowFontSize buttons {"OK"}
+		set newFontSize to text returned of the result as integer
+		if newFontSize is greater than or equal to minimumFontSize then
+			set lowFontSize to newFontSize
+		else
+			set lowFontSize to minimumFontSize
+		end if
+		
+		display dialog "Enter the maximum font size to use:" & return & return & "(Must be greater than " & (lowFontSize as string) & ")" default answer highFontSize buttons {"OK"}
+		set newFontSize to text returned of the result as integer
+		if newFontSize is greater than lowFontSize then
+			set highFontSize to newFontSize
+		else
+			set highFontSize to lowFontSize
+		end if
+		
+	else -- button returned of userInput is "Continue"
+		set theText to text returned of userInput
+		if theText is not "" then
+			set messageText to theText
+		end if
+		exit repeat
+	end if
+end repeat
+
+set fontList to {"American Typewriter", "American Typewriter Light", "American Typewriter Bold", "American Typewriter Condensed", "American Typewriter Condensed Light", "American Typewriter Condensed Bold", "Arial", "Arial Italic", "Arial Bold", "Arial Bold Italic", "Arial Black", "Baskerville", "Baskerville Italic", "Baskerville SemiBold", "Baskerville Bold", "Baskerville SemiBold Italic", "Baskerville Bold Italic", "Big Caslon Medium", "Comic Sans MS", "Comic Sans MS Bold", "Copperplate", "Copperplate Light", "Copperplate Bold", "Didot", "Didot Italic", "Didot Bold", "Futura Medium", "Futura Medium Italic", "Futura Condensed Medium", "Futura Condensed ExtraBold", "Geneva", "Gill Sans", "Gill Sans Italic", "Gill Sans Light", "Gill Sans Light Italic", "Gill Sans Bold", "Gill Sans Bold Italic", "Herculanum", "Lucida Grande", "Lucida Grande Bold", "Marker Felt Thin", "Marker Felt Wide", "Optima Regular", "Optima Italic", "Optima Bold", "Optima Bold Italic", "Optima ExtraBlack", "Papyrus", "Verdana", "Verdana Italic", "Verdana Bold", "Verdana Bold Italic", "Zapfino"}
+
+tell application "Mail"
+	activate
+	set crazyTextMessage to make new outgoing message with properties {content:messageText, visible:true}
+	
+	tell crazyTextMessage
+		repeat with eachCharacter in characters
+			set font of eachCharacter to (some item of fontList)
+			set size of eachCharacter to (random number from lowFontSize to highFontSize)
+			set color of eachCharacter to {random number from 0 to 65535, random number from 0 to 65535, random number from 0 to 65535}
+		end repeat
+	end tell
+end tell

samples/AppleScript/Get User Name.applescript

@@ -0,0 +1,41 @@
+(*
+Get User Name
+
+This script uses UI element scripting to get the name for the
+current user.
+
+If "Enable access for assistive devices" is not checked,
+this script will open the Universal Access System Preference and ask
+the user to check the checkbox.
+
+Copyright 2007 Apple Inc.
+
+You may incorporate this Apple sample code into your program(s) without
+restriction.  This Apple sample code has been provided "AS IS" and the
+responsibility for its operation is yours.  You are not permitted to
+redistribute this Apple sample code as "Apple sample code" after having
+made changes.  If you're going to redistribute the code, we require
+that you make it clear that the code was descended from Apple sample
+code, but that you've made changes.
+*)
+
+tell application "System Preferences"
+	activate
+	set current pane to pane "com.apple.preferences.users"
+end tell
+
+tell application "System Events"
+	if UI elements enabled then
+		tell tab group 1 of window "Accounts" of process "System Preferences"
+			click radio button 1
+			delay 2
+			get value of text field 1
+		end tell
+	else
+		tell application "System Preferences"
+			activate
+			set current pane to pane "com.apple.preference.universalaccess"
+			display dialog "UI element scripting is not enabled. Check \"Enable access for assistive devices\""
+		end tell
+	end if
+end tell

samples/AppleScript/Time Of Day.applescript

@@ -0,0 +1,75 @@
+(* 
+ Speaks the  date and time of day
+ 
+ Copyright 2008 Apple Inc. All rights reserved.
+ 
+ You may incorporate this Apple sample code into your program(s) without
+ restriction.  This Apple sample code has been provided "AS IS" and the
+ responsibility for its operation is yours.  You are not permitted to
+ redistribute this Apple sample code as "Apple sample code" after having
+ made changes.  If you're going to redistribute the code, we require
+ that you make it clear that the code was descended from Apple sample
+ code, but that you've made changes.
+ *)
+
+on isVoiceOverRunning()
+	set isRunning to false
+	tell application "System Events"
+		set isRunning to (name of processes) contains "VoiceOver"
+	end tell
+	return isRunning
+end isVoiceOverRunning
+
+on isVoiceOverRunningWithAppleScript()
+	if isVoiceOverRunning() then
+		set isRunningWithAppleScript to true
+		
+		-- is AppleScript enabled on VoiceOver --
+		tell application "VoiceOver"
+			try
+				set x to bounds of vo cursor
+			on error
+				set isRunningWithAppleScript to false
+			end try
+		end tell
+		return isRunningWithAppleScript
+	end if
+	return false
+end isVoiceOverRunningWithAppleScript
+
+set currentDate to current date
+set amPM to "AM"
+set currentHour to (currentDate's hours)
+set currentMinutes to currentDate's minutes
+
+if (currentHour > 12 and currentHour < 24) then
+	set amPM to "PM"
+else
+	set amPM to "AM"
+end if
+
+--  make minutes below 10 sound nice
+if currentMinutes < 10 then
+	set currentMinutes to ("0" & currentMinutes) as text
+end if
+
+--  ensure 0:nn gets set to 12:nn AM
+if currentHour is equal to 0 then
+	set currentHour to 12
+end if
+
+--  readjust for 12 hour time
+if (currentHour > 12) then
+	set currentHour to (currentHour - 12)
+end if
+
+set currentTime to ((currentDate's month) as text) & " " & ((currentDate's day) as text) & ", " & (currentHour as text) & ":" & ((currentMinutes) as text) & " " & amPM as text
+
+if isVoiceOverRunningWithAppleScript() then
+	tell application "VoiceOver"
+		output currentTime
+	end tell
+else
+	say currentTime
+	delay 2
+end if

samples/AppleScript/center.scpt

@@ -1,50 +0,0 @@
-set windowWidth to 800
-set windowHeight to 600
-delay 0.1
-
-set AppleScript's text item delimiters to "x"
-
-set res to text returned of (display dialog "Enter the width x height:" default answer ((windowWidth & windowHeight) as text))
-
-if res is "" then
-  display dialog "You need to enter a correct response"
-  return
-end if
-set {windowWidth, windowHeight} to text items of res
-
-set AppleScript's text item delimiters to ""
-
-tell application "Safari"
-  set screen_width to (do JavaScript "screen.availWidth" in document 1)
-  set screen_height to (do JavaScript "screen.availHeight" in document 1)
-end tell
-
-tell application "System Events"
-  set myFrontMost to name of first item of (processes whose frontmost is true)
-end tell
-
-tell application "Finder"
-  set {desktopTop, desktopLeft, desktopRight, desktopBottom} to bounds of desktop
-end tell
-
-try
-  tell application "System Events"
-    tell process myFrontMost
-      set {{w, h}} to size of drawer of window 1
-    end tell
-  end tell
-on error
-  set {w, h} to {0, 0}
-end try
-
-tell application "System Events"
-  tell process myFrontMost
-    try
-      set {{w, h}} to size of drawer of window 1
-    on error
-      set {w, h} to {0, 0}
-    end try
-    set position of window 1 to {((screen_width - windowWidth) / 2), ((screen_height - windowHeight) / 2.0) - desktopTop}
-    set size of window 1 to {windowWidth -w, windowHeight}
-  end tell
-end tell

samples/C++/gdsdbreader.h

@@ -0,0 +1,69 @@
+#ifndef GDSDBREADER_H
+#define GDSDBREADER_H
+
+// This file contains core structures, classes and types for the entire gds app
+// WARNING: DO NOT MODIFY UNTIL IT'S STRICTLY NECESSARY
+
+#include <QDir>
+#include "diagramwidget/qgldiagramwidget.h"
+
+#define GDS_DIR "gdsdata"
+
+enum level {LEVEL_ONE, LEVEL_TWO, LEVEL_THREE};
+
+// The internal structure of the db to store information about each node (each level)
+// this will be serialized before being written to file
+class dbDataStructure
+{
+public:
+    QString label;
+    quint32 depth;
+    quint32 userIndex;
+    QByteArray data;    // This is COMPRESSED data, optimize ram and disk space, is decompressed
+                        // just when needed (to display the comments)
+
+    // The following ID is used to create second-third level files
+    quint64 uniqueID;
+    // All the next items linked to this one
+    QVector<dbDataStructure*> nextItems;
+    // Corresponding indices vector (used to store data)
+    QVector<quint32> nextItemsIndices;
+    // The father element (or NULL if it's root)
+    dbDataStructure* father;
+    // Corresponding indices vector (used to store data)
+    quint32 fatherIndex;
+    bool noFatherRoot; // Used to tell if this node is the root (so hasn't a father)
+
+    // These fields will be useful for levels 2 and 3
+    QString fileName; // Relative filename for the associated code file
+    QByteArray firstLineData; // Compressed first line data, this will be used with the line number to retrieve info
+    QVector<quint32> linesNumbers; // First and next lines (next are relative to the first) numbers
+
+    // -- Generic system data not to be stored on disk
+    void *glPointer; // GL pointer
+
+    // These operator overrides prevent the glPointer and other non-disk-necessary data serialization
+    friend QDataStream& operator<<(QDataStream& stream, const dbDataStructure& myclass)
+    // Notice: this function has to be "friend" because it cannot be a member function, member functions
+    // have an additional parameter "this" which isn't in the argument list of an operator overload. A friend
+    // function has full access to private data of the class without having the "this" argument
+    {
+        // Don't write glPointer and every pointer-dependent structure
+        return stream << myclass.label << myclass.depth << myclass.userIndex << qCompress(myclass.data)
+                         << myclass.uniqueID << myclass.nextItemsIndices << myclass.fatherIndex << myclass.noFatherRoot
+                            << myclass.fileName << qCompress(myclass.firstLineData) << myclass.linesNumbers;
+    }
+    friend QDataStream& operator>>(QDataStream& stream, dbDataStructure& myclass)
+    {
+        //Don't read it, either
+        stream >> myclass.label >> myclass.depth >> myclass.userIndex >> myclass.data
+                      >> myclass.uniqueID >> myclass.nextItemsIndices >> myclass.fatherIndex >> myclass.noFatherRoot
+                         >> myclass.fileName >> myclass.firstLineData >> myclass.linesNumbers;
+        myclass.data = qUncompress(myclass.data);
+        myclass.firstLineData = qUncompress(myclass.firstLineData);
+        return stream;
+    }
+
+};
+
+#endif // GDSDBREADER_H

samples/C++/qscicommand.h

@@ -0,0 +1,415 @@
+// This defines the interface to the QsciCommand class.
+//
+// Copyright (c) 2011 Riverbank Computing Limited <info@riverbankcomputing.com>
+// 
+// This file is part of QScintilla.
+// 
+// This file may be used under the terms of the GNU General Public
+// License versions 2.0 or 3.0 as published by the Free Software
+// Foundation and appearing in the files LICENSE.GPL2 and LICENSE.GPL3
+// included in the packaging of this file.  Alternatively you may (at
+// your option) use any later version of the GNU General Public
+// License if such license has been publicly approved by Riverbank
+// Computing Limited (or its successors, if any) and the KDE Free Qt
+// Foundation. In addition, as a special exception, Riverbank gives you
+// certain additional rights. These rights are described in the Riverbank
+// GPL Exception version 1.1, which can be found in the file
+// GPL_EXCEPTION.txt in this package.
+// 
+// If you are unsure which license is appropriate for your use, please
+// contact the sales department at sales@riverbankcomputing.com.
+// 
+// This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
+// WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+
+
+#ifndef QSCICOMMAND_H
+#define QSCICOMMAND_H
+
+#ifdef __APPLE__
+extern "C++" {
+#endif
+
+#include <qstring.h>
+
+#include <Qsci/qsciglobal.h>
+#include <Qsci/qsciscintillabase.h>
+
+
+class QsciScintilla;
+
+
+//! \brief The QsciCommand class represents an internal editor command that may
+//! have one or two keys bound to it.
+//!
+//! Methods are provided to change the keys bound to the command and to remove
+//! a key binding.  Each command has a user friendly description of the command
+//! for use in key mapping dialogs.
+class QSCINTILLA_EXPORT QsciCommand
+{
+public:
+    //! This enum defines the different commands that can be assigned to a key.
+    enum Command {
+        //! Move down one line.
+        LineDown = QsciScintillaBase::SCI_LINEDOWN,
+
+        //! Extend the selection down one line.
+        LineDownExtend = QsciScintillaBase::SCI_LINEDOWNEXTEND,
+
+        //! Extend the rectangular selection down one line.
+        LineDownRectExtend = QsciScintillaBase::SCI_LINEDOWNRECTEXTEND,
+
+        //! Scroll the view down one line.
+        LineScrollDown = QsciScintillaBase::SCI_LINESCROLLDOWN,
+
+        //! Move up one line.
+        LineUp = QsciScintillaBase::SCI_LINEUP,
+
+        //! Extend the selection up one line.
+        LineUpExtend = QsciScintillaBase::SCI_LINEUPEXTEND,
+
+        //! Extend the rectangular selection up one line.
+        LineUpRectExtend = QsciScintillaBase::SCI_LINEUPRECTEXTEND,
+
+        //! Scroll the view up one line.
+        LineScrollUp = QsciScintillaBase::SCI_LINESCROLLUP,
+
+        //! Scroll to the start of the document.
+        ScrollToStart = QsciScintillaBase::SCI_SCROLLTOSTART,
+
+        //! Scroll to the end of the document.
+        ScrollToEnd = QsciScintillaBase::SCI_SCROLLTOEND,
+
+        //! Scroll vertically to centre the current line.
+        VerticalCentreCaret = QsciScintillaBase::SCI_VERTICALCENTRECARET,
+
+        //! Move down one paragraph.
+        ParaDown = QsciScintillaBase::SCI_PARADOWN,
+
+        //! Extend the selection down one paragraph.
+        ParaDownExtend = QsciScintillaBase::SCI_PARADOWNEXTEND,
+
+        //! Move up one paragraph.
+        ParaUp = QsciScintillaBase::SCI_PARAUP,
+
+        //! Extend the selection up one paragraph.
+        ParaUpExtend = QsciScintillaBase::SCI_PARAUPEXTEND,
+
+        //! Move left one character.
+        CharLeft = QsciScintillaBase::SCI_CHARLEFT,
+
+        //! Extend the selection left one character.
+        CharLeftExtend = QsciScintillaBase::SCI_CHARLEFTEXTEND,
+
+        //! Extend the rectangular selection left one character.
+        CharLeftRectExtend = QsciScintillaBase::SCI_CHARLEFTRECTEXTEND,
+
+        //! Move right one character.
+        CharRight = QsciScintillaBase::SCI_CHARRIGHT,
+
+        //! Extend the selection right one character.
+        CharRightExtend = QsciScintillaBase::SCI_CHARRIGHTEXTEND,
+
+        //! Extend the rectangular selection right one character.
+        CharRightRectExtend = QsciScintillaBase::SCI_CHARRIGHTRECTEXTEND,
+
+        //! Move left one word.
+        WordLeft = QsciScintillaBase::SCI_WORDLEFT,
+
+        //! Extend the selection left one word.
+        WordLeftExtend = QsciScintillaBase::SCI_WORDLEFTEXTEND,
+
+        //! Move right one word.
+        WordRight = QsciScintillaBase::SCI_WORDRIGHT,
+
+        //! Extend the selection right one word.
+        WordRightExtend = QsciScintillaBase::SCI_WORDRIGHTEXTEND,
+
+        //! Move to the end of the previous word.
+        WordLeftEnd = QsciScintillaBase::SCI_WORDLEFTEND,
+
+        //! Extend the selection to the end of the previous word.
+        WordLeftEndExtend = QsciScintillaBase::SCI_WORDLEFTENDEXTEND,
+
+        //! Move to the end of the next word.
+        WordRightEnd = QsciScintillaBase::SCI_WORDRIGHTEND,
+
+        //! Extend the selection to the end of the next word.
+        WordRightEndExtend = QsciScintillaBase::SCI_WORDRIGHTENDEXTEND,
+
+        //! Move left one word part.
+        WordPartLeft = QsciScintillaBase::SCI_WORDPARTLEFT,
+
+        //! Extend the selection left one word part.
+        WordPartLeftExtend = QsciScintillaBase::SCI_WORDPARTLEFTEXTEND,
+
+        //! Move right one word part.
+        WordPartRight = QsciScintillaBase::SCI_WORDPARTRIGHT,
+
+        //! Extend the selection right one word part.
+        WordPartRightExtend = QsciScintillaBase::SCI_WORDPARTRIGHTEXTEND,
+
+        //! Move to the start of the document line.
+        Home = QsciScintillaBase::SCI_HOME,
+
+        //! Extend the selection to the start of the document line.
+        HomeExtend = QsciScintillaBase::SCI_HOMEEXTEND,
+
+        //! Extend the rectangular selection to the start of the document line.
+        HomeRectExtend = QsciScintillaBase::SCI_HOMERECTEXTEND,
+
+        //! Move to the start of the displayed line.
+        HomeDisplay = QsciScintillaBase::SCI_HOMEDISPLAY,
+
+        //! Extend the selection to the start of the displayed line.
+        HomeDisplayExtend = QsciScintillaBase::SCI_HOMEDISPLAYEXTEND,
+
+        //! Move to the start of the displayed or document line.
+        HomeWrap = QsciScintillaBase::SCI_HOMEWRAP,
+
+        //! Extend the selection to the start of the displayed or document
+        //! line.
+        HomeWrapExtend = QsciScintillaBase::SCI_HOMEWRAPEXTEND,
+
+        //! Move to the first visible character in the document line.
+        VCHome = QsciScintillaBase::SCI_VCHOME,
+
+        //! Extend the selection to the first visible character in the document
+        //! line.
+        VCHomeExtend = QsciScintillaBase::SCI_VCHOMEEXTEND,
+
+        //! Extend the rectangular selection to the first visible character in
+        //! the document line.
+        VCHomeRectExtend = QsciScintillaBase::SCI_VCHOMERECTEXTEND,
+
+        //! Move to the first visible character of the displayed or document
+        //! line.
+        VCHomeWrap = QsciScintillaBase::SCI_VCHOMEWRAP,
+
+        //! Extend the selection to the first visible character of the
+        //! displayed or document line.
+        VCHomeWrapExtend = QsciScintillaBase::SCI_VCHOMEWRAPEXTEND,
+
+        //! Move to the end of the document line.
+        LineEnd = QsciScintillaBase::SCI_LINEEND,
+
+        //! Extend the selection to the end of the document line.
+        LineEndExtend = QsciScintillaBase::SCI_LINEENDEXTEND,
+
+        //! Extend the rectangular selection to the end of the document line.
+        LineEndRectExtend = QsciScintillaBase::SCI_LINEENDRECTEXTEND,
+
+        //! Move to the end of the displayed line.
+        LineEndDisplay = QsciScintillaBase::SCI_LINEENDDISPLAY,
+
+        //! Extend the selection to the end of the displayed line.
+        LineEndDisplayExtend = QsciScintillaBase::SCI_LINEENDDISPLAYEXTEND,
+
+        //! Move to the end of the displayed or document line.
+        LineEndWrap = QsciScintillaBase::SCI_LINEENDWRAP,
+
+        //! Extend the selection to the end of the displayed or document line.
+        LineEndWrapExtend = QsciScintillaBase::SCI_LINEENDWRAPEXTEND,
+
+        //! Move to the start of the document.
+        DocumentStart = QsciScintillaBase::SCI_DOCUMENTSTART,
+
+        //! Extend the selection to the start of the document.
+        DocumentStartExtend = QsciScintillaBase::SCI_DOCUMENTSTARTEXTEND,
+
+        //! Move to the end of the document.
+        DocumentEnd = QsciScintillaBase::SCI_DOCUMENTEND,
+
+        //! Extend the selection to the end of the document.
+        DocumentEndExtend = QsciScintillaBase::SCI_DOCUMENTENDEXTEND,
+
+        //! Move up one page.
+        PageUp = QsciScintillaBase::SCI_PAGEUP,
+
+        //! Extend the selection up one page.
+        PageUpExtend = QsciScintillaBase::SCI_PAGEUPEXTEND,
+
+        //! Extend the rectangular selection up one page.
+        PageUpRectExtend = QsciScintillaBase::SCI_PAGEUPRECTEXTEND,
+
+        //! Move down one page.
+        PageDown = QsciScintillaBase::SCI_PAGEDOWN,
+
+        //! Extend the selection down one page.
+        PageDownExtend = QsciScintillaBase::SCI_PAGEDOWNEXTEND,
+
+        //! Extend the rectangular selection down one page.
+        PageDownRectExtend = QsciScintillaBase::SCI_PAGEDOWNRECTEXTEND,
+
+        //! Stuttered move up one page.
+        StutteredPageUp = QsciScintillaBase::SCI_STUTTEREDPAGEUP,
+
+        //! Stuttered extend the selection up one page.
+        StutteredPageUpExtend = QsciScintillaBase::SCI_STUTTEREDPAGEUPEXTEND,
+
+        //! Stuttered move down one page.
+        StutteredPageDown = QsciScintillaBase::SCI_STUTTEREDPAGEDOWN,
+
+        //! Stuttered extend the selection down one page.
+        StutteredPageDownExtend = QsciScintillaBase::SCI_STUTTEREDPAGEDOWNEXTEND,
+
+        //! Delete the current character.
+        Delete = QsciScintillaBase::SCI_CLEAR,
+
+        //! Delete the previous character.
+        DeleteBack = QsciScintillaBase::SCI_DELETEBACK,
+
+        //! Delete the previous character if not at start of line.
+        DeleteBackNotLine = QsciScintillaBase::SCI_DELETEBACKNOTLINE,
+
+        //! Delete the word to the left.
+        DeleteWordLeft = QsciScintillaBase::SCI_DELWORDLEFT,
+
+        //! Delete the word to the right.
+        DeleteWordRight = QsciScintillaBase::SCI_DELWORDRIGHT,
+
+        //! Delete right to the end of the next word.
+        DeleteWordRightEnd = QsciScintillaBase::SCI_DELWORDRIGHTEND,
+
+        //! Delete the line to the left.
+        DeleteLineLeft = QsciScintillaBase::SCI_DELLINELEFT,
+
+        //! Delete the line to the right.
+        DeleteLineRight = QsciScintillaBase::SCI_DELLINERIGHT,
+
+        //! Delete the current line.
+        LineDelete = QsciScintillaBase::SCI_LINEDELETE,
+
+        //! Cut the current line to the clipboard.
+        LineCut = QsciScintillaBase::SCI_LINECUT,
+
+        //! Copy the current line to the clipboard.
+        LineCopy = QsciScintillaBase::SCI_LINECOPY,
+
+        //! Transpose the current and previous lines.
+        LineTranspose = QsciScintillaBase::SCI_LINETRANSPOSE,
+
+        //! Duplicate the current line.
+        LineDuplicate = QsciScintillaBase::SCI_LINEDUPLICATE,
+
+        //! Select the whole document.
+        SelectAll = QsciScintillaBase::SCI_SELECTALL,
+
+        //! Move the selected lines up one line.
+        MoveSelectedLinesUp = QsciScintillaBase::SCI_MOVESELECTEDLINESUP,
+
+        //! Move the selected lines down one line.
+        MoveSelectedLinesDown = QsciScintillaBase::SCI_MOVESELECTEDLINESDOWN,
+
+        //! Duplicate the selection.
+        SelectionDuplicate = QsciScintillaBase::SCI_SELECTIONDUPLICATE,
+
+        //! Convert the selection to lower case.
+        SelectionLowerCase = QsciScintillaBase::SCI_LOWERCASE,
+
+        //! Convert the selection to upper case.
+        SelectionUpperCase = QsciScintillaBase::SCI_UPPERCASE,
+
+        //! Cut the selection to the clipboard.
+        SelectionCut = QsciScintillaBase::SCI_CUT,
+
+        //! Copy the selection to the clipboard.
+        SelectionCopy = QsciScintillaBase::SCI_COPY,
+
+        //! Paste from the clipboard.
+        Paste = QsciScintillaBase::SCI_PASTE,
+
+        //! Toggle insert/overtype.
+        EditToggleOvertype = QsciScintillaBase::SCI_EDITTOGGLEOVERTYPE,
+
+        //! Insert a platform dependent newline.
+        Newline = QsciScintillaBase::SCI_NEWLINE,
+
+        //! Insert a formfeed.
+        Formfeed = QsciScintillaBase::SCI_FORMFEED,
+
+        //! Indent one level.
+        Tab = QsciScintillaBase::SCI_TAB,
+
+        //! De-indent one level.
+        Backtab = QsciScintillaBase::SCI_BACKTAB,
+
+        //! Cancel any current operation.
+        Cancel = QsciScintillaBase::SCI_CANCEL,
+
+        //! Undo the last command.
+        Undo = QsciScintillaBase::SCI_UNDO,
+
+        //! Redo the last command.
+        Redo = QsciScintillaBase::SCI_REDO,
+
+        //! Zoom in.
+        ZoomIn = QsciScintillaBase::SCI_ZOOMIN,
+
+        //! Zoom out.
+        ZoomOut = QsciScintillaBase::SCI_ZOOMOUT,
+    };
+
+    //! Return the command that will be executed by this instance.
+    Command command() const {return scicmd;}
+
+    //! Execute the command.
+    void execute();
+
+    //! Binds the key \a key to the command.  If \a key is 0 then the key
+    //! binding is removed.  If \a key is invalid then the key binding is
+    //! unchanged.  Valid keys are any visible or control character or any
+    //! of \c Key_Down, \c Key_Up, \c Key_Left, \c Key_Right, \c Key_Home,
+    //! \c Key_End, \c Key_PageUp, \c Key_PageDown, \c Key_Delete,
+    //! \c Key_Insert, \c Key_Escape, \c Key_Backspace, \c Key_Tab and
+    //! \c Key_Return.  Keys may be modified with any combination of \c SHIFT,
+    //! \c CTRL, \c ALT and \c META.
+    //!
+    //! \sa key(), setAlternateKey(), validKey()
+    void setKey(int key);
+
+    //! Binds the alternate key \a altkey to the command.  If \a key is 0
+    //! then the alternate key binding is removed.
+    //!
+    //! \sa alternateKey(), setKey(), validKey()
+    void setAlternateKey(int altkey);
+
+    //! The key that is currently bound to the command is returned.
+    //!
+    //! \sa setKey(), alternateKey()
+    int key() const {return qkey;}
+
+    //! The alternate key that is currently bound to the command is
+    //! returned.
+    //!
+    //! \sa setAlternateKey(), key()
+    int alternateKey() const {return qaltkey;}
+
+    //! If the key \a key is valid then true is returned.
+    static bool validKey(int key);
+
+    //! The user friendly description of the command is returned.
+    QString description() const;
+
+private:
+    friend class QsciCommandSet;
+
+    QsciCommand(QsciScintilla *qs, Command cmd, int key, int altkey,
+            const char *desc);
+
+    void bindKey(int key,int &qk,int &scik);
+
+    QsciScintilla *qsCmd;
+    Command scicmd;
+    int qkey, scikey, qaltkey, scialtkey;
+    const char *descCmd;
+
+    QsciCommand(const QsciCommand &);
+    QsciCommand &operator=(const QsciCommand &);
+};
+
+#ifdef __APPLE__
+}
+#endif
+
+#endif

samples/C++/qsciprinter.h

@@ -0,0 +1,116 @@
+// This module defines interface to the QsciPrinter class.
+//
+// Copyright (c) 2011 Riverbank Computing Limited <info@riverbankcomputing.com>
+// 
+// This file is part of QScintilla.
+// 
+// This file may be used under the terms of the GNU General Public
+// License versions 2.0 or 3.0 as published by the Free Software
+// Foundation and appearing in the files LICENSE.GPL2 and LICENSE.GPL3
+// included in the packaging of this file.  Alternatively you may (at
+// your option) use any later version of the GNU General Public
+// License if such license has been publicly approved by Riverbank
+// Computing Limited (or its successors, if any) and the KDE Free Qt
+// Foundation. In addition, as a special exception, Riverbank gives you
+// certain additional rights. These rights are described in the Riverbank
+// GPL Exception version 1.1, which can be found in the file
+// GPL_EXCEPTION.txt in this package.
+// 
+// If you are unsure which license is appropriate for your use, please
+// contact the sales department at sales@riverbankcomputing.com.
+// 
+// This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
+// WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+
+
+#ifndef QSCIPRINTER_H
+#define QSCIPRINTER_H
+
+#ifdef __APPLE__
+extern "C++" {
+#endif
+
+#include <qprinter.h>
+
+#include <Qsci/qsciglobal.h>
+#include <Qsci/qsciscintilla.h>
+
+
+QT_BEGIN_NAMESPACE
+class QRect;
+class QPainter;
+QT_END_NAMESPACE
+
+class QsciScintillaBase;
+
+
+//! \brief The QsciPrinter class is a sub-class of the Qt QPrinter class that
+//! is able to print the text of a Scintilla document.
+//!
+//! The class can be further sub-classed to alter to layout of the text, adding
+//! headers and footers for example.
+class QSCINTILLA_EXPORT QsciPrinter : public QPrinter
+{
+public:
+    //! Constructs a printer paint device with mode \a mode.
+    QsciPrinter(PrinterMode mode = ScreenResolution);
+
+    //! Destroys the QsciPrinter instance.
+    virtual ~QsciPrinter();
+
+    //! Format a page, by adding headers and footers for example, before the
+    //! document text is drawn on it.  \a painter is the painter to be used to
+    //! add customised text and graphics.  \a drawing is true if the page is
+    //! actually being drawn rather than being sized.  \a painter drawing
+    //! methods must only be called when \a drawing is true.  \a area is the
+    //! area of the page that will be used to draw the text.  This should be
+    //! modified if it is necessary to reserve space for any customised text or
+    //! graphics.  By default the area is relative to the printable area of the
+    //! page.  Use QPrinter::setFullPage() because calling printRange() if you
+    //! want to try and print over the whole page.  \a pagenr is the number of
+    //! the page.  The first page is numbered 1.
+    virtual void formatPage(QPainter &painter, bool drawing, QRect &area,
+            int pagenr);
+
+    //! Return the number of points to add to each font when printing.
+    //!
+    //! \sa setMagnification()
+    int magnification() const {return mag;}
+
+    //! Sets the number of points to add to each font when printing to \a
+    //! magnification.
+    //!
+    //! \sa magnification()
+    virtual void setMagnification(int magnification);
+
+    //! Print a range of lines from the Scintilla instance \a qsb.  \a from is
+    //! the first line to print and a negative value signifies the first line
+    //! of text.  \a to is the last line to print and a negative value
+    //! signifies the last line of text.  true is returned if there was no
+    //! error.
+    virtual int printRange(QsciScintillaBase *qsb, int from = -1, int to = -1);
+
+    //! Return the line wrap mode used when printing.  The default is
+    //! QsciScintilla::WrapWord.
+    //!
+    //! \sa setWrapMode()
+    QsciScintilla::WrapMode wrapMode() const {return wrap;}
+
+    //! Sets the line wrap mode used when printing to \a wmode.
+    //!
+    //! \sa wrapMode()
+    virtual void setWrapMode(QsciScintilla::WrapMode wmode);
+
+private:
+    int mag;
+    QsciScintilla::WrapMode wrap;
+
+    QsciPrinter(const QsciPrinter &);
+    QsciPrinter &operator=(const QsciPrinter &);
+};
+
+#ifdef __APPLE__
+}
+#endif
+
+#endif

samples/C/rf_io.h

@@ -0,0 +1,682 @@
+/**
+** Copyright (c) 2011-2012, Karapetsas Eleftherios
+** All rights reserved.
+**
+** Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+**  1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+**  2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in
+**     the documentation and/or other materials provided with the distribution.
+**  3. Neither the name of the Original Author of Refu nor the names of its contributors may be used to endorse or promote products derived from
+**
+**  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+**  INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+**  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+**  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+**  SERVICES;LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+**  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+**  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**/
+
+
+#ifndef REFU_IO_H
+#define REFU_IO_H
+
+#include <rf_setup.h>
+#include <stdio.h>
+
+#ifdef __cplusplus
+extern "C"
+{// opening bracket for calling from C++
+#endif
+
+// New line feed
+#define RF_LF   0xA
+// Carriage Return
+#define RF_CR   0xD
+
+#ifdef REFU_WIN32_VERSION
+    #define i_PLUSB_WIN32   "b"
+#else
+    #define i_PLUSB_WIN32   ""
+#endif
+
+// This is the type that represents the file offset
+#ifdef _MSC_VER
+typedef __int64 foff_rft;
+#else
+#include <sys/types.h>
+typedef off64_t foff_rft;
+#endif
+///Fseek and Ftelll definitions
+#ifdef _MSC_VER
+    #define rfFseek(i_FILE_,i_OFFSET_,i_WHENCE_)    _fseeki64(i_FILE_,i_OFFSET_,i_WHENCE_)
+    #define rfFtell(i_FILE_)                        _ftelli64(i_FILE_)
+#else
+    #define rfFseek(i_FILE_,i_OFFSET_,i_WHENCE_)    fseeko64(i_FILE_,i_OFFSET_,i_WHENCE_)
+    #define rfFtell(i_FILE_)                        ftello64(i_FILE_)
+#endif
+
+/**
+** @defgroup RF_IOGRP I/O
+** @addtogroup RF_IOGRP
+** @{
+**/
+
+// @brief Reads a UTF-8 file descriptor until end of line or EOF is found and returns a UTF-8 byte buffer
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// When the compile flag @c RF_NEWLINE_CRLF is defined (the default case at Windows) then this function
+// shall not be adding any CR character that is found in the file behind a newline character since this is
+// the Windows line ending scheme. Beware though that the returned  read bytes value shall still count the CR character inside.
+//
+// @param[in] f The file descriptor to read
+// @param[out] utf8 Give here a refence to an unitialized char* that will be allocated inside the function
+// and contain the utf8 byte buffer. Needs to be freed by the caller explicitly later
+// @param[out] byteLength Give an @c uint32_t here to receive the length of the @c utf8 buffer in bytes
+// @param[out] bufferSize Give an @c uint32_t here to receive the capacity of the @c utf8 buffer in bytes
+// @param[out] eof Pass a pointer to a char to receive a true or false value in case the end of file
+// with reading this line
+// @return Returns either a positive number for success that represents the number of bytes read from @c f and and error in case something goes wrong.
+// The possible errors to return are the same as rfFgets_UTF8()
+i_DECLIMEX_ int32_t rfFReadLine_UTF8(FILE* f,char** utf8,uint32_t* byteLength,uint32_t* bufferSize,char* eof);
+// @brief Reads a Big Endian UTF-16 file descriptor until end of line or EOF is found and returns a UTF-8 byte buffer
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// When the compile flag @c RF_NEWLINE_CRLF is defined (the default case at Windows) then this function
+// shall not be adding any CR character that is found in the file behind a newline character since this is
+// the Windows line ending scheme. Beware though that the returned  read bytes value shall still count the CR character inside.
+//
+// @param[in] f The file descriptor to read
+// @param[out] utf8 Give here a refence to an unitialized char* that will be allocated inside the function
+// and contain the utf8 byte buffer. Needs to be freed by the caller explicitly later
+// @param[out] byteLength Give an @c uint32_t here to receive the length of the @c utf8 buffer in bytes
+// @param[out] eof Pass a pointer to a char to receive a true or false value in case the end of file
+// with reading this line
+// @return Returns either a positive number for success that represents the number of bytes read from @c f and and error in case something goes wrong.
+// + Any error that can be returned by @ref rfFgets_UTF16BE()
+// + @c RE_UTF16_INVALID_SEQUENCE: Failed to decode the UTF-16 byte stream of the file descriptor
+// + @c RE_UTF8_ENCODING: Failed to encode the UTF-16 of the file descriptor into UTF-8
+i_DECLIMEX_ int32_t rfFReadLine_UTF16BE(FILE* f,char** utf8,uint32_t* byteLength,char* eof);
+// @brief Reads a Little Endian UTF-16 file descriptor until end of line or EOF is found and returns a UTF-8 byte buffer
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// When the compile flag @c RF_NEWLINE_CRLF is defined (the default case at Windows) then this function
+// shall not be adding any CR character that is found in the file behind a newline character since this is
+// the Windows line ending scheme. Beware though that the returned read bytes value shall still count the CR character inside.
+//
+// @param[in] f The file descriptor to read
+// @param[out] utf8 Give here a refence to an unitialized char* that will be allocated inside the function
+// and contain the utf8 byte buffer. Needs to be freed by the caller explicitly later
+// @param[out] byteLength Give an @c uint32_t here to receive the length of the @c utf8 buffer in bytes
+// @param[out] eof Pass a pointer to a char to receive a true or false value in case the end of file
+// with reading this line
+// @return Returns either a positive number for success that represents the number of bytes read from @c f and and error in case something goes wrong.
+// + Any error that can be returned by @ref rfFgets_UTF16LE()
+// + @c RE_UTF16_INVALID_SEQUENCE: Failed to decode the UTF-16 byte stream of the file descriptor
+// + @c RE_UTF8_ENCODING: Failed to encode the UTF-16 of the file descriptor into UTF-8
+i_DECLIMEX_ int32_t rfFReadLine_UTF16LE(FILE* f,char** utf8,uint32_t* byteLength,char* eof);
+
+// @brief Reads a Big Endian UTF-32 file descriptor until end of line or EOF is found and returns a UTF-8 byte buffer
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// When the compile flag @c RF_NEWLINE_CRLF is defined (the default case at Windows) then this function
+// shall not be adding any CR character that is found in the file behind a newline character since this is
+// the Windows line ending scheme. Beware though that the returned read bytes value shall still count the CR character inside.
+//
+// @param[in] f The file descriptor to read
+// @param[out] utf8 Give here a refence to an unitialized char* that will be allocated inside the function
+// and contain the utf8 byte buffer. Needs to be freed by the caller explicitly later
+// @param[out] byteLength Give an @c uint32_t here to receive the length of the @c utf8 buffer in bytes
+// @param[out] eof Pass a pointer to a char to receive a true or false value in case the end of file
+// with reading this line
+// @return Returns either a positive number for success that represents the number of bytes read from @c f and and error in case something goes wrong.
+// + Any error that can be returned by @ref rfFgets_UTF32BE()
+// + @c RE_UTF8_ENCODING: Failed to encode the UTF-16 of the file descriptor into UTF-8
+i_DECLIMEX_ int32_t rfFReadLine_UTF32BE(FILE* f,char** utf8,uint32_t* byteLength,char* eof);
+// @brief Reads a Little Endian UTF-32 file descriptor until end of line or EOF is found and returns a UTF-8 byte buffer
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// When the compile flag @c RF_NEWLINE_CRLF is defined (the default case at Windows) then this function
+// shall not be adding any CR character that is found in the file behind a newline character since this is
+// the Windows line ending scheme. Beware though that the returned read bytes value shall still count the CR character inside.
+//
+// @param[in] f The file descriptor to read
+// @param[out] utf8 Give here a refence to an unitialized char* that will be allocated inside the function
+// and contain the utf8 byte buffer. Needs to be freed by the caller explicitly later
+// @param[out] byteLength Give an @c uint32_t here to receive the length of the @c utf8 buffer in bytes
+// @param[out] eof Pass a pointer to a char to receive a true or false value in case the end of file
+// with reading this line
+// @return Returns either a positive number for success that represents the number of bytes read from @c f and and error in case something goes wrong.
+// + Any error that can be returned by @ref rfFgets_UTF32LE()
+// + @c RE_UTF8_ENCODING: Failed to encode the UTF-16 of the file descriptor into UTF-8
+i_DECLIMEX_ int32_t rfFReadLine_UTF32LE(FILE* f,char** utf8,uint32_t* byteLength,char* eof);
+
+// @brief Gets a number of bytes from a BIG endian UTF-32 file descriptor
+//
+// This is a function that's similar to c library fgets but it also returns the number of bytes read. Reads in from the file until @c num bytes
+// have been read or new line or EOF character has been encountered.
+//
+// The function will read until @c num characters are read and if @c num
+// would take us to the middle of a UTF32 character then the next character shall also be read
+// and the function will return the number of bytes read.
+// Since the function null terminates the buffer the given @c buff needs to be of at least
+// @c num+7 size to cater for the worst case.
+//
+// The final bytestream stored inside @c buff is in the endianess of the system.
+//
+// If right after the last character read comes the EOF, the function
+// shall detect so and assign @c true to @c eof.
+//
+// In Windows where file endings are in the form of 2 bytes CR-LF (Carriage return - NewLine) this function
+// shall just ignore the carriage returns and not return it inside the return buffer at @c buff.
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param[in] buff A buffer to be filled with the contents of the file. Should be of size at least @c num+7
+// @param[in] num The maximum number of bytes to read from within the file NOT including the null terminating character(which in itelf is 4 bytes). Should be a multiple of 4
+// @param[in] f A valid FILE descriptor from which to read the bytes
+// @param[out] eof Pass a reference to a char to receive a true/false value for whether EOF has been reached.
+// @return Returns the actual number of bytes read or an error if there was a problem.
+// The possible errors are:
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgets_UTF32BE(char* buff,uint32_t num,FILE* f,char* eof);
+// @brief Gets a number of bytes from a Little endian UTF-32 file descriptor
+//
+// This is a function that's similar to c library fgets but it also returns the number of bytes read. Reads in from the file until @c num bytes
+// have been read or new line or EOF character has been encountered.
+//
+// The function will read until @c num characters are read and if @c num
+// would take us to the middle of a UTF32 character then the next character shall also be read
+// and the function will return the number of bytes read.
+// Since the function null terminates the buffer the given @c buff needs to be of at least
+// @c num+7 size to cater for the worst case.
+//
+// The final bytestream stored inside @c buff is in the endianess of the system.
+//
+// If right after the last character read comes the EOF, the function
+// shall detect so and assign @c true to @c eof.
+//
+// In Windows where file endings are in the form of 2 bytes CR-LF (Carriage return - NewLine) this function
+// shall just ignore the carriage returns and not return it inside the return buffer at @c buff.
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param[in] buff A buffer to be filled with the contents of the file. Should be of size at least @c num+7
+// @param[in] num The maximum number of bytes to read from within the file NOT including the null terminating character(which in itelf is 4 bytes). Should be a multiple of 4
+// @param[in] f A valid FILE descriptor from which to read the bytes
+// @param[out] eof Pass a reference to a char to receive a true/false value for whether EOF has been reached.
+// @return Returns the actual number of bytes read or an error if there was a problem.
+// The possible errors are:
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgets_UTF32LE(char* buff,uint32_t num,FILE* f,char* eof);
+
+// @brief Gets a number of bytes from a BIG endian UTF-16 file descriptor
+//
+// This is a function that's similar to c library fgets but it also returns the number of bytes read. Reads in from the file until @c num bytes
+// have been read or new line or EOF character has been encountered.
+//
+// The function will read until @c num characters are read and if @c num
+// would take us to the middle of a UTF16 character then the next character shall also be read
+// and the function will return the number of bytes read.
+// Since the function null terminates the buffer the given @c buff needs to be of at least
+// @c num+5 size to cater for the worst case.
+//
+// The final bytestream stored inside @c buff is in the endianess of the system.
+//
+// If right after the last character read comes the EOF, the function
+// shall detect so and assign @c true to @c eof.
+//
+// In Windows where file endings are in the form of 2 bytes CR-LF (Carriage return - NewLine) this function
+// shall just ignore the carriage returns and not return it inside the return buffer at @c buff.
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param[in] buff A buffer to be filled with the contents of the file. Should be of size at least @c num+5
+// @param[in] num The maximum number of bytes to read from within the file NOT including the null terminating character(which in itelf is 2 bytes). Should be a multiple of 2
+// @param[in] f A valid FILE descriptor from which to read the bytes
+// @param[out] eof Pass a reference to a char to receive a true/false value for whether EOF has been reached.
+// @return Returns the actual number of bytes read or an error if there was a problem.
+// The possible errors are:
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgets_UTF16BE(char* buff,uint32_t num,FILE* f,char* eof);
+// @brief Gets a number of bytes from a Little endian UTF-16 file descriptor
+//
+// This is a function that's similar to c library fgets but it also returns the number of bytes read. Reads in from the file until @c num bytes
+// have been read or new line or EOF character has been encountered.
+//
+// The function will read until @c num characters are read and if @c num
+// would take us to the middle of a UTF16 character then the next character shall also be read
+// and the function will return the number of bytes read.
+// Since the function null terminates the buffer the given @c buff needs to be of at least
+// @c num+5 size to cater for the worst case.
+//
+// The final bytestream stored inside @c buff is in the endianess of the system.
+//
+// If right after the last character read comes the EOF, the function
+// shall detect so and assign @c true to @c eof.
+//
+// In Windows where file endings are in the form of 2 bytes CR-LF (Carriage return - NewLine) this function
+// shall just ignore the carriage returns and not return it inside the return buffer at @c buff.
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param[in] buff A buffer to be filled with the contents of the file. Should be of size at least @c num+2
+// @param[in] num The maximum number of bytes to read from within the file NOT including the null terminating character(which in itelf is 2 bytes). Should be a multiple of 2
+// @param[in] f A valid FILE descriptor from which to read the bytes
+// @param[out] eof Pass a reference to a char to receive a true/false value for whether EOF has been reached.
+// @return Returns the actual number of bytes read or an error if there was a problem.
+// The possible errors are:
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgets_UTF16LE(char* buff,uint32_t num,FILE* f,char* eof);
+// @brief Gets a number of bytes from a UTF-8 file descriptor
+//
+// This is a function that's similar to c library fgets but it also returns the number of bytes read. Reads in from the file until @c num characters
+// have been read or new line or EOF character has been encountered.
+//
+// The function  automatically adds a null termination character at the end of
+// @c buff but this character is not included in the returned actual number of bytes.
+//
+// The function will read until @c num characters are read and if @c num
+// would take us to the middle of a UTF8 character then the next character shall also be read
+// and the function will return the number of bytes read.
+// Since the function null terminates the buffer the given @c buff needs to be of at least
+// @c num+4 size to cater for the worst case.
+//
+// If right after the last character read comes the EOF, the function
+// shall detect so and assign @c true to @c eof.
+//
+// In Windows where file endings are in the form of 2 bytes CR-LF (Carriage return - NewLine) this function
+// shall just ignore the carriage returns and not return it inside the return buffer at @c buff.
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param[in] buff A buffer to be filled with the contents of the file. Should of size at least @c num+4
+// @param[in] num The maximum number of bytes to read from within the file NOT including the null terminating character(which in itelf is 1 byte)
+// @param[in] f A valid FILE descriptor from which to read the bytes
+// @param[out] eof Pass a reference to a char to receive a true/false value for whether EOF has been reached.
+// @return Returns the actual number of bytes read or an error if there was a problem.
+// The possible errors are:
+// + @c RE_UTF8_INVALID_SEQUENCE_INVALID_BYTE: If an invalid UTF-8 byte has been found
+// + @c RE_UTF8_INVALID_SEQUENCE_CONBYTE: If during parsing the file we were expecting a continuation
+// byte and did not find it
+// + @c RE_UTF8_INVALID_SEQUENCE_END: If the null character is encountered in between bytes that should
+// have been continuation bytes
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgets_UTF8(char* buff,uint32_t num,FILE* f,char* eof);
+
+// @brief  Gets a unicode character from a UTF-8 file descriptor
+//
+// This function attempts to assume a more modern fgetc() role for UTF-8 encoded files.
+// Reads bytes from the File descriptor @c f until a full UTF-8 unicode character has been read
+//
+// After this function the file pointer will have moved either by @c 1, @c 2, @c 3 or @c 4
+// bytes if the return value is positive. You can see how much by checking the return value.
+//
+// You shall need to provide an integer at @c c to contain either the decoded Unicode
+// codepoint or the UTF-8 endoced byte depending on the value of the @c cp argument.
+//
+// @param f A valid FILE descriptor from which to read the bytes
+// @param c Pass an int that will receive either the unicode code point value or
+// the UTF8 bytes depending on the value of the @c cp flag
+// @param cp A boolean flag. If @c true then the int passed at @c c will contain the unicode code point
+// of the read character, so the UTF-8 will be decoded.
+// If @c false the int passed at @c c will contain the value of the read bytes in UTF-8 without any decoding
+// @return Returns the number of bytes read (either @c 1, @c 2, @c 3 or @c 4) or an error if the function
+// fails for some reason. Possible error values are:
+// + @c RE_FILE_EOF: The end of file has been found while reading. If the end of file is encountered
+// in the middle of a UTF-8 encoded character where we would be expecting something different
+// and @c RE_UTF8_INVALID_SEQUENCE_END error is also logged
+// + @c RE_UTF8_INVALID_SEQUENCE_INVALID_BYTE: If an invalid UTF-8 byte has been found
+// + @c RE_UTF8_INVALID_SEQUENCE_CONBYTE: If during parsing the file we were expecting a continuation
+// byte and did not find it
+// + @c RE_UTF8_INVALID_SEQUENCE_END: If the null character is encountered in between bytes that should
+// have been continuation bytes
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgetc_UTF8(FILE* f,uint32_t *c,char cp);
+// @brief  Gets a unicode character from a UTF-16 Big Endian file descriptor
+//
+// This function attempts to assume a more modern fgetc() role for UTF-16 encoded files.
+// Reads bytes from the File descriptor @c f until a full UTF-16 unicode character has been read
+//
+// After this function the file pointer will have moved either by @c 2 or @c 4
+// bytes if the return value is positive. You can see how much by checking the return value.
+//
+// You shall need to provide an integer at @c c to contain either the decoded Unicode
+// codepoint or the Bigendian encoded UTF-16 bytes depending on the value of @c the cp argument.
+//
+// @param f A valid FILE descriptor from which to read the bytes
+// @param c Pass an int that will receive either the unicode code point value or
+// the UTF16 bytes depending on the value of the @c cp flag
+// @param cp A boolean flag. If @c true then the int passed at @c c will contain the unicode code point
+// of the read character, so the UTF-16 will be decoded.
+// If @c false the int passed at @c c will contain the value of the read bytes in UTF-16 without any decoding
+// @return Returns the number of bytes read (either @c 2 or @c 4) or an error if the function
+// fails for some reason. Possible error values are:
+// + @c RE_UTF16_INVALID_SEQUENCE: Either the read word or its surrogate pair if 4 bytes were read held illegal values
+// + @c RE_UTF16_NO_SURRPAIR: According to the first read word a surrogate pair was expected but none was found
+// + @c RE_FILE_EOF: The end of file has been found while reading. If the end of file is encountered
+// while we expect a UTF-16 surrogate pair an appropriate error is logged
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgetc_UTF16BE(FILE* f,uint32_t *c,char cp);
+// @brief  Gets a unicode character from a UTF-16 Little Endian file descriptor
+//
+// This function attempts to assume a more modern fgetc() role for UTF-16 encoded files.
+// Reads bytes from the File descriptor @c f until a full UTF-16 unicode character has been read
+//
+// After this function the file pointer will have moved either by @c 2 or @c 4
+// bytes if the return value is positive. You can see how much by checking the return value.
+//
+// You shall need to provide an integer at @c c to contain either the decoded Unicode
+// codepoint or the Bigendian encoded UTF-16 bytes depending on the value of @c the cp argument.
+//
+// @param f A valid FILE descriptor from which to read the bytes
+// @param c Pass an int that will receive either the unicode code point value or
+// the UTF16 bytes depending on the value of the @c cp flag
+// @param cp A boolean flag. If @c true then the int passed at @c c will contain the unicode code point
+// of the read character, so the UTF-16 will be decoded.
+// If @c false the int passed at @c c will contain the value of the read bytes in UTF-16 without any decoding
+// @return Returns the number of bytes read (either @c 2 or @c 4) or an error if the function
+// fails for some reason. Possible error values are:
+// + @c RE_UTF16_INVALID_SEQUENCE: Either the read word or its surrogate pair if 4 bytes were read held illegal values
+// + @c RE_UTF16_NO_SURRPAIR: According to the first read word a surrogate pair was expected but none was found
+// + @c RE_FILE_EOF: The end of file has been found while reading. If the end of file is encountered
+// while we expect a UTF-16 surrogate pair an appropriate error is logged
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgetc_UTF16LE(FILE* f,uint32_t *c,char cp);
+// @brief  Gets a unicode character from a UTF-32 Little Endian file descriptor
+//
+// This function attempts to assume a more modern fgetc() role for UTF-32 encoded files.
+// Reads bytes from the File descriptor @c f until a full UTF-32 unicode character has been read
+//
+// After this function the file pointer will have moved by @c 4
+// bytes if the return value is positive.
+//
+// You shall need to provide an integer at @c to contain the UTF-32 codepoint.
+//
+// @param f A valid FILE descriptor from which to read the bytes
+// @param c Pass an int that will receive either the unicode code point value or
+// the UTF16 bytes depending on the value of the @c cp flag
+// If @c false the int passed at @c c will contain the value of the read bytes in UTF-16 without any decoding
+// @return Returns either @c RF_SUCCESS for succesfull readin or one of the following errors:
+// + @c RE_FILE_EOF: The end of file has been found while reading.
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgetc_UTF32LE(FILE* f,uint32_t *c);
+// @brief  Gets a unicode character from a UTF-32 Big Endian file descriptor
+//
+// This function attempts to assume a more modern fgetc() role for UTF-32 encoded files.
+// Reads bytes from the File descriptor @c f until a full UTF-32 unicode character has been read
+//
+// After this function the file pointer will have moved by @c 4
+// bytes if the return value is positive.
+//
+// You shall need to provide an integer at @c to contain the UTF-32 codepoint.
+//
+// @param f A valid FILE descriptor from which to read the bytes
+// @param c Pass an int that will receive either the unicode code point value or
+// the UTF16 bytes depending on the value of the @c cp flag
+// If @c false the int passed at @c c will contain the value of the read bytes in UTF-16 without any decoding
+// @return Returns either @c RF_SUCCESS for succesfull readin or one of the following errors:
+// + @c RE_FILE_EOF: The end of file has been found while reading.
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgetc_UTF32BE(FILE* f,uint32_t *c);
+
+// @brief Moves a unicode character backwards in a big endian UTF-32 file stream
+//
+// @param f The file stream
+// @param c Returns the character we moved back to as a unicode codepoint
+// @return Returns either @c RF_SUCCESS for success or one of the following errors:
+// + @c RE_FILE_POS_OVERFLOW: If during trying to read the current file's position it can't be represented by the system
+// + @c RE_FILE_BAD: If The file descriptor is corrupt/illegal
+// + @c RE_FILE_NOTFILE: If the file descriptor is not a file but something else. e.g. socket.
+// + @c RE_FILE_GETFILEPOS: If the file's position could not be retrieved for some unknown reason
+// + @c RE_FILE_WRITE_BLOCK: While attempting to move the file pointer, it was occupied by another thread, and the no block flag was set
+// + @c RE_INTERRUPT: Operating on the file failed due to a system interrupt
+// + @c RE_FILE_IO: There was a physical I/O error
+// + @c RE_FILE_NOSPACE: There was no space on the device holding the file
+// + @c RE_FILE_NOTFILE: The device we attempted to manipulate is non-existent
+// + @c RE_FILE_READ: If during reading the file there was an error
+// + @c RE_FILE_READ_BLOCK: If during reading the file the read operation failed due to the file being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the underlying file descriptor's mode was not correctly set for reading
+i_DECLIMEX_ int32_t rfFback_UTF32BE(FILE* f,uint32_t *c);
+// @brief Moves a unicode character backwards in a little endian UTF-32 file stream
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param f The file stream
+// @param c Returns the character we moved back to as a unicode codepoint
+// @return Returns either @c RF_SUCCESS for success or one of the following errors:
+// + @c RE_FILE_POS_OVERFLOW: If during trying to read the current file's position it can't be represented by the system
+// + @c RE_FILE_BAD: If The file descriptor is corrupt/illegal
+// + @c RE_FILE_NOTFILE: If the file descriptor is not a file but something else. e.g. socket.
+// + @c RE_FILE_GETFILEPOS: If the file's position could not be retrieved for some unknown reason
+// + @c RE_FILE_WRITE_BLOCK: While attempting to move the file pointer, it was occupied by another thread, and the no block flag was set
+// + @c RE_INTERRUPT: Operating on the file failed due to a system interrupt
+// + @c RE_FILE_IO: There was a physical I/O error
+// + @c RE_FILE_NOSPACE: There was no space on the device holding the file
+// + @c RE_FILE_NOTFILE: The device we attempted to manipulate is non-existent
+// + @c RE_FILE_READ: If during reading the file there was an error
+// + @c RE_FILE_READ_BLOCK: If during reading the file the read operation failed due to the file being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the underlying file descriptor's mode was not correctly set for reading
+i_DECLIMEX_ int32_t rfFback_UTF32LE(FILE* f,uint32_t *c);
+// @brief Moves a unicode character backwards in a big endian UTF-16 file stream
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param f The file stream
+// @param c Returns the character we moved back to as a unicode codepoint
+// @return Returns either the number of bytes moved backwards (either @c 4 or @c 2) for success or one of the following errors:
+// + @c RE_UTF16_INVALID_SEQUENCE: Either the read word or its surrogate pair if 4 bytes were read held illegal values
+// + @c RE_FILE_POS_OVERFLOW: If during trying to read the current file's position it can't be represented by the system
+// + @c RE_FILE_BAD: If The file descriptor is corrupt/illegal
+// + @c RE_FILE_NOTFILE: If the file descriptor is not a file but something else. e.g. socket.
+// + @c RE_FILE_GETFILEPOS: If the file's position could not be retrieved for some unknown reason
+// + @c RE_FILE_WRITE_BLOCK: While attempting to move the file pointer, it was occupied by another thread, and the no block flag was set
+// + @c RE_INTERRUPT: Operating on the file failed due to a system interrupt
+// + @c RE_FILE_IO: There was a physical I/O error
+// + @c RE_FILE_NOSPACE: There was no space on the device holding the file
+// + @c RE_FILE_NOTFILE: The device we attempted to manipulate is non-existent
+// + @c RE_FILE_READ: If during reading the file there was an error
+// + @c RE_FILE_READ_BLOCK: If during reading the file the read operation failed due to the file being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the underlying file descriptor's mode was not correctly set for reading
+i_DECLIMEX_ int32_t rfFback_UTF16BE(FILE* f,uint32_t *c);
+// @brief Moves a unicode character backwards in a little endian UTF-16 file stream
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param f The file stream
+// @param c Returns the character we moved back to as a unicode codepoint
+// @return Returns either the number of bytes moved backwards (either @c 4 or @c 2) for success or one of the following errors:
+// + @c RE_UTF16_INVALID_SEQUENCE: Either the read word or its surrogate pair if 4 bytes were read held illegal values
+// + @c RE_FILE_POS_OVERFLOW: If during trying to read the current file's position it can't be represented by the system
+// + @c RE_FILE_BAD: If The file descriptor is corrupt/illegal
+// + @c RE_FILE_NOTFILE: If the file descriptor is not a file but something else. e.g. socket.
+// + @c RE_FILE_GETFILEPOS: If the file's position could not be retrieved for some unknown reason
+// + @c RE_FILE_WRITE_BLOCK: While attempting to move the file pointer, it was occupied by another thread, and the no block flag was set
+// + @c RE_INTERRUPT: Operating on the file failed due to a system interrupt
+// + @c RE_FILE_IO: There was a physical I/O error
+// + @c RE_FILE_NOSPACE: There was no space on the device holding the file
+// + @c RE_FILE_NOTFILE: The device we attempted to manipulate is non-existent
+// + @c RE_FILE_READ: If during reading the file there was an error
+// + @c RE_FILE_READ_BLOCK: If during reading the file the read operation failed due to the file being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the underlying file descriptor's mode was not correctly set for reading
+i_DECLIMEX_ int32_t rfFback_UTF16LE(FILE* f,uint32_t *c);
+// @brief Moves a unicode character backwards in a UTF-8 file stream
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param f The file stream
+// @param c Returns the character we moved back to as a unicode codepoint
+// @return Returns either the number of bytes moved backwards for success (either @c 4, @c 3, @c 2 or @c 1) or one of the following errors:
+// + @c RE_UTF8_INVALID_SEQUENCE: If during moving bacwards in the file unexpected UTF-8 bytes were found
+// + @c RE_FILE_POS_OVERFLOW: If during trying to read the current file's position it can't be represented by the system
+// + @c RE_FILE_BAD: If The file descriptor is corrupt/illegal
+// + @c RE_FILE_NOTFILE: If the file descriptor is not a file but something else. e.g. socket.
+// + @c RE_FILE_GETFILEPOS: If the file's position could not be retrieved for some unknown reason
+// + @c RE_FILE_WRITE_BLOCK: While attempting to move the file pointer, it was occupied by another thread, and the no block flag was set
+// + @c RE_INTERRUPT: Operating on the file failed due to a system interrupt
+// + @c RE_FILE_IO: There was a physical I/O error
+// + @c RE_FILE_NOSPACE: There was no space on the device holding the file
+// + @c RE_FILE_NOTFILE: The device we attempted to manipulate is non-existent
+// + @c RE_FILE_READ: If during reading the file there was an error
+// + @c RE_FILE_READ_BLOCK: If during reading the file the read operation failed due to the file being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the underlying file descriptor's mode was not correctly set for reading
+i_DECLIMEX_ int32_t rfFback_UTF8(FILE* f,uint32_t *c);
+
+// @brief Opens another process as a pipe
+//
+// This function is a cross-platform popen wrapper. In linux it uses popen and in Windows it uses
+// _popen.
+// @lmsFunction
+// @param command The string with the command to execute. Is basically the name of the program/process you want to spawn
+// with its full path and its parameters. @inhtype{String,StringX} @tmpSTR
+// @param mode The mode you want the pipe to work in. There are two possible values:
+// + @c "r" The calling process can read the spawned command's standard output via the returned stream.
+// + @c "w" The calling process can write to the spawned command's standard input via the returned stream.
+//
+// Anything else will result in an error
+// @return For success popen will return a FILE descriptor that can be used to either read or write from the pipe.
+// If there was an error @c 0 is returned and an error is logged.
+#ifdef RF_IAMHERE_FOR_DOXYGEN
+i_DECLIMEX_ FILE* rfPopen(void* command,const char* mode);
+#else
+i_DECLIMEX_ FILE* i_rfPopen(void* command,const char* mode);
+#define rfPopen(i_CMD_,i_MODE_) i_rfLMS_WRAP2(FILE*,i_rfPopen,i_CMD_,i_MODE_)
+#endif
+
+// @brief Closes a pipe
+//
+// This function is a cross-platform wrapper for pclose. It closes a file descriptor opened with @ref rfPopen() and
+// returns the exit code of the process that was running
+// @param stream The file descriptor of the pipe returned by @ref rfPopen() that we want to close
+// @return Returns the exit code of the process or -1 if there was an error
+i_DECLIMEX_ int rfPclose(FILE* stream);
+
+// @} End of I/O group
+
+#ifdef __cplusplus
+}///closing bracket for calling from C++
+#endif
+
+
+#endif//include guards end

samples/JSON/Hello.maxhelp

@@ -0,0 +1,367 @@
+{
+	"patcher" : 	{
+		"fileversion" : 1,
+		"appversion" : 		{
+			"major" : 5,
+			"minor" : 1,
+			"revision" : 9
+		}
+,
+		"rect" : [ 198.0, 92.0, 365.0, 407.0 ],
+		"bglocked" : 0,
+		"defrect" : [ 198.0, 92.0, 365.0, 407.0 ],
+		"openrect" : [ 0.0, 0.0, 0.0, 0.0 ],
+		"openinpresentation" : 0,
+		"default_fontsize" : 14.0,
+		"default_fontface" : 0,
+		"default_fontname" : "Arial",
+		"gridonopen" : 0,
+		"gridsize" : [ 20.0, 20.0 ],
+		"gridsnaponopen" : 0,
+		"toolbarvisible" : 1,
+		"boxanimatetime" : 200,
+		"imprint" : 0,
+		"enablehscroll" : 1,
+		"enablevscroll" : 1,
+		"devicewidth" : 0.0,
+		"boxes" : [ 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 260.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-22"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 240.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-20"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 220.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-18"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 200.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-16"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "route 0 1 2 3",
+					"patching_rect" : [ 200.0, 220.0, 99.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 5,
+					"outlettype" : [ "", "", "", "", "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-14"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "r jojo",
+					"patching_rect" : [ 200.0, 180.0, 41.0, 23.0 ],
+					"numinlets" : 0,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"color" : [ 0.827451, 0.737255, 0.835294, 1.0 ],
+					"id" : "obj-13"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "s jojo",
+					"patching_rect" : [ 20.0, 340.0, 43.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 0,
+					"fontsize" : 14.0,
+					"color" : [ 0.827451, 0.737255, 0.835294, 1.0 ],
+					"id" : "obj-12"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "append toto",
+					"patching_rect" : [ 20.0, 300.0, 84.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-11"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "% 4",
+					"patching_rect" : [ 20.0, 260.0, 35.0, 23.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-10"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "counter",
+					"patching_rect" : [ 20.0, 220.0, 73.0, 23.0 ],
+					"numinlets" : 5,
+					"fontname" : "Arial",
+					"numoutlets" : 4,
+					"outlettype" : [ "int", "", "", "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-9"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "metro 250",
+					"patching_rect" : [ 20.0, 180.0, 74.0, 23.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"fontsize" : 14.0,
+					"id" : "obj-8"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "toggle",
+					"patching_rect" : [ 20.0, 140.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"id" : "obj-7"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "t 0",
+					"patching_rect" : [ 140.0, 80.0, 26.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-5"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "t 1",
+					"patching_rect" : [ 20.0, 80.0, 26.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-4"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "message",
+					"text" : "Goodbye World !",
+					"patching_rect" : [ 140.0, 40.0, 115.0, 21.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-3"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "message",
+					"text" : "Hello World !",
+					"patching_rect" : [ 20.0, 40.0, 90.0, 21.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-2"
+				}
+
+			}
+ ],
+		"lines" : [ 			{
+				"patchline" : 				{
+					"source" : [ "obj-2", 0 ],
+					"destination" : [ "obj-4", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-3", 0 ],
+					"destination" : [ "obj-5", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-4", 0 ],
+					"destination" : [ "obj-7", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-5", 0 ],
+					"destination" : [ "obj-7", 0 ],
+					"hidden" : 0,
+					"midpoints" : [ 149.5, 121.0, 29.5, 121.0 ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-7", 0 ],
+					"destination" : [ "obj-8", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-8", 0 ],
+					"destination" : [ "obj-9", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-9", 0 ],
+					"destination" : [ "obj-10", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-10", 0 ],
+					"destination" : [ "obj-11", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-11", 0 ],
+					"destination" : [ "obj-12", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-13", 0 ],
+					"destination" : [ "obj-14", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 0 ],
+					"destination" : [ "obj-16", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 1 ],
+					"destination" : [ "obj-18", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 2 ],
+					"destination" : [ "obj-20", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 3 ],
+					"destination" : [ "obj-22", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+ ]
+	}
+
+}

samples/JSON/Hello.maxpat

@@ -0,0 +1,368 @@
+{
+	"patcher" : 	{
+		"fileversion" : 1,
+		"appversion" : 		{
+			"major" : 5,
+			"minor" : 1,
+			"revision" : 9
+		}
+,
+		"rect" : [ 198.0, 92.0, 365.0, 407.0 ],
+		"bglocked" : 0,
+		"defrect" : [ 198.0, 92.0, 365.0, 407.0 ],
+		"openrect" : [ 0.0, 0.0, 0.0, 0.0 ],
+		"openinpresentation" : 0,
+		"default_fontsize" : 14.0,
+		"default_fontface" : 0,
+		"default_fontname" : "Arial",
+		"gridonopen" : 0,
+		"gridsize" : [ 20.0, 20.0 ],
+		"gridsnaponopen" : 0,
+		"toolbarvisible" : 1,
+		"boxanimatetime" : 200,
+		"imprint" : 0,
+		"enablehscroll" : 1,
+		"enablevscroll" : 1,
+		"devicewidth" : 0.0,
+		"boxes" : [ 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 260.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-22"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 240.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-20"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 220.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-18"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "button",
+					"patching_rect" : [ 200.0, 260.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"id" : "obj-16"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "route 0 1 2 3",
+					"patching_rect" : [ 200.0, 220.0, 99.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 5,
+					"outlettype" : [ "", "", "", "", "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-14"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "r jojo",
+					"patching_rect" : [ 200.0, 180.0, 41.0, 23.0 ],
+					"numinlets" : 0,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"color" : [ 0.827451, 0.737255, 0.835294, 1.0 ],
+					"id" : "obj-13"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "s jojo",
+					"patching_rect" : [ 20.0, 340.0, 43.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 0,
+					"fontsize" : 14.0,
+					"color" : [ 0.827451, 0.737255, 0.835294, 1.0 ],
+					"id" : "obj-12"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "append toto",
+					"patching_rect" : [ 20.0, 300.0, 84.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-11"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "% 4",
+					"patching_rect" : [ 20.0, 260.0, 35.0, 23.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-10"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "counter",
+					"patching_rect" : [ 20.0, 220.0, 73.0, 23.0 ],
+					"numinlets" : 5,
+					"fontname" : "Arial",
+					"numoutlets" : 4,
+					"outlettype" : [ "int", "", "", "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-9"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "metro 250",
+					"patching_rect" : [ 20.0, 180.0, 74.0, 23.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "bang" ],
+					"fontsize" : 14.0,
+					"id" : "obj-8"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "toggle",
+					"patching_rect" : [ 20.0, 140.0, 20.0, 20.0 ],
+					"numinlets" : 1,
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"id" : "obj-7"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "t 0",
+					"patching_rect" : [ 140.0, 80.0, 26.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-5"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "newobj",
+					"text" : "t 1",
+					"patching_rect" : [ 20.0, 80.0, 26.0, 23.0 ],
+					"numinlets" : 1,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "int" ],
+					"fontsize" : 14.0,
+					"id" : "obj-4"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "message",
+					"text" : "Goodbye World !",
+					"patching_rect" : [ 140.0, 40.0, 115.0, 21.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"presentation_rect" : [ 137.0, 42.0, 0.0, 0.0 ],
+					"id" : "obj-3"
+				}
+
+			}
+, 			{
+				"box" : 				{
+					"maxclass" : "message",
+					"text" : "Hello World !",
+					"patching_rect" : [ 20.0, 40.0, 90.0, 21.0 ],
+					"numinlets" : 2,
+					"fontname" : "Arial",
+					"numoutlets" : 1,
+					"outlettype" : [ "" ],
+					"fontsize" : 14.0,
+					"id" : "obj-2"
+				}
+
+			}
+ ],
+		"lines" : [ 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 3 ],
+					"destination" : [ "obj-22", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 2 ],
+					"destination" : [ "obj-20", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 1 ],
+					"destination" : [ "obj-18", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-14", 0 ],
+					"destination" : [ "obj-16", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-13", 0 ],
+					"destination" : [ "obj-14", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-11", 0 ],
+					"destination" : [ "obj-12", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-10", 0 ],
+					"destination" : [ "obj-11", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-9", 0 ],
+					"destination" : [ "obj-10", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-8", 0 ],
+					"destination" : [ "obj-9", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-7", 0 ],
+					"destination" : [ "obj-8", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-5", 0 ],
+					"destination" : [ "obj-7", 0 ],
+					"hidden" : 0,
+					"midpoints" : [ 149.5, 121.0, 29.5, 121.0 ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-4", 0 ],
+					"destination" : [ "obj-7", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-3", 0 ],
+					"destination" : [ "obj-5", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+, 			{
+				"patchline" : 				{
+					"source" : [ "obj-2", 0 ],
+					"destination" : [ "obj-4", 0 ],
+					"hidden" : 0,
+					"midpoints" : [  ]
+				}
+
+			}
+ ]
+	}
+
+}

samples/JSON/person.json

@@ -0,0 +1,23 @@
+{
+     "firstName": "John",
+     "lastName" : "Smith",
+     "age"      : 25,
+     "address"  :
+     {
+         "streetAddress": "21 2nd Street",
+         "city"         : "New York",
+         "state"        : "NY",
+         "postalCode"   : "10021"
+     },
+     "phoneNumber":
+     [
+         {
+           "type"  : "home",
+           "number": "212 555-1234"
+         },
+         {
+           "type"  : "fax",
+           "number": "646 555-4567"
+         }
+     ]
+ }

samples/JSON/product.json

@@ -0,0 +1,7 @@
+{
+        "id": 1,
+        "name": "Foo",
+        "price": 123,
+        "tags": ["Bar","Eek"],
+        "stock": { "warehouse":300, "retail":20 }
+}

samples/JSON/schema.json

@@ -0,0 +1,47 @@
+{
+        "name":"Product",
+        "properties":
+        {
+                "id":
+                {
+                        "type":"number",
+                        "description":"Product identifier",
+                        "required":true
+                },
+                "name":
+                {
+                        "type":"string",
+                        "description":"Name of the product",
+                        "required":true
+                },
+                "price":
+                {
+                        "type":"number",
+                        "minimum":0,
+                        "required":true
+                },
+                "tags":
+                {
+                        "type":"array",
+                        "items":
+                        {
+                                "type":"string"
+                        }
+                },
+                "stock":
+                {
+                        "type":"object",
+                        "properties":
+                        {
+                                "warehouse":
+                                {
+                                        "type":"number"
+                                },
+                                "retail":
+                                {
+                                        "type":"number"
+                                }
+                        }
+                }
+        }
+}

samples/JavaScript/js2.script!

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+var http = require('http');
+http.createServer(function (req, res) {
+  res.writeHead(200, {'Content-Type': 'text/plain'});
+  res.end('Hello World\n');
+}).listen(1337, '127.0.0.1');
+console.log('Server running at http://127.0.0.1:1337/');

samples/Max/Hello.mxt

@@ -0,0 +1 @@
+max v2;

samples/Perl/perl.script!

@@ -0,0 +1,2 @@
+#!/usr/local/bin/perl
+print "Perl\n"

samples/Prolog/calc.pl

@@ -0,0 +1,68 @@
+action_module(calculator) .
+
+
+%[-,-,d1,-] --push(D)-->          [-,-,D,-]  if mode(init)
+push(D) < -
+  mode(init),
+  deny([displayed(D1),mode(init)]),
+  affirm([displayed(D),mode(cont)]).
+
+%[-,-,D1,-] --push(D)-->          [-,-,10*D1+D,-]  if mode(cont)
+push(D) < -
+  mode(cont),
+  deny(displayed(D1)),
+  New = 10*D1 + D,
+  affirm(displayed(New)).
+
+%[a,op,d,m] --push(clear)-->      [0,nop,0,0]
+push(clear) < -
+  deny([accumulator(A),op(O),displayed(D),memory(M),mode(X)]),
+  affirm([accumulator(0),op(nop),displayed(0),memory(0),mode(init)]).
+
+%[a,op,d,m] --push(mem_rec)-->    [a,op,m,m]
+push(mem_rec) < -
+  memory(M),
+  deny([displayed(D),mode(X)]),
+  affirm([displayed(M),mode(init)]).
+
+%[a,op,d,m] --push(plus)-->       [op(a,d),plus,d,m]
+push(plus) < -
+  displayed(D),
+  deny([accumulator(A),op(O),mode(X)]),
+  eval(O,A,D,V),   ; use normal arithmetic, i.e., V=O(A,D)
+  affirm([accumulator(V),op(plus),mode(init)]).
+
+%[a,op,d,m] --push(minus)-->      [op(a,d,minus,d,m]
+push(minus) lt -
+  displayed(D),
+  deny([accumulator(A),op(O),mode(X)]),
+  eval(O,A,D,V),   ; use normal arithmetic, i.e., V=O(A,D)
+  affirm([accumulator(V),op(minus),mode(init)]).
+
+%[a,op,d,m] --push(times)-->      [op(a,d),times,d,m]
+push(times) < -
+  displayed(D),
+  deny([accumulator(A),op(O),mode(X)]),
+  eval(O,A,D,V),   ; use normal arithmetic, i.e., V=O(A,D)
+  affirm([accumulator(V),op(times),mode(init)]).
+
+%[a,op,d,m] --push(equal)-->      [a,nop,op(a,d),m]
+push(equal) < -
+  accumulator(A),
+  deny([op(O),displayed(D),mode(X)]),
+  eval(O,A,D,V),
+  affirm([op(nop),displayed(V),mode(init)]).
+
+%[a,op,d,m] --push(mem_plus)-->   [a,nop,v,plus(m,v)] where v=op(a,d)
+push(mem_plus) < -
+  accumulator(A),
+  deny([op(O),displayed(D),memory(M),mode(X)]),
+  eval(O,A,D,V),
+  eval(plus,M,V,V1),
+  affirm([op(nop),displayed(V),memory(V1),mode(init)]).
+
+%[a,op,d,m] --push(plus_minus)--> [a,op,-d,m]
+push(clear) < -
+  deny([displayed(D),mode(X)]),
+  eval(minus,0,D,V),
+  affirm([displayed(V),mode(init)]).

samples/Prolog/normal_form.pl

@@ -0,0 +1,94 @@
+%%----- normalize(+Wff,-NormalClauses) ------
+normalize(Wff,NormalClauses) :-
+   conVert(Wff,[],S),
+   cnF(S,T),
+   flatten_and(T,U),
+   make_clauses(U,NormalClauses).
+
+%%-----  make a sequence out of a conjunction -----
+flatten_and(X /\ Y, F) :-
+   !,
+   flatten_and(X,A),
+   flatten_and(Y, B),
+   sequence_append(A,B,F).
+flatten_and(X,X).
+
+%%-----  make a sequence out of a disjunction -----
+flatten_or(X \/ Y, F) :-
+   !,
+   flatten_or(X,A),
+   flatten_or(Y,B),
+   sequence_append(A,B,F).
+flatten_or(X,X).
+
+
+%%----- append two sequences -------------------------------
+sequence_append((X,R),S,(X,T)) :- !, sequence_append(R,S,T).
+sequence_append((X),S,(X,S)).
+
+%%----- separate into positive and negative literals -----------
+separate((A,B),P,N) :-
+   !,
+   (A = ~X -> N=[X|N1],
+               separate(B,P,N1)
+             ;
+               P=[A|P1],
+               separate(B,P1,N) ).
+separate(A,P,N) :-
+   (A = ~X -> N=[X],
+               P = []
+            ;
+               P=[A],
+               N = [] ).
+
+%%----- tautology ----------------------------
+tautology(P,N) :- some_occurs(N,P).
+
+some_occurs([F|R],B) :-
+   occurs(F,B) | some_occurs(R,B).
+
+occurs(A,[F|_]) :-
+   A == F,
+   !.
+occurs(A,[_|R]) :-
+   occurs(A,R).
+
+make_clauses((A,B),C) :-
+   !,
+   flatten_or(A,F),
+   separate(F,P,N),
+   (tautology(P,N) ->
+      make_clauses(B,C)
+          ;
+      make_clause(P,N,D),
+      C = [D|R],
+      make_clauses(B,R) ).
+make_clauses(A,C) :-
+   flatten_or(A,F),
+   separate(F,P,N),
+   (tautology(P,N) ->
+       C = []
+        ;
+       make_clause(P,N,D),
+       C = [D] ).
+
+make_clause([],N, false :- B) :-
+   !,
+   make_sequence(N,B,',').
+make_clause(P,[],H) :-
+   !,
+   make_sequence(P,H,'|').
+make_clause(P,N, H :- T) :-
+   make_sequence(P,H,'|'),
+   make_sequence(N,T,',').
+
+make_sequence([A],A,_) :- !.
+make_sequence([F|R],(F|S),'|') :-
+   make_sequence(R,S,'|').
+make_sequence([F|R],(F,S),',') :-
+   make_sequence(R,S,',').
+
+write_list([F|R]) :-
+   write(F), write('.'), nl,
+   write_list(R).
+write_list([]).

samples/Prolog/puzzle.pl

@@ -0,0 +1,287 @@
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%
+%%%     A* Algorithm
+%%%
+%%%
+%%%     Nodes have form    S#D#F#A
+%%%            where S describes the state or configuration
+%%%                  D is the depth of the node
+%%%                  F is the evaluation function value
+%%%                  A is the ancestor list for the node
+
+:- op(400,yfx,'#').    /* Node builder notation */
+
+solve(State,Soln) :- f_function(State,0,F),
+                     search([State#0#F#[]],S), reverse(S,Soln).
+
+f_function(State,D,F) :- h_function(State,H),
+                         F is D + H.
+
+search([State#_#_#Soln|_], Soln) :- goal(State).
+search([B|R],S) :- expand(B,Children),
+                   insert_all(Children,R,Open),
+                   search(Open,S).
+
+insert_all([F|R],Open1,Open3) :- insert(F,Open1,Open2),
+                                 insert_all(R,Open2,Open3).
+insert_all([],Open,Open).
+
+insert(B,Open,Open) :- repeat_node(B,Open), ! .
+insert(B,[C|R],[B,C|R]) :- cheaper(B,C), ! .
+insert(B,[B1|R],[B1|S]) :- insert(B,R,S), !.
+insert(B,[],[B]).
+
+repeat_node(P#_#_#_, [P#_#_#_|_]).
+
+cheaper( _#_#F1#_ , _#_#F2#_ ) :- F1 < F2.
+
+expand(State#D#_#S,All_My_Children) :-
+     bagof(Child#D1#F#[Move|S],
+           (D1 is D+1,
+             move(State,Child,Move),
+             f_function(Child,D1,F)),
+           All_My_Children).
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%
+%%%     8-puzzle solver
+%%%
+%%%
+%%%     State have form    A/B/C/D/E/F/G/H/I
+%%%            where {A,...,I} = {0,...,8}
+%%%               0 represents the empty tile
+%%%
+
+goal(1/2/3/8/0/4/7/6/5).
+
+%%% The puzzle moves
+
+left( A/0/C/D/E/F/H/I/J , 0/A/C/D/E/F/H/I/J ).
+left( A/B/C/D/0/F/H/I/J , A/B/C/0/D/F/H/I/J ).
+left( A/B/C/D/E/F/H/0/J , A/B/C/D/E/F/0/H/J ).
+left( A/B/0/D/E/F/H/I/J , A/0/B/D/E/F/H/I/J ).
+left( A/B/C/D/E/0/H/I/J , A/B/C/D/0/E/H/I/J ).
+left( A/B/C/D/E/F/H/I/0 , A/B/C/D/E/F/H/0/I ).
+
+up( A/B/C/0/E/F/H/I/J , 0/B/C/A/E/F/H/I/J ).
+up( A/B/C/D/0/F/H/I/J , A/0/C/D/B/F/H/I/J ).
+up( A/B/C/D/E/0/H/I/J , A/B/0/D/E/C/H/I/J ).
+up( A/B/C/D/E/F/0/I/J , A/B/C/0/E/F/D/I/J ).
+up( A/B/C/D/E/F/H/0/J , A/B/C/D/0/F/H/E/J ).
+up( A/B/C/D/E/F/H/I/0 , A/B/C/D/E/0/H/I/F ).
+
+right( A/0/C/D/E/F/H/I/J , A/C/0/D/E/F/H/I/J ).
+right( A/B/C/D/0/F/H/I/J , A/B/C/D/F/0/H/I/J ).
+right( A/B/C/D/E/F/H/0/J , A/B/C/D/E/F/H/J/0 ).
+right( 0/B/C/D/E/F/H/I/J , B/0/C/D/E/F/H/I/J ).
+right( A/B/C/0/E/F/H/I/J , A/B/C/E/0/F/H/I/J ).
+right( A/B/C/D/E/F/0/I/J , A/B/C/D/E/F/I/0/J ).
+
+down( A/B/C/0/E/F/H/I/J , A/B/C/H/E/F/0/I/J ).
+down( A/B/C/D/0/F/H/I/J , A/B/C/D/I/F/H/0/J ).
+down( A/B/C/D/E/0/H/I/J , A/B/C/D/E/J/H/I/0 ).
+down( 0/B/C/D/E/F/H/I/J , D/B/C/0/E/F/H/I/J ).
+down( A/0/C/D/E/F/H/I/J , A/E/C/D/0/F/H/I/J ).
+down( A/B/0/D/E/F/H/I/J , A/B/F/D/E/0/H/I/J ).
+
+%%% the heuristic function
+h_function(Puzz,H) :- p_fcn(Puzz,P),
+                      s_fcn(Puzz,S),
+                      H is P + 3*S.
+
+
+%%% the move
+move(P,C,left) :-  left(P,C).
+move(P,C,up) :-  up(P,C).
+move(P,C,right) :-  right(P,C).
+move(P,C,down) :-  down(P,C).
+
+%%% the Manhattan distance function
+p_fcn(A/B/C/D/E/F/G/H/I, P) :-
+     a(A,Pa), b(B,Pb), c(C,Pc),
+     d(D,Pd), e(E,Pe), f(F,Pf),
+     g(G,Pg), h(H,Ph), i(I,Pi),
+     P is Pa+Pb+Pc+Pd+Pe+Pf+Pg+Ph+Pg+Pi.
+
+a(0,0). a(1,0). a(2,1). a(3,2). a(4,3). a(5,4). a(6,3). a(7,2). a(8,1).
+b(0,0). b(1,1). b(2,0). b(3,1). b(4,2). b(5,3). b(6,2). b(7,3). b(8,2).
+c(0,0). c(1,2). c(2,1). c(3,0). c(4,1). c(5,2). c(6,3). c(7,4). c(8,3).
+d(0,0). d(1,1). d(2,2). d(3,3). d(4,2). d(5,3). d(6,2). d(7,2). d(8,0).
+e(0,0). e(1,2). e(2,1). e(3,2). e(4,1). e(5,2). e(6,1). e(7,2). e(8,1).
+f(0,0). f(1,3). f(2,2). f(3,1). f(4,0). f(5,1). f(6,2). f(7,3). f(8,2).
+g(0,0). g(1,2). g(2,3). g(3,4). g(4,3). g(5,2). g(6,2). g(7,0). g(8,1).
+h(0,0). h(1,3). h(2,3). h(3,3). h(4,2). h(5,1). h(6,0). h(7,1). h(8,2).
+i(0,0). i(1,4). i(2,3). i(3,2). i(4,1). i(5,0). i(6,1). i(7,2). i(8,3).
+
+%%% the out-of-cycle function
+s_fcn(A/B/C/D/E/F/G/H/I, S) :-
+     s_aux(A,B,S1), s_aux(B,C,S2), s_aux(C,F,S3),
+     s_aux(F,I,S4), s_aux(I,H,S5), s_aux(H,G,S6),
+     s_aux(G,D,S7), s_aux(D,A,S8), s_aux(E,S9),
+     S is S1+S2+S3+S4+S5+S6+S7+S8+S9.
+
+s_aux(0,0) :- !.
+s_aux(_,1).
+
+s_aux(X,Y,0) :- Y is X+1, !.
+s_aux(8,1,0) :- !.
+s_aux(_,_,2).
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%
+%%%     8-puzzle animation -- using VT100 character graphics
+%%%
+%%%
+%%%
+
+puzzle(P) :- solve(P,S),
+             animate(P,S),
+             message.
+
+animate(P,S) :- initialize(P),
+                cursor(1,2), write(S),
+                cursor(1,22), write('Hit ENTER to step solver.'),
+                get0(_X),
+                play_back(S).
+
+:- dynamic location/3.
+
+initialize(A/B/C/D/E/F/H/I/J) :-
+  cls,
+  retractall(location(_,_,_)),
+  assert(location(A,20,5)),
+  assert(location(B,30,5)),
+  assert(location(C,40,5)),
+  assert(location(F,40,10)),
+  assert(location(J,40,15)),
+  assert(location(I,30,15)),
+  assert(location(H,20,15)),
+  assert(location(D,20,10)),
+  assert(location(E,30,10)), draw_all.
+
+draw_all :- draw(1), draw(2), draw(3), draw(4),
+            draw(5), draw(6), draw(7), draw(8).
+
+%%% play_back([left,right,up,...]).
+play_back([M|R]) :- call(M), get0(_X), play_back(R).
+play_back([]) :- cursor(1,24).  %%% Put cursor out of the way
+
+message :- nl,nl,
+   write('    ********************************************'), nl,
+   write('    *  Enter 8-puzzle goals in the form ...    *'), nl,
+   write('    *     ?-  puzzle(0/8/1/2/4/3/7/6/5).       *'), nl,
+   write('    *   Enter goal ''message'' to reread this.   *'), nl,
+   write('    ********************************************'), nl, nl.
+
+
+cursor(X,Y) :- put(27), put(91), %%% ESC [
+               write(Y),
+               put(59),           %%%   ;
+               write(X),
+               put(72).           %%%   M
+
+%%% clear the screen, quickly
+cls :-  put(27), put("["), put("2"), put("J").
+
+%%% video attributes -- bold and blink not working
+plain         :- put(27), put("["), put("0"), put("m").
+reverse_video :- put(27), put("["), put("7"), put("m").
+
+
+%%% Tile objects, character map(s)
+%%% Each tile should be drawn using the character map,
+%%%   drawn at 'location', which is asserted and retracted
+%%%   by 'playback'.
+character_map(N, [ [' ',' ',' ',' ',' ',' ',' '],
+                   [' ',' ',' ', N ,' ',' ',' '],
+                   [' ',' ',' ',' ',' ',' ',' '] ]).
+
+
+%%% move empty tile (spot) to the left
+left :- retract(location(0,X0,Y0)),
+        Xnew is X0 - 10,
+        location(Tile,Xnew,Y0),
+        assert(location(0,Xnew,Y0)),
+        right(Tile),right(Tile),right(Tile),
+        right(Tile),right(Tile),
+        right(Tile),right(Tile),right(Tile),
+        right(Tile),right(Tile).
+
+up :- retract(location(0,X0,Y0)),
+      Ynew is Y0 - 5,
+      location(Tile,X0,Ynew),
+      assert(location(0,X0,Ynew)),
+      down(Tile),down(Tile),down(Tile),down(Tile),down(Tile).
+
+right :- retract(location(0,X0,Y0)),
+         Xnew is X0 + 10,
+         location(Tile,Xnew,Y0),
+         assert(location(0,Xnew,Y0)),
+         left(Tile),left(Tile),left(Tile),left(Tile),left(Tile),
+         left(Tile),left(Tile),left(Tile),left(Tile),left(Tile).
+
+down :- retract(location(0,X0,Y0)),
+        Ynew is Y0 + 5,
+        location(Tile,X0,Ynew),
+        assert(location(0,X0,Ynew)),
+        up(Tile),up(Tile),up(Tile),up(Tile),up(Tile).
+
+
+draw(Obj) :- reverse_video, character_map(Obj,M),
+             location(Obj,X,Y),
+             draw(X,Y,M), plain.
+
+%%% hide tile
+hide(Obj) :- character_map(Obj,M),
+             location(Obj,X,Y),
+             hide(X,Y,M).
+
+hide(_,_,[]).
+hide(X,Y,[R|G]) :- hide_row(X,Y,R),
+                   Y1 is Y + 1,
+                   hide(X,Y1,G).
+
+hide_row(_,_,[]).
+hide_row(X,Y,[_|R]) :- cursor(X,Y),
+                       write(' '),
+                       X1 is X + 1,
+                       hide_row(X1,Y,R).
+
+%%% draw tile
+draw(_,_,[]).
+draw(X,Y,[R|G]) :- draw_row(X,Y,R),
+                   Y1 is Y + 1,
+                   draw(X,Y1,G).
+
+draw_row(_,_,[]).
+draw_row(X,Y,[P|R]) :- cursor(X,Y),
+                       write(P),
+                       X1 is X + 1,
+                       draw_row(X1,Y,R).
+
+%%% Move an Object up
+up(Obj)    :- hide(Obj),
+              retract(location(Obj,X,Y)),
+              Y1 is Y - 1,
+              assert(location(Obj,X,Y1)),
+              draw(Obj).
+
+down(Obj)  :- hide(Obj),
+              retract(location(Obj,X,Y)),
+              Y1 is Y + 1,
+              assert(location(Obj,X,Y1)),
+              draw(Obj).
+
+left(Obj)  :- hide(Obj),
+              retract(location(Obj,X,Y)),
+              X1 is X - 1,
+              assert(location(Obj,X1,Y)),
+              draw(Obj).
+
+right(Obj) :- hide(Obj),
+              retract(location(Obj,X,Y)),
+              X1 is X + 1,
+              assert(location(Obj,X1,Y)),
+              draw(Obj).
+
+:- message.

samples/Prolog/quicksort.pl

@@ -0,0 +1,13 @@
+partition([], _, [], []).
+partition([X|Xs], Pivot, Smalls, Bigs) :-
+    (   X @< Pivot ->
+        Smalls = [X|Rest],
+        partition(Xs, Pivot, Rest, Bigs)
+    ;   Bigs = [X|Rest],
+        partition(Xs, Pivot, Smalls, Rest)
+    ).
+
+quicksort([])     --> [].
+quicksort([X|Xs]) -->
+    { partition(Xs, X, Smaller, Bigger) },
+    quicksort(Smaller), [X], quicksort(Bigger).

samples/Prolog/turing.pl

@@ -0,0 +1,21 @@
+turing(Tape0, Tape) :-
+    perform(q0, [], Ls, Tape0, Rs),
+    reverse(Ls, Ls1),
+    append(Ls1, Rs, Tape).
+
+perform(qf, Ls, Ls, Rs, Rs) :- !.
+perform(Q0, Ls0, Ls, Rs0, Rs) :-
+    symbol(Rs0, Sym, RsRest),
+    once(rule(Q0, Sym, Q1, NewSym, Action)),
+    action(Action, Ls0, Ls1, [NewSym|RsRest], Rs1),
+    perform(Q1, Ls1, Ls, Rs1, Rs).
+
+symbol([], b, []).
+symbol([Sym|Rs], Sym, Rs).
+
+action(left, Ls0, Ls, Rs0, Rs) :- left(Ls0, Ls, Rs0, Rs).
+action(stay, Ls, Ls, Rs, Rs).
+action(right, Ls0, [Sym|Ls0], [Sym|Rs], Rs).
+
+left([], [], Rs0, [b|Rs0]).
+left([L|Ls], Ls, Rs, [L|Rs]).

samples/Ruby/ruby2.script!

@@ -1,2 +1,2 @@
 #! /usr/bin/env ruby -w -Ilib:test
-echo "Ruby"
+puts "Ruby"

samples/Shell/bash.script!

@@ -0,0 +1,2 @@
+#!/bin/bash
+echo "bash"

samples/Shell/plugin.script!

@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+#	Bash script to "plugin" to the dotfile repository, does a lot of fun stuff
+#		like turning the normal dotfiles (eg .bashrc) into symlinks to this
+#		repository's versions of those files (ensure that updates are just a
+#		'git pull' away), optionally moving old files so that they can be
+#		preserved, setting up a cron job to automate the aforementioned git
+#		pull, and maybe some more fun stuff
+#
+
+shopt -s nocasematch	# This makes pattern matching case-insensitive
+
+POSTFIX="local"
+URL="https://github.com/brcooley/.f.git"
+PUSHURL="git@github.com:brcooley/.f.git"
+
+overwrite=true
+
+print_help () {
+	echo -e "\nA script to keep dotfiles up to date\n"
+	echo "Options:"
+	echo "    -k, --keep-local    Keeps local copies of dotfiles by appending"
+	echo "                        \"$POSTFIX\" to them"
+	exit 0
+}
+
+
+for opt in $@; do
+	case $opt in
+		-k | --keep-local) overwrite=false;;
+		-h | --help) print_help;;
+	esac
+done
+
+
+
+for f in .*; do
+	if [ "$f" = ".git" -o "$f" = "." -o "$f" = ".." -o "$f" = ".f" ]; then continue; fi
+	if [ -f "$HOME/$f" ]; then
+		if [ $overwrite = false ]; then
+			mv "$HOME/$f" "$HOME/${f}_$POSTFIX"
+		else
+			rm "$HOME/$f"
+		fi
+		# echo "Moving ~/$f to $HOME/${f}_$POSTFIX"
+	fi
+	ln -s "$PWD/$f" "$HOME/$f"
+done
+
+# Git versions prior to 1.7.? (1.7.1 confirmed) do not have a --local option
+git config --local remote.origin.url "$URL"
+git config --local remote.origin.pushurl "$PUSHURL"
+crontab .jobs.cron
+source ~/.bashrc
+echo "Plugin succesful"

samples/Shell/sbt.script!

@@ -0,0 +1,432 @@
+#!/usr/bin/env bash
+#
+# A more capable sbt runner, coincidentally also called sbt.
+# Author: Paul Phillips <paulp@typesafe.com>
+
+# todo - make this dynamic
+declare -r sbt_release_version=0.11.3
+declare -r sbt_snapshot_version=0.13.0-SNAPSHOT
+
+unset sbt_jar sbt_dir sbt_create sbt_snapshot sbt_launch_dir
+unset scala_version java_home sbt_explicit_version
+unset verbose debug quiet
+
+build_props_sbt () {
+  if [[ -f project/build.properties ]]; then
+    versionLine=$(grep ^sbt.version project/build.properties)
+    versionString=${versionLine##sbt.version=}
+    echo "$versionString"
+  fi
+}
+
+update_build_props_sbt () {
+  local ver="$1"
+  local old=$(build_props_sbt)
+
+  if [[ $ver == $old ]]; then
+    return
+  elif [[ -f project/build.properties ]]; then
+    perl -pi -e "s/^sbt\.version=.*\$/sbt.version=${ver}/" project/build.properties
+    grep -q '^sbt.version=' project/build.properties || echo "sbt.version=${ver}" >> project/build.properties
+
+    echo !!!
+    echo !!! Updated file project/build.properties setting sbt.version to: $ver
+    echo !!! Previous value was: $old
+    echo !!!
+  fi
+}
+
+sbt_version () {
+  if [[ -n $sbt_explicit_version ]]; then
+    echo $sbt_explicit_version
+  else
+    local v=$(build_props_sbt)
+    if [[ -n $v ]]; then
+      echo $v
+    else
+      echo $sbt_release_version
+    fi
+  fi
+}
+
+echoerr () {
+  echo 1>&2 "$@"
+}
+vlog () {
+  [[ $verbose || $debug ]] && echoerr "$@"
+}
+dlog () {
+  [[ $debug ]] && echoerr "$@"
+}
+
+# this seems to cover the bases on OSX, and someone will
+# have to tell me about the others.
+get_script_path () {
+  local path="$1"
+  [[ -L "$path" ]] || { echo "$path" ; return; }
+
+  local target=$(readlink "$path")
+  if [[ "${target:0:1}" == "/" ]]; then
+    echo "$target"
+  else
+    echo "$(dirname $path)/$target"
+  fi
+}
+
+# a ham-fisted attempt to move some memory settings in concert
+# so they need not be dicked around with individually.
+get_mem_opts () {
+  local mem=${1:-1536}
+  local perm=$(( $mem / 4 ))
+  (( $perm > 256 )) || perm=256
+  (( $perm < 1024 )) || perm=1024
+  local codecache=$(( $perm / 2 ))
+  
+  echo "-Xms${mem}m -Xmx${mem}m -XX:MaxPermSize=${perm}m -XX:ReservedCodeCacheSize=${codecache}m"
+}
+
+die() {
+  echo "Aborting: $@"
+  exit 1
+}
+
+make_url () {
+  groupid="$1"
+  category="$2"
+  version="$3"
+  
+  echo "http://typesafe.artifactoryonline.com/typesafe/ivy-$category/$groupid/sbt-launch/$version/sbt-launch.jar"
+}
+
+declare -r default_jvm_opts="-Dfile.encoding=UTF8"
+declare -r default_sbt_opts="-XX:+CMSClassUnloadingEnabled"
+declare -r default_sbt_mem=1536
+declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
+declare -r sbt_opts_file=".sbtopts"
+declare -r jvm_opts_file=".jvmopts"
+declare -r latest_28="2.8.2"
+declare -r latest_29="2.9.1"
+declare -r latest_210="2.10.0-SNAPSHOT"
+
+declare -r script_path=$(get_script_path "$BASH_SOURCE")
+declare -r script_dir="$(dirname $script_path)"
+declare -r script_name="$(basename $script_path)"
+
+# some non-read-onlies set with defaults
+declare java_cmd=java
+declare sbt_launch_dir="$script_dir/.lib"
+declare sbt_mem=$default_sbt_mem
+
+# pull -J and -D options to give to java.
+declare -a residual_args
+declare -a java_args
+declare -a scalac_args
+declare -a sbt_commands
+
+build_props_scala () {
+  if [[ -f project/build.properties ]]; then
+    versionLine=$(grep ^build.scala.versions project/build.properties)
+    versionString=${versionLine##build.scala.versions=}
+    echo ${versionString%% .*}
+  fi
+}
+
+execRunner () {
+  # print the arguments one to a line, quoting any containing spaces
+  [[ $verbose || $debug ]] && echo "# Executing command line:" && {
+    for arg; do
+      if printf "%s\n" "$arg" | grep -q ' '; then
+        printf "\"%s\"\n" "$arg"
+      else
+        printf "%s\n" "$arg"
+      fi
+    done
+    echo ""
+  }
+
+  exec "$@"
+}
+
+sbt_groupid () {
+  case $(sbt_version) in
+        0.7.*) echo org.scala-tools.sbt ;;
+       0.10.*) echo org.scala-tools.sbt ;;
+    0.11.[12]) echo org.scala-tools.sbt ;;
+            *) echo org.scala-sbt ;;
+  esac
+}
+
+sbt_artifactory_list () {
+  local version0=$(sbt_version)
+  local version=${version0%-SNAPSHOT}
+  local url="http://typesafe.artifactoryonline.com/typesafe/ivy-snapshots/$(sbt_groupid)/sbt-launch/"
+  dlog "Looking for snapshot list at: $url "
+  
+  curl -s --list-only "$url" | \
+    grep -F $version | \
+    perl -e 'print reverse <>' | \
+    perl -pe 's#^<a href="([^"/]+).*#$1#;'
+}
+
+make_release_url () {
+  make_url $(sbt_groupid) releases $(sbt_version)
+}
+
+# argument is e.g. 0.13.0-SNAPSHOT
+# finds the actual version (with the build id) at artifactory
+make_snapshot_url () {
+  for ver in $(sbt_artifactory_list); do
+    local url=$(make_url $(sbt_groupid) snapshots $ver)
+    dlog "Testing $url"
+    curl -s --head "$url" >/dev/null
+    dlog "curl returned: $?"
+    echo "$url"
+    return
+  done
+}
+
+jar_url () {
+  case $(sbt_version) in
+             0.7.*) echo "http://simple-build-tool.googlecode.com/files/sbt-launch-0.7.7.jar" ;;
+        *-SNAPSHOT) make_snapshot_url ;;
+                 *) make_release_url ;;
+  esac
+}
+
+jar_file () {
+  echo "$sbt_launch_dir/$1/sbt-launch.jar"
+}
+
+download_url () {
+  local url="$1"
+  local jar="$2"
+  
+  echo "Downloading sbt launcher $(sbt_version):"
+  echo "  From  $url"
+  echo "    To  $jar"
+
+  mkdir -p $(dirname "$jar") && {
+    if which curl >/dev/null; then
+      curl --fail --silent "$url" --output "$jar"
+    elif which wget >/dev/null; then
+      wget --quiet -O "$jar" "$url"
+    fi
+  } && [[ -f "$jar" ]]
+}
+
+acquire_sbt_jar () {
+  sbt_url="$(jar_url)"
+  sbt_jar="$(jar_file $(sbt_version))"
+
+  [[ -f "$sbt_jar" ]] || download_url "$sbt_url" "$sbt_jar"
+}
+
+usage () {
+  cat <<EOM
+Usage: $script_name [options]
+
+  -h | -help         print this message
+  -v | -verbose      this runner is chattier
+  -d | -debug        set sbt log level to Debug
+  -q | -quiet        set sbt log level to Error
+  -no-colors         disable ANSI color codes
+  -sbt-create        start sbt even if current directory contains no sbt project
+  -sbt-dir   <path>  path to global settings/plugins directory (default: ~/.sbt/<version>)
+  -sbt-boot  <path>  path to shared boot directory (default: ~/.sbt/boot in 0.11 series)
+  -ivy       <path>  path to local Ivy repository (default: ~/.ivy2)
+  -mem    <integer>  set memory options (default: $sbt_mem, which is
+                       $(get_mem_opts $sbt_mem) )
+  -no-share          use all local caches; no sharing
+  -offline           put sbt in offline mode
+  -jvm-debug <port>  Turn on JVM debugging, open at the given port.
+  -batch             Disable interactive mode
+
+  # sbt version (default: from project/build.properties if present, else latest release)
+  !!! The only way to accomplish this pre-0.12.0 if there is a build.properties file which
+  !!! contains an sbt.version property is to update the file on disk.  That's what this does.
+  -sbt-version  <version>   use the specified version of sbt 
+  -sbt-jar      <path>      use the specified jar as the sbt launcher
+  -sbt-snapshot             use a snapshot version of sbt
+  -sbt-launch-dir <path>    directory to hold sbt launchers (default: $sbt_launch_dir)
+
+  # scala version (default: as chosen by sbt)
+  -28                       use $latest_28
+  -29                       use $latest_29
+  -210                      use $latest_210
+  -scala-home <path>        use the scala build at the specified directory
+  -scala-version <version>  use the specified version of scala
+
+  # java version (default: java from PATH, currently $(java -version |& grep version))
+  -java-home <path>         alternate JAVA_HOME
+
+  # jvm options and output control
+  JAVA_OPTS     environment variable holding jvm args, if unset uses "$default_jvm_opts"
+  SBT_OPTS      environment variable holding jvm args, if unset uses "$default_sbt_opts"
+  .jvmopts      if file is in sbt root, it is prepended to the args given to the jvm
+  .sbtopts      if file is in sbt root, it is prepended to the args given to **sbt**
+  -Dkey=val     pass -Dkey=val directly to the jvm
+  -J-X          pass option -X directly to the jvm (-J is stripped)
+  -S-X          add -X to sbt's scalacOptions (-S is stripped)
+
+In the case of duplicated or conflicting options, the order above
+shows precedence: JAVA_OPTS lowest, command line options highest.
+EOM
+}
+
+addJava () {
+  dlog "[addJava] arg = '$1'"
+  java_args=( "${java_args[@]}" "$1" )
+}
+addSbt () {
+  dlog "[addSbt] arg = '$1'"
+  sbt_commands=( "${sbt_commands[@]}" "$1" )
+}
+addScalac () {
+  dlog "[addScalac] arg = '$1'"
+  scalac_args=( "${scalac_args[@]}" "$1" )
+}
+addResidual () {
+  dlog "[residual] arg = '$1'"
+  residual_args=( "${residual_args[@]}" "$1" )
+}
+addResolver () {
+  addSbt "set resolvers in ThisBuild += $1"
+}
+addDebugger () {
+  addJava "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
+}
+get_jvm_opts () {
+  # echo "${JAVA_OPTS:-$default_jvm_opts}"
+  # echo "${SBT_OPTS:-$default_sbt_opts}"
+
+  [[ -f "$jvm_opts_file" ]] && cat "$jvm_opts_file"
+}
+
+process_args ()
+{
+  require_arg () {
+    local type="$1"
+    local opt="$2"
+    local arg="$3"
+    
+    if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then
+      die "$opt requires <$type> argument"
+    fi
+  }
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+       -h|-help) usage; exit 1 ;;
+    -v|-verbose) verbose=1 && shift ;;
+      -d|-debug) debug=1 && shift ;;
+      -q|-quiet) quiet=1 && shift ;;
+
+           -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;;
+           -mem) require_arg integer "$1" "$2" && sbt_mem="$2" && shift 2 ;;
+     -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;;
+      -no-share) addJava "$noshare_opts" && shift ;;
+      -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
+       -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;;
+     -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;;
+       -offline) addSbt "set offline := true" && shift ;;
+     -jvm-debug) require_arg port "$1" "$2" && addDebugger $2 && shift 2 ;;
+         -batch) exec </dev/null && shift ;;
+
+    -sbt-create) sbt_create=true && shift ;;
+  -sbt-snapshot) sbt_explicit_version=$sbt_snapshot_version && shift ;;
+       -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;;
+   -sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;;
+-sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;;
+ -scala-version) require_arg version "$1" "$2" && addSbt "set scalaVersion := \"$2\"" && shift 2 ;;
+    -scala-home) require_arg path "$1" "$2" && addSbt "set scalaHome in ThisBuild := Some(file(\"$2\"))" && shift 2 ;;
+     -java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && shift 2 ;;
+
+            -D*) addJava "$1" && shift ;;
+            -J*) addJava "${1:2}" && shift ;;
+            -S*) addScalac "${1:2}" && shift ;;
+            -28) addSbt "++ $latest_28" && shift ;;
+            -29) addSbt "++ $latest_29" && shift ;;
+           -210) addSbt "++ $latest_210" && shift ;;
+
+              *) addResidual "$1" && shift ;;
+    esac
+  done
+  
+  [[ $debug ]] && {
+    case $(sbt_version) in
+     0.7.*) addSbt "debug" ;; 
+         *) addSbt "set logLevel in Global := Level.Debug" ;;
+    esac
+  }
+  [[ $quiet ]] && {
+    case $(sbt_version) in
+     0.7.*) ;; 
+         *) addSbt "set logLevel in Global := Level.Error" ;;
+    esac
+  }
+}
+
+# if .sbtopts exists, prepend its contents to $@ so it can be processed by this runner
+[[ -f "$sbt_opts_file" ]] && {
+  sbtargs=()
+  while IFS= read -r arg; do
+    sbtargs=( "${sbtargs[@]}" "$arg" )
+  done <"$sbt_opts_file"
+
+  set -- "${sbtargs[@]}" "$@"
+}
+
+# process the combined args, then reset "$@" to the residuals
+process_args "$@"
+set -- "${residual_args[@]}"
+argumentCount=$#
+
+# set scalacOptions if we were given any -S opts
+[[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[@]}\""
+
+# Update build.properties no disk to set explicit version - sbt gives us no choice
+[[ -n "$sbt_explicit_version" ]] && update_build_props_sbt "$sbt_explicit_version"
+echo "Detected sbt version $(sbt_version)"
+
+[[ -n "$scala_version" ]] && echo "Overriding scala version to $scala_version"
+
+# no args - alert them there's stuff in here
+(( $argumentCount > 0 )) || echo "Starting $script_name: invoke with -help for other options"
+
+# verify this is an sbt dir or -create was given
+[[ -f ./build.sbt || -d ./project || -n "$sbt_create" ]] || {
+  cat <<EOM
+$(pwd) doesn't appear to be an sbt project.
+If you want to start sbt anyway, run:
+  $0 -sbt-create
+
+EOM
+  exit 1
+}
+
+# pick up completion if present; todo
+[[ -f .sbt_completion.sh ]] && source .sbt_completion.sh
+
+# no jar? download it.
+[[ -f "$sbt_jar" ]] || acquire_sbt_jar || {
+  # still no jar? uh-oh.
+  echo "Download failed. Obtain the jar manually and place it at $sbt_jar"
+  exit 1
+}
+
+[[ -n "$sbt_dir" ]] || {
+  sbt_dir=~/.sbt/$(sbt_version)
+  addJava "-Dsbt.global.base=$sbt_dir"
+  echo "Using $sbt_dir as sbt dir, -sbt-dir to override."
+}
+
+# since sbt 0.7 doesn't understand iflast
+(( ${#residual_args[@]} == 0 )) && residual_args=( "shell" )
+
+# run sbt
+execRunner "$java_cmd" \
+  $(get_mem_opts $sbt_mem) \
+  $(get_jvm_opts) \
+  ${java_args[@]} \
+  -jar "$sbt_jar" \
+  "${sbt_commands[@]}" \
+  "${residual_args[@]}"

samples/Shell/sh.script!

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "sh"

samples/Shell/zsh.script!

@@ -0,0 +1,2 @@
+#!/bin/zsh
+echo "zsh"

samples/Text/script.foo

@@ -1,2 +0,0 @@
-#!/bin/foo
-???

test/test_blob.rb

@@ -2,6 +2,7 @@ require 'linguist/file_blob'
 require 'linguist/samples'
 
 require 'test/unit'
+require 'mocha'
 require 'mime/types'
 require 'pygments'
 
@@ -30,25 +31,19 @@ class TestBlob < Test::Unit::TestCase
   end
 
   def test_mime_type
-    assert_equal "application/octet-stream", blob("Binary/dog.o").mime_type
-    assert_equal "application/ogg", blob("Binary/foo.ogg").mime_type
     assert_equal "application/postscript", blob("Binary/octocat.ai").mime_type
     assert_equal "application/x-ruby", blob("Ruby/grit.rb").mime_type
     assert_equal "application/x-sh", blob("Shell/script.sh").mime_type
     assert_equal "application/xml", blob("XML/bar.xml").mime_type
+    assert_equal "audio/ogg", blob("Binary/foo.ogg").mime_type
     assert_equal "text/plain", blob("Text/README").mime_type
   end
 
   def test_content_type
-    assert_equal "application/octet-stream", blob("Binary/dog.o").content_type
-    assert_equal "application/ogg", blob("Binary/foo.ogg").content_type
     assert_equal "application/pdf", blob("Binary/foo.pdf").content_type
+    assert_equal "audio/ogg", blob("Binary/foo.ogg").content_type
     assert_equal "image/png", blob("Binary/foo.png").content_type
     assert_equal "text/plain; charset=iso-8859-2", blob("Text/README").content_type
-    assert_equal "text/plain; charset=iso-8859-1", blob("Perl/script.pl").content_type
-    assert_equal "text/plain; charset=iso-8859-1", blob("Python/script.py").content_type
-    assert_equal "text/plain; charset=iso-8859-1", blob("Ruby/script.rb").content_type
-    assert_equal "text/plain; charset=iso-8859-1", blob("Shell/script.sh").content_type
   end
 
   def test_disposition
@@ -262,17 +257,23 @@ class TestBlob < Test::Unit::TestCase
   end
 
   def test_indexable
-    assert blob("Text/file.txt").indexable?
     assert blob("Ruby/foo.rb").indexable?
     assert !blob("Text/defu.nkt").indexable?
     assert !blob("Text/dump.sql").indexable?
     assert !blob("Binary/github.po").indexable?
     assert !blob("Binary/linguist.gem").indexable?
+
+    # large binary blobs should fail on size check first, not call 
+    # into charlock_holmes and alloc big buffers for testing encoding
+    b = blob("Binary/octocat.ai")
+    b.expects(:binary?).never
+    assert !b.indexable?
   end
 
   def test_language
     Samples.each do |sample|
       blob = blob(sample[:path])
+      assert blob.language, "No language for #{sample[:path]}"
       assert_equal sample[:language], blob.language.name, blob.name
     end
   end
@@ -281,25 +282,6 @@ class TestBlob < Test::Unit::TestCase
     assert_equal Lexer['Ruby'], blob("Ruby/foo.rb").lexer
   end
 
-  def test_shebang_script
-    assert_equal 'sh', script_blob("Shell/script.sh").shebang_script
-    assert_equal 'bash', script_blob("Shell/script.bash").shebang_script
-    assert_equal 'zsh', script_blob("Shell/script.zsh").shebang_script
-    assert_equal 'perl', script_blob("Perl/script.pl").shebang_script
-    assert_equal 'ruby', script_blob("Ruby/script.rb").shebang_script
-    assert_equal 'ruby', script_blob("Ruby/script2.rb").shebang_script
-    assert_equal 'python', script_blob("Python/script.py").shebang_script
-    assert_equal 'node', script_blob("JavaScript/script.js").shebang_script
-    assert_equal 'groovy', script_blob("Groovy/script.groovy").shebang_script
-    assert_equal 'macruby', script_blob("Ruby/macruby-script").shebang_script
-    assert_equal 'rake', script_blob("Ruby/script.rake").shebang_script
-    assert_equal 'foo', script_blob("Text/script.foo").shebang_script
-    assert_equal 'nush', script_blob("Nu/script.nu").shebang_script
-    assert_equal 'scala', script_blob("Scala/script.scala").shebang_script
-    assert_equal 'racket', script_blob("Racket/script.rkt").shebang_script
-    assert_equal nil, script_blob("Ruby/foo.rb").shebang_script
-  end
-
   def test_colorize
     assert_equal <<-HTML, blob("Ruby/foo.rb").colorize
 <div class="highlight"><pre><span class="k">module</span> <span class="nn">Foo</span>

test/test_classifier.rb

@@ -54,11 +54,8 @@ class TestClassifier < Test::Unit::TestCase
 
   def test_classify_ambiguous_languages
     Samples.each do |sample|
-      language = Linguist::Language.find_by_name(sample[:language])
-      next unless language.overrides.any?
-
-      extname   = File.extname(sample[:path])
-      languages = Language.all.select { |l| l.extensions.include?(extname) }.map(&:name)
+      language  = Linguist::Language.find_by_name(sample[:language])
+      languages = Language.find_by_filename(sample[:path]).map(&:name)
       next unless languages.length > 1
 
       results = Classifier.classify(Samples::DATA, File.read(sample[:path]), languages)

test/test_language.rb

@@ -8,29 +8,6 @@ class TestLanguage < Test::Unit::TestCase
 
   Lexer = Pygments::Lexer
 
-  def test_ambiguous_extensions
-    assert Language.ambiguous?('.cls')
-    assert_equal Language['Apex'], Language.find_by_extension('cls')
-
-    assert Language.ambiguous?('.h')
-    assert_equal Language['C'], Language.find_by_extension('h')
-
-    assert Language.ambiguous?('.m')
-    assert_equal Language['Objective-C'], Language.find_by_extension('m')
-
-    assert Language.ambiguous?('.pl')
-    assert_equal Language['Perl'], Language.find_by_extension('pl')
-
-    assert Language.ambiguous?('.r')
-    assert_equal Language['R'], Language.find_by_extension('r')
-
-    assert Language.ambiguous?('.t')
-    assert_equal Language['Turing'], Language.find_by_extension('t')
-
-    assert Language.ambiguous?('.v')
-    assert_equal Language['Verilog'], Language.find_by_extension('v')
-  end
-
   def test_lexer
     assert_equal Lexer['ActionScript 3'], Language['ActionScript'].lexer
     assert_equal Lexer['Bash'], Language['Gentoo Ebuild'].lexer
@@ -71,7 +48,6 @@ class TestLanguage < Test::Unit::TestCase
     assert_equal Lexer['Scheme'], Language['Scheme'].lexer
     assert_equal Lexer['Standard ML'], Language['Standard ML'].lexer
     assert_equal Lexer['TeX'], Language['TeX'].lexer
-    assert_equal Lexer['Text only'], Language['Text'].lexer
     assert_equal Lexer['Verilog'], Language['Verilog'].lexer
     assert_equal Lexer['XSLT'], Language['XSLT'].lexer
     assert_equal Lexer['aspx-vb'], Language['ASP'].lexer
@@ -165,7 +141,7 @@ class TestLanguage < Test::Unit::TestCase
     assert_equal 'ruby',        Language['Ruby'].search_term
     assert_equal 'common-lisp', Language['Common Lisp'].search_term
     assert_equal 'html+erb',    Language['HTML+ERB'].search_term
-    assert_equal 'max/msp',     Language['Max/MSP'].search_term
+    assert_equal 'max/msp',     Language['Max'].search_term
     assert_equal 'puppet',      Language['Puppet'].search_term
     assert_equal 'pure-data',   Language['Pure Data'].search_term
 
@@ -242,48 +218,16 @@ class TestLanguage < Test::Unit::TestCase
     end
   end
 
-  def test_find_by_extension
-    assert_equal Language['Ruby'], Language.find_by_extension('.rb')
-    assert_equal Language['Ruby'], Language.find_by_extension('rb')
-    assert_equal Language['Dart'], Language.find_by_extension('dart')
-    assert_equal Language['Groff'], Language.find_by_extension('man')
-    assert_equal Language['Groff'], Language.find_by_extension('1')
-    assert_equal Language['Groff'], Language.find_by_extension('2')
-    assert_equal Language['Groff'], Language.find_by_extension('3')
-    assert_equal Language['PHP'], Language.find_by_extension('php')
-    assert_equal Language['PHP'], Language.find_by_extension('php3')
-    assert_equal Language['PHP'], Language.find_by_extension('php4')
-    assert_equal Language['PHP'], Language.find_by_extension('php5')
-    assert_equal Language['PowerShell'], Language.find_by_extension('psm1')
-    assert_equal Language['PowerShell'], Language.find_by_extension('ps1')
-
-    # Aliases for Streamline.js ( https://github.com/Sage/streamlinejs )
-    assert_equal Language['JavaScript'], Language.find_by_extension('_js')
-    assert_equal Language['CoffeeScript'], Language.find_by_extension('_coffee')
-
-    assert_nil Language.find_by_extension('.nkt')
-  end
-
-  def test_find_all_by_extension
-    Language.all.each do |language|
-      assert_equal language, Language.find_by_extension(language.primary_extension)
-
-      language.extensions.each do |extension|
-        unless Language.ambiguous?(extension)
-          assert_equal language, Language.find_by_extension(extension)
-        end
-      end
-    end
-  end
-
   def test_find_by_filename
-    assert_equal Language['Shell'], Language.find_by_filename('PKGBUILD')
-    assert_equal Language['Ruby'], Language.find_by_filename('foo.rb')
-    assert_equal Language['Ruby'], Language.find_by_filename('foo/bar.rb')
-    assert_equal Language['Ruby'], Language.find_by_filename('Rakefile')
-    assert_nil Language.find_by_filename('rb')
-    assert_nil Language.find_by_filename('.rb')
-    assert_nil Language.find_by_filename('.nkt')
+    assert_equal [Language['Shell']], Language.find_by_filename('PKGBUILD')
+    assert_equal [Language['Ruby']], Language.find_by_filename('foo.rb')
+    assert_equal [Language['Ruby']], Language.find_by_filename('foo/bar.rb')
+    assert_equal [Language['Ruby']], Language.find_by_filename('Rakefile')
+    assert_equal [Language['Ruby']], Language.find_by_filename('PKGBUILD.rb')
+    assert_equal ['C', 'C++', 'Objective-C'], Language.find_by_filename('foo.h').map(&:name).sort
+    assert_equal [], Language.find_by_filename('rb')
+    assert_equal [], Language.find_by_filename('.rb')
+    assert_equal [], Language.find_by_filename('.nkt')
   end
 
   def test_find
@@ -310,7 +254,6 @@ class TestLanguage < Test::Unit::TestCase
     assert_equal 'C%2B%2B', Language['C++'].escaped_name
     assert_equal 'Objective-C', Language['Objective-C'].escaped_name
     assert_equal 'Common%20Lisp', Language['Common Lisp'].escaped_name
-    assert_equal 'Max%2FMSP', Language['Max/MSP'].escaped_name
   end
 
   def test_error_without_name
@@ -336,11 +279,9 @@ class TestLanguage < Test::Unit::TestCase
     assert_equal 'csharp', Language['C#'].ace_mode
     assert_equal 'css', Language['CSS'].ace_mode
     assert_equal 'javascript', Language['JavaScript'].ace_mode
-    assert_equal 'text', Language['Text'].ace_mode
   end
 
   def test_ace_modes
-    assert Language.ace_modes.include?(Language['Text'])
     assert Language.ace_modes.include?(Language['Ruby'])
     assert !Language.ace_modes.include?(Language['FORTRAN'])
   end
@@ -373,12 +314,6 @@ class TestLanguage < Test::Unit::TestCase
 
 
   def test_colorize
-    assert_equal <<-HTML, Language['Text'].colorize("Hello")
-<div class="highlight"><pre>Hello
-</pre>
-</div>
-    HTML
-
     assert_equal <<-HTML, Language['Ruby'].colorize("def foo\n  'foo'\nend\n")
 <div class="highlight"><pre><span class="k">def</span> <span class="nf">foo</span>
   <span class="s1">&#39;foo&#39;</span>

test/test_mime.rb

@@ -1,71 +0,0 @@
-require 'linguist/mime'
-
-require 'test/unit'
-
-class TestMime < Test::Unit::TestCase
-  include Linguist
-
-  def test_extension_lookup
-    # Default to plain text if we have no idea.
-    assert_equal 'text/plain', Mime.mime_for(nil)
-    assert_equal 'text/plain', Mime.mime_for('')
-
-    # Add an assertion to this list if you add/change any extensions
-    # in mimes.yml. Its still useful to test even trivial cases since
-    # MIME::Type's extension lookup may return multiple matches and we
-    # only pick one of them. Please keep this list alphabetized.
-    assert_equal 'application/javascript', Mime.mime_for('.js')
-    assert_equal 'application/octet-stream', Mime.mime_for('.dll')
-    assert_equal 'application/octet-stream', Mime.mime_for('.dmg')
-    assert_equal 'application/octet-stream', Mime.mime_for('.exe')
-    assert_equal 'application/ogg', Mime.mime_for('.ogg')
-    assert_equal 'application/postscript', Mime.mime_for('.ai')
-    assert_equal 'application/postscript', Mime.mime_for('.eps')
-    assert_equal 'application/postscript', Mime.mime_for('.ps')
-    assert_equal 'application/vnd.adobe.air-application-installer-package+zip', Mime.mime_for('.air')
-    assert_equal 'application/vnd.oasis.opendocument.presentation', Mime.mime_for('.odp')
-    assert_equal 'application/vnd.oasis.opendocument.spreadsheet', Mime.mime_for('.ods')
-    assert_equal 'application/vnd.oasis.opendocument.text', Mime.mime_for('.odt')
-    assert_equal 'application/vnd.openofficeorg.extension', Mime.mime_for('.oxt')
-    assert_equal 'application/vnd.openxmlformats-officedocument.presentationml.presentation', Mime.mime_for('.pptx')
-    assert_equal 'application/x-chrome-extension', Mime.mime_for('.crx')
-    assert_equal 'application/x-debian-package', Mime.mime_for('.deb')
-    assert_equal 'application/x-iwork-keynote-sffkey', Mime.mime_for('.key')
-    assert_equal 'application/x-iwork-numbers-sffnumbers', Mime.mime_for('.numbers')
-    assert_equal 'application/x-iwork-pages-sffpages', Mime.mime_for('.pages')
-    assert_equal 'application/x-java-archive', Mime.mime_for('.ear')
-    assert_equal 'application/x-java-archive', Mime.mime_for('.jar')
-    assert_equal 'application/x-java-archive', Mime.mime_for('.war')
-    assert_equal 'application/x-latex', Mime.mime_for('.latex')
-    assert_equal 'application/x-ms-xbap', Mime.mime_for('.xbap')
-    assert_equal 'application/x-perl', Mime.mime_for('.pl')
-    assert_equal 'application/x-perl', Mime.mime_for('.pm')
-    assert_equal 'application/x-python', Mime.mime_for('.py')
-    assert_equal 'application/x-ruby', Mime.mime_for('.rb')
-    assert_equal 'application/x-sh', Mime.mime_for('.sh')
-    assert_equal 'application/x-shockwave-flash', Mime.mime_for('.swf')
-    assert_equal 'application/x-silverlight-app', Mime.mime_for('.xap')
-    assert_equal 'application/x-supercollider', Mime.mime_for('.sc')
-    assert_equal 'application/xaml+xml', Mime.mime_for('.xaml')
-    assert_equal 'text/cache-manifest', Mime.mime_for('.manifest')
-    assert_equal 'text/html', Mime.mime_for('.html')
-    assert_equal 'text/plain', Mime.mime_for('.c')
-    assert_equal 'text/plain', Mime.mime_for('.cc')
-    assert_equal 'text/plain', Mime.mime_for('.cpp')
-    assert_equal 'text/plain', Mime.mime_for('.cu')
-    assert_equal 'text/plain', Mime.mime_for('.cxx')
-    assert_equal 'text/plain', Mime.mime_for('.h')
-    assert_equal 'text/plain', Mime.mime_for('.hh')
-    assert_equal 'text/plain', Mime.mime_for('.hpp')
-    assert_equal 'text/plain', Mime.mime_for('.kt')
-    assert_equal 'text/x-logtalk', Mime.mime_for('.lgt')
-    assert_equal 'text/x-nemerle', Mime.mime_for('.n')
-    assert_equal 'text/x-nimrod', Mime.mime_for('.nim')
-    assert_equal 'text/x-ocaml', Mime.mime_for('.ml')
-    assert_equal 'text/x-ocaml', Mime.mime_for('.sig')
-    assert_equal 'text/x-ocaml', Mime.mime_for('.sml')
-    assert_equal 'text/x-rust', Mime.mime_for('.rc')
-    assert_equal 'text/x-rust', Mime.mime_for('.rs')
-    assert_equal 'video/quicktime', Mime.mime_for('.mov')
-  end
-end

test/test_tokenizer.rb

@@ -20,24 +20,29 @@ class TestTokenizer < Test::Unit::TestCase
     assert_equal %w(print), tokenize("print 'Josh'")
     assert_equal %w(print), tokenize('print "Hello \"Josh\""')
     assert_equal %w(print), tokenize("print 'Hello \\'Josh\\''")
+    assert_equal %w(print), tokenize("print \"Hello\", \"Josh\"")
+    assert_equal %w(print), tokenize("print 'Hello', 'Josh'")
+    assert_equal %w(print), tokenize("print \"Hello\", \"\", \"Josh\"")
+    assert_equal %w(print), tokenize("print 'Hello', '', 'Josh'")
   end
 
   def test_skip_number_literals
     assert_equal %w(+), tokenize('1 + 1')
     assert_equal %w(add \( \)), tokenize('add(123, 456)')
     assert_equal %w(|), tokenize('0x01 | 0x10')
+    assert_equal %w(*), tokenize('500.42 * 1.0')
   end
 
   def test_skip_comments
-    assert_equal %w(foo #), tokenize("foo\n# Comment")
-    assert_equal %w(foo # bar), tokenize("foo\n# Comment\nbar")
-    assert_equal %w(foo //), tokenize("foo\n// Comment")
-    assert_equal %w(foo /* */), tokenize("foo /* Comment */")
-    assert_equal %w(foo /* */), tokenize("foo /* \nComment\n */")
-    assert_equal %w(foo <!-- -->), tokenize("foo <!-- Comment -->")
-    assert_equal %w(foo {- -}), tokenize("foo {- Comment -}")
-    assert_equal %w(foo \(* *\)), tokenize("foo (* Comment *)")
-    assert_equal %w(% %), tokenize("2 % 10\n% Comment")
+    assert_equal %w(foo), tokenize("foo\n# Comment")
+    assert_equal %w(foo bar), tokenize("foo\n# Comment\nbar")
+    assert_equal %w(foo), tokenize("foo\n// Comment")
+    assert_equal %w(foo), tokenize("foo /* Comment */")
+    assert_equal %w(foo), tokenize("foo /* \nComment\n */")
+    assert_equal %w(foo), tokenize("foo <!-- Comment -->")
+    assert_equal %w(foo), tokenize("foo {- Comment -}")
+    assert_equal %w(foo), tokenize("foo (* Comment *)")
+    assert_equal %w(%), tokenize("2 % 10\n% Comment")
   end
 
   def test_sgml_tags
@@ -77,16 +82,30 @@ class TestTokenizer < Test::Unit::TestCase
   def test_objective_c_tokens
     assert_equal %w(#import <Foundation/Foundation.h> @interface Foo NSObject { } @end), tokenize(:"Objective-C/Foo.h")
     assert_equal %w(#import @implementation Foo @end), tokenize(:"Objective-C/Foo.m")
-    assert_equal %w(#import <Cocoa/Cocoa.h> int main \( int argc char *argv \) { NSLog \( @ \) ; return ; }), tokenize(:"Objective-C/hello.m")
+    assert_equal %w(#import <Cocoa/Cocoa.h> int main \( int argc char *argv [ ] \) { NSLog \( @ \) ; return ; }), tokenize(:"Objective-C/hello.m")
+  end
+
+  def test_shebang
+    assert_equal "SHEBANG#!sh", tokenize(:"Shell/sh.script!")[0]
+    assert_equal "SHEBANG#!bash", tokenize(:"Shell/bash.script!")[0]
+    assert_equal "SHEBANG#!zsh", tokenize(:"Shell/zsh.script!")[0]
+    assert_equal "SHEBANG#!perl", tokenize(:"Perl/perl.script!")[0]
+    assert_equal "SHEBANG#!python", tokenize(:"Python/python.script!")[0]
+    assert_equal "SHEBANG#!ruby", tokenize(:"Ruby/ruby.script!")[0]
+    assert_equal "SHEBANG#!ruby", tokenize(:"Ruby/ruby2.script!")[0]
+    assert_equal "SHEBANG#!node", tokenize(:"JavaScript/js.script!")[0]
   end
 
   def test_javascript_tokens
     assert_equal %w( \( function \( \) { console.log \( \) ; } \) .call \( this \) ;), tokenize(:"JavaScript/hello.js")
   end
 
+  def test_json_tokens
+    assert_equal %w( { [ ] { } } ), tokenize(:"JSON/product.json")
+  end
+
   def test_ruby_tokens
     assert_equal %w(module Foo end), tokenize(:"Ruby/foo.rb")
-    assert_equal %w(# /usr/bin/env ruby puts), tokenize(:"Ruby/script.rb")
     assert_equal %w(task default do puts end), tokenize(:"Ruby/filenames/Rakefile")
   end
 end