Linguist 2.3.3

When testing if a blob is indexable or safe to colorize, check size first

bin/linguist

@@ -23,7 +23,6 @@ elsif File.file?(path)
 
   puts "#{blob.name}: #{blob.loc} lines (#{blob.sloc} sloc)"
   puts "  type:      #{type}"
-  puts "  extension: #{blob.pathname.extname}"
   puts "  mime type: #{blob.mime_type}"
   puts "  language:  #{blob.language}"
 

github-linguist.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name    = 'github-linguist'
-  s.version = '2.2.1'
+  s.version = '2.3.3'
   s.summary = "GitHub Language detection"
 
   s.authors = "GitHub"
@@ -10,8 +10,9 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'charlock_holmes', '~> 0.6.6'
   s.add_dependency 'escape_utils',    '~> 0.2.3'
-  s.add_dependency 'mime-types',      '~> 1.18'
+  s.add_dependency 'mime-types',      '~> 1.19'
   s.add_dependency 'pygments.rb',     '>= 0.2.13'
+  s.add_development_dependency 'mocha'
   s.add_development_dependency 'json'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'yajl-ruby'

lib/linguist.rb

@@ -1,6 +1,5 @@
 require 'linguist/blob_helper'
 require 'linguist/generated'
 require 'linguist/language'
-require 'linguist/mime'
 require 'linguist/repository'
 require 'linguist/samples'

lib/linguist/blob_helper.rb

@@ -1,9 +1,9 @@
 require 'linguist/generated'
 require 'linguist/language'
-require 'linguist/mime'
 
 require 'charlock_holmes'
 require 'escape_utils'
+require 'mime/types'
 require 'pygments'
 require 'yaml'
 
@@ -23,6 +23,22 @@ module Linguist
       File.extname(name.to_s)
     end
 
+    # Internal: Lookup mime type for extension.
+    #
+    # Returns a MIME::Type
+    def _mime_type
+      if defined? @_mime_type
+        @_mime_type
+      else
+        guesses = ::MIME::Types.type_for(extname.to_s)
+
+        # Prefer text mime types over binary
+        @_mime_type = guesses.detect { |type| type.ascii? } ||
+          # Otherwise use the first guess
+          guesses.first
+      end
+    end
+
     # Public: Get the actual blob mime type
     #
     # Examples
@@ -32,7 +48,14 @@ module Linguist
     #
     # Returns a mime type String.
     def mime_type
-      @mime_type ||= Mime.mime_for(extname.to_s)
+      _mime_type ? _mime_type.to_s : 'text/plain'
+    end
+
+    # Internal: Is the blob binary according to its mime type
+    #
+    # Return true or false
+    def binary_mime_type?
+      _mime_type ? _mime_type.binary? : false
     end
 
     # Public: Get the Content-Type header value
@@ -83,15 +106,6 @@ module Linguist
       @detect_encoding ||= CharlockHolmes::EncodingDetector.new.detect(data) if data
     end
 
-    # Public: Is the blob binary according to its mime type
-    #
-    # Return true or false
-    def binary_mime_type?
-      if mime_type = Mime.lookup_mime_type_for(extname)
-        mime_type.binary?
-      end
-    end
-
     # Public: Is the blob binary?
     #
     # Return true or false
@@ -146,7 +160,7 @@ module Linguist
     #
     # Return true or false
     def safe_to_colorize?
-      text? && !large? && !high_ratio_of_long_lines?
+      !large? && text? && !high_ratio_of_long_lines?
     end
 
     # Internal: Does the blob have a ratio of long lines?
@@ -236,7 +250,9 @@ module Linguist
     #
     # Return true or false
     def indexable?
-      if binary?
+      if size > 100 * 1024
+        false
+      elsif binary?
         false
       elsif extname == '.txt'
         true
@@ -246,8 +262,6 @@ module Linguist
         false
       elsif generated?
         false
-      elsif size > 100 * 1024
-        false
       else
         true
       end
@@ -259,11 +273,15 @@ module Linguist
     #
     # Returns a Language or nil if none is detected
     def language
-      if defined? @language
-        @language
-      elsif !binary_mime_type?
-        @language = Language.detect(name.to_s, lambda { data }, mode)
+      return @language if defined? @language
+
+      if defined?(@data) && @data.is_a?(String)
+        data = @data
+      else
+        data = lambda { binary_mime_type? ? "" : self.data }
       end
+
+      @language = Language.detect(name.to_s, data, mode)
     end
 
     # Internal: Get the lexer of the blob.

lib/linguist/mime.rb

@@ -1,91 +0,0 @@
-require 'mime/types'
-require 'yaml'
-
-class MIME::Type
-  attr_accessor :override
-end
-
-# Register additional mime type extensions
-#
-# Follows same format as mime-types data file
-#   https://github.com/halostatue/mime-types/blob/master/lib/mime/types.rb.data
-File.read(File.expand_path("../mimes.yml", __FILE__)).lines.each do |line|
-  # Regexp was cargo culted from mime-types lib
-  next unless line =~ %r{^
-    #{MIME::Type::MEDIA_TYPE_RE}
-    (?:\s@([^\s]+))?
-    (?:\s:(#{MIME::Type::ENCODING_RE}))?
-  }x
-
-  mediatype  = $1
-  subtype    = $2
-  extensions = $3
-  encoding   = $4
-
-  # Lookup existing mime type
-  mime_type = MIME::Types["#{mediatype}/#{subtype}"].first ||
-    # Or create a new instance
-    MIME::Type.new("#{mediatype}/#{subtype}")
-
-  if extensions
-    extensions.split(/,/).each do |extension|
-      mime_type.extensions << extension
-    end
-  end
-
-  if encoding
-    mime_type.encoding = encoding
-  end
-
-  mime_type.override = true
-
-  # Kind of hacky, but we need to reindex the mime type after making changes
-  MIME::Types.add_type_variant(mime_type)
-  MIME::Types.index_extensions(mime_type)
-end
-
-module Linguist
-  module Mime
-    # Internal: Look up mime type for extension.
-    #
-    # ext - The extension String. May include leading "."
-    #
-    # Examples
-    #
-    #   Mime.mime_for('.html')
-    #   # => 'text/html'
-    #
-    #   Mime.mime_for('txt')
-    #   # => 'text/plain'
-    #
-    # Return mime type String otherwise falls back to 'text/plain'.
-    def self.mime_for(ext)
-      mime_type = lookup_mime_type_for(ext)
-      mime_type ? mime_type.to_s : 'text/plain'
-    end
-
-    # Internal: Lookup mime type for extension or mime type
-    #
-    # ext_or_mime_type - A file extension ".txt" or mime type "text/plain".
-    #
-    # Returns a MIME::Type
-    def self.lookup_mime_type_for(ext_or_mime_type)
-      ext_or_mime_type ||= ''
-
-      if ext_or_mime_type =~ /\w+\/\w+/
-        guesses = ::MIME::Types[ext_or_mime_type]
-      else
-        guesses = ::MIME::Types.type_for(ext_or_mime_type)
-      end
-
-      # Use custom override first
-      guesses.detect { |type| type.override } ||
-
-        # Prefer text mime types over binary
-        guesses.detect { |type| type.ascii? } ||
-
-        # Otherwise use the first guess
-        guesses.first
-    end
-  end
-end

lib/linguist/mimes.yml

@@ -1,62 +0,0 @@
-# Additional types to add to MIME::Types
-#
-# MIME types are used to set the Content-Type of raw binary blobs. All text
-# blobs are served as text/plain regardless of their type to ensure they
-# open in the browser rather than downloading.
-#
-# The encoding helps determine whether a file should be treated as plain
-# text or binary. By default, a mime type's encoding is base64 (binary).
-# These types will show a "View Raw" link. To force a type to render as
-# plain text, set it to 8bit for UTF-8. text/* types will be treated as
-# text by default.
-#
-#   <type> @<extensions> :<encoding>
-#
-# type       - mediatype/subtype
-# extensions - comma seperated extension list
-# encoding   - base64 (binary), 7bit (ASCII), 8bit (UTF-8), or
-#              quoted-printable (Printable ASCII).
-#
-# Follows same format as mime-types data file
-#   https://github.com/halostatue/mime-types/blob/master/lib/mime/types.rb.data
-#
-# Any additions or modifications (even trivial) should have corresponding
-# test change in `test/test_mime.rb`.
-
-# TODO: Lookup actual types
-application/octet-stream @a,blend,gem,graffle,ipa,lib,mcz,nib,o,ogv,otf,pfx,pigx,plgx,psd,sib,spl,sqlite3,swc,ucode,xpi
-
-# Please keep this list alphabetized
-application/java-archive @ear,war
-application/netcdf :8bit
-application/ogg @ogg
-application/postscript :base64
-application/vnd.adobe.air-application-installer-package+zip @air
-application/vnd.mozilla.xul+xml :8bit
-application/vnd.oasis.opendocument.presentation @odp
-application/vnd.oasis.opendocument.spreadsheet @ods
-application/vnd.oasis.opendocument.text @odt
-application/vnd.openofficeorg.extension @oxt
-application/vnd.openxmlformats-officedocument.presentationml.presentation @pptx
-application/x-chrome-extension @crx
-application/x-iwork-keynote-sffkey @key
-application/x-iwork-numbers-sffnumbers @numbers
-application/x-iwork-pages-sffpages @pages
-application/x-ms-xbap @xbap :8bit
-application/x-parrot-bytecode @pbc
-application/x-shockwave-flash @swf
-application/x-silverlight-app @xap
-application/x-supercollider @sc :8bit
-application/x-troff-ms :8bit
-application/x-wais-source :8bit
-application/xaml+xml @xaml :8bit
-application/xslt+xml @xslt :8bit
-image/x-icns @icns
-text/cache-manifest @manifest
-text/plain @cu,cxx
-text/x-logtalk @lgt
-text/x-nemerle @n
-text/x-nimrod @nim
-text/x-ocaml @ml,mli,mll,mly,sig,sml
-text/x-rust @rs,rc
-text/x-scheme @rkt,scm,sls,sps,ss

lib/linguist/tokenizer.rb

@@ -16,12 +16,18 @@ module Linguist
       new.extract_tokens(data)
     end
 
+    # Read up to 100KB
+    BYTE_LIMIT = 100_000
+
+    # Start state on token, ignore anything till the next newline
     SINGLE_LINE_COMMENTS = [
       '//', # C
       '#',  # Ruby
       '%',  # Tex
     ]
 
+    # Start state on opening token, ignore anything until the closing
+    # token is reached.
     MULTI_LINE_COMMENTS = [
       ['/*', '*/'],    # C
       ['<!--', '-->'], # XML
@@ -30,7 +36,7 @@ module Linguist
     ]
 
     START_SINGLE_LINE_COMMENT =  Regexp.compile(SINGLE_LINE_COMMENTS.map { |c|
-      "^\s*#{Regexp.escape(c)} "
+      "\s*#{Regexp.escape(c)} "
     }.join("|"))
 
     START_MULTI_LINE_COMMENT =  Regexp.compile(MULTI_LINE_COMMENTS.map { |c|
@@ -52,22 +58,24 @@ module Linguist
 
       tokens = []
       until s.eos?
+        break if s.pos >= BYTE_LIMIT
+
         if token = s.scan(/^#!.+$/)
           if name = extract_shebang(token)
             tokens << "SHEBANG#!#{name}"
           end
 
         # Single line comment
-        elsif token = s.scan(START_SINGLE_LINE_COMMENT)
-          tokens << token.strip
+        elsif s.beginning_of_line? && token = s.scan(START_SINGLE_LINE_COMMENT)
+          # tokens << token.strip
           s.skip_until(/\n|\Z/)
 
         # Multiline comments
         elsif token = s.scan(START_MULTI_LINE_COMMENT)
-          tokens << token
+          # tokens << token
           close_token = MULTI_LINE_COMMENTS.assoc(token)[1]
           s.skip_until(Regexp.compile(Regexp.escape(close_token)))
-          tokens << close_token
+          # tokens << close_token
 
         # Skip single or double quoted strings
         elsif s.scan(/"/)

samples/C++/gdsdbreader.h

@@ -0,0 +1,69 @@
+#ifndef GDSDBREADER_H
+#define GDSDBREADER_H
+
+// This file contains core structures, classes and types for the entire gds app
+// WARNING: DO NOT MODIFY UNTIL IT'S STRICTLY NECESSARY
+
+#include <QDir>
+#include "diagramwidget/qgldiagramwidget.h"
+
+#define GDS_DIR "gdsdata"
+
+enum level {LEVEL_ONE, LEVEL_TWO, LEVEL_THREE};
+
+// The internal structure of the db to store information about each node (each level)
+// this will be serialized before being written to file
+class dbDataStructure
+{
+public:
+    QString label;
+    quint32 depth;
+    quint32 userIndex;
+    QByteArray data;    // This is COMPRESSED data, optimize ram and disk space, is decompressed
+                        // just when needed (to display the comments)
+
+    // The following ID is used to create second-third level files
+    quint64 uniqueID;
+    // All the next items linked to this one
+    QVector<dbDataStructure*> nextItems;
+    // Corresponding indices vector (used to store data)
+    QVector<quint32> nextItemsIndices;
+    // The father element (or NULL if it's root)
+    dbDataStructure* father;
+    // Corresponding indices vector (used to store data)
+    quint32 fatherIndex;
+    bool noFatherRoot; // Used to tell if this node is the root (so hasn't a father)
+
+    // These fields will be useful for levels 2 and 3
+    QString fileName; // Relative filename for the associated code file
+    QByteArray firstLineData; // Compressed first line data, this will be used with the line number to retrieve info
+    QVector<quint32> linesNumbers; // First and next lines (next are relative to the first) numbers
+
+    // -- Generic system data not to be stored on disk
+    void *glPointer; // GL pointer
+
+    // These operator overrides prevent the glPointer and other non-disk-necessary data serialization
+    friend QDataStream& operator<<(QDataStream& stream, const dbDataStructure& myclass)
+    // Notice: this function has to be "friend" because it cannot be a member function, member functions
+    // have an additional parameter "this" which isn't in the argument list of an operator overload. A friend
+    // function has full access to private data of the class without having the "this" argument
+    {
+        // Don't write glPointer and every pointer-dependent structure
+        return stream << myclass.label << myclass.depth << myclass.userIndex << qCompress(myclass.data)
+                         << myclass.uniqueID << myclass.nextItemsIndices << myclass.fatherIndex << myclass.noFatherRoot
+                            << myclass.fileName << qCompress(myclass.firstLineData) << myclass.linesNumbers;
+    }
+    friend QDataStream& operator>>(QDataStream& stream, dbDataStructure& myclass)
+    {
+        //Don't read it, either
+        stream >> myclass.label >> myclass.depth >> myclass.userIndex >> myclass.data
+                      >> myclass.uniqueID >> myclass.nextItemsIndices >> myclass.fatherIndex >> myclass.noFatherRoot
+                         >> myclass.fileName >> myclass.firstLineData >> myclass.linesNumbers;
+        myclass.data = qUncompress(myclass.data);
+        myclass.firstLineData = qUncompress(myclass.firstLineData);
+        return stream;
+    }
+
+};
+
+#endif // GDSDBREADER_H

samples/C++/qscicommand.h

@@ -0,0 +1,415 @@
+// This defines the interface to the QsciCommand class.
+//
+// Copyright (c) 2011 Riverbank Computing Limited <info@riverbankcomputing.com>
+// 
+// This file is part of QScintilla.
+// 
+// This file may be used under the terms of the GNU General Public
+// License versions 2.0 or 3.0 as published by the Free Software
+// Foundation and appearing in the files LICENSE.GPL2 and LICENSE.GPL3
+// included in the packaging of this file.  Alternatively you may (at
+// your option) use any later version of the GNU General Public
+// License if such license has been publicly approved by Riverbank
+// Computing Limited (or its successors, if any) and the KDE Free Qt
+// Foundation. In addition, as a special exception, Riverbank gives you
+// certain additional rights. These rights are described in the Riverbank
+// GPL Exception version 1.1, which can be found in the file
+// GPL_EXCEPTION.txt in this package.
+// 
+// If you are unsure which license is appropriate for your use, please
+// contact the sales department at sales@riverbankcomputing.com.
+// 
+// This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
+// WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+
+
+#ifndef QSCICOMMAND_H
+#define QSCICOMMAND_H
+
+#ifdef __APPLE__
+extern "C++" {
+#endif
+
+#include <qstring.h>
+
+#include <Qsci/qsciglobal.h>
+#include <Qsci/qsciscintillabase.h>
+
+
+class QsciScintilla;
+
+
+//! \brief The QsciCommand class represents an internal editor command that may
+//! have one or two keys bound to it.
+//!
+//! Methods are provided to change the keys bound to the command and to remove
+//! a key binding.  Each command has a user friendly description of the command
+//! for use in key mapping dialogs.
+class QSCINTILLA_EXPORT QsciCommand
+{
+public:
+    //! This enum defines the different commands that can be assigned to a key.
+    enum Command {
+        //! Move down one line.
+        LineDown = QsciScintillaBase::SCI_LINEDOWN,
+
+        //! Extend the selection down one line.
+        LineDownExtend = QsciScintillaBase::SCI_LINEDOWNEXTEND,
+
+        //! Extend the rectangular selection down one line.
+        LineDownRectExtend = QsciScintillaBase::SCI_LINEDOWNRECTEXTEND,
+
+        //! Scroll the view down one line.
+        LineScrollDown = QsciScintillaBase::SCI_LINESCROLLDOWN,
+
+        //! Move up one line.
+        LineUp = QsciScintillaBase::SCI_LINEUP,
+
+        //! Extend the selection up one line.
+        LineUpExtend = QsciScintillaBase::SCI_LINEUPEXTEND,
+
+        //! Extend the rectangular selection up one line.
+        LineUpRectExtend = QsciScintillaBase::SCI_LINEUPRECTEXTEND,
+
+        //! Scroll the view up one line.
+        LineScrollUp = QsciScintillaBase::SCI_LINESCROLLUP,
+
+        //! Scroll to the start of the document.
+        ScrollToStart = QsciScintillaBase::SCI_SCROLLTOSTART,
+
+        //! Scroll to the end of the document.
+        ScrollToEnd = QsciScintillaBase::SCI_SCROLLTOEND,
+
+        //! Scroll vertically to centre the current line.
+        VerticalCentreCaret = QsciScintillaBase::SCI_VERTICALCENTRECARET,
+
+        //! Move down one paragraph.
+        ParaDown = QsciScintillaBase::SCI_PARADOWN,
+
+        //! Extend the selection down one paragraph.
+        ParaDownExtend = QsciScintillaBase::SCI_PARADOWNEXTEND,
+
+        //! Move up one paragraph.
+        ParaUp = QsciScintillaBase::SCI_PARAUP,
+
+        //! Extend the selection up one paragraph.
+        ParaUpExtend = QsciScintillaBase::SCI_PARAUPEXTEND,
+
+        //! Move left one character.
+        CharLeft = QsciScintillaBase::SCI_CHARLEFT,
+
+        //! Extend the selection left one character.
+        CharLeftExtend = QsciScintillaBase::SCI_CHARLEFTEXTEND,
+
+        //! Extend the rectangular selection left one character.
+        CharLeftRectExtend = QsciScintillaBase::SCI_CHARLEFTRECTEXTEND,
+
+        //! Move right one character.
+        CharRight = QsciScintillaBase::SCI_CHARRIGHT,
+
+        //! Extend the selection right one character.
+        CharRightExtend = QsciScintillaBase::SCI_CHARRIGHTEXTEND,
+
+        //! Extend the rectangular selection right one character.
+        CharRightRectExtend = QsciScintillaBase::SCI_CHARRIGHTRECTEXTEND,
+
+        //! Move left one word.
+        WordLeft = QsciScintillaBase::SCI_WORDLEFT,
+
+        //! Extend the selection left one word.
+        WordLeftExtend = QsciScintillaBase::SCI_WORDLEFTEXTEND,
+
+        //! Move right one word.
+        WordRight = QsciScintillaBase::SCI_WORDRIGHT,
+
+        //! Extend the selection right one word.
+        WordRightExtend = QsciScintillaBase::SCI_WORDRIGHTEXTEND,
+
+        //! Move to the end of the previous word.
+        WordLeftEnd = QsciScintillaBase::SCI_WORDLEFTEND,
+
+        //! Extend the selection to the end of the previous word.
+        WordLeftEndExtend = QsciScintillaBase::SCI_WORDLEFTENDEXTEND,
+
+        //! Move to the end of the next word.
+        WordRightEnd = QsciScintillaBase::SCI_WORDRIGHTEND,
+
+        //! Extend the selection to the end of the next word.
+        WordRightEndExtend = QsciScintillaBase::SCI_WORDRIGHTENDEXTEND,
+
+        //! Move left one word part.
+        WordPartLeft = QsciScintillaBase::SCI_WORDPARTLEFT,
+
+        //! Extend the selection left one word part.
+        WordPartLeftExtend = QsciScintillaBase::SCI_WORDPARTLEFTEXTEND,
+
+        //! Move right one word part.
+        WordPartRight = QsciScintillaBase::SCI_WORDPARTRIGHT,
+
+        //! Extend the selection right one word part.
+        WordPartRightExtend = QsciScintillaBase::SCI_WORDPARTRIGHTEXTEND,
+
+        //! Move to the start of the document line.
+        Home = QsciScintillaBase::SCI_HOME,
+
+        //! Extend the selection to the start of the document line.
+        HomeExtend = QsciScintillaBase::SCI_HOMEEXTEND,
+
+        //! Extend the rectangular selection to the start of the document line.
+        HomeRectExtend = QsciScintillaBase::SCI_HOMERECTEXTEND,
+
+        //! Move to the start of the displayed line.
+        HomeDisplay = QsciScintillaBase::SCI_HOMEDISPLAY,
+
+        //! Extend the selection to the start of the displayed line.
+        HomeDisplayExtend = QsciScintillaBase::SCI_HOMEDISPLAYEXTEND,
+
+        //! Move to the start of the displayed or document line.
+        HomeWrap = QsciScintillaBase::SCI_HOMEWRAP,
+
+        //! Extend the selection to the start of the displayed or document
+        //! line.
+        HomeWrapExtend = QsciScintillaBase::SCI_HOMEWRAPEXTEND,
+
+        //! Move to the first visible character in the document line.
+        VCHome = QsciScintillaBase::SCI_VCHOME,
+
+        //! Extend the selection to the first visible character in the document
+        //! line.
+        VCHomeExtend = QsciScintillaBase::SCI_VCHOMEEXTEND,
+
+        //! Extend the rectangular selection to the first visible character in
+        //! the document line.
+        VCHomeRectExtend = QsciScintillaBase::SCI_VCHOMERECTEXTEND,
+
+        //! Move to the first visible character of the displayed or document
+        //! line.
+        VCHomeWrap = QsciScintillaBase::SCI_VCHOMEWRAP,
+
+        //! Extend the selection to the first visible character of the
+        //! displayed or document line.
+        VCHomeWrapExtend = QsciScintillaBase::SCI_VCHOMEWRAPEXTEND,
+
+        //! Move to the end of the document line.
+        LineEnd = QsciScintillaBase::SCI_LINEEND,
+
+        //! Extend the selection to the end of the document line.
+        LineEndExtend = QsciScintillaBase::SCI_LINEENDEXTEND,
+
+        //! Extend the rectangular selection to the end of the document line.
+        LineEndRectExtend = QsciScintillaBase::SCI_LINEENDRECTEXTEND,
+
+        //! Move to the end of the displayed line.
+        LineEndDisplay = QsciScintillaBase::SCI_LINEENDDISPLAY,
+
+        //! Extend the selection to the end of the displayed line.
+        LineEndDisplayExtend = QsciScintillaBase::SCI_LINEENDDISPLAYEXTEND,
+
+        //! Move to the end of the displayed or document line.
+        LineEndWrap = QsciScintillaBase::SCI_LINEENDWRAP,
+
+        //! Extend the selection to the end of the displayed or document line.
+        LineEndWrapExtend = QsciScintillaBase::SCI_LINEENDWRAPEXTEND,
+
+        //! Move to the start of the document.
+        DocumentStart = QsciScintillaBase::SCI_DOCUMENTSTART,
+
+        //! Extend the selection to the start of the document.
+        DocumentStartExtend = QsciScintillaBase::SCI_DOCUMENTSTARTEXTEND,
+
+        //! Move to the end of the document.
+        DocumentEnd = QsciScintillaBase::SCI_DOCUMENTEND,
+
+        //! Extend the selection to the end of the document.
+        DocumentEndExtend = QsciScintillaBase::SCI_DOCUMENTENDEXTEND,
+
+        //! Move up one page.
+        PageUp = QsciScintillaBase::SCI_PAGEUP,
+
+        //! Extend the selection up one page.
+        PageUpExtend = QsciScintillaBase::SCI_PAGEUPEXTEND,
+
+        //! Extend the rectangular selection up one page.
+        PageUpRectExtend = QsciScintillaBase::SCI_PAGEUPRECTEXTEND,
+
+        //! Move down one page.
+        PageDown = QsciScintillaBase::SCI_PAGEDOWN,
+
+        //! Extend the selection down one page.
+        PageDownExtend = QsciScintillaBase::SCI_PAGEDOWNEXTEND,
+
+        //! Extend the rectangular selection down one page.
+        PageDownRectExtend = QsciScintillaBase::SCI_PAGEDOWNRECTEXTEND,
+
+        //! Stuttered move up one page.
+        StutteredPageUp = QsciScintillaBase::SCI_STUTTEREDPAGEUP,
+
+        //! Stuttered extend the selection up one page.
+        StutteredPageUpExtend = QsciScintillaBase::SCI_STUTTEREDPAGEUPEXTEND,
+
+        //! Stuttered move down one page.
+        StutteredPageDown = QsciScintillaBase::SCI_STUTTEREDPAGEDOWN,
+
+        //! Stuttered extend the selection down one page.
+        StutteredPageDownExtend = QsciScintillaBase::SCI_STUTTEREDPAGEDOWNEXTEND,
+
+        //! Delete the current character.
+        Delete = QsciScintillaBase::SCI_CLEAR,
+
+        //! Delete the previous character.
+        DeleteBack = QsciScintillaBase::SCI_DELETEBACK,
+
+        //! Delete the previous character if not at start of line.
+        DeleteBackNotLine = QsciScintillaBase::SCI_DELETEBACKNOTLINE,
+
+        //! Delete the word to the left.
+        DeleteWordLeft = QsciScintillaBase::SCI_DELWORDLEFT,
+
+        //! Delete the word to the right.
+        DeleteWordRight = QsciScintillaBase::SCI_DELWORDRIGHT,
+
+        //! Delete right to the end of the next word.
+        DeleteWordRightEnd = QsciScintillaBase::SCI_DELWORDRIGHTEND,
+
+        //! Delete the line to the left.
+        DeleteLineLeft = QsciScintillaBase::SCI_DELLINELEFT,
+
+        //! Delete the line to the right.
+        DeleteLineRight = QsciScintillaBase::SCI_DELLINERIGHT,
+
+        //! Delete the current line.
+        LineDelete = QsciScintillaBase::SCI_LINEDELETE,
+
+        //! Cut the current line to the clipboard.
+        LineCut = QsciScintillaBase::SCI_LINECUT,
+
+        //! Copy the current line to the clipboard.
+        LineCopy = QsciScintillaBase::SCI_LINECOPY,
+
+        //! Transpose the current and previous lines.
+        LineTranspose = QsciScintillaBase::SCI_LINETRANSPOSE,
+
+        //! Duplicate the current line.
+        LineDuplicate = QsciScintillaBase::SCI_LINEDUPLICATE,
+
+        //! Select the whole document.
+        SelectAll = QsciScintillaBase::SCI_SELECTALL,
+
+        //! Move the selected lines up one line.
+        MoveSelectedLinesUp = QsciScintillaBase::SCI_MOVESELECTEDLINESUP,
+
+        //! Move the selected lines down one line.
+        MoveSelectedLinesDown = QsciScintillaBase::SCI_MOVESELECTEDLINESDOWN,
+
+        //! Duplicate the selection.
+        SelectionDuplicate = QsciScintillaBase::SCI_SELECTIONDUPLICATE,
+
+        //! Convert the selection to lower case.
+        SelectionLowerCase = QsciScintillaBase::SCI_LOWERCASE,
+
+        //! Convert the selection to upper case.
+        SelectionUpperCase = QsciScintillaBase::SCI_UPPERCASE,
+
+        //! Cut the selection to the clipboard.
+        SelectionCut = QsciScintillaBase::SCI_CUT,
+
+        //! Copy the selection to the clipboard.
+        SelectionCopy = QsciScintillaBase::SCI_COPY,
+
+        //! Paste from the clipboard.
+        Paste = QsciScintillaBase::SCI_PASTE,
+
+        //! Toggle insert/overtype.
+        EditToggleOvertype = QsciScintillaBase::SCI_EDITTOGGLEOVERTYPE,
+
+        //! Insert a platform dependent newline.
+        Newline = QsciScintillaBase::SCI_NEWLINE,
+
+        //! Insert a formfeed.
+        Formfeed = QsciScintillaBase::SCI_FORMFEED,
+
+        //! Indent one level.
+        Tab = QsciScintillaBase::SCI_TAB,
+
+        //! De-indent one level.
+        Backtab = QsciScintillaBase::SCI_BACKTAB,
+
+        //! Cancel any current operation.
+        Cancel = QsciScintillaBase::SCI_CANCEL,
+
+        //! Undo the last command.
+        Undo = QsciScintillaBase::SCI_UNDO,
+
+        //! Redo the last command.
+        Redo = QsciScintillaBase::SCI_REDO,
+
+        //! Zoom in.
+        ZoomIn = QsciScintillaBase::SCI_ZOOMIN,
+
+        //! Zoom out.
+        ZoomOut = QsciScintillaBase::SCI_ZOOMOUT,
+    };
+
+    //! Return the command that will be executed by this instance.
+    Command command() const {return scicmd;}
+
+    //! Execute the command.
+    void execute();
+
+    //! Binds the key \a key to the command.  If \a key is 0 then the key
+    //! binding is removed.  If \a key is invalid then the key binding is
+    //! unchanged.  Valid keys are any visible or control character or any
+    //! of \c Key_Down, \c Key_Up, \c Key_Left, \c Key_Right, \c Key_Home,
+    //! \c Key_End, \c Key_PageUp, \c Key_PageDown, \c Key_Delete,
+    //! \c Key_Insert, \c Key_Escape, \c Key_Backspace, \c Key_Tab and
+    //! \c Key_Return.  Keys may be modified with any combination of \c SHIFT,
+    //! \c CTRL, \c ALT and \c META.
+    //!
+    //! \sa key(), setAlternateKey(), validKey()
+    void setKey(int key);
+
+    //! Binds the alternate key \a altkey to the command.  If \a key is 0
+    //! then the alternate key binding is removed.
+    //!
+    //! \sa alternateKey(), setKey(), validKey()
+    void setAlternateKey(int altkey);
+
+    //! The key that is currently bound to the command is returned.
+    //!
+    //! \sa setKey(), alternateKey()
+    int key() const {return qkey;}
+
+    //! The alternate key that is currently bound to the command is
+    //! returned.
+    //!
+    //! \sa setAlternateKey(), key()
+    int alternateKey() const {return qaltkey;}
+
+    //! If the key \a key is valid then true is returned.
+    static bool validKey(int key);
+
+    //! The user friendly description of the command is returned.
+    QString description() const;
+
+private:
+    friend class QsciCommandSet;
+
+    QsciCommand(QsciScintilla *qs, Command cmd, int key, int altkey,
+            const char *desc);
+
+    void bindKey(int key,int &qk,int &scik);
+
+    QsciScintilla *qsCmd;
+    Command scicmd;
+    int qkey, scikey, qaltkey, scialtkey;
+    const char *descCmd;
+
+    QsciCommand(const QsciCommand &);
+    QsciCommand &operator=(const QsciCommand &);
+};
+
+#ifdef __APPLE__
+}
+#endif
+
+#endif

samples/C++/qsciprinter.h

@@ -0,0 +1,116 @@
+// This module defines interface to the QsciPrinter class.
+//
+// Copyright (c) 2011 Riverbank Computing Limited <info@riverbankcomputing.com>
+// 
+// This file is part of QScintilla.
+// 
+// This file may be used under the terms of the GNU General Public
+// License versions 2.0 or 3.0 as published by the Free Software
+// Foundation and appearing in the files LICENSE.GPL2 and LICENSE.GPL3
+// included in the packaging of this file.  Alternatively you may (at
+// your option) use any later version of the GNU General Public
+// License if such license has been publicly approved by Riverbank
+// Computing Limited (or its successors, if any) and the KDE Free Qt
+// Foundation. In addition, as a special exception, Riverbank gives you
+// certain additional rights. These rights are described in the Riverbank
+// GPL Exception version 1.1, which can be found in the file
+// GPL_EXCEPTION.txt in this package.
+// 
+// If you are unsure which license is appropriate for your use, please
+// contact the sales department at sales@riverbankcomputing.com.
+// 
+// This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
+// WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+
+
+#ifndef QSCIPRINTER_H
+#define QSCIPRINTER_H
+
+#ifdef __APPLE__
+extern "C++" {
+#endif
+
+#include <qprinter.h>
+
+#include <Qsci/qsciglobal.h>
+#include <Qsci/qsciscintilla.h>
+
+
+QT_BEGIN_NAMESPACE
+class QRect;
+class QPainter;
+QT_END_NAMESPACE
+
+class QsciScintillaBase;
+
+
+//! \brief The QsciPrinter class is a sub-class of the Qt QPrinter class that
+//! is able to print the text of a Scintilla document.
+//!
+//! The class can be further sub-classed to alter to layout of the text, adding
+//! headers and footers for example.
+class QSCINTILLA_EXPORT QsciPrinter : public QPrinter
+{
+public:
+    //! Constructs a printer paint device with mode \a mode.
+    QsciPrinter(PrinterMode mode = ScreenResolution);
+
+    //! Destroys the QsciPrinter instance.
+    virtual ~QsciPrinter();
+
+    //! Format a page, by adding headers and footers for example, before the
+    //! document text is drawn on it.  \a painter is the painter to be used to
+    //! add customised text and graphics.  \a drawing is true if the page is
+    //! actually being drawn rather than being sized.  \a painter drawing
+    //! methods must only be called when \a drawing is true.  \a area is the
+    //! area of the page that will be used to draw the text.  This should be
+    //! modified if it is necessary to reserve space for any customised text or
+    //! graphics.  By default the area is relative to the printable area of the
+    //! page.  Use QPrinter::setFullPage() because calling printRange() if you
+    //! want to try and print over the whole page.  \a pagenr is the number of
+    //! the page.  The first page is numbered 1.
+    virtual void formatPage(QPainter &painter, bool drawing, QRect &area,
+            int pagenr);
+
+    //! Return the number of points to add to each font when printing.
+    //!
+    //! \sa setMagnification()
+    int magnification() const {return mag;}
+
+    //! Sets the number of points to add to each font when printing to \a
+    //! magnification.
+    //!
+    //! \sa magnification()
+    virtual void setMagnification(int magnification);
+
+    //! Print a range of lines from the Scintilla instance \a qsb.  \a from is
+    //! the first line to print and a negative value signifies the first line
+    //! of text.  \a to is the last line to print and a negative value
+    //! signifies the last line of text.  true is returned if there was no
+    //! error.
+    virtual int printRange(QsciScintillaBase *qsb, int from = -1, int to = -1);
+
+    //! Return the line wrap mode used when printing.  The default is
+    //! QsciScintilla::WrapWord.
+    //!
+    //! \sa setWrapMode()
+    QsciScintilla::WrapMode wrapMode() const {return wrap;}
+
+    //! Sets the line wrap mode used when printing to \a wmode.
+    //!
+    //! \sa wrapMode()
+    virtual void setWrapMode(QsciScintilla::WrapMode wmode);
+
+private:
+    int mag;
+    QsciScintilla::WrapMode wrap;
+
+    QsciPrinter(const QsciPrinter &);
+    QsciPrinter &operator=(const QsciPrinter &);
+};
+
+#ifdef __APPLE__
+}
+#endif
+
+#endif

samples/C/rf_io.h

@@ -0,0 +1,682 @@
+/**
+** Copyright (c) 2011-2012, Karapetsas Eleftherios
+** All rights reserved.
+**
+** Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+**  1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+**  2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in
+**     the documentation and/or other materials provided with the distribution.
+**  3. Neither the name of the Original Author of Refu nor the names of its contributors may be used to endorse or promote products derived from
+**
+**  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+**  INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+**  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+**  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+**  SERVICES;LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+**  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+**  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**/
+
+
+#ifndef REFU_IO_H
+#define REFU_IO_H
+
+#include <rf_setup.h>
+#include <stdio.h>
+
+#ifdef __cplusplus
+extern "C"
+{// opening bracket for calling from C++
+#endif
+
+// New line feed
+#define RF_LF   0xA
+// Carriage Return
+#define RF_CR   0xD
+
+#ifdef REFU_WIN32_VERSION
+    #define i_PLUSB_WIN32   "b"
+#else
+    #define i_PLUSB_WIN32   ""
+#endif
+
+// This is the type that represents the file offset
+#ifdef _MSC_VER
+typedef __int64 foff_rft;
+#else
+#include <sys/types.h>
+typedef off64_t foff_rft;
+#endif
+///Fseek and Ftelll definitions
+#ifdef _MSC_VER
+    #define rfFseek(i_FILE_,i_OFFSET_,i_WHENCE_)    _fseeki64(i_FILE_,i_OFFSET_,i_WHENCE_)
+    #define rfFtell(i_FILE_)                        _ftelli64(i_FILE_)
+#else
+    #define rfFseek(i_FILE_,i_OFFSET_,i_WHENCE_)    fseeko64(i_FILE_,i_OFFSET_,i_WHENCE_)
+    #define rfFtell(i_FILE_)                        ftello64(i_FILE_)
+#endif
+
+/**
+** @defgroup RF_IOGRP I/O
+** @addtogroup RF_IOGRP
+** @{
+**/
+
+// @brief Reads a UTF-8 file descriptor until end of line or EOF is found and returns a UTF-8 byte buffer
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// When the compile flag @c RF_NEWLINE_CRLF is defined (the default case at Windows) then this function
+// shall not be adding any CR character that is found in the file behind a newline character since this is
+// the Windows line ending scheme. Beware though that the returned  read bytes value shall still count the CR character inside.
+//
+// @param[in] f The file descriptor to read
+// @param[out] utf8 Give here a refence to an unitialized char* that will be allocated inside the function
+// and contain the utf8 byte buffer. Needs to be freed by the caller explicitly later
+// @param[out] byteLength Give an @c uint32_t here to receive the length of the @c utf8 buffer in bytes
+// @param[out] bufferSize Give an @c uint32_t here to receive the capacity of the @c utf8 buffer in bytes
+// @param[out] eof Pass a pointer to a char to receive a true or false value in case the end of file
+// with reading this line
+// @return Returns either a positive number for success that represents the number of bytes read from @c f and and error in case something goes wrong.
+// The possible errors to return are the same as rfFgets_UTF8()
+i_DECLIMEX_ int32_t rfFReadLine_UTF8(FILE* f,char** utf8,uint32_t* byteLength,uint32_t* bufferSize,char* eof);
+// @brief Reads a Big Endian UTF-16 file descriptor until end of line or EOF is found and returns a UTF-8 byte buffer
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// When the compile flag @c RF_NEWLINE_CRLF is defined (the default case at Windows) then this function
+// shall not be adding any CR character that is found in the file behind a newline character since this is
+// the Windows line ending scheme. Beware though that the returned  read bytes value shall still count the CR character inside.
+//
+// @param[in] f The file descriptor to read
+// @param[out] utf8 Give here a refence to an unitialized char* that will be allocated inside the function
+// and contain the utf8 byte buffer. Needs to be freed by the caller explicitly later
+// @param[out] byteLength Give an @c uint32_t here to receive the length of the @c utf8 buffer in bytes
+// @param[out] eof Pass a pointer to a char to receive a true or false value in case the end of file
+// with reading this line
+// @return Returns either a positive number for success that represents the number of bytes read from @c f and and error in case something goes wrong.
+// + Any error that can be returned by @ref rfFgets_UTF16BE()
+// + @c RE_UTF16_INVALID_SEQUENCE: Failed to decode the UTF-16 byte stream of the file descriptor
+// + @c RE_UTF8_ENCODING: Failed to encode the UTF-16 of the file descriptor into UTF-8
+i_DECLIMEX_ int32_t rfFReadLine_UTF16BE(FILE* f,char** utf8,uint32_t* byteLength,char* eof);
+// @brief Reads a Little Endian UTF-16 file descriptor until end of line or EOF is found and returns a UTF-8 byte buffer
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// When the compile flag @c RF_NEWLINE_CRLF is defined (the default case at Windows) then this function
+// shall not be adding any CR character that is found in the file behind a newline character since this is
+// the Windows line ending scheme. Beware though that the returned read bytes value shall still count the CR character inside.
+//
+// @param[in] f The file descriptor to read
+// @param[out] utf8 Give here a refence to an unitialized char* that will be allocated inside the function
+// and contain the utf8 byte buffer. Needs to be freed by the caller explicitly later
+// @param[out] byteLength Give an @c uint32_t here to receive the length of the @c utf8 buffer in bytes
+// @param[out] eof Pass a pointer to a char to receive a true or false value in case the end of file
+// with reading this line
+// @return Returns either a positive number for success that represents the number of bytes read from @c f and and error in case something goes wrong.
+// + Any error that can be returned by @ref rfFgets_UTF16LE()
+// + @c RE_UTF16_INVALID_SEQUENCE: Failed to decode the UTF-16 byte stream of the file descriptor
+// + @c RE_UTF8_ENCODING: Failed to encode the UTF-16 of the file descriptor into UTF-8
+i_DECLIMEX_ int32_t rfFReadLine_UTF16LE(FILE* f,char** utf8,uint32_t* byteLength,char* eof);
+
+// @brief Reads a Big Endian UTF-32 file descriptor until end of line or EOF is found and returns a UTF-8 byte buffer
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// When the compile flag @c RF_NEWLINE_CRLF is defined (the default case at Windows) then this function
+// shall not be adding any CR character that is found in the file behind a newline character since this is
+// the Windows line ending scheme. Beware though that the returned read bytes value shall still count the CR character inside.
+//
+// @param[in] f The file descriptor to read
+// @param[out] utf8 Give here a refence to an unitialized char* that will be allocated inside the function
+// and contain the utf8 byte buffer. Needs to be freed by the caller explicitly later
+// @param[out] byteLength Give an @c uint32_t here to receive the length of the @c utf8 buffer in bytes
+// @param[out] eof Pass a pointer to a char to receive a true or false value in case the end of file
+// with reading this line
+// @return Returns either a positive number for success that represents the number of bytes read from @c f and and error in case something goes wrong.
+// + Any error that can be returned by @ref rfFgets_UTF32BE()
+// + @c RE_UTF8_ENCODING: Failed to encode the UTF-16 of the file descriptor into UTF-8
+i_DECLIMEX_ int32_t rfFReadLine_UTF32BE(FILE* f,char** utf8,uint32_t* byteLength,char* eof);
+// @brief Reads a Little Endian UTF-32 file descriptor until end of line or EOF is found and returns a UTF-8 byte buffer
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// When the compile flag @c RF_NEWLINE_CRLF is defined (the default case at Windows) then this function
+// shall not be adding any CR character that is found in the file behind a newline character since this is
+// the Windows line ending scheme. Beware though that the returned read bytes value shall still count the CR character inside.
+//
+// @param[in] f The file descriptor to read
+// @param[out] utf8 Give here a refence to an unitialized char* that will be allocated inside the function
+// and contain the utf8 byte buffer. Needs to be freed by the caller explicitly later
+// @param[out] byteLength Give an @c uint32_t here to receive the length of the @c utf8 buffer in bytes
+// @param[out] eof Pass a pointer to a char to receive a true or false value in case the end of file
+// with reading this line
+// @return Returns either a positive number for success that represents the number of bytes read from @c f and and error in case something goes wrong.
+// + Any error that can be returned by @ref rfFgets_UTF32LE()
+// + @c RE_UTF8_ENCODING: Failed to encode the UTF-16 of the file descriptor into UTF-8
+i_DECLIMEX_ int32_t rfFReadLine_UTF32LE(FILE* f,char** utf8,uint32_t* byteLength,char* eof);
+
+// @brief Gets a number of bytes from a BIG endian UTF-32 file descriptor
+//
+// This is a function that's similar to c library fgets but it also returns the number of bytes read. Reads in from the file until @c num bytes
+// have been read or new line or EOF character has been encountered.
+//
+// The function will read until @c num characters are read and if @c num
+// would take us to the middle of a UTF32 character then the next character shall also be read
+// and the function will return the number of bytes read.
+// Since the function null terminates the buffer the given @c buff needs to be of at least
+// @c num+7 size to cater for the worst case.
+//
+// The final bytestream stored inside @c buff is in the endianess of the system.
+//
+// If right after the last character read comes the EOF, the function
+// shall detect so and assign @c true to @c eof.
+//
+// In Windows where file endings are in the form of 2 bytes CR-LF (Carriage return - NewLine) this function
+// shall just ignore the carriage returns and not return it inside the return buffer at @c buff.
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param[in] buff A buffer to be filled with the contents of the file. Should be of size at least @c num+7
+// @param[in] num The maximum number of bytes to read from within the file NOT including the null terminating character(which in itelf is 4 bytes). Should be a multiple of 4
+// @param[in] f A valid FILE descriptor from which to read the bytes
+// @param[out] eof Pass a reference to a char to receive a true/false value for whether EOF has been reached.
+// @return Returns the actual number of bytes read or an error if there was a problem.
+// The possible errors are:
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgets_UTF32BE(char* buff,uint32_t num,FILE* f,char* eof);
+// @brief Gets a number of bytes from a Little endian UTF-32 file descriptor
+//
+// This is a function that's similar to c library fgets but it also returns the number of bytes read. Reads in from the file until @c num bytes
+// have been read or new line or EOF character has been encountered.
+//
+// The function will read until @c num characters are read and if @c num
+// would take us to the middle of a UTF32 character then the next character shall also be read
+// and the function will return the number of bytes read.
+// Since the function null terminates the buffer the given @c buff needs to be of at least
+// @c num+7 size to cater for the worst case.
+//
+// The final bytestream stored inside @c buff is in the endianess of the system.
+//
+// If right after the last character read comes the EOF, the function
+// shall detect so and assign @c true to @c eof.
+//
+// In Windows where file endings are in the form of 2 bytes CR-LF (Carriage return - NewLine) this function
+// shall just ignore the carriage returns and not return it inside the return buffer at @c buff.
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param[in] buff A buffer to be filled with the contents of the file. Should be of size at least @c num+7
+// @param[in] num The maximum number of bytes to read from within the file NOT including the null terminating character(which in itelf is 4 bytes). Should be a multiple of 4
+// @param[in] f A valid FILE descriptor from which to read the bytes
+// @param[out] eof Pass a reference to a char to receive a true/false value for whether EOF has been reached.
+// @return Returns the actual number of bytes read or an error if there was a problem.
+// The possible errors are:
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgets_UTF32LE(char* buff,uint32_t num,FILE* f,char* eof);
+
+// @brief Gets a number of bytes from a BIG endian UTF-16 file descriptor
+//
+// This is a function that's similar to c library fgets but it also returns the number of bytes read. Reads in from the file until @c num bytes
+// have been read or new line or EOF character has been encountered.
+//
+// The function will read until @c num characters are read and if @c num
+// would take us to the middle of a UTF16 character then the next character shall also be read
+// and the function will return the number of bytes read.
+// Since the function null terminates the buffer the given @c buff needs to be of at least
+// @c num+5 size to cater for the worst case.
+//
+// The final bytestream stored inside @c buff is in the endianess of the system.
+//
+// If right after the last character read comes the EOF, the function
+// shall detect so and assign @c true to @c eof.
+//
+// In Windows where file endings are in the form of 2 bytes CR-LF (Carriage return - NewLine) this function
+// shall just ignore the carriage returns and not return it inside the return buffer at @c buff.
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param[in] buff A buffer to be filled with the contents of the file. Should be of size at least @c num+5
+// @param[in] num The maximum number of bytes to read from within the file NOT including the null terminating character(which in itelf is 2 bytes). Should be a multiple of 2
+// @param[in] f A valid FILE descriptor from which to read the bytes
+// @param[out] eof Pass a reference to a char to receive a true/false value for whether EOF has been reached.
+// @return Returns the actual number of bytes read or an error if there was a problem.
+// The possible errors are:
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgets_UTF16BE(char* buff,uint32_t num,FILE* f,char* eof);
+// @brief Gets a number of bytes from a Little endian UTF-16 file descriptor
+//
+// This is a function that's similar to c library fgets but it also returns the number of bytes read. Reads in from the file until @c num bytes
+// have been read or new line or EOF character has been encountered.
+//
+// The function will read until @c num characters are read and if @c num
+// would take us to the middle of a UTF16 character then the next character shall also be read
+// and the function will return the number of bytes read.
+// Since the function null terminates the buffer the given @c buff needs to be of at least
+// @c num+5 size to cater for the worst case.
+//
+// The final bytestream stored inside @c buff is in the endianess of the system.
+//
+// If right after the last character read comes the EOF, the function
+// shall detect so and assign @c true to @c eof.
+//
+// In Windows where file endings are in the form of 2 bytes CR-LF (Carriage return - NewLine) this function
+// shall just ignore the carriage returns and not return it inside the return buffer at @c buff.
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param[in] buff A buffer to be filled with the contents of the file. Should be of size at least @c num+2
+// @param[in] num The maximum number of bytes to read from within the file NOT including the null terminating character(which in itelf is 2 bytes). Should be a multiple of 2
+// @param[in] f A valid FILE descriptor from which to read the bytes
+// @param[out] eof Pass a reference to a char to receive a true/false value for whether EOF has been reached.
+// @return Returns the actual number of bytes read or an error if there was a problem.
+// The possible errors are:
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgets_UTF16LE(char* buff,uint32_t num,FILE* f,char* eof);
+// @brief Gets a number of bytes from a UTF-8 file descriptor
+//
+// This is a function that's similar to c library fgets but it also returns the number of bytes read. Reads in from the file until @c num characters
+// have been read or new line or EOF character has been encountered.
+//
+// The function  automatically adds a null termination character at the end of
+// @c buff but this character is not included in the returned actual number of bytes.
+//
+// The function will read until @c num characters are read and if @c num
+// would take us to the middle of a UTF8 character then the next character shall also be read
+// and the function will return the number of bytes read.
+// Since the function null terminates the buffer the given @c buff needs to be of at least
+// @c num+4 size to cater for the worst case.
+//
+// If right after the last character read comes the EOF, the function
+// shall detect so and assign @c true to @c eof.
+//
+// In Windows where file endings are in the form of 2 bytes CR-LF (Carriage return - NewLine) this function
+// shall just ignore the carriage returns and not return it inside the return buffer at @c buff.
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param[in] buff A buffer to be filled with the contents of the file. Should of size at least @c num+4
+// @param[in] num The maximum number of bytes to read from within the file NOT including the null terminating character(which in itelf is 1 byte)
+// @param[in] f A valid FILE descriptor from which to read the bytes
+// @param[out] eof Pass a reference to a char to receive a true/false value for whether EOF has been reached.
+// @return Returns the actual number of bytes read or an error if there was a problem.
+// The possible errors are:
+// + @c RE_UTF8_INVALID_SEQUENCE_INVALID_BYTE: If an invalid UTF-8 byte has been found
+// + @c RE_UTF8_INVALID_SEQUENCE_CONBYTE: If during parsing the file we were expecting a continuation
+// byte and did not find it
+// + @c RE_UTF8_INVALID_SEQUENCE_END: If the null character is encountered in between bytes that should
+// have been continuation bytes
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgets_UTF8(char* buff,uint32_t num,FILE* f,char* eof);
+
+// @brief  Gets a unicode character from a UTF-8 file descriptor
+//
+// This function attempts to assume a more modern fgetc() role for UTF-8 encoded files.
+// Reads bytes from the File descriptor @c f until a full UTF-8 unicode character has been read
+//
+// After this function the file pointer will have moved either by @c 1, @c 2, @c 3 or @c 4
+// bytes if the return value is positive. You can see how much by checking the return value.
+//
+// You shall need to provide an integer at @c c to contain either the decoded Unicode
+// codepoint or the UTF-8 endoced byte depending on the value of the @c cp argument.
+//
+// @param f A valid FILE descriptor from which to read the bytes
+// @param c Pass an int that will receive either the unicode code point value or
+// the UTF8 bytes depending on the value of the @c cp flag
+// @param cp A boolean flag. If @c true then the int passed at @c c will contain the unicode code point
+// of the read character, so the UTF-8 will be decoded.
+// If @c false the int passed at @c c will contain the value of the read bytes in UTF-8 without any decoding
+// @return Returns the number of bytes read (either @c 1, @c 2, @c 3 or @c 4) or an error if the function
+// fails for some reason. Possible error values are:
+// + @c RE_FILE_EOF: The end of file has been found while reading. If the end of file is encountered
+// in the middle of a UTF-8 encoded character where we would be expecting something different
+// and @c RE_UTF8_INVALID_SEQUENCE_END error is also logged
+// + @c RE_UTF8_INVALID_SEQUENCE_INVALID_BYTE: If an invalid UTF-8 byte has been found
+// + @c RE_UTF8_INVALID_SEQUENCE_CONBYTE: If during parsing the file we were expecting a continuation
+// byte and did not find it
+// + @c RE_UTF8_INVALID_SEQUENCE_END: If the null character is encountered in between bytes that should
+// have been continuation bytes
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgetc_UTF8(FILE* f,uint32_t *c,char cp);
+// @brief  Gets a unicode character from a UTF-16 Big Endian file descriptor
+//
+// This function attempts to assume a more modern fgetc() role for UTF-16 encoded files.
+// Reads bytes from the File descriptor @c f until a full UTF-16 unicode character has been read
+//
+// After this function the file pointer will have moved either by @c 2 or @c 4
+// bytes if the return value is positive. You can see how much by checking the return value.
+//
+// You shall need to provide an integer at @c c to contain either the decoded Unicode
+// codepoint or the Bigendian encoded UTF-16 bytes depending on the value of @c the cp argument.
+//
+// @param f A valid FILE descriptor from which to read the bytes
+// @param c Pass an int that will receive either the unicode code point value or
+// the UTF16 bytes depending on the value of the @c cp flag
+// @param cp A boolean flag. If @c true then the int passed at @c c will contain the unicode code point
+// of the read character, so the UTF-16 will be decoded.
+// If @c false the int passed at @c c will contain the value of the read bytes in UTF-16 without any decoding
+// @return Returns the number of bytes read (either @c 2 or @c 4) or an error if the function
+// fails for some reason. Possible error values are:
+// + @c RE_UTF16_INVALID_SEQUENCE: Either the read word or its surrogate pair if 4 bytes were read held illegal values
+// + @c RE_UTF16_NO_SURRPAIR: According to the first read word a surrogate pair was expected but none was found
+// + @c RE_FILE_EOF: The end of file has been found while reading. If the end of file is encountered
+// while we expect a UTF-16 surrogate pair an appropriate error is logged
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgetc_UTF16BE(FILE* f,uint32_t *c,char cp);
+// @brief  Gets a unicode character from a UTF-16 Little Endian file descriptor
+//
+// This function attempts to assume a more modern fgetc() role for UTF-16 encoded files.
+// Reads bytes from the File descriptor @c f until a full UTF-16 unicode character has been read
+//
+// After this function the file pointer will have moved either by @c 2 or @c 4
+// bytes if the return value is positive. You can see how much by checking the return value.
+//
+// You shall need to provide an integer at @c c to contain either the decoded Unicode
+// codepoint or the Bigendian encoded UTF-16 bytes depending on the value of @c the cp argument.
+//
+// @param f A valid FILE descriptor from which to read the bytes
+// @param c Pass an int that will receive either the unicode code point value or
+// the UTF16 bytes depending on the value of the @c cp flag
+// @param cp A boolean flag. If @c true then the int passed at @c c will contain the unicode code point
+// of the read character, so the UTF-16 will be decoded.
+// If @c false the int passed at @c c will contain the value of the read bytes in UTF-16 without any decoding
+// @return Returns the number of bytes read (either @c 2 or @c 4) or an error if the function
+// fails for some reason. Possible error values are:
+// + @c RE_UTF16_INVALID_SEQUENCE: Either the read word or its surrogate pair if 4 bytes were read held illegal values
+// + @c RE_UTF16_NO_SURRPAIR: According to the first read word a surrogate pair was expected but none was found
+// + @c RE_FILE_EOF: The end of file has been found while reading. If the end of file is encountered
+// while we expect a UTF-16 surrogate pair an appropriate error is logged
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgetc_UTF16LE(FILE* f,uint32_t *c,char cp);
+// @brief  Gets a unicode character from a UTF-32 Little Endian file descriptor
+//
+// This function attempts to assume a more modern fgetc() role for UTF-32 encoded files.
+// Reads bytes from the File descriptor @c f until a full UTF-32 unicode character has been read
+//
+// After this function the file pointer will have moved by @c 4
+// bytes if the return value is positive.
+//
+// You shall need to provide an integer at @c to contain the UTF-32 codepoint.
+//
+// @param f A valid FILE descriptor from which to read the bytes
+// @param c Pass an int that will receive either the unicode code point value or
+// the UTF16 bytes depending on the value of the @c cp flag
+// If @c false the int passed at @c c will contain the value of the read bytes in UTF-16 without any decoding
+// @return Returns either @c RF_SUCCESS for succesfull readin or one of the following errors:
+// + @c RE_FILE_EOF: The end of file has been found while reading.
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgetc_UTF32LE(FILE* f,uint32_t *c);
+// @brief  Gets a unicode character from a UTF-32 Big Endian file descriptor
+//
+// This function attempts to assume a more modern fgetc() role for UTF-32 encoded files.
+// Reads bytes from the File descriptor @c f until a full UTF-32 unicode character has been read
+//
+// After this function the file pointer will have moved by @c 4
+// bytes if the return value is positive.
+//
+// You shall need to provide an integer at @c to contain the UTF-32 codepoint.
+//
+// @param f A valid FILE descriptor from which to read the bytes
+// @param c Pass an int that will receive either the unicode code point value or
+// the UTF16 bytes depending on the value of the @c cp flag
+// If @c false the int passed at @c c will contain the value of the read bytes in UTF-16 without any decoding
+// @return Returns either @c RF_SUCCESS for succesfull readin or one of the following errors:
+// + @c RE_FILE_EOF: The end of file has been found while reading.
+// + @c RE_FILE_READ: If during reading the file there was an unknown read error
+// + @c RE_FILE_READ_BLOCK: If the read operation failed due to the file descriptor being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the file descriptor's mode was not correctly set for reading
+// + @c RE_FILE_POS_OVERFLOW: If during reading, the current file position can't be represented by the system
+// + @c RE_INTERRUPT: If during reading, there was a system interrupt
+// + @c RE_FILE_IO: If there was a physical I/O error
+// + @c RE_FILE_NOSPACE: If reading failed due to insufficient storage space
+i_DECLIMEX_ int32_t rfFgetc_UTF32BE(FILE* f,uint32_t *c);
+
+// @brief Moves a unicode character backwards in a big endian UTF-32 file stream
+//
+// @param f The file stream
+// @param c Returns the character we moved back to as a unicode codepoint
+// @return Returns either @c RF_SUCCESS for success or one of the following errors:
+// + @c RE_FILE_POS_OVERFLOW: If during trying to read the current file's position it can't be represented by the system
+// + @c RE_FILE_BAD: If The file descriptor is corrupt/illegal
+// + @c RE_FILE_NOTFILE: If the file descriptor is not a file but something else. e.g. socket.
+// + @c RE_FILE_GETFILEPOS: If the file's position could not be retrieved for some unknown reason
+// + @c RE_FILE_WRITE_BLOCK: While attempting to move the file pointer, it was occupied by another thread, and the no block flag was set
+// + @c RE_INTERRUPT: Operating on the file failed due to a system interrupt
+// + @c RE_FILE_IO: There was a physical I/O error
+// + @c RE_FILE_NOSPACE: There was no space on the device holding the file
+// + @c RE_FILE_NOTFILE: The device we attempted to manipulate is non-existent
+// + @c RE_FILE_READ: If during reading the file there was an error
+// + @c RE_FILE_READ_BLOCK: If during reading the file the read operation failed due to the file being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the underlying file descriptor's mode was not correctly set for reading
+i_DECLIMEX_ int32_t rfFback_UTF32BE(FILE* f,uint32_t *c);
+// @brief Moves a unicode character backwards in a little endian UTF-32 file stream
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param f The file stream
+// @param c Returns the character we moved back to as a unicode codepoint
+// @return Returns either @c RF_SUCCESS for success or one of the following errors:
+// + @c RE_FILE_POS_OVERFLOW: If during trying to read the current file's position it can't be represented by the system
+// + @c RE_FILE_BAD: If The file descriptor is corrupt/illegal
+// + @c RE_FILE_NOTFILE: If the file descriptor is not a file but something else. e.g. socket.
+// + @c RE_FILE_GETFILEPOS: If the file's position could not be retrieved for some unknown reason
+// + @c RE_FILE_WRITE_BLOCK: While attempting to move the file pointer, it was occupied by another thread, and the no block flag was set
+// + @c RE_INTERRUPT: Operating on the file failed due to a system interrupt
+// + @c RE_FILE_IO: There was a physical I/O error
+// + @c RE_FILE_NOSPACE: There was no space on the device holding the file
+// + @c RE_FILE_NOTFILE: The device we attempted to manipulate is non-existent
+// + @c RE_FILE_READ: If during reading the file there was an error
+// + @c RE_FILE_READ_BLOCK: If during reading the file the read operation failed due to the file being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the underlying file descriptor's mode was not correctly set for reading
+i_DECLIMEX_ int32_t rfFback_UTF32LE(FILE* f,uint32_t *c);
+// @brief Moves a unicode character backwards in a big endian UTF-16 file stream
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param f The file stream
+// @param c Returns the character we moved back to as a unicode codepoint
+// @return Returns either the number of bytes moved backwards (either @c 4 or @c 2) for success or one of the following errors:
+// + @c RE_UTF16_INVALID_SEQUENCE: Either the read word or its surrogate pair if 4 bytes were read held illegal values
+// + @c RE_FILE_POS_OVERFLOW: If during trying to read the current file's position it can't be represented by the system
+// + @c RE_FILE_BAD: If The file descriptor is corrupt/illegal
+// + @c RE_FILE_NOTFILE: If the file descriptor is not a file but something else. e.g. socket.
+// + @c RE_FILE_GETFILEPOS: If the file's position could not be retrieved for some unknown reason
+// + @c RE_FILE_WRITE_BLOCK: While attempting to move the file pointer, it was occupied by another thread, and the no block flag was set
+// + @c RE_INTERRUPT: Operating on the file failed due to a system interrupt
+// + @c RE_FILE_IO: There was a physical I/O error
+// + @c RE_FILE_NOSPACE: There was no space on the device holding the file
+// + @c RE_FILE_NOTFILE: The device we attempted to manipulate is non-existent
+// + @c RE_FILE_READ: If during reading the file there was an error
+// + @c RE_FILE_READ_BLOCK: If during reading the file the read operation failed due to the file being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the underlying file descriptor's mode was not correctly set for reading
+i_DECLIMEX_ int32_t rfFback_UTF16BE(FILE* f,uint32_t *c);
+// @brief Moves a unicode character backwards in a little endian UTF-16 file stream
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param f The file stream
+// @param c Returns the character we moved back to as a unicode codepoint
+// @return Returns either the number of bytes moved backwards (either @c 4 or @c 2) for success or one of the following errors:
+// + @c RE_UTF16_INVALID_SEQUENCE: Either the read word or its surrogate pair if 4 bytes were read held illegal values
+// + @c RE_FILE_POS_OVERFLOW: If during trying to read the current file's position it can't be represented by the system
+// + @c RE_FILE_BAD: If The file descriptor is corrupt/illegal
+// + @c RE_FILE_NOTFILE: If the file descriptor is not a file but something else. e.g. socket.
+// + @c RE_FILE_GETFILEPOS: If the file's position could not be retrieved for some unknown reason
+// + @c RE_FILE_WRITE_BLOCK: While attempting to move the file pointer, it was occupied by another thread, and the no block flag was set
+// + @c RE_INTERRUPT: Operating on the file failed due to a system interrupt
+// + @c RE_FILE_IO: There was a physical I/O error
+// + @c RE_FILE_NOSPACE: There was no space on the device holding the file
+// + @c RE_FILE_NOTFILE: The device we attempted to manipulate is non-existent
+// + @c RE_FILE_READ: If during reading the file there was an error
+// + @c RE_FILE_READ_BLOCK: If during reading the file the read operation failed due to the file being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the underlying file descriptor's mode was not correctly set for reading
+i_DECLIMEX_ int32_t rfFback_UTF16LE(FILE* f,uint32_t *c);
+// @brief Moves a unicode character backwards in a UTF-8 file stream
+//
+// The file descriptor at @c f must have been opened in <b>binary</b> and not text mode. That means that if under
+// Windows make sure to call fopen with "wb", "rb" e.t.c. instead of the simple "w", "r" e.t.c. since the initial
+// default value under Windows is text mode. Alternatively you can set the initial value using _get_fmode() and
+// _set_fmode(). For more information take a look at the msdn pages here:
+// http://msdn.microsoft.com/en-us/library/ktss1a9b.aspx
+//
+// @param f The file stream
+// @param c Returns the character we moved back to as a unicode codepoint
+// @return Returns either the number of bytes moved backwards for success (either @c 4, @c 3, @c 2 or @c 1) or one of the following errors:
+// + @c RE_UTF8_INVALID_SEQUENCE: If during moving bacwards in the file unexpected UTF-8 bytes were found
+// + @c RE_FILE_POS_OVERFLOW: If during trying to read the current file's position it can't be represented by the system
+// + @c RE_FILE_BAD: If The file descriptor is corrupt/illegal
+// + @c RE_FILE_NOTFILE: If the file descriptor is not a file but something else. e.g. socket.
+// + @c RE_FILE_GETFILEPOS: If the file's position could not be retrieved for some unknown reason
+// + @c RE_FILE_WRITE_BLOCK: While attempting to move the file pointer, it was occupied by another thread, and the no block flag was set
+// + @c RE_INTERRUPT: Operating on the file failed due to a system interrupt
+// + @c RE_FILE_IO: There was a physical I/O error
+// + @c RE_FILE_NOSPACE: There was no space on the device holding the file
+// + @c RE_FILE_NOTFILE: The device we attempted to manipulate is non-existent
+// + @c RE_FILE_READ: If during reading the file there was an error
+// + @c RE_FILE_READ_BLOCK: If during reading the file the read operation failed due to the file being occupied by another thread
+// + @c RE_FILE_MODE: If during reading the file the underlying file descriptor's mode was not correctly set for reading
+i_DECLIMEX_ int32_t rfFback_UTF8(FILE* f,uint32_t *c);
+
+// @brief Opens another process as a pipe
+//
+// This function is a cross-platform popen wrapper. In linux it uses popen and in Windows it uses
+// _popen.
+// @lmsFunction
+// @param command The string with the command to execute. Is basically the name of the program/process you want to spawn
+// with its full path and its parameters. @inhtype{String,StringX} @tmpSTR
+// @param mode The mode you want the pipe to work in. There are two possible values:
+// + @c "r" The calling process can read the spawned command's standard output via the returned stream.
+// + @c "w" The calling process can write to the spawned command's standard input via the returned stream.
+//
+// Anything else will result in an error
+// @return For success popen will return a FILE descriptor that can be used to either read or write from the pipe.
+// If there was an error @c 0 is returned and an error is logged.
+#ifdef RF_IAMHERE_FOR_DOXYGEN
+i_DECLIMEX_ FILE* rfPopen(void* command,const char* mode);
+#else
+i_DECLIMEX_ FILE* i_rfPopen(void* command,const char* mode);
+#define rfPopen(i_CMD_,i_MODE_) i_rfLMS_WRAP2(FILE*,i_rfPopen,i_CMD_,i_MODE_)
+#endif
+
+// @brief Closes a pipe
+//
+// This function is a cross-platform wrapper for pclose. It closes a file descriptor opened with @ref rfPopen() and
+// returns the exit code of the process that was running
+// @param stream The file descriptor of the pipe returned by @ref rfPopen() that we want to close
+// @return Returns the exit code of the process or -1 if there was an error
+i_DECLIMEX_ int rfPclose(FILE* stream);
+
+// @} End of I/O group
+
+#ifdef __cplusplus
+}///closing bracket for calling from C++
+#endif
+
+
+#endif//include guards end

samples/Prolog/calc.pl

@@ -0,0 +1,68 @@
+action_module(calculator) .
+
+
+%[-,-,d1,-] --push(D)-->          [-,-,D,-]  if mode(init)
+push(D) < -
+  mode(init),
+  deny([displayed(D1),mode(init)]),
+  affirm([displayed(D),mode(cont)]).
+
+%[-,-,D1,-] --push(D)-->          [-,-,10*D1+D,-]  if mode(cont)
+push(D) < -
+  mode(cont),
+  deny(displayed(D1)),
+  New = 10*D1 + D,
+  affirm(displayed(New)).
+
+%[a,op,d,m] --push(clear)-->      [0,nop,0,0]
+push(clear) < -
+  deny([accumulator(A),op(O),displayed(D),memory(M),mode(X)]),
+  affirm([accumulator(0),op(nop),displayed(0),memory(0),mode(init)]).
+
+%[a,op,d,m] --push(mem_rec)-->    [a,op,m,m]
+push(mem_rec) < -
+  memory(M),
+  deny([displayed(D),mode(X)]),
+  affirm([displayed(M),mode(init)]).
+
+%[a,op,d,m] --push(plus)-->       [op(a,d),plus,d,m]
+push(plus) < -
+  displayed(D),
+  deny([accumulator(A),op(O),mode(X)]),
+  eval(O,A,D,V),   ; use normal arithmetic, i.e., V=O(A,D)
+  affirm([accumulator(V),op(plus),mode(init)]).
+
+%[a,op,d,m] --push(minus)-->      [op(a,d,minus,d,m]
+push(minus) lt -
+  displayed(D),
+  deny([accumulator(A),op(O),mode(X)]),
+  eval(O,A,D,V),   ; use normal arithmetic, i.e., V=O(A,D)
+  affirm([accumulator(V),op(minus),mode(init)]).
+
+%[a,op,d,m] --push(times)-->      [op(a,d),times,d,m]
+push(times) < -
+  displayed(D),
+  deny([accumulator(A),op(O),mode(X)]),
+  eval(O,A,D,V),   ; use normal arithmetic, i.e., V=O(A,D)
+  affirm([accumulator(V),op(times),mode(init)]).
+
+%[a,op,d,m] --push(equal)-->      [a,nop,op(a,d),m]
+push(equal) < -
+  accumulator(A),
+  deny([op(O),displayed(D),mode(X)]),
+  eval(O,A,D,V),
+  affirm([op(nop),displayed(V),mode(init)]).
+
+%[a,op,d,m] --push(mem_plus)-->   [a,nop,v,plus(m,v)] where v=op(a,d)
+push(mem_plus) < -
+  accumulator(A),
+  deny([op(O),displayed(D),memory(M),mode(X)]),
+  eval(O,A,D,V),
+  eval(plus,M,V,V1),
+  affirm([op(nop),displayed(V),memory(V1),mode(init)]).
+
+%[a,op,d,m] --push(plus_minus)--> [a,op,-d,m]
+push(clear) < -
+  deny([displayed(D),mode(X)]),
+  eval(minus,0,D,V),
+  affirm([displayed(V),mode(init)]).

samples/Prolog/normal_form.pl

@@ -0,0 +1,94 @@
+%%----- normalize(+Wff,-NormalClauses) ------
+normalize(Wff,NormalClauses) :-
+   conVert(Wff,[],S),
+   cnF(S,T),
+   flatten_and(T,U),
+   make_clauses(U,NormalClauses).
+
+%%-----  make a sequence out of a conjunction -----
+flatten_and(X /\ Y, F) :-
+   !,
+   flatten_and(X,A),
+   flatten_and(Y, B),
+   sequence_append(A,B,F).
+flatten_and(X,X).
+
+%%-----  make a sequence out of a disjunction -----
+flatten_or(X \/ Y, F) :-
+   !,
+   flatten_or(X,A),
+   flatten_or(Y,B),
+   sequence_append(A,B,F).
+flatten_or(X,X).
+
+
+%%----- append two sequences -------------------------------
+sequence_append((X,R),S,(X,T)) :- !, sequence_append(R,S,T).
+sequence_append((X),S,(X,S)).
+
+%%----- separate into positive and negative literals -----------
+separate((A,B),P,N) :-
+   !,
+   (A = ~X -> N=[X|N1],
+               separate(B,P,N1)
+             ;
+               P=[A|P1],
+               separate(B,P1,N) ).
+separate(A,P,N) :-
+   (A = ~X -> N=[X],
+               P = []
+            ;
+               P=[A],
+               N = [] ).
+
+%%----- tautology ----------------------------
+tautology(P,N) :- some_occurs(N,P).
+
+some_occurs([F|R],B) :-
+   occurs(F,B) | some_occurs(R,B).
+
+occurs(A,[F|_]) :-
+   A == F,
+   !.
+occurs(A,[_|R]) :-
+   occurs(A,R).
+
+make_clauses((A,B),C) :-
+   !,
+   flatten_or(A,F),
+   separate(F,P,N),
+   (tautology(P,N) ->
+      make_clauses(B,C)
+          ;
+      make_clause(P,N,D),
+      C = [D|R],
+      make_clauses(B,R) ).
+make_clauses(A,C) :-
+   flatten_or(A,F),
+   separate(F,P,N),
+   (tautology(P,N) ->
+       C = []
+        ;
+       make_clause(P,N,D),
+       C = [D] ).
+
+make_clause([],N, false :- B) :-
+   !,
+   make_sequence(N,B,',').
+make_clause(P,[],H) :-
+   !,
+   make_sequence(P,H,'|').
+make_clause(P,N, H :- T) :-
+   make_sequence(P,H,'|'),
+   make_sequence(N,T,',').
+
+make_sequence([A],A,_) :- !.
+make_sequence([F|R],(F|S),'|') :-
+   make_sequence(R,S,'|').
+make_sequence([F|R],(F,S),',') :-
+   make_sequence(R,S,',').
+
+write_list([F|R]) :-
+   write(F), write('.'), nl,
+   write_list(R).
+write_list([]).

samples/Prolog/puzzle.pl

@@ -0,0 +1,287 @@
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%
+%%%     A* Algorithm
+%%%
+%%%
+%%%     Nodes have form    S#D#F#A
+%%%            where S describes the state or configuration
+%%%                  D is the depth of the node
+%%%                  F is the evaluation function value
+%%%                  A is the ancestor list for the node
+
+:- op(400,yfx,'#').    /* Node builder notation */
+
+solve(State,Soln) :- f_function(State,0,F),
+                     search([State#0#F#[]],S), reverse(S,Soln).
+
+f_function(State,D,F) :- h_function(State,H),
+                         F is D + H.
+
+search([State#_#_#Soln|_], Soln) :- goal(State).
+search([B|R],S) :- expand(B,Children),
+                   insert_all(Children,R,Open),
+                   search(Open,S).
+
+insert_all([F|R],Open1,Open3) :- insert(F,Open1,Open2),
+                                 insert_all(R,Open2,Open3).
+insert_all([],Open,Open).
+
+insert(B,Open,Open) :- repeat_node(B,Open), ! .
+insert(B,[C|R],[B,C|R]) :- cheaper(B,C), ! .
+insert(B,[B1|R],[B1|S]) :- insert(B,R,S), !.
+insert(B,[],[B]).
+
+repeat_node(P#_#_#_, [P#_#_#_|_]).
+
+cheaper( _#_#F1#_ , _#_#F2#_ ) :- F1 < F2.
+
+expand(State#D#_#S,All_My_Children) :-
+     bagof(Child#D1#F#[Move|S],
+           (D1 is D+1,
+             move(State,Child,Move),
+             f_function(Child,D1,F)),
+           All_My_Children).
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%
+%%%     8-puzzle solver
+%%%
+%%%
+%%%     State have form    A/B/C/D/E/F/G/H/I
+%%%            where {A,...,I} = {0,...,8}
+%%%               0 represents the empty tile
+%%%
+
+goal(1/2/3/8/0/4/7/6/5).
+
+%%% The puzzle moves
+
+left( A/0/C/D/E/F/H/I/J , 0/A/C/D/E/F/H/I/J ).
+left( A/B/C/D/0/F/H/I/J , A/B/C/0/D/F/H/I/J ).
+left( A/B/C/D/E/F/H/0/J , A/B/C/D/E/F/0/H/J ).
+left( A/B/0/D/E/F/H/I/J , A/0/B/D/E/F/H/I/J ).
+left( A/B/C/D/E/0/H/I/J , A/B/C/D/0/E/H/I/J ).
+left( A/B/C/D/E/F/H/I/0 , A/B/C/D/E/F/H/0/I ).
+
+up( A/B/C/0/E/F/H/I/J , 0/B/C/A/E/F/H/I/J ).
+up( A/B/C/D/0/F/H/I/J , A/0/C/D/B/F/H/I/J ).
+up( A/B/C/D/E/0/H/I/J , A/B/0/D/E/C/H/I/J ).
+up( A/B/C/D/E/F/0/I/J , A/B/C/0/E/F/D/I/J ).
+up( A/B/C/D/E/F/H/0/J , A/B/C/D/0/F/H/E/J ).
+up( A/B/C/D/E/F/H/I/0 , A/B/C/D/E/0/H/I/F ).
+
+right( A/0/C/D/E/F/H/I/J , A/C/0/D/E/F/H/I/J ).
+right( A/B/C/D/0/F/H/I/J , A/B/C/D/F/0/H/I/J ).
+right( A/B/C/D/E/F/H/0/J , A/B/C/D/E/F/H/J/0 ).
+right( 0/B/C/D/E/F/H/I/J , B/0/C/D/E/F/H/I/J ).
+right( A/B/C/0/E/F/H/I/J , A/B/C/E/0/F/H/I/J ).
+right( A/B/C/D/E/F/0/I/J , A/B/C/D/E/F/I/0/J ).
+
+down( A/B/C/0/E/F/H/I/J , A/B/C/H/E/F/0/I/J ).
+down( A/B/C/D/0/F/H/I/J , A/B/C/D/I/F/H/0/J ).
+down( A/B/C/D/E/0/H/I/J , A/B/C/D/E/J/H/I/0 ).
+down( 0/B/C/D/E/F/H/I/J , D/B/C/0/E/F/H/I/J ).
+down( A/0/C/D/E/F/H/I/J , A/E/C/D/0/F/H/I/J ).
+down( A/B/0/D/E/F/H/I/J , A/B/F/D/E/0/H/I/J ).
+
+%%% the heuristic function
+h_function(Puzz,H) :- p_fcn(Puzz,P),
+                      s_fcn(Puzz,S),
+                      H is P + 3*S.
+
+
+%%% the move
+move(P,C,left) :-  left(P,C).
+move(P,C,up) :-  up(P,C).
+move(P,C,right) :-  right(P,C).
+move(P,C,down) :-  down(P,C).
+
+%%% the Manhattan distance function
+p_fcn(A/B/C/D/E/F/G/H/I, P) :-
+     a(A,Pa), b(B,Pb), c(C,Pc),
+     d(D,Pd), e(E,Pe), f(F,Pf),
+     g(G,Pg), h(H,Ph), i(I,Pi),
+     P is Pa+Pb+Pc+Pd+Pe+Pf+Pg+Ph+Pg+Pi.
+
+a(0,0). a(1,0). a(2,1). a(3,2). a(4,3). a(5,4). a(6,3). a(7,2). a(8,1).
+b(0,0). b(1,1). b(2,0). b(3,1). b(4,2). b(5,3). b(6,2). b(7,3). b(8,2).
+c(0,0). c(1,2). c(2,1). c(3,0). c(4,1). c(5,2). c(6,3). c(7,4). c(8,3).
+d(0,0). d(1,1). d(2,2). d(3,3). d(4,2). d(5,3). d(6,2). d(7,2). d(8,0).
+e(0,0). e(1,2). e(2,1). e(3,2). e(4,1). e(5,2). e(6,1). e(7,2). e(8,1).
+f(0,0). f(1,3). f(2,2). f(3,1). f(4,0). f(5,1). f(6,2). f(7,3). f(8,2).
+g(0,0). g(1,2). g(2,3). g(3,4). g(4,3). g(5,2). g(6,2). g(7,0). g(8,1).
+h(0,0). h(1,3). h(2,3). h(3,3). h(4,2). h(5,1). h(6,0). h(7,1). h(8,2).
+i(0,0). i(1,4). i(2,3). i(3,2). i(4,1). i(5,0). i(6,1). i(7,2). i(8,3).
+
+%%% the out-of-cycle function
+s_fcn(A/B/C/D/E/F/G/H/I, S) :-
+     s_aux(A,B,S1), s_aux(B,C,S2), s_aux(C,F,S3),
+     s_aux(F,I,S4), s_aux(I,H,S5), s_aux(H,G,S6),
+     s_aux(G,D,S7), s_aux(D,A,S8), s_aux(E,S9),
+     S is S1+S2+S3+S4+S5+S6+S7+S8+S9.
+
+s_aux(0,0) :- !.
+s_aux(_,1).
+
+s_aux(X,Y,0) :- Y is X+1, !.
+s_aux(8,1,0) :- !.
+s_aux(_,_,2).
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%
+%%%     8-puzzle animation -- using VT100 character graphics
+%%%
+%%%
+%%%
+
+puzzle(P) :- solve(P,S),
+             animate(P,S),
+             message.
+
+animate(P,S) :- initialize(P),
+                cursor(1,2), write(S),
+                cursor(1,22), write('Hit ENTER to step solver.'),
+                get0(_X),
+                play_back(S).
+
+:- dynamic location/3.
+
+initialize(A/B/C/D/E/F/H/I/J) :-
+  cls,
+  retractall(location(_,_,_)),
+  assert(location(A,20,5)),
+  assert(location(B,30,5)),
+  assert(location(C,40,5)),
+  assert(location(F,40,10)),
+  assert(location(J,40,15)),
+  assert(location(I,30,15)),
+  assert(location(H,20,15)),
+  assert(location(D,20,10)),
+  assert(location(E,30,10)), draw_all.
+
+draw_all :- draw(1), draw(2), draw(3), draw(4),
+            draw(5), draw(6), draw(7), draw(8).
+
+%%% play_back([left,right,up,...]).
+play_back([M|R]) :- call(M), get0(_X), play_back(R).
+play_back([]) :- cursor(1,24).  %%% Put cursor out of the way
+
+message :- nl,nl,
+   write('    ********************************************'), nl,
+   write('    *  Enter 8-puzzle goals in the form ...    *'), nl,
+   write('    *     ?-  puzzle(0/8/1/2/4/3/7/6/5).       *'), nl,
+   write('    *   Enter goal ''message'' to reread this.   *'), nl,
+   write('    ********************************************'), nl, nl.
+
+
+cursor(X,Y) :- put(27), put(91), %%% ESC [
+               write(Y),
+               put(59),           %%%   ;
+               write(X),
+               put(72).           %%%   M
+
+%%% clear the screen, quickly
+cls :-  put(27), put("["), put("2"), put("J").
+
+%%% video attributes -- bold and blink not working
+plain         :- put(27), put("["), put("0"), put("m").
+reverse_video :- put(27), put("["), put("7"), put("m").
+
+
+%%% Tile objects, character map(s)
+%%% Each tile should be drawn using the character map,
+%%%   drawn at 'location', which is asserted and retracted
+%%%   by 'playback'.
+character_map(N, [ [' ',' ',' ',' ',' ',' ',' '],
+                   [' ',' ',' ', N ,' ',' ',' '],
+                   [' ',' ',' ',' ',' ',' ',' '] ]).
+
+
+%%% move empty tile (spot) to the left
+left :- retract(location(0,X0,Y0)),
+        Xnew is X0 - 10,
+        location(Tile,Xnew,Y0),
+        assert(location(0,Xnew,Y0)),
+        right(Tile),right(Tile),right(Tile),
+        right(Tile),right(Tile),
+        right(Tile),right(Tile),right(Tile),
+        right(Tile),right(Tile).
+
+up :- retract(location(0,X0,Y0)),
+      Ynew is Y0 - 5,
+      location(Tile,X0,Ynew),
+      assert(location(0,X0,Ynew)),
+      down(Tile),down(Tile),down(Tile),down(Tile),down(Tile).
+
+right :- retract(location(0,X0,Y0)),
+         Xnew is X0 + 10,
+         location(Tile,Xnew,Y0),
+         assert(location(0,Xnew,Y0)),
+         left(Tile),left(Tile),left(Tile),left(Tile),left(Tile),
+         left(Tile),left(Tile),left(Tile),left(Tile),left(Tile).
+
+down :- retract(location(0,X0,Y0)),
+        Ynew is Y0 + 5,
+        location(Tile,X0,Ynew),
+        assert(location(0,X0,Ynew)),
+        up(Tile),up(Tile),up(Tile),up(Tile),up(Tile).
+
+
+draw(Obj) :- reverse_video, character_map(Obj,M),
+             location(Obj,X,Y),
+             draw(X,Y,M), plain.
+
+%%% hide tile
+hide(Obj) :- character_map(Obj,M),
+             location(Obj,X,Y),
+             hide(X,Y,M).
+
+hide(_,_,[]).
+hide(X,Y,[R|G]) :- hide_row(X,Y,R),
+                   Y1 is Y + 1,
+                   hide(X,Y1,G).
+
+hide_row(_,_,[]).
+hide_row(X,Y,[_|R]) :- cursor(X,Y),
+                       write(' '),
+                       X1 is X + 1,
+                       hide_row(X1,Y,R).
+
+%%% draw tile
+draw(_,_,[]).
+draw(X,Y,[R|G]) :- draw_row(X,Y,R),
+                   Y1 is Y + 1,
+                   draw(X,Y1,G).
+
+draw_row(_,_,[]).
+draw_row(X,Y,[P|R]) :- cursor(X,Y),
+                       write(P),
+                       X1 is X + 1,
+                       draw_row(X1,Y,R).
+
+%%% Move an Object up
+up(Obj)    :- hide(Obj),
+              retract(location(Obj,X,Y)),
+              Y1 is Y - 1,
+              assert(location(Obj,X,Y1)),
+              draw(Obj).
+
+down(Obj)  :- hide(Obj),
+              retract(location(Obj,X,Y)),
+              Y1 is Y + 1,
+              assert(location(Obj,X,Y1)),
+              draw(Obj).
+
+left(Obj)  :- hide(Obj),
+              retract(location(Obj,X,Y)),
+              X1 is X - 1,
+              assert(location(Obj,X1,Y)),
+              draw(Obj).
+
+right(Obj) :- hide(Obj),
+              retract(location(Obj,X,Y)),
+              X1 is X + 1,
+              assert(location(Obj,X1,Y)),
+              draw(Obj).
+
+:- message.

samples/Prolog/quicksort.pl

@@ -0,0 +1,13 @@
+partition([], _, [], []).
+partition([X|Xs], Pivot, Smalls, Bigs) :-
+    (   X @< Pivot ->
+        Smalls = [X|Rest],
+        partition(Xs, Pivot, Rest, Bigs)
+    ;   Bigs = [X|Rest],
+        partition(Xs, Pivot, Smalls, Rest)
+    ).
+
+quicksort([])     --> [].
+quicksort([X|Xs]) -->
+    { partition(Xs, X, Smaller, Bigger) },
+    quicksort(Smaller), [X], quicksort(Bigger).

samples/Prolog/turing.pl

@@ -0,0 +1,21 @@
+turing(Tape0, Tape) :-
+    perform(q0, [], Ls, Tape0, Rs),
+    reverse(Ls, Ls1),
+    append(Ls1, Rs, Tape).
+
+perform(qf, Ls, Ls, Rs, Rs) :- !.
+perform(Q0, Ls0, Ls, Rs0, Rs) :-
+    symbol(Rs0, Sym, RsRest),
+    once(rule(Q0, Sym, Q1, NewSym, Action)),
+    action(Action, Ls0, Ls1, [NewSym|RsRest], Rs1),
+    perform(Q1, Ls1, Ls, Rs1, Rs).
+
+symbol([], b, []).
+symbol([Sym|Rs], Sym, Rs).
+
+action(left, Ls0, Ls, Rs0, Rs) :- left(Ls0, Ls, Rs0, Rs).
+action(stay, Ls, Ls, Rs, Rs).
+action(right, Ls0, [Sym|Ls0], [Sym|Rs], Rs).
+
+left([], [], Rs0, [b|Rs0]).
+left([L|Ls], Ls, Rs, [L|Rs]).

samples/Shell/plugin.script!

@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+#	Bash script to "plugin" to the dotfile repository, does a lot of fun stuff
+#		like turning the normal dotfiles (eg .bashrc) into symlinks to this
+#		repository's versions of those files (ensure that updates are just a
+#		'git pull' away), optionally moving old files so that they can be
+#		preserved, setting up a cron job to automate the aforementioned git
+#		pull, and maybe some more fun stuff
+#
+
+shopt -s nocasematch	# This makes pattern matching case-insensitive
+
+POSTFIX="local"
+URL="https://github.com/brcooley/.f.git"
+PUSHURL="git@github.com:brcooley/.f.git"
+
+overwrite=true
+
+print_help () {
+	echo -e "\nA script to keep dotfiles up to date\n"
+	echo "Options:"
+	echo "    -k, --keep-local    Keeps local copies of dotfiles by appending"
+	echo "                        \"$POSTFIX\" to them"
+	exit 0
+}
+
+
+for opt in $@; do
+	case $opt in
+		-k | --keep-local) overwrite=false;;
+		-h | --help) print_help;;
+	esac
+done
+
+
+
+for f in .*; do
+	if [ "$f" = ".git" -o "$f" = "." -o "$f" = ".." -o "$f" = ".f" ]; then continue; fi
+	if [ -f "$HOME/$f" ]; then
+		if [ $overwrite = false ]; then
+			mv "$HOME/$f" "$HOME/${f}_$POSTFIX"
+		else
+			rm "$HOME/$f"
+		fi
+		# echo "Moving ~/$f to $HOME/${f}_$POSTFIX"
+	fi
+	ln -s "$PWD/$f" "$HOME/$f"
+done
+
+# Git versions prior to 1.7.? (1.7.1 confirmed) do not have a --local option
+git config --local remote.origin.url "$URL"
+git config --local remote.origin.pushurl "$PUSHURL"
+crontab .jobs.cron
+source ~/.bashrc
+echo "Plugin succesful"

samples/Shell/sbt.script!

@@ -0,0 +1,432 @@
+#!/usr/bin/env bash
+#
+# A more capable sbt runner, coincidentally also called sbt.
+# Author: Paul Phillips <paulp@typesafe.com>
+
+# todo - make this dynamic
+declare -r sbt_release_version=0.11.3
+declare -r sbt_snapshot_version=0.13.0-SNAPSHOT
+
+unset sbt_jar sbt_dir sbt_create sbt_snapshot sbt_launch_dir
+unset scala_version java_home sbt_explicit_version
+unset verbose debug quiet
+
+build_props_sbt () {
+  if [[ -f project/build.properties ]]; then
+    versionLine=$(grep ^sbt.version project/build.properties)
+    versionString=${versionLine##sbt.version=}
+    echo "$versionString"
+  fi
+}
+
+update_build_props_sbt () {
+  local ver="$1"
+  local old=$(build_props_sbt)
+
+  if [[ $ver == $old ]]; then
+    return
+  elif [[ -f project/build.properties ]]; then
+    perl -pi -e "s/^sbt\.version=.*\$/sbt.version=${ver}/" project/build.properties
+    grep -q '^sbt.version=' project/build.properties || echo "sbt.version=${ver}" >> project/build.properties
+
+    echo !!!
+    echo !!! Updated file project/build.properties setting sbt.version to: $ver
+    echo !!! Previous value was: $old
+    echo !!!
+  fi
+}
+
+sbt_version () {
+  if [[ -n $sbt_explicit_version ]]; then
+    echo $sbt_explicit_version
+  else
+    local v=$(build_props_sbt)
+    if [[ -n $v ]]; then
+      echo $v
+    else
+      echo $sbt_release_version
+    fi
+  fi
+}
+
+echoerr () {
+  echo 1>&2 "$@"
+}
+vlog () {
+  [[ $verbose || $debug ]] && echoerr "$@"
+}
+dlog () {
+  [[ $debug ]] && echoerr "$@"
+}
+
+# this seems to cover the bases on OSX, and someone will
+# have to tell me about the others.
+get_script_path () {
+  local path="$1"
+  [[ -L "$path" ]] || { echo "$path" ; return; }
+
+  local target=$(readlink "$path")
+  if [[ "${target:0:1}" == "/" ]]; then
+    echo "$target"
+  else
+    echo "$(dirname $path)/$target"
+  fi
+}
+
+# a ham-fisted attempt to move some memory settings in concert
+# so they need not be dicked around with individually.
+get_mem_opts () {
+  local mem=${1:-1536}
+  local perm=$(( $mem / 4 ))
+  (( $perm > 256 )) || perm=256
+  (( $perm < 1024 )) || perm=1024
+  local codecache=$(( $perm / 2 ))
+  
+  echo "-Xms${mem}m -Xmx${mem}m -XX:MaxPermSize=${perm}m -XX:ReservedCodeCacheSize=${codecache}m"
+}
+
+die() {
+  echo "Aborting: $@"
+  exit 1
+}
+
+make_url () {
+  groupid="$1"
+  category="$2"
+  version="$3"
+  
+  echo "http://typesafe.artifactoryonline.com/typesafe/ivy-$category/$groupid/sbt-launch/$version/sbt-launch.jar"
+}
+
+declare -r default_jvm_opts="-Dfile.encoding=UTF8"
+declare -r default_sbt_opts="-XX:+CMSClassUnloadingEnabled"
+declare -r default_sbt_mem=1536
+declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
+declare -r sbt_opts_file=".sbtopts"
+declare -r jvm_opts_file=".jvmopts"
+declare -r latest_28="2.8.2"
+declare -r latest_29="2.9.1"
+declare -r latest_210="2.10.0-SNAPSHOT"
+
+declare -r script_path=$(get_script_path "$BASH_SOURCE")
+declare -r script_dir="$(dirname $script_path)"
+declare -r script_name="$(basename $script_path)"
+
+# some non-read-onlies set with defaults
+declare java_cmd=java
+declare sbt_launch_dir="$script_dir/.lib"
+declare sbt_mem=$default_sbt_mem
+
+# pull -J and -D options to give to java.
+declare -a residual_args
+declare -a java_args
+declare -a scalac_args
+declare -a sbt_commands
+
+build_props_scala () {
+  if [[ -f project/build.properties ]]; then
+    versionLine=$(grep ^build.scala.versions project/build.properties)
+    versionString=${versionLine##build.scala.versions=}
+    echo ${versionString%% .*}
+  fi
+}
+
+execRunner () {
+  # print the arguments one to a line, quoting any containing spaces
+  [[ $verbose || $debug ]] && echo "# Executing command line:" && {
+    for arg; do
+      if printf "%s\n" "$arg" | grep -q ' '; then
+        printf "\"%s\"\n" "$arg"
+      else
+        printf "%s\n" "$arg"
+      fi
+    done
+    echo ""
+  }
+
+  exec "$@"
+}
+
+sbt_groupid () {
+  case $(sbt_version) in
+        0.7.*) echo org.scala-tools.sbt ;;
+       0.10.*) echo org.scala-tools.sbt ;;
+    0.11.[12]) echo org.scala-tools.sbt ;;
+            *) echo org.scala-sbt ;;
+  esac
+}
+
+sbt_artifactory_list () {
+  local version0=$(sbt_version)
+  local version=${version0%-SNAPSHOT}
+  local url="http://typesafe.artifactoryonline.com/typesafe/ivy-snapshots/$(sbt_groupid)/sbt-launch/"
+  dlog "Looking for snapshot list at: $url "
+  
+  curl -s --list-only "$url" | \
+    grep -F $version | \
+    perl -e 'print reverse <>' | \
+    perl -pe 's#^<a href="([^"/]+).*#$1#;'
+}
+
+make_release_url () {
+  make_url $(sbt_groupid) releases $(sbt_version)
+}
+
+# argument is e.g. 0.13.0-SNAPSHOT
+# finds the actual version (with the build id) at artifactory
+make_snapshot_url () {
+  for ver in $(sbt_artifactory_list); do
+    local url=$(make_url $(sbt_groupid) snapshots $ver)
+    dlog "Testing $url"
+    curl -s --head "$url" >/dev/null
+    dlog "curl returned: $?"
+    echo "$url"
+    return
+  done
+}
+
+jar_url () {
+  case $(sbt_version) in
+             0.7.*) echo "http://simple-build-tool.googlecode.com/files/sbt-launch-0.7.7.jar" ;;
+        *-SNAPSHOT) make_snapshot_url ;;
+                 *) make_release_url ;;
+  esac
+}
+
+jar_file () {
+  echo "$sbt_launch_dir/$1/sbt-launch.jar"
+}
+
+download_url () {
+  local url="$1"
+  local jar="$2"
+  
+  echo "Downloading sbt launcher $(sbt_version):"
+  echo "  From  $url"
+  echo "    To  $jar"
+
+  mkdir -p $(dirname "$jar") && {
+    if which curl >/dev/null; then
+      curl --fail --silent "$url" --output "$jar"
+    elif which wget >/dev/null; then
+      wget --quiet -O "$jar" "$url"
+    fi
+  } && [[ -f "$jar" ]]
+}
+
+acquire_sbt_jar () {
+  sbt_url="$(jar_url)"
+  sbt_jar="$(jar_file $(sbt_version))"
+
+  [[ -f "$sbt_jar" ]] || download_url "$sbt_url" "$sbt_jar"
+}
+
+usage () {
+  cat <<EOM
+Usage: $script_name [options]
+
+  -h | -help         print this message
+  -v | -verbose      this runner is chattier
+  -d | -debug        set sbt log level to Debug
+  -q | -quiet        set sbt log level to Error
+  -no-colors         disable ANSI color codes
+  -sbt-create        start sbt even if current directory contains no sbt project
+  -sbt-dir   <path>  path to global settings/plugins directory (default: ~/.sbt/<version>)
+  -sbt-boot  <path>  path to shared boot directory (default: ~/.sbt/boot in 0.11 series)
+  -ivy       <path>  path to local Ivy repository (default: ~/.ivy2)
+  -mem    <integer>  set memory options (default: $sbt_mem, which is
+                       $(get_mem_opts $sbt_mem) )
+  -no-share          use all local caches; no sharing
+  -offline           put sbt in offline mode
+  -jvm-debug <port>  Turn on JVM debugging, open at the given port.
+  -batch             Disable interactive mode
+
+  # sbt version (default: from project/build.properties if present, else latest release)
+  !!! The only way to accomplish this pre-0.12.0 if there is a build.properties file which
+  !!! contains an sbt.version property is to update the file on disk.  That's what this does.
+  -sbt-version  <version>   use the specified version of sbt 
+  -sbt-jar      <path>      use the specified jar as the sbt launcher
+  -sbt-snapshot             use a snapshot version of sbt
+  -sbt-launch-dir <path>    directory to hold sbt launchers (default: $sbt_launch_dir)
+
+  # scala version (default: as chosen by sbt)
+  -28                       use $latest_28
+  -29                       use $latest_29
+  -210                      use $latest_210
+  -scala-home <path>        use the scala build at the specified directory
+  -scala-version <version>  use the specified version of scala
+
+  # java version (default: java from PATH, currently $(java -version |& grep version))
+  -java-home <path>         alternate JAVA_HOME
+
+  # jvm options and output control
+  JAVA_OPTS     environment variable holding jvm args, if unset uses "$default_jvm_opts"
+  SBT_OPTS      environment variable holding jvm args, if unset uses "$default_sbt_opts"
+  .jvmopts      if file is in sbt root, it is prepended to the args given to the jvm
+  .sbtopts      if file is in sbt root, it is prepended to the args given to **sbt**
+  -Dkey=val     pass -Dkey=val directly to the jvm
+  -J-X          pass option -X directly to the jvm (-J is stripped)
+  -S-X          add -X to sbt's scalacOptions (-S is stripped)
+
+In the case of duplicated or conflicting options, the order above
+shows precedence: JAVA_OPTS lowest, command line options highest.
+EOM
+}
+
+addJava () {
+  dlog "[addJava] arg = '$1'"
+  java_args=( "${java_args[@]}" "$1" )
+}
+addSbt () {
+  dlog "[addSbt] arg = '$1'"
+  sbt_commands=( "${sbt_commands[@]}" "$1" )
+}
+addScalac () {
+  dlog "[addScalac] arg = '$1'"
+  scalac_args=( "${scalac_args[@]}" "$1" )
+}
+addResidual () {
+  dlog "[residual] arg = '$1'"
+  residual_args=( "${residual_args[@]}" "$1" )
+}
+addResolver () {
+  addSbt "set resolvers in ThisBuild += $1"
+}
+addDebugger () {
+  addJava "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
+}
+get_jvm_opts () {
+  # echo "${JAVA_OPTS:-$default_jvm_opts}"
+  # echo "${SBT_OPTS:-$default_sbt_opts}"
+
+  [[ -f "$jvm_opts_file" ]] && cat "$jvm_opts_file"
+}
+
+process_args ()
+{
+  require_arg () {
+    local type="$1"
+    local opt="$2"
+    local arg="$3"
+    
+    if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then
+      die "$opt requires <$type> argument"
+    fi
+  }
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+       -h|-help) usage; exit 1 ;;
+    -v|-verbose) verbose=1 && shift ;;
+      -d|-debug) debug=1 && shift ;;
+      -q|-quiet) quiet=1 && shift ;;
+
+           -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;;
+           -mem) require_arg integer "$1" "$2" && sbt_mem="$2" && shift 2 ;;
+     -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;;
+      -no-share) addJava "$noshare_opts" && shift ;;
+      -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
+       -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;;
+     -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;;
+       -offline) addSbt "set offline := true" && shift ;;
+     -jvm-debug) require_arg port "$1" "$2" && addDebugger $2 && shift 2 ;;
+         -batch) exec </dev/null && shift ;;
+
+    -sbt-create) sbt_create=true && shift ;;
+  -sbt-snapshot) sbt_explicit_version=$sbt_snapshot_version && shift ;;
+       -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;;
+   -sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;;
+-sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;;
+ -scala-version) require_arg version "$1" "$2" && addSbt "set scalaVersion := \"$2\"" && shift 2 ;;
+    -scala-home) require_arg path "$1" "$2" && addSbt "set scalaHome in ThisBuild := Some(file(\"$2\"))" && shift 2 ;;
+     -java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && shift 2 ;;
+
+            -D*) addJava "$1" && shift ;;
+            -J*) addJava "${1:2}" && shift ;;
+            -S*) addScalac "${1:2}" && shift ;;
+            -28) addSbt "++ $latest_28" && shift ;;
+            -29) addSbt "++ $latest_29" && shift ;;
+           -210) addSbt "++ $latest_210" && shift ;;
+
+              *) addResidual "$1" && shift ;;
+    esac
+  done
+  
+  [[ $debug ]] && {
+    case $(sbt_version) in
+     0.7.*) addSbt "debug" ;; 
+         *) addSbt "set logLevel in Global := Level.Debug" ;;
+    esac
+  }
+  [[ $quiet ]] && {
+    case $(sbt_version) in
+     0.7.*) ;; 
+         *) addSbt "set logLevel in Global := Level.Error" ;;
+    esac
+  }
+}
+
+# if .sbtopts exists, prepend its contents to $@ so it can be processed by this runner
+[[ -f "$sbt_opts_file" ]] && {
+  sbtargs=()
+  while IFS= read -r arg; do
+    sbtargs=( "${sbtargs[@]}" "$arg" )
+  done <"$sbt_opts_file"
+
+  set -- "${sbtargs[@]}" "$@"
+}
+
+# process the combined args, then reset "$@" to the residuals
+process_args "$@"
+set -- "${residual_args[@]}"
+argumentCount=$#
+
+# set scalacOptions if we were given any -S opts
+[[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[@]}\""
+
+# Update build.properties no disk to set explicit version - sbt gives us no choice
+[[ -n "$sbt_explicit_version" ]] && update_build_props_sbt "$sbt_explicit_version"
+echo "Detected sbt version $(sbt_version)"
+
+[[ -n "$scala_version" ]] && echo "Overriding scala version to $scala_version"
+
+# no args - alert them there's stuff in here
+(( $argumentCount > 0 )) || echo "Starting $script_name: invoke with -help for other options"
+
+# verify this is an sbt dir or -create was given
+[[ -f ./build.sbt || -d ./project || -n "$sbt_create" ]] || {
+  cat <<EOM
+$(pwd) doesn't appear to be an sbt project.
+If you want to start sbt anyway, run:
+  $0 -sbt-create
+
+EOM
+  exit 1
+}
+
+# pick up completion if present; todo
+[[ -f .sbt_completion.sh ]] && source .sbt_completion.sh
+
+# no jar? download it.
+[[ -f "$sbt_jar" ]] || acquire_sbt_jar || {
+  # still no jar? uh-oh.
+  echo "Download failed. Obtain the jar manually and place it at $sbt_jar"
+  exit 1
+}
+
+[[ -n "$sbt_dir" ]] || {
+  sbt_dir=~/.sbt/$(sbt_version)
+  addJava "-Dsbt.global.base=$sbt_dir"
+  echo "Using $sbt_dir as sbt dir, -sbt-dir to override."
+}
+
+# since sbt 0.7 doesn't understand iflast
+(( ${#residual_args[@]} == 0 )) && residual_args=( "shell" )
+
+# run sbt
+execRunner "$java_cmd" \
+  $(get_mem_opts $sbt_mem) \
+  $(get_jvm_opts) \
+  ${java_args[@]} \
+  -jar "$sbt_jar" \
+  "${sbt_commands[@]}" \
+  "${residual_args[@]}"

test/test_blob.rb

@@ -2,6 +2,7 @@ require 'linguist/file_blob'
 require 'linguist/samples'
 
 require 'test/unit'
+require 'mocha'
 require 'mime/types'
 require 'pygments'
 
@@ -30,19 +31,17 @@ class TestBlob < Test::Unit::TestCase
   end
 
   def test_mime_type
-    assert_equal "application/octet-stream", blob("Binary/dog.o").mime_type
-    assert_equal "application/ogg", blob("Binary/foo.ogg").mime_type
     assert_equal "application/postscript", blob("Binary/octocat.ai").mime_type
     assert_equal "application/x-ruby", blob("Ruby/grit.rb").mime_type
     assert_equal "application/x-sh", blob("Shell/script.sh").mime_type
     assert_equal "application/xml", blob("XML/bar.xml").mime_type
+    assert_equal "audio/ogg", blob("Binary/foo.ogg").mime_type
     assert_equal "text/plain", blob("Text/README").mime_type
   end
 
   def test_content_type
-    assert_equal "application/octet-stream", blob("Binary/dog.o").content_type
-    assert_equal "application/ogg", blob("Binary/foo.ogg").content_type
     assert_equal "application/pdf", blob("Binary/foo.pdf").content_type
+    assert_equal "audio/ogg", blob("Binary/foo.ogg").content_type
     assert_equal "image/png", blob("Binary/foo.png").content_type
     assert_equal "text/plain; charset=iso-8859-2", blob("Text/README").content_type
   end
@@ -263,11 +262,18 @@ class TestBlob < Test::Unit::TestCase
     assert !blob("Text/dump.sql").indexable?
     assert !blob("Binary/github.po").indexable?
     assert !blob("Binary/linguist.gem").indexable?
+
+    # large binary blobs should fail on size check first, not call 
+    # into charlock_holmes and alloc big buffers for testing encoding
+    b = blob("Binary/octocat.ai")
+    b.expects(:binary?).never
+    assert !b.indexable?
   end
 
   def test_language
     Samples.each do |sample|
       blob = blob(sample[:path])
+      assert blob.language, "No language for #{sample[:path]}"
       assert_equal sample[:language], blob.language.name, blob.name
     end
   end

test/test_mime.rb

@@ -1,71 +0,0 @@
-require 'linguist/mime'
-
-require 'test/unit'
-
-class TestMime < Test::Unit::TestCase
-  include Linguist
-
-  def test_extension_lookup
-    # Default to plain text if we have no idea.
-    assert_equal 'text/plain', Mime.mime_for(nil)
-    assert_equal 'text/plain', Mime.mime_for('')
-
-    # Add an assertion to this list if you add/change any extensions
-    # in mimes.yml. Its still useful to test even trivial cases since
-    # MIME::Type's extension lookup may return multiple matches and we
-    # only pick one of them. Please keep this list alphabetized.
-    assert_equal 'application/javascript', Mime.mime_for('.js')
-    assert_equal 'application/octet-stream', Mime.mime_for('.dll')
-    assert_equal 'application/octet-stream', Mime.mime_for('.dmg')
-    assert_equal 'application/octet-stream', Mime.mime_for('.exe')
-    assert_equal 'application/ogg', Mime.mime_for('.ogg')
-    assert_equal 'application/postscript', Mime.mime_for('.ai')
-    assert_equal 'application/postscript', Mime.mime_for('.eps')
-    assert_equal 'application/postscript', Mime.mime_for('.ps')
-    assert_equal 'application/vnd.adobe.air-application-installer-package+zip', Mime.mime_for('.air')
-    assert_equal 'application/vnd.oasis.opendocument.presentation', Mime.mime_for('.odp')
-    assert_equal 'application/vnd.oasis.opendocument.spreadsheet', Mime.mime_for('.ods')
-    assert_equal 'application/vnd.oasis.opendocument.text', Mime.mime_for('.odt')
-    assert_equal 'application/vnd.openofficeorg.extension', Mime.mime_for('.oxt')
-    assert_equal 'application/vnd.openxmlformats-officedocument.presentationml.presentation', Mime.mime_for('.pptx')
-    assert_equal 'application/x-chrome-extension', Mime.mime_for('.crx')
-    assert_equal 'application/x-debian-package', Mime.mime_for('.deb')
-    assert_equal 'application/x-iwork-keynote-sffkey', Mime.mime_for('.key')
-    assert_equal 'application/x-iwork-numbers-sffnumbers', Mime.mime_for('.numbers')
-    assert_equal 'application/x-iwork-pages-sffpages', Mime.mime_for('.pages')
-    assert_equal 'application/x-java-archive', Mime.mime_for('.ear')
-    assert_equal 'application/x-java-archive', Mime.mime_for('.jar')
-    assert_equal 'application/x-java-archive', Mime.mime_for('.war')
-    assert_equal 'application/x-latex', Mime.mime_for('.latex')
-    assert_equal 'application/x-ms-xbap', Mime.mime_for('.xbap')
-    assert_equal 'application/x-perl', Mime.mime_for('.pl')
-    assert_equal 'application/x-perl', Mime.mime_for('.pm')
-    assert_equal 'application/x-python', Mime.mime_for('.py')
-    assert_equal 'application/x-ruby', Mime.mime_for('.rb')
-    assert_equal 'application/x-sh', Mime.mime_for('.sh')
-    assert_equal 'application/x-shockwave-flash', Mime.mime_for('.swf')
-    assert_equal 'application/x-silverlight-app', Mime.mime_for('.xap')
-    assert_equal 'application/x-supercollider', Mime.mime_for('.sc')
-    assert_equal 'application/xaml+xml', Mime.mime_for('.xaml')
-    assert_equal 'text/cache-manifest', Mime.mime_for('.manifest')
-    assert_equal 'text/html', Mime.mime_for('.html')
-    assert_equal 'text/plain', Mime.mime_for('.c')
-    assert_equal 'text/plain', Mime.mime_for('.cc')
-    assert_equal 'text/plain', Mime.mime_for('.cpp')
-    assert_equal 'text/plain', Mime.mime_for('.cu')
-    assert_equal 'text/plain', Mime.mime_for('.cxx')
-    assert_equal 'text/plain', Mime.mime_for('.h')
-    assert_equal 'text/plain', Mime.mime_for('.hh')
-    assert_equal 'text/plain', Mime.mime_for('.hpp')
-    assert_equal 'text/plain', Mime.mime_for('.kt')
-    assert_equal 'text/x-logtalk', Mime.mime_for('.lgt')
-    assert_equal 'text/x-nemerle', Mime.mime_for('.n')
-    assert_equal 'text/x-nimrod', Mime.mime_for('.nim')
-    assert_equal 'text/x-ocaml', Mime.mime_for('.ml')
-    assert_equal 'text/x-ocaml', Mime.mime_for('.sig')
-    assert_equal 'text/x-ocaml', Mime.mime_for('.sml')
-    assert_equal 'text/x-rust', Mime.mime_for('.rc')
-    assert_equal 'text/x-rust', Mime.mime_for('.rs')
-    assert_equal 'video/quicktime', Mime.mime_for('.mov')
-  end
-end

test/test_tokenizer.rb

@@ -34,15 +34,15 @@ class TestTokenizer < Test::Unit::TestCase
   end
 
   def test_skip_comments
-    assert_equal %w(foo #), tokenize("foo\n# Comment")
-    assert_equal %w(foo # bar), tokenize("foo\n# Comment\nbar")
-    assert_equal %w(foo //), tokenize("foo\n// Comment")
-    assert_equal %w(foo /* */), tokenize("foo /* Comment */")
-    assert_equal %w(foo /* */), tokenize("foo /* \nComment\n */")
-    assert_equal %w(foo <!-- -->), tokenize("foo <!-- Comment -->")
-    assert_equal %w(foo {- -}), tokenize("foo {- Comment -}")
-    assert_equal %w(foo \(* *\)), tokenize("foo (* Comment *)")
-    assert_equal %w(% %), tokenize("2 % 10\n% Comment")
+    assert_equal %w(foo), tokenize("foo\n# Comment")
+    assert_equal %w(foo bar), tokenize("foo\n# Comment\nbar")
+    assert_equal %w(foo), tokenize("foo\n// Comment")
+    assert_equal %w(foo), tokenize("foo /* Comment */")
+    assert_equal %w(foo), tokenize("foo /* \nComment\n */")
+    assert_equal %w(foo), tokenize("foo <!-- Comment -->")
+    assert_equal %w(foo), tokenize("foo {- Comment -}")
+    assert_equal %w(foo), tokenize("foo (* Comment *)")
+    assert_equal %w(%), tokenize("2 % 10\n% Comment")
   end
 
   def test_sgml_tags