Merge pull request #2911 from github/cut-release-v4.8.1

Cut release v4.8.1

.gitignore

@@ -4,3 +4,5 @@ benchmark/
 lib/linguist/samples.json
 /grammars
 /node_modules
+test/fixtures/ace_modes.json
+/vendor/gems/

.gitmodules

@@ -0,0 +1,700 @@
+[submodule "vendor/grammars/go-tmbundle"]
+	path = vendor/grammars/go-tmbundle
+	url = https://github.com/AlanQuatermain/go-tmbundle
+[submodule "vendor/grammars/PHP-Twig.tmbundle"]
+	path = vendor/grammars/PHP-Twig.tmbundle
+	url = https://github.com/Anomareh/PHP-Twig.tmbundle
+[submodule "vendor/grammars/sublime-cirru"]
+	path = vendor/grammars/sublime-cirru
+	url = https://github.com/Cirru/sublime-cirru
+[submodule "vendor/grammars/SublimeBrainfuck"]
+	path = vendor/grammars/SublimeBrainfuck
+	url = https://github.com/Drako/SublimeBrainfuck
+[submodule "vendor/grammars/awk-sublime"]
+	path = vendor/grammars/awk-sublime
+	url = https://github.com/JohnNilsson/awk-sublime
+[submodule "vendor/grammars/Sublime-SQF-Language"]
+	path = vendor/grammars/Sublime-SQF-Language
+	url = https://github.com/JonBons/Sublime-SQF-Language
+[submodule "vendor/grammars/SCSS.tmbundle"]
+	path = vendor/grammars/SCSS.tmbundle
+	url = https://github.com/MarioRicalde/SCSS.tmbundle
+[submodule "vendor/grammars/Sublime-REBOL"]
+	path = vendor/grammars/Sublime-REBOL
+	url = https://github.com/Oldes/Sublime-REBOL
+[submodule "vendor/grammars/Sublime-VimL"]
+	path = vendor/grammars/Sublime-VimL
+	url = https://github.com/SalGnt/Sublime-VimL
+[submodule "vendor/grammars/ColdFusion"]
+	path = vendor/grammars/ColdFusion
+	url = https://github.com/SublimeText/ColdFusion
+[submodule "vendor/grammars/NSIS"]
+	path = vendor/grammars/NSIS
+	url = https://github.com/SublimeText/NSIS
+[submodule "vendor/grammars/NimLime"]
+	path = vendor/grammars/NimLime
+	url = https://github.com/Varriount/NimLime
+[submodule "vendor/grammars/gradle.tmbundle"]
+	path = vendor/grammars/gradle.tmbundle
+	url = https://github.com/alkemist/gradle.tmbundle
+[submodule "vendor/grammars/Sublime-Loom"]
+	path = vendor/grammars/Sublime-Loom
+	url = https://github.com/ambethia/Sublime-Loom
+[submodule "vendor/grammars/VBDotNetSyntax"]
+	path = vendor/grammars/VBDotNetSyntax
+	url = https://github.com/angryant0007/VBDotNetSyntax
+[submodule "vendor/grammars/cool-tmbundle"]
+	path = vendor/grammars/cool-tmbundle
+	url = https://github.com/anunayk/cool-tmbundle
+[submodule "vendor/grammars/Docker.tmbundle"]
+	path = vendor/grammars/Docker.tmbundle
+	url = https://github.com/asbjornenge/Docker.tmbundle
+[submodule "vendor/grammars/jasmin-sublime"]
+	path = vendor/grammars/jasmin-sublime
+	url = https://github.com/atmarksharp/jasmin-sublime
+[submodule "vendor/grammars/language-clojure"]
+	path = vendor/grammars/language-clojure
+	url = https://github.com/atom/language-clojure
+[submodule "vendor/grammars/language-coffee-script"]
+	path = vendor/grammars/language-coffee-script
+	url = https://github.com/atom/language-coffee-script
+[submodule "vendor/grammars/language-csharp"]
+	path = vendor/grammars/language-csharp
+	url = https://github.com/atom/language-csharp
+[submodule "vendor/grammars/language-gfm"]
+	path = vendor/grammars/language-gfm
+	url = https://github.com/atom/language-gfm
+[submodule "vendor/grammars/language-javascript"]
+	path = vendor/grammars/language-javascript
+	url = https://github.com/atom/language-javascript
+[submodule "vendor/grammars/language-python"]
+	path = vendor/grammars/language-python
+	url = https://github.com/atom/language-python
+[submodule "vendor/grammars/language-shellscript"]
+	path = vendor/grammars/language-shellscript
+	url = https://github.com/atom/language-shellscript
+[submodule "vendor/grammars/language-supercollider"]
+	path = vendor/grammars/language-supercollider
+	url = https://github.com/supercollider/language-supercollider
+[submodule "vendor/grammars/language-yaml"]
+	path = vendor/grammars/language-yaml
+	url = https://github.com/atom/language-yaml
+[submodule "vendor/grammars/Sublime-Lasso"]
+	path = vendor/grammars/Sublime-Lasso
+	url = https://github.com/bfad/Sublime-Lasso
+[submodule "vendor/grammars/chapel-tmbundle"]
+	path = vendor/grammars/chapel-tmbundle
+	url = https://github.com/chapel-lang/chapel-tmbundle
+[submodule "vendor/grammars/sublime-nginx"]
+	path = vendor/grammars/sublime-nginx
+	url = https://github.com/brandonwamboldt/sublime-nginx
+[submodule "vendor/grammars/bro-sublime"]
+	path = vendor/grammars/bro-sublime
+	url = https://github.com/bro/bro-sublime
+[submodule "vendor/grammars/sublime_man_page_support"]
+	path = vendor/grammars/sublime_man_page_support
+	url = https://github.com/carsonoid/sublime_man_page_support
+[submodule "vendor/grammars/sublime-MuPAD"]
+	path = vendor/grammars/sublime-MuPAD
+	url = https://github.com/ccreutzig/sublime-MuPAD
+[submodule "vendor/grammars/haxe-sublime-bundle"]
+	path = vendor/grammars/haxe-sublime-bundle
+	url = https://github.com/clemos/haxe-sublime-bundle
+[submodule "vendor/grammars/cucumber-tmbundle"]
+	path = vendor/grammars/cucumber-tmbundle
+	url = https://github.com/cucumber/cucumber-tmbundle
+[submodule "vendor/grammars/powershell"]
+	path = vendor/grammars/powershell
+	url = https://github.com/SublimeText/PowerShell
+[submodule "vendor/grammars/jade-tmbundle"]
+	path = vendor/grammars/jade-tmbundle
+	url = https://github.com/davidrios/jade-tmbundle
+[submodule "vendor/grammars/elixir-tmbundle"]
+	path = vendor/grammars/elixir-tmbundle
+	url = https://github.com/elixir-lang/elixir-tmbundle
+[submodule "vendor/grammars/sublime-glsl"]
+	path = vendor/grammars/sublime-glsl
+	url = https://github.com/euler0/sublime-glsl
+[submodule "vendor/grammars/fancy-tmbundle"]
+	path = vendor/grammars/fancy-tmbundle
+	url = https://github.com/fancy-lang/fancy-tmbundle
+[submodule "vendor/grammars/dart-sublime-bundle"]
+	path = vendor/grammars/dart-sublime-bundle
+	url = https://github.com/guillermooo/dart-sublime-bundle
+[submodule "vendor/grammars/sublimetext-cuda-cpp"]
+	path = vendor/grammars/sublimetext-cuda-cpp
+	url = https://github.com/harrism/sublimetext-cuda-cpp
+[submodule "vendor/grammars/pike-textmate"]
+	path = vendor/grammars/pike-textmate
+	url = https://github.com/hww3/pike-textmate
+[submodule "vendor/grammars/ceylon-sublimetext"]
+	path = vendor/grammars/ceylon-sublimetext
+	url = https://github.com/jeancharles-roger/ceylon-sublimetext
+[submodule "vendor/grammars/Sublime-Text-2-OpenEdge-ABL"]
+	path = vendor/grammars/Sublime-Text-2-OpenEdge-ABL
+	url = https://github.com/jfairbank/Sublime-Text-2-OpenEdge-ABL
+[submodule "vendor/grammars/sublime-rust"]
+	path = vendor/grammars/sublime-rust
+	url = https://github.com/jhasse/sublime-rust
+[submodule "vendor/grammars/sublime-befunge"]
+	path = vendor/grammars/sublime-befunge
+	url = https://github.com/johanasplund/sublime-befunge
+[submodule "vendor/grammars/RDoc.tmbundle"]
+	path = vendor/grammars/RDoc.tmbundle
+	url = https://github.com/joshaven/RDoc.tmbundle
+[submodule "vendor/grammars/Textmate-Gosu-Bundle"]
+	path = vendor/grammars/Textmate-Gosu-Bundle
+	url = https://github.com/jpcamara/Textmate-Gosu-Bundle
+[submodule "vendor/grammars/fish-tmbundle"]
+	path = vendor/grammars/fish-tmbundle
+	url = https://github.com/l15n/fish-tmbundle
+[submodule "vendor/grammars/moonscript-tmbundle"]
+	path = vendor/grammars/moonscript-tmbundle
+	url = https://github.com/leafo/moonscript-tmbundle
+[submodule "vendor/grammars/Isabelle.tmbundle"]
+	path = vendor/grammars/Isabelle.tmbundle
+	url = https://github.com/lsf37/Isabelle.tmbundle
+[submodule "vendor/grammars/Alloy.tmbundle"]
+	path = vendor/grammars/Alloy.tmbundle
+	url = https://github.com/macekond/Alloy.tmbundle
+[submodule "vendor/grammars/opa.tmbundle"]
+	path = vendor/grammars/opa.tmbundle
+	url = https://github.com/mads379/opa.tmbundle
+[submodule "vendor/grammars/scala.tmbundle"]
+	path = vendor/grammars/scala.tmbundle
+	url = https://github.com/mads379/scala.tmbundle
+[submodule "vendor/grammars/mako-tmbundle"]
+	path = vendor/grammars/mako-tmbundle
+	url = https://github.com/marconi/mako-tmbundle
+[submodule "vendor/grammars/gnuplot-tmbundle"]
+	path = vendor/grammars/gnuplot-tmbundle
+	url = https://github.com/mattfoster/gnuplot-tmbundle
+[submodule "vendor/grammars/idl.tmbundle"]
+	path = vendor/grammars/idl.tmbundle
+	url = https://github.com/mgalloy/idl.tmbundle
+[submodule "vendor/grammars/protobuf-tmbundle"]
+	path = vendor/grammars/protobuf-tmbundle
+	url = https://github.com/michaeledgar/protobuf-tmbundle
+[submodule "vendor/grammars/Sublime-Coq"]
+	path = vendor/grammars/Sublime-Coq
+	url = https://github.com/mkolosick/Sublime-Coq
+[submodule "vendor/grammars/Agda.tmbundle"]
+	path = vendor/grammars/Agda.tmbundle
+	url = https://github.com/mokus0/Agda.tmbundle
+[submodule "vendor/grammars/Julia.tmbundle"]
+	path = vendor/grammars/Julia.tmbundle
+	url = https://github.com/nanoant/Julia.tmbundle
+[submodule "vendor/grammars/ooc.tmbundle"]
+	path = vendor/grammars/ooc.tmbundle
+	url = https://github.com/nilium/ooc.tmbundle
+[submodule "vendor/grammars/LiveScript.tmbundle"]
+	path = vendor/grammars/LiveScript.tmbundle
+	url = https://github.com/paulmillr/LiveScript.tmbundle
+[submodule "vendor/grammars/sublime-tea"]
+	path = vendor/grammars/sublime-tea
+	url = https://github.com/pferruggiaro/sublime-tea
+[submodule "vendor/grammars/abap.tmbundle"]
+	path = vendor/grammars/abap.tmbundle
+	url = https://github.com/pvl/abap.tmbundle
+[submodule "vendor/grammars/mercury-tmlanguage"]
+	path = vendor/grammars/mercury-tmlanguage
+	url = https://github.com/sebgod/mercury-tmlanguage
+[submodule "vendor/grammars/mathematica-tmbundle"]
+	path = vendor/grammars/mathematica-tmbundle
+	url = https://github.com/shadanan/mathematica-tmbundle
+[submodule "vendor/grammars/sublime-robot-plugin"]
+	path = vendor/grammars/sublime-robot-plugin
+	url = https://github.com/shellderp/sublime-robot-plugin
+[submodule "vendor/grammars/actionscript3-tmbundle"]
+	path = vendor/grammars/actionscript3-tmbundle
+	url = https://github.com/honzabrecka/actionscript3-tmbundle
+[submodule "vendor/grammars/Sublime-QML"]
+	path = vendor/grammars/Sublime-QML
+	url = https://github.com/skozlovf/Sublime-QML
+[submodule "vendor/grammars/Slash.tmbundle"]
+	path = vendor/grammars/Slash.tmbundle
+	url = https://github.com/slash-lang/Slash.tmbundle
+[submodule "vendor/grammars/factor"]
+	path = vendor/grammars/factor
+	url = https://github.com/slavapestov/factor
+[submodule "vendor/grammars/ruby-slim.tmbundle"]
+	path = vendor/grammars/ruby-slim.tmbundle
+	url = https://github.com/slim-template/ruby-slim.tmbundle
+[submodule "vendor/grammars/SublimeXtend"]
+	path = vendor/grammars/SublimeXtend
+	url = https://github.com/staltz/SublimeXtend
+[submodule "vendor/grammars/Vala-TMBundle"]
+	path = vendor/grammars/Vala-TMBundle
+	url = https://github.com/technosophos/Vala-TMBundle
+[submodule "vendor/grammars/ant.tmbundle"]
+	path = vendor/grammars/ant.tmbundle
+	url = https://github.com/textmate/ant.tmbundle
+[submodule "vendor/grammars/antlr.tmbundle"]
+	path = vendor/grammars/antlr.tmbundle
+	url = https://github.com/textmate/antlr.tmbundle
+[submodule "vendor/grammars/apache.tmbundle"]
+	path = vendor/grammars/apache.tmbundle
+	url = https://github.com/textmate/apache.tmbundle
+[submodule "vendor/grammars/applescript.tmbundle"]
+	path = vendor/grammars/applescript.tmbundle
+	url = https://github.com/textmate/applescript.tmbundle
+[submodule "vendor/grammars/asp.tmbundle"]
+	path = vendor/grammars/asp.tmbundle
+	url = https://github.com/textmate/asp.tmbundle
+[submodule "vendor/grammars/bison.tmbundle"]
+	path = vendor/grammars/bison.tmbundle
+	url = https://github.com/textmate/bison.tmbundle
+[submodule "vendor/grammars/capnproto.tmbundle"]
+	path = vendor/grammars/capnproto.tmbundle
+	url = https://github.com/textmate/capnproto.tmbundle
+[submodule "vendor/grammars/cmake.tmbundle"]
+	path = vendor/grammars/cmake.tmbundle
+	url = https://github.com/textmate/cmake.tmbundle
+[submodule "vendor/grammars/cpp-qt.tmbundle"]
+	path = vendor/grammars/cpp-qt.tmbundle
+	url = https://github.com/textmate/cpp-qt.tmbundle
+[submodule "vendor/grammars/css.tmbundle"]
+	path = vendor/grammars/css.tmbundle
+	url = https://github.com/textmate/css.tmbundle
+[submodule "vendor/grammars/d.tmbundle"]
+	path = vendor/grammars/d.tmbundle
+	url = https://github.com/textmate/d.tmbundle
+[submodule "vendor/grammars/diff.tmbundle"]
+	path = vendor/grammars/diff.tmbundle
+	url = https://github.com/textmate/diff.tmbundle
+[submodule "vendor/grammars/dylan.tmbundle"]
+	path = vendor/grammars/dylan.tmbundle
+	url = https://github.com/textmate/dylan.tmbundle
+[submodule "vendor/grammars/eiffel.tmbundle"]
+	path = vendor/grammars/eiffel.tmbundle
+	url = https://github.com/textmate/eiffel.tmbundle
+[submodule "vendor/grammars/erlang.tmbundle"]
+	path = vendor/grammars/erlang.tmbundle
+	url = https://github.com/textmate/erlang.tmbundle
+[submodule "vendor/grammars/fortran.tmbundle"]
+	path = vendor/grammars/fortran.tmbundle
+	url = https://github.com/textmate/fortran.tmbundle
+[submodule "vendor/grammars/gettext.tmbundle"]
+	path = vendor/grammars/gettext.tmbundle
+	url = https://github.com/textmate/gettext.tmbundle
+[submodule "vendor/grammars/graphviz.tmbundle"]
+	path = vendor/grammars/graphviz.tmbundle
+	url = https://github.com/textmate/graphviz.tmbundle
+[submodule "vendor/grammars/groovy.tmbundle"]
+	path = vendor/grammars/groovy.tmbundle
+	url = https://github.com/textmate/groovy.tmbundle
+[submodule "vendor/grammars/haskell.tmbundle"]
+	path = vendor/grammars/haskell.tmbundle
+	url = https://github.com/textmate/haskell.tmbundle
+[submodule "vendor/grammars/html.tmbundle"]
+	path = vendor/grammars/html.tmbundle
+	url = https://github.com/textmate/html.tmbundle
+[submodule "vendor/grammars/ini.tmbundle"]
+	path = vendor/grammars/ini.tmbundle
+	url = https://github.com/textmate/ini.tmbundle
+[submodule "vendor/grammars/desktop.tmbundle"]
+	path = vendor/grammars/desktop.tmbundle
+	url = https://github.com/Mailaender/desktop.tmbundle.git
+[submodule "vendor/grammars/io.tmbundle"]
+	path = vendor/grammars/io.tmbundle
+	url = https://github.com/textmate/io.tmbundle
+[submodule "vendor/grammars/java.tmbundle"]
+	path = vendor/grammars/java.tmbundle
+	url = https://github.com/textmate/java.tmbundle
+[submodule "vendor/grammars/javascript-objective-j.tmbundle"]
+	path = vendor/grammars/javascript-objective-j.tmbundle
+	url = https://github.com/textmate/javascript-objective-j.tmbundle
+[submodule "vendor/grammars/json.tmbundle"]
+	path = vendor/grammars/json.tmbundle
+	url = https://github.com/textmate/json.tmbundle
+[submodule "vendor/grammars/latex.tmbundle"]
+	path = vendor/grammars/latex.tmbundle
+	url = https://github.com/textmate/latex.tmbundle
+[submodule "vendor/grammars/lilypond.tmbundle"]
+	path = vendor/grammars/lilypond.tmbundle
+	url = https://github.com/textmate/lilypond.tmbundle
+[submodule "vendor/grammars/lisp.tmbundle"]
+	path = vendor/grammars/lisp.tmbundle
+	url = https://github.com/textmate/lisp.tmbundle
+[submodule "vendor/grammars/logtalk.tmbundle"]
+	path = vendor/grammars/logtalk.tmbundle
+	url = https://github.com/textmate/logtalk.tmbundle
+[submodule "vendor/grammars/lua.tmbundle"]
+	path = vendor/grammars/lua.tmbundle
+	url = https://github.com/textmate/lua.tmbundle
+[submodule "vendor/grammars/make.tmbundle"]
+	path = vendor/grammars/make.tmbundle
+	url = https://github.com/textmate/make.tmbundle
+[submodule "vendor/grammars/matlab.tmbundle"]
+	path = vendor/grammars/matlab.tmbundle
+	url = https://github.com/textmate/matlab.tmbundle
+[submodule "vendor/grammars/maven.tmbundle"]
+	path = vendor/grammars/maven.tmbundle
+	url = https://github.com/textmate/maven.tmbundle
+[submodule "vendor/grammars/nemerle.tmbundle"]
+	path = vendor/grammars/nemerle.tmbundle
+	url = https://github.com/textmate/nemerle.tmbundle
+[submodule "vendor/grammars/ninja.tmbundle"]
+	path = vendor/grammars/ninja.tmbundle
+	url = https://github.com/textmate/ninja.tmbundle
+[submodule "vendor/grammars/objective-c.tmbundle"]
+	path = vendor/grammars/objective-c.tmbundle
+	url = https://github.com/textmate/objective-c.tmbundle
+[submodule "vendor/grammars/ocaml.tmbundle"]
+	path = vendor/grammars/ocaml.tmbundle
+	url = https://github.com/textmate/ocaml.tmbundle
+[submodule "vendor/grammars/pascal.tmbundle"]
+	path = vendor/grammars/pascal.tmbundle
+	url = https://github.com/textmate/pascal.tmbundle
+[submodule "vendor/grammars/php-smarty.tmbundle"]
+	path = vendor/grammars/php-smarty.tmbundle
+	url = https://github.com/textmate/php-smarty.tmbundle
+[submodule "vendor/grammars/php.tmbundle"]
+	path = vendor/grammars/php.tmbundle
+	url = https://github.com/textmate/php.tmbundle
+[submodule "vendor/grammars/postscript.tmbundle"]
+	path = vendor/grammars/postscript.tmbundle
+	url = https://github.com/textmate/postscript.tmbundle
+[submodule "vendor/grammars/processing.tmbundle"]
+	path = vendor/grammars/processing.tmbundle
+	url = https://github.com/textmate/processing.tmbundle
+[submodule "vendor/grammars/python-django.tmbundle"]
+	path = vendor/grammars/python-django.tmbundle
+	url = https://github.com/textmate/python-django.tmbundle
+[submodule "vendor/grammars/r.tmbundle"]
+	path = vendor/grammars/r.tmbundle
+	url = https://github.com/textmate/r.tmbundle
+[submodule "vendor/grammars/restructuredtext.tmbundle"]
+	path = vendor/grammars/restructuredtext.tmbundle
+	url = https://github.com/textmate/restructuredtext.tmbundle
+[submodule "vendor/grammars/ruby-haml.tmbundle"]
+	path = vendor/grammars/ruby-haml.tmbundle
+	url = https://github.com/textmate/ruby-haml.tmbundle
+[submodule "vendor/grammars/scheme.tmbundle"]
+	path = vendor/grammars/scheme.tmbundle
+	url = https://github.com/textmate/scheme.tmbundle
+[submodule "vendor/grammars/scilab.tmbundle"]
+	path = vendor/grammars/scilab.tmbundle
+	url = https://github.com/textmate/scilab.tmbundle
+[submodule "vendor/grammars/sql.tmbundle"]
+	path = vendor/grammars/sql.tmbundle
+	url = https://github.com/textmate/sql.tmbundle
+[submodule "vendor/grammars/standard-ml.tmbundle"]
+	path = vendor/grammars/standard-ml.tmbundle
+	url = https://github.com/textmate/standard-ml.tmbundle
+[submodule "vendor/grammars/swift.tmbundle"]
+	path = vendor/grammars/swift.tmbundle
+	url = https://github.com/textmate/swift.tmbundle
+[submodule "vendor/grammars/tcl.tmbundle"]
+	path = vendor/grammars/tcl.tmbundle
+	url = https://github.com/textmate/tcl.tmbundle
+[submodule "vendor/grammars/thrift.tmbundle"]
+	path = vendor/grammars/thrift.tmbundle
+	url = https://github.com/textmate/thrift.tmbundle
+[submodule "vendor/grammars/toml.tmbundle"]
+	path = vendor/grammars/toml.tmbundle
+	url = https://github.com/textmate/toml.tmbundle
+[submodule "vendor/grammars/verilog.tmbundle"]
+	path = vendor/grammars/verilog.tmbundle
+	url = https://github.com/textmate/verilog.tmbundle
+[submodule "vendor/grammars/xml.tmbundle"]
+	path = vendor/grammars/xml.tmbundle
+	url = https://github.com/textmate/xml.tmbundle
+[submodule "vendor/grammars/smalltalk-tmbundle"]
+	path = vendor/grammars/smalltalk-tmbundle
+	url = https://github.com/tomas-stefano/smalltalk-tmbundle
+[submodule "vendor/grammars/ioke-outdated"]
+	path = vendor/grammars/ioke-outdated
+	url = https://github.com/vic/ioke-outdated
+[submodule "vendor/grammars/kotlin-sublime-package"]
+	path = vendor/grammars/kotlin-sublime-package
+	url = https://github.com/vkostyukov/kotlin-sublime-package
+[submodule "vendor/grammars/c.tmbundle"]
+	path = vendor/grammars/c.tmbundle
+	url = https://github.com/textmate/c.tmbundle
+[submodule "vendor/grammars/zephir-sublime"]
+	path = vendor/grammars/zephir-sublime
+	url = https://github.com/vmg/zephir-sublime
+[submodule "vendor/grammars/llvm.tmbundle"]
+	path = vendor/grammars/llvm.tmbundle
+	url = https://github.com/whitequark/llvm.tmbundle
+[submodule "vendor/grammars/oz-tmbundle"]
+	path = vendor/grammars/oz-tmbundle
+	url = https://github.com/eregon/oz-tmbundle
+[submodule "vendor/grammars/ebundles"]
+	path = vendor/grammars/ebundles
+	url = https://github.com/ericzou/ebundles
+[submodule "vendor/grammars/sublime-mask"]
+	path = vendor/grammars/sublime-mask
+	url = https://github.com/tenbits/sublime-mask
+[submodule "vendor/grammars/sublime_cobol"]
+	path = vendor/grammars/sublime_cobol
+	url = https://bitbucket.org/bitlang/sublime_cobol
+[submodule "vendor/grammars/ruby.tmbundle"]
+	path = vendor/grammars/ruby.tmbundle
+	url = https://github.com/aroben/ruby.tmbundle
+	branch = pl
+[submodule "vendor/grammars/IDL-Syntax"]
+	path = vendor/grammars/IDL-Syntax
+	url = https://github.com/andik/IDL-Syntax
+[submodule "vendor/grammars/sas.tmbundle"]
+	path = vendor/grammars/sas.tmbundle
+	url = https://github.com/rpardee/sas.tmbundle
+[submodule "vendor/grammars/atom-salt"]
+	path = vendor/grammars/atom-salt
+	url = https://github.com/saltstack/atom-salt
+[submodule "vendor/grammars/Scalate.tmbundle"]
+	path = vendor/grammars/Scalate.tmbundle
+	url = https://github.com/scalate/Scalate.tmbundle
+[submodule "vendor/grammars/Elm.tmLanguage"]
+	path = vendor/grammars/Elm.tmLanguage
+	url = https://github.com/deadfoxygrandpa/Elm.tmLanguage
+[submodule "vendor/grammars/sublime-bsv"]
+	path = vendor/grammars/sublime-bsv
+	url = https://github.com/thotypous/sublime-bsv
+[submodule "vendor/grammars/Sublime-HTTP"]
+	path = vendor/grammars/Sublime-HTTP
+	url = https://github.com/httpspec/sublime-highlighting
+[submodule "vendor/grammars/sass-textmate-bundle"]
+	path = vendor/grammars/sass-textmate-bundle
+	url = https://github.com/nathos/sass-textmate-bundle
+[submodule "vendor/grammars/carto-atom"]
+	path = vendor/grammars/carto-atom
+	url = https://github.com/yohanboniface/carto-atom
+[submodule "vendor/grammars/Sublime-Nit"]
+	path = vendor/grammars/Sublime-Nit
+	url = https://github.com/R4PaSs/Sublime-Nit
+[submodule "vendor/grammars/language-hy"]
+	path = vendor/grammars/language-hy
+	url = https://github.com/rwtolbert/language-hy
+[submodule "vendor/grammars/Racket"]
+	path = vendor/grammars/Racket
+	url = https://github.com/soegaard/racket-highlight-for-github
+[submodule "vendor/grammars/turtle.tmbundle"]
+	path = vendor/grammars/turtle.tmbundle
+	url = https://github.com/peta/turtle.tmbundle
+[submodule "vendor/grammars/liquid.tmbundle"]
+	path = vendor/grammars/liquid.tmbundle
+	url = https://github.com/bastilian/validcode-textmate-bundles
+[submodule "vendor/grammars/Modelica"]
+	path = vendor/grammars/Modelica
+	url = https://github.com/BorisChumichev/modelicaSublimeTextPackage
+[submodule "vendor/grammars/sublime-apl"]
+	path = vendor/grammars/sublime-apl
+	url = https://github.com/StoneCypher/sublime-apl
+[submodule "vendor/grammars/sublime-golo"]
+	path = vendor/grammars/sublime-golo
+	url = https://github.com/TypeUnsafe/sublime-golo
+[submodule "vendor/grammars/JSyntax"]
+	path = vendor/grammars/JSyntax
+	url = https://github.com/bcj/JSyntax
+[submodule "vendor/grammars/TXL"]
+	path = vendor/grammars/TXL
+	url = https://github.com/MikeHoffert/Sublime-Text-TXL-syntax
+[submodule "vendor/grammars/G-Code"]
+	path = vendor/grammars/G-Code
+	url = https://github.com/robotmaster/sublime-text-syntax-highlighting
+[submodule "vendor/grammars/sublime-text-ox"]
+	path = vendor/grammars/sublime-text-ox
+	url = https://github.com/andreashetland/sublime-text-ox
+[submodule "vendor/grammars/AutoHotkey"]
+	path = vendor/grammars/AutoHotkey
+	url = https://github.com/ahkscript/SublimeAutoHotkey
+[submodule "vendor/grammars/ec.tmbundle"]
+	path = vendor/grammars/ec.tmbundle
+	url = https://github.com/ecere/ec.tmbundle
+[submodule "vendor/grammars/gap-tmbundle"]
+	path = vendor/grammars/gap-tmbundle
+	url = https://github.com/dhowden/gap-tmbundle
+[submodule "vendor/grammars/SublimePapyrus"]
+	path = vendor/grammars/SublimePapyrus
+	url = https://github.com/Kapiainen/SublimePapyrus
+[submodule "vendor/grammars/sublime-spintools"]
+	path = vendor/grammars/sublime-spintools
+	url = https://github.com/bitbased/sublime-spintools
+[submodule "vendor/grammars/PogoScript.tmbundle"]
+	path = vendor/grammars/PogoScript.tmbundle
+	url = https://github.com/featurist/PogoScript.tmbundle
+[submodule "vendor/grammars/sublime-opal"]
+	path = vendor/grammars/sublime-opal
+	url = https://github.com/artifactz/sublime-opal
+[submodule "vendor/grammars/mediawiki.tmbundle"]
+	path = vendor/grammars/mediawiki.tmbundle
+	url = https://github.com/textmate/mediawiki.tmbundle
+[submodule "vendor/grammars/SublimeClarion"]
+	path = vendor/grammars/SublimeClarion
+	url = https://github.com/fushnisoft/SublimeClarion
+[submodule "vendor/grammars/BrightScript.tmbundle"]
+	path = vendor/grammars/BrightScript.tmbundle
+	url = https://github.com/cmink/BrightScript.tmbundle
+[submodule "vendor/grammars/Stylus"]
+	path = vendor/grammars/Stylus
+	url = https://github.com/billymoon/Stylus
+[submodule "vendor/grammars/asciidoc.tmbundle"]
+	path = vendor/grammars/asciidoc.tmbundle
+	url = https://github.com/zuckschwerdt/asciidoc.tmbundle
+[submodule "vendor/grammars/Lean.tmbundle"]
+	path = vendor/grammars/Lean.tmbundle
+	url = https://github.com/leanprover/Lean.tmbundle
+[submodule "vendor/grammars/ampl"]
+	path = vendor/grammars/ampl
+	url = https://github.com/ampl/sublime-ampl
+[submodule "vendor/grammars/sublime-varnish"]
+	path = vendor/grammars/sublime-varnish
+	url = https://github.com/brandonwamboldt/sublime-varnish
+[submodule "vendor/grammars/xc.tmbundle"]
+	path = vendor/grammars/xc.tmbundle
+	url = https://github.com/graymalkin/xc.tmbundle
+[submodule "vendor/grammars/perl.tmbundle"]
+	path = vendor/grammars/perl.tmbundle
+	url = https://github.com/textmate/perl.tmbundle
+[submodule "vendor/grammars/sublime-netlinx"]
+	path = vendor/grammars/sublime-netlinx
+	url = https://github.com/amclain/sublime-netlinx
+[submodule "vendor/grammars/Sublime-Red"]
+	path = vendor/grammars/Sublime-Red
+	url = https://github.com/Oldes/Sublime-Red
+[submodule "vendor/grammars/jflex.tmbundle"]
+	path = vendor/grammars/jflex.tmbundle
+	url = https://github.com/jflex-de/jflex.tmbundle.git
+[submodule "vendor/grammars/Sublime-Modula-2"]
+	path = vendor/grammars/Sublime-Modula-2
+	url = https://github.com/harogaston/Sublime-Modula-2
+[submodule "vendor/grammars/ada.tmbundle"]
+	path = vendor/grammars/ada.tmbundle
+	url = https://github.com/textmate/ada.tmbundle
+[submodule "vendor/grammars/api-blueprint-sublime-plugin"]
+	path = vendor/grammars/api-blueprint-sublime-plugin
+	url = https://github.com/apiaryio/api-blueprint-sublime-plugin
+[submodule "vendor/grammars/Handlebars"]
+	path = vendor/grammars/Handlebars
+	url = https://github.com/daaain/Handlebars
+[submodule "vendor/grammars/smali-sublime"]
+	path = vendor/grammars/smali-sublime
+	url = https://github.com/ShaneWilton/sublime-smali
+[submodule "vendor/grammars/language-jsoniq"]
+	path = vendor/grammars/language-jsoniq
+	url = http://github.com/wcandillon/language-jsoniq
+[submodule "vendor/grammars/atom-fsharp"]
+	path = vendor/grammars/atom-fsharp
+	url = https://github.com/fsprojects/atom-fsharp
+[submodule "vendor/grammars/SMT.tmbundle"]
+	path = vendor/grammars/SMT.tmbundle
+	url = https://github.com/SRI-CSL/SMT.tmbundle.git
+[submodule "vendor/grammars/language-crystal"]
+	path = vendor/grammars/language-crystal
+	url = https://github.com/atom-crystal/language-crystal
+[submodule "vendor/grammars/language-xbase"]
+	path = vendor/grammars/language-xbase
+	url = https://github.com/hernad/atom-language-harbour
+[submodule "vendor/grammars/language-ncl"]
+	path = vendor/grammars/language-ncl
+	url = https://github.com/rpavlick/language-ncl.git
+[submodule "vendor/grammars/pawn-sublime-language"]
+	path = vendor/grammars/pawn-sublime-language
+	url = https://github.com/Southclaw/pawn-sublime-language.git
+[submodule "vendor/grammars/atom-language-purescript"]
+	path = vendor/grammars/atom-language-purescript
+	url = https://github.com/purescript-contrib/atom-language-purescript
+[submodule "vendor/grammars/vue-syntax-highlight"]
+	path = vendor/grammars/vue-syntax-highlight
+	url = https://github.com/vuejs/vue-syntax-highlight
+[submodule "vendor/grammars/st2-zonefile"]
+	path = vendor/grammars/st2-zonefile
+	url = https://github.com/sixty4k/st2-zonefile
+[submodule "vendor/grammars/sublimeprolog"]
+	path = vendor/grammars/sublimeprolog
+	url = https://github.com/alnkpa/sublimeprolog
+[submodule "vendor/grammars/sublime-aspectj"]
+	path = vendor/grammars/sublime-aspectj
+	url = https://github.com/pchaigno/sublime-aspectj
+[submodule "vendor/grammars/sublime-typescript"]
+	path = vendor/grammars/sublime-typescript
+	url = https://github.com/Microsoft/TypeScript-Sublime-Plugin
+[submodule "vendor/grammars/sublime-pony"]
+	path = vendor/grammars/sublime-pony
+	url = https://github.com/CausalityLtd/sublime-pony
+[submodule "vendor/grammars/X10"]
+	path = vendor/grammars/X10
+	url = https://github.com/x10-lang/x10-highlighting
+[submodule "vendor/grammars/language-babel"]
+	path = vendor/grammars/language-babel
+	url = https://github.com/gandm/language-babel
+[submodule "vendor/grammars/UrWeb-Language-Definition"]
+	path = vendor/grammars/UrWeb-Language-Definition
+	url = https://github.com/gwalborn/UrWeb-Language-Definition.git
+[submodule "vendor/grammars/Stata.tmbundle"]
+	path = vendor/grammars/Stata.tmbundle
+	url = https://github.com/pschumm/Stata.tmbundle
+[submodule "vendor/grammars/FreeMarker.tmbundle"]
+	path = vendor/grammars/FreeMarker.tmbundle
+	url = https://github.com/freemarker/FreeMarker.tmbundle
+[submodule "vendor/grammars/MagicPython"]
+	path = vendor/grammars/MagicPython
+	url = https://github.com/MagicStack/MagicPython
+[submodule "vendor/grammars/language-click"]
+	path = vendor/grammars/language-click
+	url = https://github.com/stenverbois/language-click.git
+[submodule "vendor/grammars/language-maxscript"]
+	path = vendor/grammars/language-maxscript
+	url = https://github.com/Alhadis/language-maxscript
+[submodule "vendor/grammars/language-renpy"]
+	path = vendor/grammars/language-renpy
+	url = https://github.com/williamd1k0/language-renpy.git
+[submodule "vendor/grammars/language-inform7"]
+	path = vendor/grammars/language-inform7
+	url = https://github.com/erkyrath/language-inform7
+[submodule "vendor/grammars/atom-language-stan"]
+	path = vendor/grammars/atom-language-stan
+	url = https://github.com/jrnold/atom-language-stan
+[submodule "vendor/grammars/language-yang"]
+	path = vendor/grammars/language-yang
+	url = https://github.com/DzonyKalafut/language-yang.git
+[submodule "vendor/grammars/perl6fe"]
+	path = vendor/grammars/perl6fe
+	url = https://github.com/MadcapJake/language-perl6fe.git
+[submodule "vendor/grammars/language-less"]
+	path = vendor/grammars/language-less
+	url = https://github.com/atom/language-less.git
+[submodule "vendor/grammars/language-povray"]
+	path = vendor/grammars/language-povray
+	url = https://github.com/c-lipka/language-povray
+[submodule "vendor/grammars/sublime-terra"]
+	path = vendor/grammars/sublime-terra
+	url = https://github.com/pyk/sublime-terra
+[submodule "vendor/grammars/SublimePuppet"]
+	path = vendor/grammars/SublimePuppet
+	url = https://github.com/russCloak/SublimePuppet
+[submodule "vendor/grammars/sublimeassembly"]
+	path = vendor/grammars/sublimeassembly
+	url = https://github.com/Nessphoro/sublimeassembly
+[submodule "vendor/grammars/monkey"]
+	path = vendor/grammars/monkey
+	url = https://github.com/gingerbeardman/monkey.tmbundle
+[submodule "vendor/grammars/assembly"]
+	path = vendor/grammars/assembly
+	url = https://github.com/nanoant/assembly.tmbundle
+[submodule "vendor/grammars/boo"]
+	path = vendor/grammars/boo
+	url = https://github.com/Shammah/boo-sublime
+[submodule "vendor/grammars/logos"]
+	path = vendor/grammars/logos
+	url = https://github.com/Cykey/Sublime-Logos
+[submodule "vendor/grammars/pig-latin"]
+	path = vendor/grammars/pig-latin
+	url = https://github.com/goblindegook/sublime-text-pig-latin
+[submodule "vendor/grammars/sourcepawn"]
+	path = vendor/grammars/sourcepawn
+	url = https://github.com/austinwagner/sublime-sourcepawn
+[submodule "vendor/grammars/gdscript"]
+	path = vendor/grammars/gdscript
+	url = https://github.com/beefsack/GDScript-sublime
+[submodule "vendor/grammars/nesC"]
+	path = vendor/grammars/nesC
+	url = https://github.com/cdwilson/nesC.tmbundle
+[submodule "vendor/grammars/ats"]
+	path = vendor/grammars/ats
+	url = https://github.com/steinwaywhw/ats-mode-sublimetext
+[submodule "vendor/grammars/grace"]
+	path = vendor/grammars/grace
+	url = https://github.com/zmthy/grace-tmbundle

.travis.yml

@@ -1,13 +1,17 @@
-before_install:
-  - git fetch origin master:master
-  - git fetch origin v2.0.0:v2.0.0
-  - git fetch origin test/attributes:test/attributes
-  - git fetch origin test/master:test/master
-  - sudo apt-get install libicu-dev -y
+language: ruby
+sudo: false
+addons:
+  apt:
+    packages:
+    - libicu-dev
+    - libicu48
+before_install: script/travis/before_install
 rvm:
-  - 1.9.3
   - 2.0.0
   - 2.1
   - 2.2
 notifications:
   disabled: true
+git:
+  submodules: false
+cache: bundler

CONTRIBUTING.md

@@ -1,31 +1,105 @@
-## Contributing
+# Contributing
 
-The majority of contributions won't need to touch any Ruby code at all. The [master language list][languages] is just a YAML configuration file.
+[code-of-conduct]: http://todogroup.org/opencodeofconduct/#Linguist/opensource@github.com
 
-Almost all bug fixes or new language additions should come with some additional code samples. Just drop them under [`samples/`][samples] in the correct subdirectory and our test suite will automatically test them. In most cases you shouldn't need to add any new assertions.
+Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great. This project adheres to the [Open Code of Conduct][code-of-conduct]. By participating, you are expected to uphold this code.
 
-### My code is detected as the wrong language
+The majority of contributions won't need to touch any Ruby code at all.
 
-This can usually be solved either by adding a new filename or file name extension to the language's entry in [`languages.yml`][languages] or adding more [samples][samples] for your language to the repository to make Linguist's classifier smarter.
+## Adding an extension to a language
 
-### Syntax highlighting looks wrong
+We try only to add new extensions once they have some usage on GitHub. In most cases we prefer that extensions be in use in hundreds of repositories before supporting them in Linguist.
 
-Assuming your code is being detected as the right language (see above), in most cases this is due to a bug in the language grammar rather than a bug in Linguist. [`grammars.yml`][grammars] lists all the grammars we use for syntax highlighting on github.com. Find the one corresponding to your code's programming language and submit a bug report upstream.
+To add support for a new extension:
 
-You can also try to fix the bug yourself and submit a Pull Request. [This piece from TextMate's documentation](http://manual.macromates.com/en/language_grammars) offers a good introduction on how to work with TextMate-compatible grammars. You can test grammars using [Lightshow](https://lightshow.githubapp.com).
+0. Add your extension to the language entry in [`languages.yml`][languages], keeping the extensions in alphabetical order.
+0. Add at least one sample for your extension to the [samples directory][samples] in the correct subdirectory.
+0. Open a pull request, linking to a [GitHub search result](https://github.com/search?utf8=%E2%9C%93&q=extension%3Aboot+NOT+nothack&type=Code&ref=searchresults) showing in-the-wild usage.
 
-Once the bug has been fixed upstream, please let us know and we'll pick it up for GitHub.
+In addition, if this extension is already listed in [`languages.yml`][languages] then sometimes a few more steps will need to be taken:
 
-### I want to add support for the `X` programming language
+0. Make sure that example `.yourextension` files are present in the [samples directory][samples] for each language that uses `.yourextension`.
+0. Test the performance of the Bayesian classifier with a relatively large number (1000s) of sample `.yourextension` files. (ping @arfon or @bkeepers to help with this) to ensure we're not misclassifying files.
+0. If the Bayesian classifier does a bad job with the sample `.yourextension` files then a [heuristic](https://github.com/github/linguist/blob/master/lib/linguist/heuristics.rb) may need to be written to help.
 
-Great! You'll need to:
+
+## Adding a language
+
+We try only to add languages once they have some usage on GitHub. In most cases we prefer that each new file extension be in use in hundreds of repositories before supporting them in Linguist.
+
+To add support for a new language:
 
 0. Add an entry for your language to [`languages.yml`][languages].
-0. Add a grammar for your language to [`grammars.yml`][grammars] by running `script/download-grammars --add URL`. Please only add grammars that have a license that permits redistribution.
-0. Add samples for your language to the [samples directory][samples].
+0. Add a grammar for your language. Please only add grammars that have a license that permits redistribution.
+  0. Add your grammar as a submodule: `git submodule add https://github.com/JaneSmith/MyGrammar vendor/grammars/MyGrammar`.
+  0. Add your grammar to [`grammars.yml`][grammars] by running `script/convert-grammars --add vendor/grammars/MyGrammar`.
+0. Add samples for your language to the [samples directory][samples] in the correct subdirectory.
+0. Open a pull request, linking to a [GitHub search result](https://github.com/search?utf8=%E2%9C%93&q=extension%3Aboot+NOT+nothack&type=Code&ref=searchresults) showing in-the-wild usage.
 
-We try only to add languages once they have some usage on GitHub, so please note in-the-wild usage examples in your pull request. In most cases we prefer that languages already be in use in hundreds of repositories before supporting them in Linguist.
+In addition, if your new language defines an extension that's already listed in [`languages.yml`][languages] (such as `.foo`) then sometimes a few more steps will need to be taken:
+
+0. Make sure that example `.foo` files are present in the [samples directory][samples] for each language that uses `.foo`.
+0. Test the performance of the Bayesian classifier with a relatively large number (1000s) of sample `.foo` files. (ping @arfon or @bkeepers to help with this) to ensure we're not misclassifying files.
+0. If the Bayesian classifier does a bad job with the sample `.foo` files then a [heuristic](https://github.com/github/linguist/blob/master/lib/linguist/heuristics.rb) may need to be written to help.
+
+Remember, the goal here is to try and avoid false positives!
+
+
+## Fixing a misclassified language
+
+Most languages are detected by their file extension defined in [languages.yml][languages].  For disambiguating between files with common extensions, linguist applies some [heuristics](/lib/linguist/heuristics.rb) and a [statistical classifier](lib/linguist/classifier.rb). This process can help differentiate between, for example, `.h` files which could be either C, C++, or Obj-C.
+
+Misclassifications can often be solved by either adding a new filename or extension for the language or adding more [samples][samples] to make the classifier smarter.
+
+
+## Fixing syntax highlighting
+
+Syntax highlighting in GitHub is performed using TextMate-compatible grammars. These are the same grammars that TextMate, Sublime Text and Atom use. Every language in [languages.yml][languages] is mapped to its corresponding TM `scope`. This scope will be used when picking up a grammar for highlighting.
+
+Assuming your code is being detected as the right language, in most cases this is due to a bug in the language grammar rather than a bug in Linguist. [`grammars.yml`][grammars] lists all the grammars we use for syntax highlighting on github.com. Find the one corresponding to your code's programming language and submit a bug report upstream. If you can, try to reproduce the highlighting problem in the text editor that the grammar is designed for (TextMate, Sublime Text, or Atom) and include that information in your bug report.
+
+You can also try to fix the bug yourself and submit a Pull Request. [TextMate's documentation](https://manual.macromates.com/en/language_grammars) offers a good introduction on how to work with TextMate-compatible grammars. You can test grammars using [Lightshow](https://github-lightshow.herokuapp.com).
+
+Once the bug has been fixed upstream, we'll pick it up for GitHub in the next release of Linguist.
+
+## Testing
+
+For development you are going to want to checkout out the source. To get it, clone the repo and run [Bundler](http://gembundler.com/) to install its dependencies.
+
+    git clone https://github.com/github/linguist.git
+    cd linguist/
+    script/bootstrap
+
+To run the tests:
+
+    bundle exec rake test
+
+Sometimes getting the tests running can be too much work, especially if you don't have much Ruby experience. It's okay: be lazy and let our build bot [Travis](https://travis-ci.org/#!/github/linguist) run the tests for you. Just open a pull request and the bot will start cranking away.
+
+Here's our current build status: [![Build Status](https://api.travis-ci.org/github/linguist.svg?branch=master)](https://travis-ci.org/github/linguist)
+
+
+## Releasing
+
+If you are the current maintainer of this gem:
+
+0. Create a branch for the release: `git checkout -b cut-release-vxx.xx.xx`
+0. Make sure your local dependencies are up to date: `script/bootstrap`
+0. If grammar submodules have not been updated recently, update them: `git submodule update --remote && git commit -a`
+0. Ensure that samples are updated: `bundle exec rake samples`
+0. Ensure that tests are green: `bundle exec rake test`
+0. Bump gem version in `lib/linguist/version.rb`, [like this](https://github.com/github/linguist/commit/8d2ea90a5ba3b2fe6e1508b7155aa4632eea2985).
+0. Make a PR to github/linguist, [like this](https://github.com/github/linguist/pull/1238).
+0. Build a local gem: `bundle exec rake build_gem`
+0. Test the gem:
+  0. Bump the Gemfile and Gemfile.lock versions for an app which relies on this gem
+  0. Install the new gem locally
+  0. Test behavior locally, branch deploy, whatever needs to happen
+0. Merge github/linguist PR
+0. Tag and push: `git tag vx.xx.xx; git push --tags`
+0. Push to rubygems.org -- `gem push github-linguist-3.0.0.gem`
 
 [grammars]: /grammars.yml
 [languages]: /lib/linguist/languages.yml
 [samples]: /samples
+[new-issue]: https://github.com/github/linguist/issues/new

Gemfile

@@ -1,5 +1,3 @@
 source 'https://rubygems.org'
 gemspec :name => "github-linguist"
-gemspec :name => "github-linguist-grammars"
-gem 'test-unit', require: false if RUBY_VERSION >= '2.2'
 gem 'byebug' if RUBY_VERSION >= '2.0'

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2011-2014 GitHub, Inc.
+Copyright (c) 2011-2016 GitHub, Inc.
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation

README.md

@@ -1,45 +1,85 @@
 # Linguist
 
-We use this library at GitHub to detect blob languages, ignore binary files, suppress generated files in diffs, and generate language breakdown graphs.
+[issues]: https://github.com/github/linguist/issues
+[new-issue]: https://github.com/github/linguist/issues/new
 
-Tips for filing issues and creating pull requests can be found in [`CONTRIBUTING.md`](/CONTRIBUTING.md).
+This library is used on GitHub.com to detect blob languages, ignore binary or vendored files, suppress generated files in diffs, and generate language breakdown graphs.
 
-## Features
+See [Troubleshooting](#troubleshooting) and [`CONTRIBUTING.md`](/CONTRIBUTING.md) before filing an issue or creating a pull request.
 
-### Language detection
+## Troubleshooting
 
-Linguist defines a list of all languages known to GitHub in a [yaml file](https://github.com/github/linguist/blob/master/lib/linguist/languages.yml).
+### My repository is detected as the wrong language
 
-Most languages are detected by their file extension. For disambiguating between files with common extensions, we first apply some common-sense heuristics to pick out obvious languages. After that, we use a
-[statistical
-classifier](https://github.com/github/linguist/blob/master/lib/linguist/classifier.rb).
-This process can help us tell the difference between, for example, `.h` files which could be either C, C++, or Obj-C.
+![language stats bar](https://cloud.githubusercontent.com/assets/173/5562290/48e24654-8ddf-11e4-8fe7-735b0ce3a0d3.png)
 
-```ruby
+The Language stats bar displays languages percentages for the files in the repository. The percentages are calculated based on the bytes of code for each language as reported by the [List Languages](https://developer.github.com/v3/repos/#list-languages) API. If the bar is reporting a language that you don't expect:
 
-Linguist::FileBlob.new("lib/linguist.rb").language.name #=> "Ruby"
+0. Click on the name of the language in the stats bar to see a list of the files that are identified as that language.
+0. If you see files that you didn't write, consider moving the files into one of the [paths for vendored  code](/lib/linguist/vendor.yml), or use the [manual overrides](#overrides) feature to ignore them.
+0. If the files are being misclassified, search for [open issues][issues] to see if anyone else has already reported the issue. Any information you can add, especially links to public repositories, is helpful.
+0. If there are no reported issues of this misclassification, [open an issue][new-issue] and include a link to the repository or a sample of the code that is being misclassified.
 
-Linguist::FileBlob.new("bin/linguist").language.name #=> "Ruby"
+## Overrides
+
+Linguist supports a number of different custom overrides strategies for language definitions and vendored paths.
+
+### Using gitattributes
+
+Add a `.gitattributes` file to your project and use standard git-style path matchers for the files you want to override to set `linguist-documentation`, `linguist-language`, and `linguist-vendored`. `.gitattributes` will be used to determine language statistics, but will not be used to syntax highlight files. To manually set syntax highlighting, use [Vim or Emacs modelines](#using-emacs-or-vim-modelines).
+
+```
+$ cat .gitattributes
+*.rb linguist-language=Java
 ```
 
-See [lib/linguist/language.rb](https://github.com/github/linguist/blob/master/lib/linguist/language.rb) and [lib/linguist/languages.yml](https://github.com/github/linguist/blob/master/lib/linguist/languages.yml).
+Checking code you didn't write, such as JavaScript libraries, into your git repo is a common practice, but this often inflates your project's language stats and may even cause your project to be labeled as another language. By default, Linguist treats all of the paths defined in [lib/linguist/vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml) as vendored and therefore doesn't include them in the language statistics for a repository.
 
-### Syntax Highlighting
+Use the `linguist-vendored` attribute to vendor or un-vendor paths.
 
-Syntax highlighting in GitHub is performed using TextMate-compatible grammars. These are the same grammars that TextMate, Sublime Text and Atom use.
+```
+$ cat .gitattributes
+special-vendored-path/* linguist-vendored
+jquery.js linguist-vendored=false
+```
 
-Every language in `languages.yml` is mapped to its corresponding TM `scope`. This scope will be used when picking up a grammar for highlighting. **When adding a new language to Linguist, please add its corresponding scope too (assuming there's an existing TextMate bundle, Sublime Text package, or Atom package) so syntax highlighting works for it**.
+Similar to vendored files, Linguist excludes documentation files from your project's language stats. (Unlike vendored files, documentation files are displayed in diffs on github.com.) [lib/linguist/documentation.yml](lib/linguist/documentation.yml) lists common documentation paths and excludes them from the language statistics for your repository.
 
-### Stats
+Use the `linguist-documentation` attribute to mark or unmark paths as documentation.
 
-The Language stats bar that you see on every repository is built by aggregating the languages of each file in that repository. The top language in the graph determines the project's primary language.
+```
+$ cat .gitattributes
+project-docs/* linguist-documentation
+docs/formatter.rb linguist-documentation=false
+```
 
-The repository stats API, accessed through `#languages`, can be used on a directory:
+### Using Emacs or Vim modelines
 
-***API UPDATE***
+Alternatively, you can use Vim or Emacs style modelines to set the language for a single file. Modelines can be placed anywhere within a file and are respected when determining how to syntax-highlight a file on GitHub.com
 
-Since [Version 3.0.0](https://github.com/github/linguist/releases/tag/v3.0.0) Linguist expects a git repository (in the form of a [Rugged::Repository](https://github.com/libgit2/rugged#repositories)) to be passed when initializing `Linguist::Repository`.
+##### Vim
+```
+# Some examples of various styles:
+vim: syntax=java
+vim: set syntax=ruby:
+vim: set filetype=prolog:
+vim: set ft=cpp:
+```
 
+##### Emacs
+```
+-*- mode: php;-*-
+```
+
+## Usage
+
+Install the gem:
+
+```
+$ gem install github-linguist
+```
+
+Then use it in your application:
 
 ```ruby
 require 'rugged'
@@ -51,146 +91,32 @@ project.language       #=> "Ruby"
 project.languages      #=> { "Ruby" => 119387 }
 ```
 
-These stats are also printed out by the `linguist` binary. You can use the
+These stats are also printed out by the `linguist` executable. You can use the
 `--breakdown` flag, and the binary will also output the breakdown of files by language.
 
 You can try running `linguist` on the root directory in this repository itself:
 
-    $ bundle exec linguist --breakdown
+```
+$ bundle exec linguist --breakdown
 
-    100.00% Ruby
+100.00% Ruby
 
-    Ruby:
-    Gemfile
-    Rakefile
-    bin/linguist
-    github-linguist.gemspec
-    lib/linguist.rb
-    lib/linguist/blob_helper.rb
-    lib/linguist/classifier.rb
-    lib/linguist/file_blob.rb
-    lib/linguist/generated.rb
-    lib/linguist/heuristics.rb
-    lib/linguist/language.rb
-    lib/linguist/lazy_blob.rb
-    lib/linguist/md5.rb
-    lib/linguist/repository.rb
-    lib/linguist/samples.rb
-    lib/linguist/tokenizer.rb
-    lib/linguist/version.rb
-    test/test_blob.rb
-    test/test_classifier.rb
-    test/test_heuristics.rb
-    test/test_language.rb
-    test/test_md5.rb
-    test/test_pedantic.rb
-    test/test_repository.rb
-    test/test_samples.rb
-    test/test_tokenizer.rb
-
-#### Ignore vendored files
-
-Checking other code into your git repo is a common practice. But this often inflates your project's language stats and may even cause your project to be labeled as another language. We are able to identify some of these files and directories and exclude them.
-
-```ruby
-Linguist::FileBlob.new("vendor/plugins/foo.rb").vendored? # => true
+Ruby:
+Gemfile
+Rakefile
+bin/linguist
+github-linguist.gemspec
+lib/linguist.rb
+…
 ```
 
-See [Linguist::BlobHelper#vendored?](https://github.com/github/linguist/blob/master/lib/linguist/blob_helper.rb) and [lib/linguist/vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml).
+## Contributing
 
-#### Generated file detection
+Please check out our [contributing guidelines](CONTRIBUTING.md).
 
-Not all plain text files are true source files. Generated files like minified js and compiled CoffeeScript can be detected and excluded from language stats. As an extra bonus, these files are suppressed in diffs.
+## License
 
-```ruby
-Linguist::FileBlob.new("underscore.min.js").generated? # => true
-```
+The language grammars included in this gem are covered by their repositories'
+respective licenses. `grammars.yml` specifies the repository for each grammar.
 
-See [Linguist::Generated#generated?](https://github.com/github/linguist/blob/master/lib/linguist/generated.rb).
-
-## Overrides
-
-Linguist supports custom overrides for language definitions and vendored paths. Add a `.gitattributes` file to your project using the keys `linguist-language` and `linguist-vendored` with the standard git-style path matchers for the files you want to override.
-
-Please note that the overrides currently only affect the language statistics for a repository and not the syntax-highlighting of files.
-
-```
-$ cat .gitattributes
-*.rb linguist-language=Java
-
-$ linguist --breakdown
-100.00% Java
-
-Java:
-ruby_file.rb
-```
-
-By default, Linguist treats all of the paths defined in [lib/linguist/vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml) as vendored and therefore doesn't include them in the language statistics for a repository. Use the `linguist-vendored` attribute to vendor or un-vendor paths.
-
-```
-$ cat .gitattributes
-special-vendored-path/* linguist-vendored
-jquery.js linguist-vendored=false
-```
-
-## Installation
-
-Github.com is usually running the latest version of the `github-linguist` gem that is released on [RubyGems.org](http://rubygems.org/gems/github-linguist).
-
-But for development you are going to want to checkout out the source. To get it, clone the repo and run [Bundler](http://gembundler.com/) to install its dependencies.
-
-    git clone https://github.com/github/linguist.git
-    cd linguist/
-    bundle install
-
-To run the tests:
-
-    bundle exec rake test
-
-### A note on language extensions
-
-Linguist has a number of methods available to it for identifying the language of a particular file. The initial lookup is based upon the extension of the file, possible file extensions are defined in an array called `extensions`. Take a look at this example for example for `Perl`:
-
-```
-Perl:
-  type: programming
-  ace_mode: perl
-  color: "#0298c3"
-  extensions:
-  - .pl
-  - .PL
-  - .perl
-  - .ph
-  - .plx
-  - .pm
-  - .pod
-  - .psgi
-  interpreters:
-  - perl
-```
-Any of the extensions defined are valid but the first in this array should be the most popular.
-
-### Testing
-
-Sometimes getting the tests running can be too much work, especially if you don't have much Ruby experience. It's okay: be lazy and let our build bot [Travis](http://travis-ci.org/#!/github/linguist) run the tests for you. Just open a pull request and the bot will start cranking away.
-
-Here's our current build status, which is hopefully green: [![Build Status](https://secure.travis-ci.org/github/linguist.png?branch=master)](http://travis-ci.org/github/linguist)
-
-### Releasing
-
-If you are the current maintainer of this gem:
-
- 0. Create a branch for the release: `git checkout -b cut-release-vxx.xx.xx`
- 0. Make sure your local dependencies are up to date: `bundle install`
- 0. Ensure that samples are updated: `bundle exec rake samples`
- 0. Ensure that tests are green: `bundle exec rake test`
- 0. Bump gem version in `lib/linguist/version.rb`.  For example, [like this](https://github.com/github/linguist/commit/8d2ea90a5ba3b2fe6e1508b7155aa4632eea2985).
- 0. Make a PR to github/linguist.  For example, [#1238](https://github.com/github/linguist/pull/1238).
- 0. Build a local gem: `bundle exec rake build_gem`
- 0. Testing:
-   0. Bump the Gemfile and Gemfile.lock versions for an app which relies on this gem
-   0. Install the new gem locally
-   0. Test behavior locally, branch deploy, whatever needs to happen
- 0. Merge github/linguist PR
- 0. Tag and push: `git tag vx.xx.xx; git push --tags`
- 0. Push to rubygems.org -- `gem push github-linguist-3.0.0.gem`
+All other files are covered by the MIT license, see `LICENSE`.

Rakefile

@@ -3,13 +3,14 @@ require 'rake/clean'
 require 'rake/testtask'
 require 'yaml'
 require 'yajl'
+require 'open-uri'
 
 task :default => :test
 
 Rake::TestTask.new
 
-# Extend test task to check for samples
-task :test => :check_samples
+# Extend test task to check for samples and fetch latest Ace modes
+task :test => [:check_samples, :fetch_ace_modes]
 
 desc "Check that we have samples.json generated"
 task :check_samples do
@@ -18,6 +19,20 @@ task :check_samples do
   end
 end
 
+desc "Fetch the latest Ace modes from its GitHub repository"
+task :fetch_ace_modes do
+  ACE_FIXTURE_PATH = File.join('test', 'fixtures', 'ace_modes.json')
+
+  File.delete(ACE_FIXTURE_PATH) if File.exist?(ACE_FIXTURE_PATH)
+
+  begin
+    ace_github_modes = open("https://api.github.com/repos/ajaxorg/ace/contents/lib/ace/mode").read
+    File.write(ACE_FIXTURE_PATH, ace_github_modes)
+  rescue OpenURI::HTTPError, SocketError
+      # no internet? no problem.
+  end
+end
+
 task :samples do
   require 'linguist/samples'
   json = Yajl.dump(Linguist::Samples.data, :pretty => true)
@@ -25,18 +40,14 @@ task :samples do
 end
 
 task :build_gem => :samples do
+  rm_rf "grammars"
+  sh "script/convert-grammars"
   languages = YAML.load_file("lib/linguist/languages.yml")
   File.write("lib/linguist/languages.json", Yajl.dump(languages))
   `gem build github-linguist.gemspec`
   File.delete("lib/linguist/languages.json")
 end
 
-task :build_grammars_gem do
-  rm_rf "grammars"
-  sh "script/download-grammars"
-  sh "gem", "build", "github-linguist-grammars.gemspec"
-end
-
 namespace :benchmark do
   benchmark_path = "benchmark/results"
 
@@ -47,7 +58,7 @@ namespace :benchmark do
 
     corpus = File.expand_path(ENV["CORPUS"] || "samples")
 
-    require 'linguist/language'
+    require 'linguist'
 
     results = Hash.new
     Dir.glob("#{corpus}/**/*").each do |file|

bin/git-linguist

@@ -0,0 +1,138 @@
+#!/usr/bin/env ruby
+
+require 'linguist'
+require 'rugged'
+require 'optparse'
+require 'json'
+require 'tmpdir'
+require 'zlib'
+
+class GitLinguist
+  def initialize(path, commit_oid, incremental = true)
+    @repo_path = path
+    @commit_oid = commit_oid
+    @incremental = incremental
+  end
+
+  def linguist
+    if @commit_oid.nil?
+      raise "git-linguist must be called with a specific commit OID to perform language computation"
+    end
+    repo = Linguist::Repository.new(rugged, @commit_oid)
+
+    if @incremental && stats = load_language_stats
+      old_commit_oid, old_stats = stats
+
+      # A cache with NULL oid means that we want to froze
+      # these language stats in place and stop computing
+      # them (for performance reasons)
+      return old_stats if old_commit_oid == NULL_OID
+      repo.load_existing_stats(old_commit_oid, old_stats)
+    end
+
+    result = yield repo
+
+    save_language_stats(@commit_oid, repo.cache)
+    result
+  end
+
+  def load_language_stats
+    version, oid, stats = load_cache
+    if version == LANGUAGE_STATS_CACHE_VERSION && oid && stats
+      [oid, stats]
+    end
+  end
+
+  def save_language_stats(oid, stats)
+    cache = [LANGUAGE_STATS_CACHE_VERSION, oid, stats]
+    write_cache(cache)
+  end
+
+  def clear_language_stats
+    File.unlink(cache_file)
+  rescue Errno::ENOENT
+  end
+
+  def disable_language_stats
+    save_language_stats(NULL_OID, {})
+  end
+
+  protected
+  NULL_OID = ("0" * 40).freeze
+
+  LANGUAGE_STATS_CACHE = 'language-stats.cache'
+  LANGUAGE_STATS_CACHE_VERSION = "v3:#{Linguist::VERSION}"
+
+  def rugged
+    @rugged ||= Rugged::Repository.bare(@repo_path)
+  end
+
+  def cache_file
+    File.join(@repo_path, LANGUAGE_STATS_CACHE)
+  end
+
+  def write_cache(object)
+    return unless File.directory? @repo_path
+
+    begin
+      tmp_path = Dir::Tmpname.make_tmpname(cache_file, nil)
+      File.open(tmp_path, "wb") do |f|
+        marshal = Marshal.dump(object)
+        f.write(Zlib::Deflate.deflate(marshal))
+      end
+
+      File.rename(tmp_path, cache_file)
+    rescue => e
+      (File.unlink(tmp_path) rescue nil)
+      raise e
+    end
+  end
+
+  def load_cache
+    marshal = File.open(cache_file, "rb") { |f| Zlib::Inflate.inflate(f.read) }
+    Marshal.load(marshal)
+  rescue SystemCallError, ::Zlib::DataError, ::Zlib::BufError, TypeError
+    nil
+  end
+end
+
+
+def git_linguist(args)
+  incremental = true
+  commit = nil
+
+  parser = OptionParser.new do |opts|
+    opts.banner = "Usage: git-linguist [OPTIONS] stats|breakdown|dump-cache|clear|disable"
+
+    opts.on("-f", "--force", "Force a full rescan") { incremental = false }
+    opts.on("--commit=COMMIT", "Commit to index") { |v| commit = v}
+  end
+
+  parser.parse!(args)
+
+  git_dir = `git rev-parse --git-dir`.strip
+  raise "git-linguist must be ran in a Git repository (#{Dir.pwd})" unless $?.success?
+  wrapper = GitLinguist.new(git_dir, commit, incremental)
+
+  case args.pop
+  when "stats"
+    wrapper.linguist do |linguist|
+      puts JSON.dump(linguist.languages)
+    end
+  when "breakdown"
+    wrapper.linguist do |linguist|
+      puts JSON.dump(linguist.breakdown_by_file)
+    end
+  when "dump-cache"
+    puts JSON.dump(wrapper.load_language_stats)
+  when "clear"
+    wrapper.clear_language_stats
+  when "disable"
+    wrapper.disable_language_stats
+  else
+    $stderr.print(parser.help)
+    exit 1
+  end
+end
+
+git_linguist(ARGV)

bin/linguist

@@ -2,10 +2,8 @@
 
 # linguist — detect language type for a file, or, given a directory, determine language breakdown
 #     usage: linguist <path> [<--breakdown>]
-
-require 'linguist/file_blob'
-require 'linguist/language'
-require 'linguist/repository'
+#
+require 'linguist'
 require 'rugged'
 
 path = ARGV[0] || Dir.pwd

github-linguist-grammars.gemspec

@@ -1,14 +0,0 @@
-require File.expand_path('../lib/linguist/version', __FILE__)
-
-Gem::Specification.new do |s|
-  s.name    = 'github-linguist-grammars'
-  s.version = Linguist::VERSION
-  s.summary = "Language grammars for use with github-linguist"
-
-  s.authors  = "GitHub"
-  s.homepage = "https://github.com/github/linguist"
-
-  s.files = ['lib/linguist/grammars.rb'] + Dir['grammars/*']
-
-  s.add_development_dependency 'plist', '~>3.1'
-end

github-linguist.gemspec

@@ -10,16 +10,21 @@ Gem::Specification.new do |s|
   s.homepage = "https://github.com/github/linguist"
   s.license  = "MIT"
 
-  s.files = Dir['lib/**/*'] - ['lib/linguist/grammars.rb']
-  s.executables << 'linguist'
+  s.files = Dir['lib/**/*'] + Dir['grammars/*'] + ['LICENSE']
+  s.executables = ['linguist', 'git-linguist']
 
   s.add_dependency 'charlock_holmes', '~> 0.7.3'
-  s.add_dependency 'escape_utils',    '~> 1.0.1'
+  s.add_dependency 'escape_utils',    '~> 1.1.0'
   s.add_dependency 'mime-types',      '>= 1.19'
-  s.add_dependency 'rugged',          '~> 0.22.0b4'
+  s.add_dependency 'rugged',          '>= 0.23.0b'
 
+  s.add_development_dependency 'minitest', '>= 5.0'
   s.add_development_dependency 'mocha'
+  s.add_development_dependency 'plist', '~>3.1'
   s.add_development_dependency 'pry'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'yajl-ruby'
+  s.add_development_dependency 'color-proximity', '~> 0.2.1'
+  s.add_development_dependency 'licensed'
+
 end

grammars.yml

@@ -9,8 +9,6 @@ http://svn.textmate.org/trunk/Review/Bundles/Forth.tmbundle:
 - source.forth
 http://svn.textmate.org/trunk/Review/Bundles/Parrot.tmbundle:
 - source.parrot.pir
-http://svn.textmate.org/trunk/Review/Bundles/Ruby%20Sass.tmbundle:
-- source.sass
 http://svn.textmate.org/trunk/Review/Bundles/SecondLife%20LSL.tmbundle:
 - source.lsl
 http://svn.textmate.org/trunk/Review/Bundles/VHDL.tmbundle:
@@ -20,403 +18,567 @@ http://svn.textmate.org/trunk/Review/Bundles/XQuery.tmbundle:
 https://bitbucket.org/Clams/sublimesystemverilog/get/default.tar.gz:
 - source.systemverilog
 - source.ucfconstraints
-https://bitbucket.org/bitlang/sublime_cobol/raw/b0e9c44ac5f7a2fb553421aa986b35854cbfda4a/COBOL.tmLanguage:
-- source.cobol
 https://fan.googlecode.com/hg-history/Build%201.0.55/adm/tools/textmate/Fan.tmbundle/Syntaxes/Fan.tmLanguage:
 - source.fan
-https://github.com/AlanQuatermain/go-tmbundle:
-- source.go
-https://github.com/Anomareh/PHP-Twig.tmbundle:
-- text.html.twig
-https://github.com/Cirru/sublime-cirru/raw/master/Cirru.tmLanguage:
-- source.cirru
-https://github.com/Cykey/Sublime-Logos:
-- source.logos
-https://github.com/Drako/SublimeBrainfuck/raw/master/Brainfuck.tmLanguage:
-- source.bf
-https://github.com/JohnNilsson/awk-sublime/raw/master/AWK.tmLanguage:
-- source.awk
-https://github.com/JonBons/Sublime-SQF-Language:
-- source.sqf
-https://github.com/MarioRicalde/SCSS.tmbundle:
-- source.scss
-https://github.com/Oldes/Sublime-REBOL:
-- source.rebol
-https://github.com/PogiNate/Sublime-Inform:
-- source.Inform7
-https://github.com/Red-Nova-Technologies/autoitv3-tmbundle:
-- source.autoit.3
-https://github.com/SalGnt/Sublime-VimL:
-- source.viml
-https://github.com/Shammah/boo-sublime/raw/master/Boo.tmLanguage:
-- source.boo
-https://github.com/SublimeText/ColdFusion:
+vendor/grammars/Agda.tmbundle:
+- source.agda
+vendor/grammars/Alloy.tmbundle:
+- source.alloy
+vendor/grammars/AutoHotkey/:
+- source.ahk
+vendor/grammars/BrightScript.tmbundle/:
+- source.brightauthorproject
+- source.brightscript
+vendor/grammars/ColdFusion:
 - source.cfscript
 - source.cfscript.cfc
 - text.cfml.basic
 - text.html.cfm
-https://github.com/SublimeText/NSIS:
-- source.nsis
-https://github.com/Varriount/NimLime:
-- source.nimrod
-- source.nimrod_filter
-- source.nimrodcfg
-https://github.com/alkemist/gradle.tmbundle:
-- source.groovy.gradle
-https://github.com/ambethia/Sublime-Loom:
-- source.loomscript
-https://github.com/angryant0007/VBDotNetSyntax:
-- source.vbnet
-https://github.com/anunayk/cool-tmbundle:
-- source.cool
-https://github.com/aroben/ada.tmbundle/raw/c45eed4d5f98fe3bcbbffbb9e436601ab5bbde4b/Syntaxes/Ada.plist:
-- source.ada
-https://github.com/aroben/ruby.tmbundle@4636a3023153c3034eb6ffc613899ba9cf33b41f:
-- source.ruby
-- text.html.erb
-https://github.com/asbjornenge/Docker.tmbundle:
+vendor/grammars/Docker.tmbundle:
 - source.dockerfile
-https://github.com/atom/language-clojure:
-- source.clojure
-https://github.com/atom/language-coffee-script:
-- source.coffee
-- source.litcoffee
-https://github.com/atom/language-csharp:
-- source.cs
-- source.csx
-- source.nant-build
-https://github.com/atom/language-javascript:
-- source.js
-- source.js.regexp
-https://github.com/atom/language-python:
+vendor/grammars/Elm.tmLanguage:
+- source.elm
+vendor/grammars/FreeMarker.tmbundle:
+- text.html.ftl
+vendor/grammars/G-Code/:
+- source.LS
+- source.MCPOST
+- source.MOD
+- source.apt
+- source.gcode
+vendor/grammars/Handlebars:
+- text.html.handlebars
+vendor/grammars/IDL-Syntax:
+- source.webidl
+vendor/grammars/Isabelle.tmbundle:
+- source.isabelle.root
+- source.isabelle.theory
+vendor/grammars/JSyntax/:
+- source.j
+vendor/grammars/Julia.tmbundle:
+- source.julia
+vendor/grammars/Lean.tmbundle:
+- source.lean
+vendor/grammars/LiveScript.tmbundle:
+- source.livescript
+vendor/grammars/MagicPython:
 - source.python
 - source.regexp.python
-- text.python.traceback
-https://github.com/atom/language-shellscript:
-- source.shell
-- text.shell-session
-https://github.com/austinwagner/sublime-sourcepawn:
-- source.sp
-https://github.com/bfad/Sublime-Lasso:
-- file.lasso
-https://github.com/bholt/chapel-tmbundle:
-- source.chapel
-https://github.com/brandonwamboldt/sublime-nginx:
-- source.nginx
-https://github.com/bro/bro-sublime:
-- source.bro
-https://github.com/carsonoid/sublime_man_page_support/raw/master/man-groff.tmLanguage:
-- text.groff
-https://github.com/ccreutzig/sublime-MuPAD:
-- source.mupad
-https://github.com/cdwilson/nesC.tmbundle:
-- source.nesc
-https://github.com/christophevg/racket-tmbundle:
+vendor/grammars/Modelica/:
+- source.modelica
+vendor/grammars/NSIS:
+- source.nsis
+vendor/grammars/NimLime:
+- source.nim
+- source.nim_filter
+- source.nimcfg
+vendor/grammars/PHP-Twig.tmbundle:
+- text.html.twig
+vendor/grammars/PogoScript.tmbundle/:
+- source.pogoscript
+vendor/grammars/RDoc.tmbundle:
+- text.rdoc
+vendor/grammars/Racket:
 - source.racket
-https://github.com/clemos/haxe-sublime-bundle:
+vendor/grammars/SCSS.tmbundle:
+- source.scss
+vendor/grammars/SMT.tmbundle:
+- source.smt
+vendor/grammars/Scalate.tmbundle:
+- source.scaml
+- text.html.ssp
+vendor/grammars/Slash.tmbundle:
+- text.html.slash
+vendor/grammars/Stata.tmbundle/:
+- source.mata
+- source.stata
+vendor/grammars/Stylus/:
+- source.stylus
+vendor/grammars/Sublime-Coq:
+- source.coq
+vendor/grammars/Sublime-HTTP:
+- source.httpspec
+vendor/grammars/Sublime-Lasso:
+- file.lasso
+vendor/grammars/Sublime-Loom:
+- source.loomscript
+vendor/grammars/Sublime-Modula-2/:
+- source.modula2
+vendor/grammars/Sublime-Nit:
+- source.nit
+vendor/grammars/Sublime-QML:
+- source.qml
+vendor/grammars/Sublime-REBOL:
+- source.rebol
+vendor/grammars/Sublime-Red:
+- source.red
+vendor/grammars/Sublime-SQF-Language:
+- source.sqf
+vendor/grammars/Sublime-Text-2-OpenEdge-ABL:
+- source.abl
+- text.html.abl
+vendor/grammars/Sublime-VimL:
+- source.viml
+vendor/grammars/SublimeBrainfuck:
+- source.bf
+vendor/grammars/SublimeClarion/:
+- source.clarion
+vendor/grammars/SublimePapyrus/:
+- source.papyrus.skyrim
+vendor/grammars/SublimePuppet/:
+- source.puppet
+vendor/grammars/SublimeXtend:
+- source.xtend
+vendor/grammars/TXL/:
+- source.txl
+vendor/grammars/Textmate-Gosu-Bundle:
+- source.gosu.2
+vendor/grammars/UrWeb-Language-Definition:
+- source.ur
+vendor/grammars/VBDotNetSyntax:
+- source.vbnet
+vendor/grammars/Vala-TMBundle:
+- source.vala
+vendor/grammars/X10:
+- source.x10
+vendor/grammars/abap.tmbundle:
+- source.abap
+vendor/grammars/actionscript3-tmbundle:
+- source.actionscript.3
+- text.html.asdoc
+- text.xml.flex-config
+vendor/grammars/ada.tmbundle:
+- source.ada
+vendor/grammars/ampl:
+- source.ampl
+vendor/grammars/ant.tmbundle:
+- text.xml.ant
+vendor/grammars/antlr.tmbundle:
+- source.antlr
+vendor/grammars/apache.tmbundle:
+- source.apache-config
+- source.apache-config.mod_perl
+vendor/grammars/api-blueprint-sublime-plugin/:
+- text.html.markdown.source.gfm.apib
+- text.html.markdown.source.gfm.mson
+vendor/grammars/applescript.tmbundle:
+- source.applescript
+vendor/grammars/asciidoc.tmbundle/:
+- text.html.asciidoc
+vendor/grammars/asp.tmbundle:
+- source.asp
+- text.html.asp
+vendor/grammars/assembly/:
+- objdump.x86asm
+- source.x86asm
+vendor/grammars/atom-fsharp/:
+- source.fsharp
+- source.fsharp.fsi
+- source.fsharp.fsl
+- source.fsharp.fsx
+vendor/grammars/atom-language-purescript/:
+- source.purescript
+vendor/grammars/atom-language-stan/:
+- source.stan
+vendor/grammars/atom-salt:
+- source.python.salt
+- source.yaml.salt
+vendor/grammars/ats:
+- source.ats
+vendor/grammars/awk-sublime:
+- source.awk
+vendor/grammars/bison.tmbundle:
+- source.bison
+vendor/grammars/boo/:
+- source.boo
+vendor/grammars/bro-sublime:
+- source.bro
+vendor/grammars/c.tmbundle:
+- source.c
+- source.c++
+- source.c.platform
+vendor/grammars/capnproto.tmbundle:
+- source.capnp
+vendor/grammars/carto-atom:
+- source.css.mss
+vendor/grammars/ceylon-sublimetext:
+- module.ceylon
+- source.ceylon
+vendor/grammars/chapel-tmbundle:
+- source.chapel
+vendor/grammars/cmake.tmbundle:
+- source.cache.cmake
+- source.cmake
+vendor/grammars/cool-tmbundle:
+- source.cool
+vendor/grammars/cpp-qt.tmbundle:
+- source.c++.qt
+- source.qmake
+vendor/grammars/css.tmbundle:
+- source.css
+vendor/grammars/cucumber-tmbundle:
+- source.ruby.rspec.cucumber.steps
+- text.gherkin.feature
+vendor/grammars/d.tmbundle:
+- source.d
+vendor/grammars/dart-sublime-bundle:
+- source.dart
+- source.pubspec
+- text.dart-analysis-output
+- text.dart-doccomments
+vendor/grammars/desktop.tmbundle:
+- source.desktop
+vendor/grammars/diff.tmbundle:
+- source.diff
+vendor/grammars/dylan.tmbundle:
+- source.dylan
+- source.lid
+- source.makegen
+vendor/grammars/ebundles/Bundles/MSDOS batch file.tmbundle:
+- source.dosbatch
+vendor/grammars/ec.tmbundle/:
+- source.c.ec
+vendor/grammars/eiffel.tmbundle:
+- source.eiffel
+vendor/grammars/elixir-tmbundle:
+- source.elixir
+- text.elixir
+- text.html.elixir
+vendor/grammars/erlang.tmbundle:
+- source.erlang
+- text.html.erlang.yaws
+vendor/grammars/factor:
+- source.factor
+- text.html.factor
+vendor/grammars/fancy-tmbundle:
+- source.fancy
+vendor/grammars/fish-tmbundle:
+- source.fish
+vendor/grammars/fortran.tmbundle:
+- source.fortran
+- source.fortran.modern
+vendor/grammars/gap-tmbundle/:
+- source.gap
+vendor/grammars/gdscript/:
+- source.gdscript
+vendor/grammars/gettext.tmbundle:
+- source.po
+vendor/grammars/gnuplot-tmbundle:
+- source.gnuplot
+vendor/grammars/go-tmbundle:
+- source.go
+vendor/grammars/grace:
+- source.grace
+vendor/grammars/gradle.tmbundle:
+- source.groovy.gradle
+vendor/grammars/graphviz.tmbundle:
+- source.dot
+vendor/grammars/groovy.tmbundle:
+- source.groovy
+vendor/grammars/haskell.tmbundle:
+- source.haskell
+- text.tex.latex.haskell
+vendor/grammars/haxe-sublime-bundle:
 - source.erazor
 - source.haxe.2
 - source.hss.1
 - source.hxml
 - source.nmml
-https://github.com/cucumber/cucumber-tmbundle:
-- source.ruby.rspec.cucumber.steps
-- text.gherkin.feature
-https://github.com/daaain/Handlebars/raw/master/Handlebars.tmLanguage:
-- text.html.handlebars
-https://github.com/davidpeckham/powershell.tmbundle:
-- source.powershell
-https://github.com/davidrios/jade-tmbundle:
-- source.jade
-- source.pyjade
-https://github.com/elixir-lang/elixir-tmbundle:
-- source.elixir
-- text.elixir
-- text.html.elixir
-https://github.com/ericzou/ebundles/raw/master/Bundles/MSDOS%20batch%20file.tmbundle/Syntaxes/MSDOS%20batch%20file.tmLanguage:
-- source.dosbatch
-https://github.com/euler0/sublime-glsl/raw/master/GLSL.tmLanguage:
-- source.glsl
-https://github.com/fancy-lang/fancy-tmbundle:
-- source.fancy
-https://github.com/fsharp/fsharpbinding:
-- source.fsharp
-https://github.com/gingerbeardman/monkey.tmbundle:
-- source.monkey
-https://github.com/guillermooo/dart-sublime-bundle/raw/master/Dart.tmLanguage:
-- source.dart
-https://github.com/harrism/sublimetext-cuda-cpp/raw/master/cuda-c%2B%2B.tmLanguage:
-- source.cuda-c++
-https://github.com/hww3/pike-textmate:
-- source.pike
-https://github.com/jeancharles-roger/ceylon-sublimetext/raw/master/Ceylon.tmLanguage:
-- source.ceylon
-https://github.com/jfairbank/Sublime-Text-2-OpenEdge-ABL:
-- source.abl
-https://github.com/jhasse/sublime-rust:
-- source.rust
-https://github.com/johanasplund/sublime-befunge/raw/master/Befunge-93.tmLanguage:
-- source.befunge
-https://github.com/joshaven/RDoc.tmbundle:
-- text.rdoc
-https://github.com/jpcamara/Textmate-Gosu-Bundle/raw/master/Gosu.tmbundle/Syntaxes/Gosu.tmLanguage:
-- source.gosu.2
-https://github.com/kswedberg/jquery-tmbundle:
-- source.js.jquery
-https://github.com/laughedelic/sublime-idris/raw/master/Idris.tmLanguage:
-- source.idris
-https://github.com/lavrton/sublime-better-typescript:
-- source.ts
-https://github.com/leafo/moonscript-tmbundle:
-- source.moonscript
-https://github.com/lsf37/Isabelle.tmbundle:
-- source.isabelle.theory
-https://github.com/lunixbochs/x86-assembly-textmate-bundle:
-- source.asm.x86
-https://github.com/macekond/Alloy.tmbundle:
-- source.alloy
-https://github.com/mads379/opa.tmbundle:
-- source.opa
-https://github.com/mads379/scala.tmbundle:
-- source.sbt
-- source.scala
-https://github.com/marconi/mako-tmbundle:
-- text.html.mako
-https://github.com/mattfoster/gnuplot-tmbundle:
-- source.gnuplot
-https://github.com/mgalloy/idl.tmbundle:
+vendor/grammars/html.tmbundle:
+- text.html.basic
+vendor/grammars/idl.tmbundle:
 - source.idl
 - source.idl-dlm
 - text.idl-idldoc
-https://github.com/michaeledgar/protobuf-tmbundle:
-- source.protobuf
-https://github.com/mkolosick/Sublime-Coq/raw/master/Coq.tmLanguage:
-- source.coq
-https://github.com/mokus0/Agda.tmbundle:
-- source.agda
-https://github.com/nanoant/Julia.tmbundle:
-- source.julia
-https://github.com/nanoant/assembly.tmbundle/raw/master/Syntaxes/objdump%20C%2B%2B.tmLanguage:
-- objdump.x86asm
-https://github.com/nilium/ooc.tmbundle:
-- source.ooc
-https://github.com/paulmillr/LiveScript.tmbundle:
-- source.livescript
-https://github.com/pferruggiaro/sublime-tea:
-- source.tea
-https://github.com/puppet-textmate-bundle/puppet-textmate-bundle:
-- source.puppet
-https://github.com/pvl/abap.tmbundle:
-- source.abap
-https://github.com/scalate/Scalate.tmbundle:
-- source.scaml
-- text.html.ssp
-https://github.com/shadanan/mathematica-tmbundle:
-- source.mathematica
-https://github.com/shellderp/sublime-robot-plugin:
-- text.robot
-https://github.com/simongregory/actionscript3-tmbundle:
-- source.actionscript.3
-- text.html.asdoc
-- text.xml.flex-config
-https://github.com/skozlovf/Sublime-QML:
-- source.qml
-https://github.com/slash-lang/Slash.tmbundle:
-- text.html.slash
-https://github.com/slavapestov/factor/raw/master/misc/Factor.tmbundle/Syntaxes/Factor.tmLanguage:
-- source.factor
-https://github.com/slim-template/ruby-slim.tmbundle:
-- text.slim
-https://github.com/staltz/SublimeXtend:
-- source.xtend
-https://github.com/statatmbundle/Stata.tmbundle:
-- source.mata
-- source.stata
-https://github.com/technosophos/Vala-TMBundle:
-- source.vala
-https://github.com/textmate/ant.tmbundle:
-- text.xml.ant
-https://github.com/textmate/antlr.tmbundle:
-- source.antlr
-https://github.com/textmate/apache.tmbundle:
-- source.apache-config
-- source.apache-config.mod_perl
-https://github.com/textmate/applescript.tmbundle:
-- source.applescript
-https://github.com/textmate/asp.tmbundle:
-- source.asp
-- text.html.asp
-https://github.com/textmate/bison.tmbundle:
-- source.bison
-https://github.com/textmate/c.tmbundle:
-- source.c
-- source.c++
-- source.c.platform
-https://github.com/textmate/capnproto.tmbundle:
-- source.capnp
-https://github.com/textmate/cmake.tmbundle:
-- source.cache.cmake
-- source.cmake
-https://github.com/textmate/cpp-qt.tmbundle:
-- source.c++.qt
-- source.qmake
-https://github.com/textmate/css.tmbundle:
-- source.css
-https://github.com/textmate/d.tmbundle:
-- source.d
-https://github.com/textmate/diff.tmbundle:
-- source.diff
-https://github.com/textmate/dylan.tmbundle:
-- source.dylan
-- source.lid
-- source.makegen
-https://github.com/textmate/eiffel.tmbundle:
-- source.eiffel
-https://github.com/textmate/erlang.tmbundle:
-- source.erlang
-- text.html.erlang.yaws
-https://github.com/textmate/fortran.tmbundle:
-- source.fortran
-- source.fortran.modern
-https://github.com/textmate/gettext.tmbundle:
-- source.po
-https://github.com/textmate/graphviz.tmbundle:
-- source.dot
-https://github.com/textmate/groovy.tmbundle:
-- source.groovy
-https://github.com/textmate/haskell.tmbundle:
-- source.haskell
-- text.tex.latex.haskell
-https://github.com/textmate/html.tmbundle:
-- text.html.basic
-https://github.com/textmate/ini.tmbundle:
+vendor/grammars/ini.tmbundle:
 - source.ini
-https://github.com/textmate/io.tmbundle:
+vendor/grammars/io.tmbundle:
 - source.io
-https://github.com/textmate/java.tmbundle:
+vendor/grammars/ioke-outdated:
+- source.ioke
+vendor/grammars/jade-tmbundle:
+- source.pyjade
+- text.jade
+vendor/grammars/jasmin-sublime:
+- source.jasmin
+vendor/grammars/java.tmbundle:
 - source.java
 - source.java-properties
 - text.html.jsp
 - text.junit-test-report
-https://github.com/textmate/javadoc.tmbundle:
-- text.html.javadoc
-https://github.com/textmate/javascript-objective-j.tmbundle:
+vendor/grammars/javascript-objective-j.tmbundle:
 - source.js.objj
-https://github.com/textmate/json.tmbundle:
+vendor/grammars/jflex.tmbundle:
+- source.jflex
+vendor/grammars/json.tmbundle:
 - source.json
-https://github.com/textmate/latex.tmbundle:
+vendor/grammars/kotlin-sublime-package:
+- source.Kotlin
+vendor/grammars/language-babel/:
+- source.js.jsx
+- source.regexp.babel
+vendor/grammars/language-click/:
+- source.click
+vendor/grammars/language-clojure:
+- source.clojure
+vendor/grammars/language-coffee-script:
+- source.coffee
+- source.litcoffee
+vendor/grammars/language-crystal:
+- source.crystal
+vendor/grammars/language-csharp:
+- source.cake
+- source.cs
+- source.csx
+- source.nant-build
+vendor/grammars/language-gfm:
+- source.gfm
+vendor/grammars/language-hy:
+- source.hy
+vendor/grammars/language-inform7:
+- source.inform7
+vendor/grammars/language-javascript:
+- source.js
+- source.js.regexp
+- source.js.regexp.replacement
+vendor/grammars/language-jsoniq/:
+- source.jq
+- source.xq
+vendor/grammars/language-less/:
+- source.css.less
+vendor/grammars/language-maxscript:
+- source.maxscript
+vendor/grammars/language-ncl:
+- source.ncl
+vendor/grammars/language-povray:
+- source.pov-ray sdl
+vendor/grammars/language-python:
+- text.python.console
+- text.python.traceback
+vendor/grammars/language-renpy:
+- source.renpy
+vendor/grammars/language-shellscript:
+- source.shell
+- text.shell-session
+vendor/grammars/language-supercollider:
+- source.supercollider
+vendor/grammars/language-xbase:
+- source.harbour
+vendor/grammars/language-yaml:
+- source.yaml
+vendor/grammars/language-yang/:
+- source.yang
+vendor/grammars/latex.tmbundle:
 - text.bibtex
 - text.log.latex
 - text.tex
 - text.tex.latex
 - text.tex.latex.beamer
 - text.tex.latex.memoir
-https://github.com/textmate/less.tmbundle:
-- source.css.less
-https://github.com/textmate/lilypond.tmbundle:
+vendor/grammars/lilypond.tmbundle:
 - source.lilypond
-https://github.com/textmate/lisp.tmbundle:
+vendor/grammars/liquid.tmbundle:
+- text.html.liquid
+vendor/grammars/lisp.tmbundle:
 - source.lisp
-https://github.com/textmate/logtalk.tmbundle:
+vendor/grammars/llvm.tmbundle:
+- source.llvm
+vendor/grammars/logos:
+- source.logos
+vendor/grammars/logtalk.tmbundle:
 - source.logtalk
-https://github.com/textmate/lua.tmbundle:
+vendor/grammars/lua.tmbundle:
 - source.lua
-https://github.com/textmate/make.tmbundle:
+vendor/grammars/make.tmbundle:
 - source.makefile
-https://github.com/textmate/markdown.tmbundle:
-- text.html.markdown
-https://github.com/textmate/matlab.tmbundle:
+vendor/grammars/mako-tmbundle:
+- text.html.mako
+vendor/grammars/mathematica-tmbundle:
+- source.mathematica
+vendor/grammars/matlab.tmbundle:
 - source.matlab
 - source.octave
-https://github.com/textmate/maven.tmbundle:
+vendor/grammars/maven.tmbundle:
 - text.xml.pom
-https://github.com/textmate/nemerle.tmbundle:
+vendor/grammars/mediawiki.tmbundle/:
+- text.html.mediawiki
+vendor/grammars/mercury-tmlanguage:
+- source.mercury
+vendor/grammars/monkey/:
+- source.monkey
+vendor/grammars/moonscript-tmbundle:
+- source.moonscript
+vendor/grammars/nemerle.tmbundle:
 - source.nemerle
-https://github.com/textmate/ninja.tmbundle:
+vendor/grammars/nesC:
+- source.nesc
+vendor/grammars/ninja.tmbundle:
 - source.ninja
-https://github.com/textmate/objective-c.tmbundle:
+vendor/grammars/objective-c.tmbundle:
 - source.objc
 - source.objc++
 - source.objc.platform
 - source.strings
-https://github.com/textmate/ocaml.tmbundle:
+vendor/grammars/ocaml.tmbundle:
 - source.camlp4.ocaml
 - source.ocaml
 - source.ocamllex
 - source.ocamlyacc
-https://github.com/textmate/pascal.tmbundle:
+vendor/grammars/ooc.tmbundle:
+- source.ooc
+vendor/grammars/opa.tmbundle:
+- source.opa
+vendor/grammars/oz-tmbundle/Syntaxes/Oz.tmLanguage:
+- source.oz
+vendor/grammars/pascal.tmbundle:
 - source.pascal
-https://github.com/textmate/perl.tmbundle:
+vendor/grammars/pawn-sublime-language/:
+- source.pawn
+vendor/grammars/perl.tmbundle/:
 - source.perl
-https://github.com/textmate/php-smarty.tmbundle:
-- source.smarty
-https://github.com/textmate/php.tmbundle:
+- source.perl.6
+vendor/grammars/perl6fe:
+- source.meta-info
+- source.perl6fe
+- source.regexp.perl6fe
+vendor/grammars/php-smarty.tmbundle:
+- text.html.smarty
+vendor/grammars/php.tmbundle:
 - text.html.php
-https://github.com/textmate/postscript.tmbundle:
+vendor/grammars/pig-latin/:
+- source.pig_latin
+vendor/grammars/pike-textmate:
+- source.pike
+vendor/grammars/postscript.tmbundle:
 - source.postscript
-https://github.com/textmate/processing.tmbundle:
+vendor/grammars/powershell:
+- source.powershell
+vendor/grammars/processing.tmbundle:
 - source.processing
-https://github.com/textmate/prolog.tmbundle:
-- source.prolog
-https://github.com/textmate/python-django.tmbundle:
+vendor/grammars/protobuf-tmbundle:
+- source.protobuf
+vendor/grammars/python-django.tmbundle:
 - source.python.django
 - text.html.django
-https://github.com/textmate/r.tmbundle:
+vendor/grammars/r.tmbundle:
 - source.r
 - text.tex.latex.rd
-https://github.com/textmate/restructuredtext.tmbundle:
+vendor/grammars/restructuredtext.tmbundle:
 - text.restructuredtext
-https://github.com/textmate/ruby-haml.tmbundle:
+vendor/grammars/ruby-haml.tmbundle:
 - text.haml
-https://github.com/textmate/ruby-on-rails-tmbundle:
-- source.js.erb.rails
-- source.ruby.rails
-- source.ruby.rails.rjs
-- source.sql.ruby
-- text.html.erb.rails
-https://github.com/textmate/scheme.tmbundle:
+vendor/grammars/ruby-slim.tmbundle:
+- text.slim
+vendor/grammars/ruby.tmbundle:
+- source.ruby
+- text.html.erb
+vendor/grammars/sas.tmbundle:
+- source.sas
+- source.sas_log
+vendor/grammars/sass-textmate-bundle:
+- source.sass
+vendor/grammars/scala.tmbundle:
+- source.sbt
+- source.scala
+vendor/grammars/scheme.tmbundle:
 - source.scheme
-https://github.com/textmate/scilab.tmbundle:
+vendor/grammars/scilab.tmbundle:
 - source.scilab
-https://github.com/textmate/sql.tmbundle:
+vendor/grammars/smali-sublime/smali.tmLanguage:
+- source.smali
+vendor/grammars/smalltalk-tmbundle:
+- source.smalltalk
+vendor/grammars/sourcepawn/:
+- source.sp
+vendor/grammars/sql.tmbundle:
 - source.sql
-https://github.com/textmate/standard-ml.tmbundle:
+vendor/grammars/st2-zonefile:
+- text.zone_file
+vendor/grammars/standard-ml.tmbundle:
 - source.cm
 - source.ml
-https://github.com/textmate/swift.tmbundle:
+vendor/grammars/sublime-MuPAD:
+- source.mupad
+vendor/grammars/sublime-apl/:
+- source.apl
+vendor/grammars/sublime-aspectj/:
+- source.aspectj
+vendor/grammars/sublime-befunge:
+- source.befunge
+vendor/grammars/sublime-bsv:
+- source.bsv
+vendor/grammars/sublime-cirru:
+- source.cirru
+vendor/grammars/sublime-glsl:
+- source.essl
+- source.glsl
+vendor/grammars/sublime-golo/:
+- source.golo
+vendor/grammars/sublime-mask:
+- source.mask
+vendor/grammars/sublime-netlinx:
+- source.netlinx
+- source.netlinx.erb
+vendor/grammars/sublime-nginx:
+- source.nginx
+vendor/grammars/sublime-opal/:
+- source.opal
+- source.opalsysdefs
+vendor/grammars/sublime-pony:
+- source.pony
+vendor/grammars/sublime-robot-plugin:
+- text.robot
+vendor/grammars/sublime-rust:
+- source.rust
+vendor/grammars/sublime-spintools/:
+- source.regexp.spin
+- source.spin
+vendor/grammars/sublime-tea:
+- source.tea
+vendor/grammars/sublime-terra:
+- source.terra
+vendor/grammars/sublime-text-ox/:
+- source.ox
+vendor/grammars/sublime-typescript/:
+- source.ts
+- source.tsx
+vendor/grammars/sublime-varnish:
+- source.varnish.vcl
+vendor/grammars/sublime_cobol:
+- source.acucobol
+- source.cobol
+- source.jcl
+- source.opencobol
+vendor/grammars/sublime_man_page_support:
+- source.man
+- text.groff
+vendor/grammars/sublimeassembly/:
+- source.assembly
+vendor/grammars/sublimeprolog/:
+- source.prolog
+- source.prolog.eclipse
+vendor/grammars/sublimetext-cuda-cpp:
+- source.cuda-c++
+vendor/grammars/swift.tmbundle:
 - source.swift
-https://github.com/textmate/tcl.tmbundle:
+vendor/grammars/tcl.tmbundle:
 - source.tcl
 - text.html.tcl
-https://github.com/textmate/text.tmbundle:
-- text.plain
-https://github.com/textmate/textile.tmbundle:
-- text.html.textile
-https://github.com/textmate/textmate.tmbundle:
-- source.regexp.oniguruma
-- source.tm-properties
-https://github.com/textmate/thrift.tmbundle:
+vendor/grammars/thrift.tmbundle:
 - source.thrift
-https://github.com/textmate/toml.tmbundle:
+vendor/grammars/toml.tmbundle:
 - source.toml
-https://github.com/textmate/verilog.tmbundle:
+vendor/grammars/turtle.tmbundle:
+- source.sparql
+- source.turtle
+vendor/grammars/verilog.tmbundle:
 - source.verilog
-https://github.com/textmate/xml.tmbundle:
+vendor/grammars/vue-syntax-highlight:
+- text.html.vue
+vendor/grammars/xc.tmbundle/:
+- source.xc
+vendor/grammars/xml.tmbundle:
 - text.xml
 - text.xml.xsl
-https://github.com/textmate/yaml.tmbundle:
-- source.yaml
-https://github.com/tomas-stefano/smalltalk-tmbundle:
-- source.smalltalk
-https://github.com/vic/ioke-outdated/raw/master/share/TextMate/Ioke.tmbundle/Syntaxes/Ioke.tmLanguage:
-- source.ioke
-https://github.com/vkostyukov/kotlin-sublime-package:
-- source.Kotlin
-https://github.com/vmg/zephir-sublime:
+vendor/grammars/zephir-sublime:
 - source.php.zephir
-https://github.com/whitequark/llvm.tmbundle:
-- source.llvm
-https://github.com/wmertens/sublime-nix:
-- source.nix
-https://raw.githubusercontent.com/eregon/oz-tmbundle/master/Syntaxes/Oz.tmLanguage:
-- source.oz

lib/linguist.rb

@@ -1,8 +1,99 @@
 require 'linguist/blob_helper'
 require 'linguist/generated'
+require 'linguist/grammars'
 require 'linguist/heuristics'
 require 'linguist/language'
 require 'linguist/repository'
 require 'linguist/samples'
 require 'linguist/shebang'
 require 'linguist/version'
+
+class << Linguist
+  # Public: Detects the Language of the blob.
+  #
+  # blob - an object that includes the Linguist `BlobHelper` interface;
+  #       see Linguist::LazyBlob and Linguist::FileBlob for examples
+  #
+  # Returns Language or nil.
+  def detect(blob)
+    # Bail early if the blob is binary or empty.
+    return nil if blob.likely_binary? || blob.binary? || blob.empty?
+
+    Linguist.instrument("linguist.detection", :blob => blob) do
+      # Call each strategy until one candidate is returned.
+      languages = []
+      returning_strategy = nil
+
+      STRATEGIES.each do |strategy|
+        returning_strategy = strategy
+        candidates = Linguist.instrument("linguist.strategy", :blob => blob, :strategy => strategy, :candidates => languages) do
+          strategy.call(blob, languages)
+        end
+        if candidates.size == 1
+          languages = candidates
+          break
+        elsif candidates.size > 1
+          # More than one candidate was found, pass them to the next strategy.
+          languages = candidates
+        else
+          # No candidates, try the next strategy
+        end
+      end
+
+      Linguist.instrument("linguist.detected", :blob => blob, :strategy => returning_strategy, :language => languages.first)
+
+      languages.first
+    end
+  end
+
+  # Internal: The strategies used to detect the language of a file.
+  #
+  # A strategy is an object that has a `.call` method that takes two arguments:
+  #
+  #   blob - An object that quacks like a blob.
+  #   languages - An Array of candidate Language objects that were returned by the
+  #               previous strategy.
+  #
+  # A strategy should return an Array of Language candidates.
+  #
+  # Strategies are called in turn until a single Language is returned.
+  STRATEGIES = [
+    Linguist::Strategy::Modeline,
+    Linguist::Shebang,
+    Linguist::Strategy::Filename,
+    Linguist::Heuristics,
+    Linguist::Classifier
+  ]
+
+  # Public: Set an instrumenter.
+  #
+  #     class CustomInstrumenter
+  #       def instrument(name, payload = {})
+  #         warn "Instrumenting #{name}: #{payload[:blob]}"
+  #       end
+  #     end
+  #
+  #     Linguist.instrumenter = CustomInstrumenter
+  #
+  # The instrumenter must conform to the `ActiveSupport::Notifications`
+  # interface, which defines `#instrument` and accepts:
+  #
+  # name    - the String name of the event (e.g. "linguist.detected")
+  # payload - a Hash of the exception context.
+  attr_accessor :instrumenter
+
+  # Internal: Perform instrumentation on a block
+  #
+  #     Linguist.instrument("linguist.dosomething", :blob => blob) do
+  #       # logic to instrument here.
+  #     end
+  #
+  def instrument(*args, &bk)
+    if instrumenter
+      instrumenter.instrument(*args, &bk)
+    elsif block_given?
+      yield
+    end
+  end
+
+end

lib/linguist/blob.rb

@@ -0,0 +1,73 @@
+require 'linguist/blob_helper'
+
+module Linguist
+  # A Blob is a wrapper around the content of a file to make it quack
+  # like a Grit::Blob. It provides the basic interface: `name`,
+  # `data`, `path` and `size`.
+  class Blob
+    include BlobHelper
+
+    # Public: Initialize a new Blob.
+    #
+    # path    - A path String (does not necessarily exists on the file system).
+    # content - Content of the file.
+    #
+    # Returns a Blob.
+    def initialize(path, content)
+      @path = path
+      @content = content
+    end
+
+    # Public: Filename
+    #
+    # Examples
+    #
+    #   Blob.new("/path/to/linguist/lib/linguist.rb", "").path
+    #   # =>  "/path/to/linguist/lib/linguist.rb"
+    #
+    # Returns a String
+    attr_reader :path
+
+    # Public: File name
+    #
+    # Returns a String
+    def name
+      File.basename(@path)
+    end
+
+    # Public: File contents.
+    #
+    # Returns a String.
+    def data
+      @content
+    end
+
+    # Public: Get byte size
+    #
+    # Returns an Integer.
+    def size
+      @content.bytesize
+    end
+
+    # Public: Get file extension.
+    #
+    # Returns a String.
+    def extension
+      extensions.last || ""
+    end
+
+    # Public: Return an array of the file extensions
+    #
+    #     >> Linguist::Blob.new("app/views/things/index.html.erb").extensions
+    #     => [".html.erb", ".erb"]
+    #
+    # Returns an Array
+    def extensions
+      _, *segments = name.downcase.split(".")
+
+      segments.map.with_index do |segment, index|
+        "." + segments[index..-1].join(".")
+      end
+    end
+  end
+end

lib/linguist/blob_helper.rb

@@ -6,7 +6,7 @@ require 'yaml'
 
 module Linguist
   # DEPRECATED Avoid mixing into Blob classes. Prefer functional interfaces
-  # like `Language.detect` over `Blob#language`. Functions are much easier to
+  # like `Linguist.detect` over `Blob#language`. Functions are much easier to
   # cache and compose.
   #
   # Avoid adding additional bloat to this module.
@@ -99,7 +99,7 @@ module Linguist
       elsif name.nil?
         "attachment"
       else
-        "attachment; filename=#{EscapeUtils.escape_url(File.basename(name))}"
+        "attachment; filename=#{EscapeUtils.escape_url(name)}"
       end
     end
 
@@ -233,7 +233,22 @@ module Linguist
     #
     # Return true or false
     def vendored?
-      name =~ VendoredRegexp ? true : false
+      path =~ VendoredRegexp ? true : false
+    end
+
+    documentation_paths = YAML.load_file(File.expand_path("../documentation.yml", __FILE__))
+    DocumentationRegexp = Regexp.new(documentation_paths.join('|'))
+
+    # Public: Is the blob in a documentation directory?
+    #
+    # Documentation files are ignored by language statistics.
+    #
+    # See "documentation.yml" for a list of documentation conventions that match
+    # this pattern.
+    #
+    # Return true or false
+    def documentation?
+      path =~ DocumentationRegexp ? true : false
     end
 
     # Public: Get each line of data
@@ -301,7 +316,7 @@ module Linguist
     #
     # Return true or false
     def generated?
-      @_generated ||= Generated.generated?(name, lambda { data })
+      @_generated ||= Generated.generated?(path, lambda { data })
     end
 
     # Public: Detects the Language of the blob.
@@ -310,12 +325,22 @@ module Linguist
     #
     # Returns a Language or nil if none is detected
     def language
-      @language ||= Language.detect(self)
+      @language ||= Linguist.detect(self)
     end
 
     # Internal: Get the TextMate compatible scope for the blob
     def tm_scope
       language && language.tm_scope
     end
+
+    DETECTABLE_TYPES = [:programming, :markup].freeze
+
+    # Internal: Should this blob be included in repository language statistics?
+    def include_in_language_stats?
+      !vendored? &&
+      !documentation? &&
+      !generated? &&
+      language && DETECTABLE_TYPES.include?(language.type)
+    end
   end
 end

lib/linguist/documentation.yml

@@ -0,0 +1,30 @@
+# Documentation files and directories are excluded from language
+# statistics.
+#
+# Lines in this file are Regexps that are matched against the file
+# pathname.
+#
+# Please add additional test coverage to
+# `test/test_blob.rb#test_documentation` if you make any changes.
+
+## Documentation directories ##
+
+- ^docs?/
+- (^|/)[Dd]ocumentation/
+- (^|/)javadoc/
+- ^man/
+- ^[Ee]xamples/
+
+## Documentation files ##
+
+- (^|/)CHANGE(S|LOG)?(\.|$)
+- (^|/)CONTRIBUTING(\.|$)
+- (^|/)COPYING(\.|$)
+- (^|/)INSTALL(\.|$)
+- (^|/)LICEN[CS]E(\.|$)
+- (^|/)[Ll]icen[cs]e(\.|$)
+- (^|/)README(\.|$)
+- (^|/)[Rr]eadme(\.|$)
+
+# Samples folders
+- ^[Ss]amples/

lib/linguist/file_blob.rb

@@ -1,10 +1,11 @@
 require 'linguist/blob_helper'
+require 'linguist/blob'
 
 module Linguist
   # A FileBlob is a wrapper around a File object to make it quack
   # like a Grit::Blob. It provides the basic interface: `name`,
-  # `data`, and `size`.
-  class FileBlob
+  # `data`, `path` and `size`.
+  class FileBlob < Blob
     include BlobHelper
 
     # Public: Initialize a new FileBlob from a path
@@ -14,64 +15,29 @@ module Linguist
     #
     # Returns a FileBlob.
     def initialize(path, base_path = nil)
-      @path = path
-      @name = base_path ? path.sub("#{base_path}/", '') : path
+      @fullpath = path
+      @path = base_path ? path.sub("#{base_path}/", '') : path
     end
 
-    # Public: Filename
-    #
-    # Examples
-    #
-    #   FileBlob.new("/path/to/linguist/lib/linguist.rb").name
-    #   # =>  "/path/to/linguist/lib/linguist.rb"
-    #
-    #   FileBlob.new("/path/to/linguist/lib/linguist.rb",
-    #                "/path/to/linguist").name
-    #   # =>  "lib/linguist.rb"
-    #
-    # Returns a String
-    attr_reader :name
-
     # Public: Read file permissions
     #
     # Returns a String like '100644'
     def mode
-      File.stat(@path).mode.to_s(8)
+      File.stat(@fullpath).mode.to_s(8)
     end
 
     # Public: Read file contents.
     #
     # Returns a String.
     def data
-      File.read(@path)
+      File.read(@fullpath)
     end
 
     # Public: Get byte size
     #
     # Returns an Integer.
     def size
-      File.size(@path)
-    end
-
-    # Public: Get file extension.
-    #
-    # Returns a String.
-    def extension
-      extensions.last || ""
-    end
-
-    # Public: Return an array of the file extensions
-    #
-    #     >> Linguist::FileBlob.new("app/views/things/index.html.erb").extensions
-    #     => [".html.erb", ".erb"]
-    #
-    # Returns an Array
-    def extensions
-      basename, *segments = File.basename(name).split(".")
-
-      segments.map.with_index do |segment, index|
-        "." + segments[index..-1].join(".")
-      end
+      File.size(@fullpath)
     end
   end
 end

lib/linguist/generated.rb

@@ -51,20 +51,30 @@ module Linguist
     #
     # Return true or false
     def generated?
-      minified_files? ||
-      compiled_coffeescript? ||
       xcode_file? ||
-      generated_parser? ||
-      generated_net_docfile? ||
       generated_net_designer_file? ||
-      generated_postscript? ||
-      generated_protocol_buffer? ||
-      generated_jni_header? ||
+      generated_net_specflow_feature_file? ||
       composer_lock? ||
       node_modules? ||
       godeps? ||
+      generated_by_zephir? ||
+      minified_files? ||
+      source_map? ||
+      compiled_coffeescript? ||
+      generated_parser? ||
+      generated_net_docfile? ||
+      generated_postscript? ||
+      compiled_cython_file? ||
+      generated_go? ||
+      generated_protocol_buffer? ||
+      generated_apache_thrift? ||
+      generated_jni_header? ||
       vcr_cassette? ||
-      generated_by_zephir?
+      generated_module? ||
+      generated_unity3d_meta? ||
+      generated_racc? ||
+      generated_jflex? ||
+      generated_grammarkit?
     end
 
     # Internal: Is the blob an Xcode file?
@@ -94,6 +104,20 @@ module Linguist
       end
     end
 
+    # Internal: Is the blob a generated source map?
+    #
+    # Source Maps usually have .css.map or .js.map extensions. In case they
+    # are not following the name convention, detect them based on the content.
+    #
+    # Returns true or false.
+    def source_map?
+      return false unless extname.downcase == '.map'
+
+      name =~ /(\.css|\.js)\.map$/i ||                 # Name convention
+      lines[0] =~ /^{"version":\d+,/ ||                # Revision 2 and later begin with the version number
+      lines[0] =~ /^\/\*\* Begin line maps\. \*\*\/{/  # Revision 1 begins with a magic comment
+    end
+
     # Internal: Is the blob of JS generated by CoffeeScript?
     #
     # CoffeeScript is meant to output JS that would be difficult to
@@ -162,6 +186,17 @@ module Linguist
       name.downcase =~ /\.designer\.cs$/
     end
 
+    # Internal: Is this a codegen file for Specflow feature file?
+    #
+    # Visual Studio's SpecFlow extension generates *.feature.cs files
+    # from *.feature files, they are not meant to be consumed by humans.
+    # Let's hide them.
+    #
+    # Returns true or false
+    def generated_net_specflow_feature_file?
+      name.downcase =~ /\.feature\.cs$/
+    end
+
     # Internal: Is the blob of JS a parser generated by PEG.js?
     #
     # PEG.js-generated parsers are not meant to be consumed by humans.
@@ -202,17 +237,38 @@ module Linguist
         creator.include?("ImageMagick")
     end
 
+    def generated_go?
+      return false unless extname == '.go'
+      return false unless lines.count > 1
+
+      return lines[0].include?("Code generated by")
+    end
+
+    PROTOBUF_EXTENSIONS = ['.py', '.java', '.h', '.cc', '.cpp']
+
     # Internal: Is the blob a C++, Java or Python source file generated by the
     # Protocol Buffer compiler?
     #
     # Returns true of false.
     def generated_protocol_buffer?
-      return false unless ['.py', '.java', '.h', '.cc', '.cpp'].include?(extname)
+      return false unless PROTOBUF_EXTENSIONS.include?(extname)
       return false unless lines.count > 1
 
       return lines[0].include?("Generated by the protocol buffer compiler.  DO NOT EDIT!")
     end
 
+    APACHE_THRIFT_EXTENSIONS = ['.rb', '.py', '.go', '.js', '.m', '.java', '.h', '.cc', '.cpp']
+
+    # Internal: Is the blob generated by Apache Thrift compiler?
+    #
+    # Returns true or false
+    def generated_apache_thrift?
+      return false unless APACHE_THRIFT_EXTENSIONS.include?(extname)
+      return false unless lines.count > 1
+
+      return lines[0].include?("Autogenerated by Thrift Compiler") || lines[1].include?("Autogenerated by Thrift Compiler")
+    end
+
     # Internal: Is the blob a C/C++ header generated by the Java JNI tool javah?
     #
     # Returns true of false.
@@ -262,5 +318,89 @@ module Linguist
       # VCR Cassettes have "recorded_with: VCR" in the second last line.
       return lines[-2].include?("recorded_with: VCR")
     end
+
+    # Internal: Is this a compiled C/C++ file from Cython?
+    #
+    # Cython-compiled C/C++ files typically contain:
+    # /* Generated by Cython x.x.x on ... */
+    # on the first line.
+    #
+    # Return true or false
+    def compiled_cython_file?
+      return false unless ['.c', '.cpp'].include? extname
+      return false unless lines.count > 1
+      return lines[0].include?("Generated by Cython")
+    end
+
+    # Internal: Is it a KiCAD or GFortran module file?
+    #
+    # KiCAD module files contain:
+    # PCBNEW-LibModule-V1  yyyy-mm-dd h:mm:ss XM
+    # on the first line.
+    #
+    # GFortran module files contain:
+    # GFORTRAN module version 'x' created from 
+    # on the first line.
+    #
+    # Return true of false
+    def generated_module?
+      return false unless extname == '.mod'
+      return false unless lines.count > 1
+      return lines[0].include?("PCBNEW-LibModule-V") || 
+              lines[0].include?("GFORTRAN module version '")
+    end
+
+    # Internal: Is this a metadata file from Unity3D?
+    #
+    # Unity3D Meta files start with:
+    #   fileFormatVersion: X
+    #   guid: XXXXXXXXXXXXXXX
+    #
+    # Return true or false
+    def generated_unity3d_meta?
+      return false unless extname == '.meta'
+      return false unless lines.count > 1
+      return lines[0].include?("fileFormatVersion: ")
+    end
+
+    # Internal: Is this a Racc-generated file?
+    #
+    # A Racc-generated file contains:
+    # # This file is automatically generated by Racc x.y.z
+    # on the third line.
+    #
+    # Return true or false
+    def generated_racc?
+      return false unless extname == '.rb'
+      return false unless lines.count > 2
+      return lines[2].start_with?("# This file is automatically generated by Racc")
+    end
+
+    # Internal: Is this a JFlex-generated file?
+    #
+    # A JFlex-generated file contains:
+    # /* The following code was generated by JFlex x.y.z on d/at/e ti:me */
+    # on the first line.
+    #
+    # Return true or false
+    def generated_jflex?
+      return false unless extname == '.java'
+      return false unless lines.count > 1
+      return lines[0].start_with?("/* The following code was generated by JFlex ")
+    end
+
+    # Internal: Is this a GrammarKit-generated file?
+    #
+    # A GrammarKit-generated file typically contain:
+    # // This is a generated file. Not intended for manual editing.
+    # on the first line. This is not always the case, as it's possible to
+    # customize the class header.
+    #
+    # Return true or false
+    def generated_grammarkit?
+      return false unless extname == '.java'
+      return false unless lines.count > 1
+      return lines[0].start_with?("// This is a generated file. Not intended for manual editing.")
+    end
   end
 end

lib/linguist/grammars.rb

@@ -1,6 +1,3 @@
-# Note: This file is included in the github-linguist-grammars gem, not the
-# github-linguist gem.
-
 module Linguist
   module Grammars
     # Get the path to the directory containing the language grammar JSON files.

lib/linguist/heuristics.rb

@@ -13,11 +13,14 @@ module Linguist
     #   ])
     #
     # Returns an Array of languages, or empty if none matched or were inconclusive.
-    def self.call(blob, languages)
+    def self.call(blob, candidates)
       data = blob.data
 
       @heuristics.each do |heuristic|
-        return Array(heuristic.call(data)) if heuristic.matches?(languages)
+        if heuristic.matches?(blob.name)
+          languages = Array(heuristic.call(data))
+          return languages if languages.any? || languages.all? { |l| candidates.include?(l) }
+        end
       end
 
       [] # No heuristics matched
@@ -30,30 +33,31 @@ module Linguist
     #
     # Examples
     #
-    #     disambiguate "Perl", "Prolog" do |data|
+    #     disambiguate ".pm" do |data|
     #       if data.include?("use strict")
     #         Language["Perl"]
-    #       elsif data.include?(":-")
+    #       elsif /^[^#]+:-/.match(data)
     #         Language["Prolog"]
     #       end
     #     end
     #
-    def self.disambiguate(*languages, &heuristic)
-      @heuristics << new(languages, &heuristic)
+    def self.disambiguate(*extensions, &heuristic)
+      @heuristics << new(extensions, &heuristic)
     end
 
     # Internal: Array of defined heuristics
     @heuristics = []
 
     # Internal
-    def initialize(languages, &heuristic)
-      @languages = languages
+    def initialize(extensions, &heuristic)
+      @extensions = extensions
       @heuristic = heuristic
     end
 
     # Internal: Check if this heuristic matches the candidate languages.
-    def matches?(candidates)
-      candidates.any? && candidates.all? { |l| @languages.include?(l.name) }
+    def matches?(filename)
+      filename = filename.downcase
+      @extensions.any? { |ext| filename.end_with?(ext) }
     end
 
     # Internal: Perform the heuristic
@@ -61,43 +65,35 @@ module Linguist
       @heuristic.call(data)
     end
 
-    disambiguate "Objective-C", "C++", "C" do |data|
-      if (/@(interface|class|protocol|property|end|synchronised|selector|implementation)\b/.match(data))
-        Language["Objective-C"]
-      elsif (/^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>/.match(data) ||
-        /^\s*template\s*</.match(data) || /^[^@]class\s+\w+/.match(data) || /^[^@](private|public|protected):$/.match(data) || /std::.+$/.match(data))
-        Language["C++"]
+    # Common heuristics
+    ObjectiveCRegex = /^\s*(@(interface|class|protocol|property|end|synchronised|selector|implementation)\b|#import\s+.+\.h[">])/
+
+    disambiguate ".asc" do |data|
+      if /^(----[- ]BEGIN|ssh-(rsa|dss)) /.match(data)
+        Language["Public Key"]
+      elsif /^[=-]+(\s|\n)|{{[A-Za-z]/.match(data)
+        Language["AsciiDoc"]
+      elsif /^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])/.match(data)
+        Language["AGS Script"]
       end
     end
 
-    disambiguate "Perl", "Perl6", "Prolog" do |data|
-      if data.include?("use v6")
-        Language["Perl6"]
-      elsif data.include?("use strict")
-        Language["Perl"]
-      elsif data.include?(":-")
-        Language["Prolog"]
+    disambiguate ".bb" do |data|
+      if /^\s*; /.match(data) || data.include?("End Function")
+        Language["BlitzBasic"]
+      elsif /^\s*(# |include|require)\b/.match(data)
+        Language["BitBake"]
       end
     end
 
-    disambiguate "ECL", "Prolog" do |data|
-      if data.include?(":-")
-        Language["Prolog"]
-      elsif data.include?(":=")
-        Language["ECL"]
+    disambiguate ".ch" do |data|
+      if /^\s*#\s*(if|ifdef|ifndef|define|command|xcommand|translate|xtranslate|include|pragma|undef)\b/i.match(data)
+        Language["xBase"]
       end
     end
 
-    disambiguate "IDL", "Prolog" do |data|
-      if data.include?(":-")
-        Language["Prolog"]
-      else
-        Language["IDL"]
-      end
-    end
-
-    disambiguate "Common Lisp", "OpenCL", "Cool" do |data|
-      if data.include?("(defun ")
+    disambiguate ".cl" do |data|
+      if /^\s*\((defun|in-package|defpackage) /i.match(data)
         Language["Common Lisp"]
       elsif /^class/x.match(data)
         Language["Cool"]
@@ -106,49 +102,88 @@ module Linguist
       end
     end
 
-    disambiguate "Hack", "PHP" do |data|
-      if data.include?("<?hh")
-        Language["Hack"]
-      elsif /<?[^h]/.match(data)
-        Language["PHP"]
+    disambiguate ".cs" do |data|
+      if /![\w\s]+methodsFor: /.match(data)
+        Language["Smalltalk"]
+      elsif /^\s*namespace\s*[\w\.]+\s*{/.match(data) || /^\s*\/\//.match(data)
+        Language["C#"]
       end
     end
 
-    disambiguate "Scala", "SuperCollider" do |data|
-      if /\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data)
-        Language["SuperCollider"]
-      elsif /^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data)
-        Language["Scala"]
+    disambiguate ".d" do |data|
+      if /^module /.match(data)
+        Language["D"]
+      elsif /^((dtrace:::)?BEGIN|provider |#pragma (D (option|attributes)|ident)\s)/.match(data)
+        Language["DTrace"]
+      elsif /(\/.*:( .* \\)$| : \\$|^ : |: \\$)/.match(data)
+        Language["Makefile"]
       end
     end
 
-    disambiguate "AsciiDoc", "AGS Script" do |data|
-      Language["AsciiDoc"] if /^=+(\s|\n)/.match(data)
+    disambiguate ".ecl" do |data|
+      if /^[^#]+:-/.match(data)
+        Language["ECLiPSe"]
+      elsif data.include?(":=")
+        Language["ECL"]
+      end
     end
 
-    disambiguate "FORTRAN", "Forth" do |data|
+    disambiguate ".for", ".f" do |data|
       if /^: /.match(data)
         Language["Forth"]
-      elsif /^([c*][^a-z]|      subroutine\s)/i.match(data)
+      elsif /^([c*][^abd-z]|      (subroutine|program|end)\s|\s*!)/i.match(data)
         Language["FORTRAN"]
       end
     end
 
-    disambiguate "F#", "Forth", "GLSL" do |data|
-      if /^(: |new-device)/.match(data)
+    disambiguate ".fr" do |data|
+      if /^(: |also |new-device|previous )/.match(data)
         Language["Forth"]
-      elsif /^(#light|import|let|module|namespace|open|type)/.match(data)
-        Language["F#"]
-      elsif /^(#include|#pragma|precision|uniform|varying|void)/.match(data)
-        Language["GLSL"]
+      elsif /^\s*(import|module|package|data|type) /.match(data)
+        Language["Frege"]
+      else
+        Language["Text"]
       end
     end
 
-    disambiguate "Gosu", "JavaScript" do |data|
+    disambiguate ".fs" do |data|
+      if /^(: |new-device)/.match(data)
+        Language["Forth"]
+      elsif /^\s*(#light|import|let|module|namespace|open|type)/.match(data)
+        Language["F#"]
+      elsif /^\s*(#version|precision|uniform|varying|vec[234])/.match(data)
+        Language["GLSL"]
+      elsif /#include|#pragma\s+(rs|version)|__attribute__/.match(data)
+        Language["Filterscript"]
+      end
+    end
+
+    disambiguate ".gs" do |data|
       Language["Gosu"] if /^uses java\./.match(data)
     end
 
-    disambiguate "LoomScript", "LiveScript" do |data|
+    disambiguate ".h" do |data|
+      if ObjectiveCRegex.match(data)
+        Language["Objective-C"]
+      elsif (/^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>/.match(data) ||
+        /^\s*template\s*</.match(data) || /^[ \t]*try/.match(data) || /^[ \t]*catch\s*\(/.match(data) || /^[ \t]*(class|(using[ \t]+)?namespace)\s+\w+/.match(data) || /^[ \t]*(private|public|protected):$/.match(data) || /std::\w+/.match(data))
+        Language["C++"]
+      end
+    end
+
+    disambiguate ".l" do |data|
+      if /\(def(un|macro)\s/.match(data)
+        Language["Common Lisp"]
+      elsif /^(%[%{}]xs|<.*>)/.match(data)
+        Language["Lex"]
+      elsif /^\.[a-z][a-z](\s|$)/i.match(data)
+        Language["Groff"]
+      elsif /^\((de|class|rel|code|data|must)\s/.match(data)
+        Language["PicoLisp"]
+      end
+    end
+
+    disambiguate ".ls" do |data|
       if /^\s*package\s*[\w\.\/\*\s]*\s*{/.match(data)
         Language["LoomScript"]
       else
@@ -156,5 +191,199 @@ module Linguist
       end
     end
 
+    disambiguate ".lsp", ".lisp" do |data|
+      if /^\s*\((defun|in-package|defpackage) /i.match(data)
+        Language["Common Lisp"]
+      elsif /^\s*\(define /.match(data)
+        Language["NewLisp"]
+      end
+    end
+
+    disambiguate ".m" do |data|
+      if ObjectiveCRegex.match(data)
+        Language["Objective-C"]
+      elsif data.include?(":- module")
+        Language["Mercury"]
+      elsif /^: /.match(data)
+        Language["MUF"]
+      elsif /^\s*;/.match(data)
+        Language["M"]
+      elsif /^\s*\(\*/.match(data)
+        Language["Mathematica"]
+      elsif /^\s*%/.match(data)
+        Language["Matlab"]
+      elsif /^\w+\s*:\s*module\s*{/.match(data)
+        Language["Limbo"]
+      end
+    end
+
+    disambiguate ".ml" do |data|
+      if /(^\s*module)|let rec |match\s+(\S+\s)+with/.match(data)
+        Language["OCaml"]
+      elsif /=> |case\s+(\S+\s)+of/.match(data)
+        Language["Standard ML"]
+      end
+    end
+
+    disambiguate ".mod" do |data|
+      if data.include?('<!ENTITY ')
+        Language["XML"]
+      elsif /MODULE\s\w+\s*;/i.match(data) || /^\s*END \w+;$/i.match(data)
+        Language["Modula-2"]
+      else
+        [Language["Linux Kernel Module"], Language["AMPL"]]
+      end
+    end
+
+    disambiguate ".ms" do |data|
+      if /^[.'][a-z][a-z](\s|$)/i.match(data)
+        Language["Groff"]
+      elsif /(?<!\S)\.(include|globa?l)\s/.match(data) || /(?<!\/\*)(\A|\n)\s*\.[A-Za-z]/.match(data.gsub(/"([^\\"]|\\.)*"|'([^\\']|\\.)*'|\\\s*(?:--.*)?\n/, ""))
+        Language["GAS"]
+      else
+        Language["MAXScript"]
+      end
+    end
+
+    disambiguate ".n" do |data|
+      if /^[.']/.match(data)
+        Language["Groff"]
+      elsif /^(module|namespace|using)\s/.match(data)
+        Language["Nemerle"]
+      end
+    end
+
+    disambiguate ".ncl" do |data|
+      if data.include?("THE_TITLE")
+        Language["Text"]
+      end
+    end
+
+    disambiguate ".nl" do |data|
+      if /^(b|g)[0-9]+ /.match(data)
+        Language["NL"]
+      else
+        Language["NewLisp"]
+      end
+    end
+
+    disambiguate ".php" do |data|
+      if data.include?("<?hh")
+        Language["Hack"]
+      elsif /<?[^h]/.match(data)
+        Language["PHP"]
+      end
+    end
+
+    disambiguate ".pl" do |data|
+      if /^[^#]+:-/.match(data)
+        Language["Prolog"]
+      elsif /use strict|use\s+v?5\./.match(data)
+        Language["Perl"]
+      elsif /^(use v6|(my )?class|module)/.match(data)
+        Language["Perl6"]
+      end
+    end
+
+    disambiguate ".pm", ".t" do |data|
+      if /use strict|use\s+v?5\./.match(data)
+        Language["Perl"]
+      elsif /^(use v6|(my )?class|module)/.match(data)
+        Language["Perl6"]
+      end
+    end
+
+    disambiguate ".pod" do |data|
+      if /^=\w+$/.match(data)
+        Language["Pod"]
+      else
+        Language["Perl"]
+      end
+    end
+
+    disambiguate ".pro" do |data|
+      if /^[^#]+:-/.match(data)
+        Language["Prolog"]
+      elsif data.include?("last_client=")
+        Language["INI"]
+      elsif data.include?("HEADERS") && data.include?("SOURCES")
+        Language["QMake"]
+      elsif /^\s*function[ \w,]+$/.match(data)
+        Language["IDL"]
+      end
+    end
+
+    disambiguate ".props" do |data|
+      if /^(\s*)(<Project|<Import|<Property|<?xml|xmlns)/i.match(data)
+        Language["XML"]
+      elsif /\w+\s*=\s*/i.match(data)
+        Language["INI"]
+      end
+    end
+
+    disambiguate ".r" do |data|
+      if /\bRebol\b/i.match(data)
+        Language["Rebol"]
+      elsif data.include?("<-")
+        Language["R"]
+      end
+    end
+
+    disambiguate ".rpy" do |data|
+      if /(^(import|from|class|def)\s)/m.match(data)
+        Language["Python"]
+      else
+        Language["Ren'Py"]
+      end
+    end
+
+    disambiguate ".rs" do |data|
+      if /^(use |fn |mod |pub |macro_rules|impl|#!?\[)/.match(data)
+        Language["Rust"]
+      elsif /#include|#pragma\s+(rs|version)|__attribute__/.match(data)
+        Language["RenderScript"]
+      end
+    end
+
+    disambiguate ".sc" do |data|
+      if /\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data)
+        Language["SuperCollider"]
+      elsif /^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data)
+        Language["Scala"]
+      end
+    end
+
+    disambiguate ".sql" do |data|
+      if /^\\i\b|AS \$\$|LANGUAGE '?plpgsql'?/i.match(data) || /SECURITY (DEFINER|INVOKER)/i.match(data) || /BEGIN( WORK| TRANSACTION)?;/i.match(data)
+        #Postgres
+        Language["PLpgSQL"]
+      elsif /(alter module)|(language sql)|(begin( NOT)+ atomic)/i.match(data)  || /signal SQLSTATE '[0-9]+'/i.match(data)
+        #IBM db2
+        Language["SQLPL"]
+      elsif /pragma|\$\$PLSQL_|XMLTYPE|sysdate|systimestamp|\.nextval|connect by|AUTHID (DEFINER|CURRENT_USER)/i.match(data) || /constructor\W+function/i.match(data)
+        #Oracle
+        Language["PLSQL"]
+      elsif ! /begin|boolean|package|exception/i.match(data)
+        #Generic SQL
+        Language["SQL"]
+      end
+    end
+
+    disambiguate ".ts" do |data|
+      if data.include?("<TS ")
+        Language["XML"]
+      else
+        Language["TypeScript"]
+      end
+    end
+
+    disambiguate ".tst" do |data|
+      if (data.include?("gap> "))
+        Language["GAP"]
+      # Heads up - we don't usually write heuristics like this (with no regex match)
+      else
+        Language["Scilab"]
+      end
+    end
   end
 end

lib/linguist/language.rb

@@ -11,6 +11,7 @@ require 'linguist/samples'
 require 'linguist/file_blob'
 require 'linguist/blob_helper'
 require 'linguist/strategy/filename'
+require 'linguist/strategy/modeline'
 require 'linguist/shebang'
 
 module Linguist
@@ -31,13 +32,6 @@ module Linguist
     # Valid Languages types
     TYPES = [:data, :markup, :programming, :prose]
 
-    # Names of non-programming languages that we will still detect
-    #
-    # Returns an array
-    def self.detectable_markup
-      ["CSS", "Less", "Sass", "SCSS", "Stylus", "TeX"]
-    end
-
     # Detect languages by a specific type
     #
     # type - A symbol that exists within TYPES
@@ -79,7 +73,7 @@ module Linguist
           raise ArgumentError, "Extension is missing a '.': #{extension.inspect}"
         end
 
-        @extension_index[extension] << language
+        @extension_index[extension.downcase] << language
       end
 
       language.interpreters.each do |interpreter|
@@ -93,13 +87,6 @@ module Linguist
       language
     end
 
-    STRATEGIES = [
-      Linguist::Strategy::Filename,
-      Linguist::Shebang,
-      Linguist::Heuristics,
-      Linguist::Classifier
-    ]
-
     # Public: Detects the Language of the blob.
     #
     # blob - an object that includes the Linguist `BlobHelper` interface;
@@ -107,22 +94,8 @@ module Linguist
     #
     # Returns Language or nil.
     def self.detect(blob)
-      # Bail early if the blob is binary or empty.
-      return nil if blob.likely_binary? || blob.binary? || blob.empty?
-
-      # Call each strategy until one candidate is returned.
-      STRATEGIES.reduce([]) do |languages, strategy|
-        candidates = strategy.call(blob, languages)
-        if candidates.size == 1
-          return candidates.first
-        elsif candidates.size > 1
-          # More than one candidate was found, pass them to the next strategy.
-          candidates
-        else
-          # No candiates were found, pass on languages from the previous strategy.
-          languages
-        end
-      end.first
+      warn "[DEPRECATED] `Linguist::Language.detect` is deprecated. Use `Linguist.detect`. #{caller[0]}"
+      Linguist.detect(blob)
     end
 
     # Public: Get all Languages
@@ -143,7 +116,8 @@ module Linguist
     #
     # Returns the Language or nil if none was found.
     def self.find_by_name(name)
-      name && @name_index[name.downcase]
+      return nil if name.to_s.empty?
+      name && (@name_index[name.downcase] || @name_index[name.split(',').first.downcase])
     end
 
     # Public: Look up Language by one of its aliases.
@@ -155,9 +129,10 @@ module Linguist
     #   Language.find_by_alias('cpp')
     #   # => #<Language name="C++">
     #
-    # Returns the Lexer or nil if none was found.
+    # Returns the Language or nil if none was found.
     def self.find_by_alias(name)
-      name && @alias_index[name.downcase]
+      return nil if name.to_s.empty?
+      name && (@alias_index[name.downcase] || @alias_index[name.split(',').first.downcase])
     end
 
     # Public: Look up Languages by filename.
@@ -196,7 +171,7 @@ module Linguist
     # Returns all matching Languages or [] if none were found.
     def self.find_by_extension(extname)
       extname = ".#{extname}" unless extname.start_with?(".")
-      @extension_index[extname]
+      @extension_index[extname.downcase]
     end
 
     # DEPRECATED
@@ -219,7 +194,7 @@ module Linguist
     end
 
 
-    # Public: Look up Language by its name or lexer.
+    # Public: Look up Language by its name.
     #
     # name - The String name of the Language
     #
@@ -233,7 +208,8 @@ module Linguist
     #
     # Returns the Language or nil if none was found.
     def self.[](name)
-      name && @index[name.downcase]
+      return nil if name.to_s.empty?
+      name && (@index[name.downcase] || @index[name.split(',').first.downcase])
     end
 
     # Public: A List of popular languages
@@ -243,7 +219,7 @@ module Linguist
     #
     # This list is configured in "popular.yml".
     #
-    # Returns an Array of Lexers.
+    # Returns an Array of Languages.
     def self.popular
       @popular ||= all.select(&:popular?).sort_by { |lang| lang.name.downcase }
     end
@@ -255,7 +231,7 @@ module Linguist
     #
     # This list is created from all the languages not listed in "popular.yml".
     #
-    # Returns an Array of Lexers.
+    # Returns an Array of Languages.
     def self.unpopular
       @unpopular ||= all.select(&:unpopular?).sort_by { |lang| lang.name.downcase }
     end
@@ -269,8 +245,12 @@ module Linguist
 
     # Public: A List of languages compatible with Ace.
     #
+    # TODO: Remove this method in a 5.x release. Every language now needs an ace_mode
+    # key, so this function isn't doing anything unique anymore.
+    #
     # Returns an Array of Languages.
     def self.ace_modes
+      warn "This method will be deprecated in a future 5.x release. Every language now has an `ace_mode` set."
       @ace_modes ||= all.select(&:ace_mode).sort_by { |lang| lang.name.downcase }
     end
 
@@ -371,11 +351,6 @@ module Linguist
     # Returns the name String
     attr_reader :search_term
 
-    # Public: Get Lexer
-    #
-    # Returns the Lexer
-    attr_reader :lexer
-
     # Public: Get the name of a TextMate-compatible scope
     #
     # Returns the scope
@@ -491,16 +466,6 @@ module Linguist
       @searchable
     end
 
-    # Public: Highlight syntax of text
-    #
-    # text    - String of code to be highlighted
-    # options - A Hash of options (defaults to {})
-    #
-    # Returns html String
-    def colorize(text, options = {})
-      lexer.highlight(text, options)
-    end
-
     # Public: Return name as String representation
     def to_s
       name
@@ -544,8 +509,8 @@ module Linguist
 
     if extnames = extensions[name]
       extnames.each do |extname|
-        if !options['extensions'].include?(extname)
-          warn "#{name} has a sample with extension (#{extname}) that isn't explicitly defined in languages.yml" unless extname == '.script!'
+        if !options['extensions'].index { |x| x.downcase.end_with? extname.downcase }
+          warn "#{name} has a sample with extension (#{extname.downcase}) that isn't explicitly defined in languages.yml"
           options['extensions'] << extname
         end
       end
@@ -576,7 +541,6 @@ module Linguist
       :color             => options['color'],
       :type              => options['type'],
       :aliases           => options['aliases'],
-      :lexer             => options['lexer'],
       :tm_scope          => options['tm_scope'],
       :ace_mode          => options['ace_mode'],
       :wrap              => options['wrap'],

lib/linguist/lazy_blob.rb

@@ -4,7 +4,11 @@ require 'rugged'
 
 module Linguist
   class LazyBlob
-    GIT_ATTR = ['linguist-language', 'linguist-vendored']
+    GIT_ATTR = ['linguist-documentation',
+                'linguist-language',
+                'linguist-vendored',
+                'linguist-generated']
+
     GIT_ATTR_OPTS = { :priority => [:index], :skip_system => true }
     GIT_ATTR_FLAGS = Rugged::Repository::Attributes.parse_opts(GIT_ATTR_OPTS)
 
@@ -14,14 +18,17 @@ module Linguist
 
     attr_reader :repository
     attr_reader :oid
-    attr_reader :name
+    attr_reader :path
     attr_reader :mode
 
-    def initialize(repo, oid, name, mode = nil)
+    alias :name :path
+
+    def initialize(repo, oid, path, mode = nil)
       @repository = repo
       @oid = oid
-      @name = name
+      @path = path
       @mode = mode
+      @data = nil
     end
 
     def git_attributes
@@ -29,11 +36,27 @@ module Linguist
         name, GIT_ATTR, GIT_ATTR_FLAGS)
     end
 
+    def documentation?
+      if attr = git_attributes['linguist-documentation']
+        boolean_attribute(attr)
+      else
+        super
+      end
+    end
+
+    def generated?
+      if attr = git_attributes['linguist-generated']
+        boolean_attribute(attr)
+      else
+        super
+      end
+    end
+
     def vendored?
       if attr = git_attributes['linguist-vendored']
         return boolean_attribute(attr)
       else
-        return super
+        super
       end
     end
 
@@ -41,7 +64,7 @@ module Linguist
       return @language if defined?(@language)
 
       @language = if lang = git_attributes['linguist-language']
-        Language.find_by_name(lang)
+        Language.find_by_alias(lang)
       else
         super
       end
@@ -57,11 +80,15 @@ module Linguist
       @size
     end
 
+    def cleanup!
+      @data.clear if @data
+    end
+
     protected
 
     # Returns true if the attribute is present and not the string "false".
-    def boolean_attribute(attr)
-      attr != "false"
+    def boolean_attribute(attribute)
+      attribute != "false"
     end
 
     def load_blob!

lib/linguist/popular.yml

@@ -3,27 +3,27 @@
 # This file should only be edited by GitHub staff
 
 - ActionScript
-- Bash
 - C
 - C#
 - C++
 - CSS
 - Clojure
 - CoffeeScript
-- Common Lisp
-- Diff
-- Emacs Lisp
-- Erlang
+- Go
 - HTML
 - Haskell
 - Java
 - JavaScript
 - Lua
+- Matlab
 - Objective-C
 - PHP
 - Perl
 - Python
+- R
 - Ruby
-- SQL
 - Scala
-- Scheme
+- Shell
+- Swift
+- TeX
+- VimL

lib/linguist/repository.rb

@@ -30,6 +30,9 @@ module Linguist
       @repository = repo
       @commit_oid = commit_oid
 
+      @old_commit_oid = nil
+      @old_stats = nil
+
       raise TypeError, 'commit_oid must be a commit SHA1' unless commit_oid.is_a?(String)
     end
 
@@ -126,12 +129,13 @@ module Linguist
     end
 
     protected
+    MAX_TREE_SIZE = 100_000
 
     def compute_stats(old_commit_oid, cache = nil)
+      return {} if current_tree.count_recursive(MAX_TREE_SIZE) >= MAX_TREE_SIZE
+
       old_tree = old_commit_oid && Rugged::Commit.lookup(repository, old_commit_oid).tree
-
       read_index
-
       diff = Rugged::Tree.diff(repository, old_tree, current_tree)
 
       # Clear file map and fetch full diff if any .gitattributes files are changed
@@ -150,19 +154,18 @@ module Linguist
         next if delta.binary
 
         if [:added, :modified].include? delta.status
-          # Skip submodules
+          # Skip submodules and symlinks
           mode = delta.new_file[:mode]
-          next if (mode & 040000) != 0
+          mode_format = (mode & 0170000)
+          next if mode_format == 0120000 || mode_format == 040000 || mode_format == 0160000
 
           blob = Linguist::LazyBlob.new(repository, delta.new_file[:oid], new, mode.to_s(8))
 
-          # Skip vendored or generated blobs
-          next if blob.vendored? || blob.generated? || blob.language.nil?
-
-          # Only include programming languages and acceptable markup languages
-          if blob.language.type == :programming || Language.detectable_markup.include?(blob.language.name)
+          if blob.include_in_language_stats?
             file_map[new] = [blob.language.group.name, blob.size]
           end
+
+          blob.cleanup!
         end
       end
 

lib/linguist/samples.rb

@@ -34,10 +34,6 @@ module Linguist
       Dir.entries(ROOT).sort!.each do |category|
         next if category == '.' || category == '..'
 
-        # Skip text and binary for now
-        # Possibly reconsider this later
-        next if category == 'Text' || category == 'Binary'
-
         dirname = File.join(ROOT, category)
         Dir.entries(dirname).each do |filename|
           next if filename == '.' || filename == '..'
@@ -54,16 +50,13 @@ module Linguist
             end
           else
             path = File.join(dirname, filename)
-
-            if File.extname(filename) == ""
-              raise "#{path} is missing an extension, maybe it belongs in filenames/ subdir"
-            end
+            extname = File.extname(filename)
 
             yield({
               :path     => path,
               :language => category,
               :interpreter => Shebang.interpreter(File.read(path)),
-              :extname  => File.extname(filename)
+              :extname  => extname.empty? ? nil : extname
             })
           end
         end

lib/linguist/shebang.rb

@@ -18,23 +18,38 @@ module Linguist
     #
     # Returns a String or nil
     def self.interpreter(data)
-      lines = data.lines
-      return unless match = /^#! ?(.+)$/.match(lines.first)
+      shebang = data.lines.first
 
-      tokens = match[1].split(' ')
-      script = tokens.first.split('/').last
+      # First line must start with #!
+      return unless shebang && shebang.start_with?("#!")
 
-      script = tokens[1] if script == 'env'
+      s = StringScanner.new(shebang)
 
-      # If script has an invalid shebang, we might get here
+      # There was nothing after the #!
+      return unless path = s.scan(/^#!\s*\S+/)
+
+      # Keep going
+      script = path.split('/').last
+
+      # if /usr/bin/env type shebang then walk the string
+      if script == 'env'
+        s.scan(/\s+/)
+        s.scan(/.*=[^\s]+\s+/) # skip over variable arguments e.g. foo=bar
+        script = s.scan(/\S+/)
+      end
+
+      # Interpreter was /usr/bin/env with no arguments
       return unless script
 
       # "python2.6" -> "python2"
-      script.sub! $1, '' if script =~ /(\.\d+)$/
+      script.sub!(/(\.\d+)$/, '')
+
+      # #! perl -> perl
+      script.sub!(/^#!\s*/, '')
 
       # Check for multiline shebang hacks that call `exec`
       if script == 'sh' &&
-        lines.first(5).any? { |l| l.match(/exec (\w+).+\$0.+\$@/) }
+        data.lines.first(5).any? { |l| l.match(/exec (\w+).+\$0.+\$@/) }
         script = $1
       end
 

lib/linguist/strategy/filename.rb

@@ -3,17 +3,7 @@ module Linguist
     # Detects language based on filename and/or extension
     class Filename
       def self.call(blob, _)
-        name = blob.name.to_s
-
-        # A bit of an elegant hack. If the file is executable but extensionless,
-        # append a "magic" extension so it can be classified with other
-        # languages that have shebang scripts.
-        extensions = FileBlob.new(name).extensions
-        if extensions.empty? && blob.mode && (blob.mode.to_i(8) & 05) == 05
-          name += ".script!"
-        end
-
-        Language.find_by_filename(name)
+        Language.find_by_filename(blob.name.to_s)
       end
     end
   end

lib/linguist/strategy/modeline.rb

@@ -0,0 +1,41 @@
+module Linguist
+  module Strategy
+    class Modeline
+      EMACS_MODELINE = /-\*-\s*(?:(?!mode)[\w-]+\s*:\s*(?:[\w+-]+)\s*;?\s*)*(?:mode\s*:)?\s*([\w+-]+)\s*(?:;\s*(?!mode)[\w-]+\s*:\s*[\w+-]+\s*)*;?\s*-\*-/i
+
+      # First form vim modeline
+      # [text]{white}{vi:|vim:|ex:}[white]{options}
+      # ex: 'vim: syntax=ruby'
+      VIM_MODELINE_1 = /(?:vim|vi|ex):\s*(?:ft|filetype|syntax)=(\w+)\s?/i
+
+      # Second form vim modeline (compatible with some versions of Vi)
+      # [text]{white}{vi:|vim:|Vim:|ex:}[white]se[t] {options}:[text]
+      # ex: 'vim set syntax=ruby:'
+      VIM_MODELINE_2 = /(?:vim|vi|Vim|ex):\s*se(?:t)?.*\s(?:ft|filetype|syntax)=(\w+)\s?.*:/i
+
+      MODELINES = [EMACS_MODELINE, VIM_MODELINE_1, VIM_MODELINE_2]
+
+      # Public: Detects language based on Vim and Emacs modelines
+      #
+      # blob               - An object that quacks like a blob.
+      #
+      # Examples
+      #
+      #   Modeline.call(FileBlob.new("path/to/file"))
+      #
+      # Returns an Array with one Language if the blob has a Vim or Emacs modeline
+      # that matches a Language name or alias. Returns an empty array if no match.
+      def self.call(blob, _ = nil)
+        Array(Language.find_by_alias(modeline(blob.data)))
+      end
+
+      # Public: Get the modeline from the first n-lines of the file
+      #
+      # Returns a String or nil
+      def self.modeline(data)
+        match = MODELINES.map { |regex| data.match(regex) }.reject(&:nil?).first
+        match[1] if match
+      end
+    end
+  end
+end

lib/linguist/tokenizer.rb

@@ -22,8 +22,10 @@ module Linguist
     # Start state on token, ignore anything till the next newline
     SINGLE_LINE_COMMENTS = [
       '//', # C
+      '--', # Ada, Haskell, AppleScript
       '#',  # Ruby
       '%',  # Tex
+      '"',  # Vim
     ]
 
     # Start state on opening token, ignore anything until the closing
@@ -33,7 +35,8 @@ module Linguist
       ['<!--', '-->'], # XML
       ['{-', '-}'],    # Haskell
       ['(*', '*)'],    # Coq
-      ['"""', '"""']   # Python
+      ['"""', '"""'],  # Python
+      ["'''", "'''"]   # Python
     ]
 
     START_SINGLE_LINE_COMMENT =  Regexp.compile(SINGLE_LINE_COMMENTS.map { |c|
@@ -83,17 +86,17 @@ module Linguist
           if s.peek(1) == "\""
             s.getch
           else
-            s.skip_until(/[^\\]"/)
+            s.skip_until(/(?<!\\)"/)
           end
         elsif s.scan(/'/)
           if s.peek(1) == "'"
             s.getch
           else
-            s.skip_until(/[^\\]'/)
+            s.skip_until(/(?<!\\)'/)
           end
 
         # Skip number literals
-        elsif s.scan(/(0x)?\d(\d|\.)*/)
+        elsif s.scan(/(0x\h(\h|\.)*|\d(\d|\.)*)([uU][lL]{0,2}|([eE][-+]\d*)?[fFlL]*)/)
 
         # SGML style brackets
         elsif token = s.scan(/<[^\s<>][^<>]*>/)
@@ -129,6 +132,9 @@ module Linguist
     #   extract_shebang("#!/usr/bin/env node")
     #   # => "node"
     #
+    #   extract_shebang("#!/usr/bin/env A=B foo=bar awk -f")
+    #   # => "awk"
+    #
     # Returns String token or nil it couldn't be parsed.
     def extract_shebang(data)
       s = StringScanner.new(data)
@@ -137,6 +143,7 @@ module Linguist
         script = path.split('/').last
         if script == 'env'
           s.scan(/\s+/)
+          s.scan(/.*=[^\s]+\s+/)
           script = s.scan(/\S+/)
         end
         script = script[/[^\d]+/, 0] if script

lib/linguist/vendor.yml

@@ -20,10 +20,20 @@
 - ^deps/
 - ^tools/
 - (^|/)configure$
-- (^|/)configure.ac$
 - (^|/)config.guess$
 - (^|/)config.sub$
 
+# stuff autogenerated by autoconf - still C deps
+- (^|/)aclocal.m4
+- (^|/)libtool.m4
+- (^|/)ltoptions.m4
+- (^|/)ltsugar.m4
+- (^|/)ltversion.m4
+- (^|/)lt~obsolete.m4
+
+# Linters
+- cpplint.py
+
 # Node dependencies
 - node_modules/
 
@@ -32,6 +42,7 @@
 
 # Erlang bundles
 - ^rebar$
+- erlang.mk
 
 # Go dependencies
 - Godeps/_workspace/
@@ -39,30 +50,34 @@
 # Minified JavaScript and CSS
 - (\.|-)min\.(js|css)$
 
+# Stylesheets imported from packages
+- ([^\s]*)import\.(css|less|scss|styl)$
+
 # Bootstrap css and js
-- (^|/)bootstrap([^.]*)\.(js|css)$
+- (^|/)bootstrap([^.]*)\.(js|css|less|scss|styl)$
+- (^|/)custom\.bootstrap([^\s]*)(js|css|less|scss|styl)$
 
 # Font Awesome
-- font-awesome.css
+- (^|/)font-awesome\.(css|less|scss|styl)$
 
 # Foundation css
-- foundation.css
+- (^|/)foundation\.(css|less|scss|styl)$
 
 # Normalize.css
-- normalize.css
+- (^|/)normalize\.(css|less|scss|styl)$
 
-# Bourbon SCSS
-- (^|/)[Bb]ourbon/.*\.css$
-- (^|/)[Bb]ourbon/.*\.scss$
+# Bourbon css
+- (^|/)[Bb]ourbon/.*\.(css|less|scss|styl)$
 
 # Animate.css
-- animate.css
+- (^|/)animate\.(css|less|scss|styl)$
 
 # Vendored dependencies
 - third[-_]?party/
 - 3rd[-_]?party/
 - vendors?/
 - extern(al)?/
+- (^|/)[Vv]+endor/
 
 # Debian packaging
 - ^debian/
@@ -70,6 +85,9 @@
 # Haxelib projects often contain a neko bytecode file named run.n
 - run.n$
 
+# Bootstrap Datepicker
+- bootstrap-datepicker/
+
 ## Commonly Bundled JavaScript frameworks ##
 
 # jQuery
@@ -80,6 +98,34 @@
 - (^|/)jquery\-ui(\-\d\.\d+(\.\d+)?)?(\.\w+)?\.(js|css)$
 - (^|/)jquery\.(ui|effects)\.([^.]*)\.(js|css)$
 
+# jQuery Gantt
+- jquery.fn.gantt.js
+
+# jQuery fancyBox
+- jquery.fancybox.(js|css)
+
+# Fuel UX
+- fuelux.js
+
+# jQuery File Upload
+- (^|/)jquery\.fileupload(-\w+)?\.js$
+
+# Slick
+- (^|/)slick\.\w+.js$
+
+# Leaflet plugins
+- (^|/)Leaflet\.Coordinates-\d+\.\d+\.\d+\.src\.js$
+- leaflet.draw-src.js
+- leaflet.draw.css
+- Control.FullScreen.css
+- Control.FullScreen.js
+- leaflet.spin.js
+- wicket-leaflet.js
+
+# Sublime Text workspace files
+- .sublime-project
+- .sublime-workspace
+
 # Prototype
 - (^|/)prototype(.*)\.js$
 - (^|/)effects\.js$
@@ -114,7 +160,7 @@
 - (^|/)Chart\.js$
 
 # Codemirror
-- (^|/)[Cc]ode[Mm]irror/(lib|mode|theme|addon|keymap)
+- (^|/)[Cc]ode[Mm]irror/(\d+\.\d+/)?(lib|mode|theme|addon|keymap|demo)
 
 # SyntaxHighlighter - http://alexgorbatchev.com/
 - (^|/)shBrush([^.]*)\.js$
@@ -139,6 +185,9 @@
 
 ## Python ##
 
+# Sphinx
+- (^|/)docs?/_?(build|themes?|templates?|static)/
+
 # django
 - (^|/)admin_media/
 
@@ -153,12 +202,31 @@
 
 ## Obj-C ##
 
+# Xcode
+
+- \.xctemplate/
+- \.imageset/
+
+# Carthage
+- ^Carthage/
+
 # Cocoapods
 - ^Pods/
 
 # Sparkle
 - (^|/)Sparkle/
 
+# Crashlytics
+- Crashlytics.framework/
+
+# Fabric
+- Fabric.framework/
+
+# git config files
+- gitattributes$
+- gitignore$
+- gitmodules$
+
 ## Groovy ##
 
 # Gradle
@@ -203,21 +271,9 @@
 # Html5shiv
 - (^|/)html5shiv\.js$
 
-# Samples folders
-- ^[Ss]amples/
-
-# LICENSE, README, git config files
-- ^COPYING$
-- LICENSE$
-- License$
-- gitattributes$
-- gitignore$
-- gitmodules$
-- ^README$
-- ^readme$
-
 # Test fixtures
-- ^[Tt]est/fixtures/
+- ^[Tt]ests?/fixtures/
+- ^[Ss]pecs?/fixtures/
 
 # PhoneGap/Cordova
 - (^|/)cordova([^.]*)\.js$
@@ -229,7 +285,7 @@
 # Vagrant
 - ^Vagrantfile$
 
-# .DS_Store's
+# .DS_Stores
 - .[Dd][Ss]_[Ss]tore$
 
 # R packages
@@ -247,3 +303,12 @@
 # ProGuard
 - proguard.pro
 - proguard-rules.pro
+
+# PuPHPet
+- ^puphpet/
+
+# Android Google APIs
+- (^|/)\.google_apis/
+
+# Jenkins Pipeline
+- ^Jenkinsfile$

lib/linguist/version.rb

@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "4.2.2"
+  VERSION = "4.8.1"
 end

package.json

@@ -1,6 +1,7 @@
 {
   "repository": "https://github.com/github/linguist",
   "dependencies": {
-    "season": "~>3.0"
-  }
+    "season": "~>5.0"
+  },
+  "license": "MIT"
 }

samples/AMPL/CT2.mod

@@ -0,0 +1,58 @@
+param num_beams;                  # number of beams
+
+param num_rows >= 1, integer;     # number of rows
+param num_cols >= 1, integer;     # number of columns 
+
+set BEAMS   := 1 .. num_beams;    # set of beams
+
+set ROWS    := 1 .. num_rows;	  # set of rows
+set COLUMNS := 1 .. num_cols;	  # set of columns
+
+# values for entries of each beam
+param beam_values {BEAMS, ROWS, COLUMNS} >= 0; 
+
+# values of tumor
+param tumor_values {ROWS, COLUMNS} >= 0; 
+
+# values of critical area
+param critical_values {ROWS, COLUMNS} >= 0; 
+
+# critical maximum dosage requirement
+param critical_max;
+
+# tumor minimum dosage requirement
+param tumor_min;
+
+# dosage scalar of each beam
+var X {i in BEAMS} >= 0;
+
+
+# define the tumor area which includes the locations where tumor exists
+set tumor_area := {k in ROWS, h in COLUMNS: tumor_values[k,h] > 0};
+
+# define critical area 
+set critical_area := {k in ROWS, h in COLUMNS: critical_values[k,h] > 0};
+
+var S {(k,h) in tumor_area} >= 0;
+var T {(k,h) in critical_area} >= 0;
+
+# maximize total dosage in tumor area
+maximize total_tumor_dosage: sum {i in BEAMS} sum {(k,h) in tumor_area} X[i] * beam_values[i,k,h];
+
+# minimize total dosage in critical area
+minimize total_critical_dosage: sum {i in BEAMS} sum {(k,h) in critical_area} X[i] * beam_values[i,k,h];
+
+# minimize total tumor slack
+minimize total_tumor_slack: sum {(k,h) in tumor_area} S[k,h];
+
+# minimize total critical area slack
+minimize total_critical_slack: sum {(k,h) in critical_area} T[k,h];
+
+# total dosage at each tumor location [k,h] should be >= min tumor dosage with slack variable
+subject to tumor_limit {(k,h) in tumor_area} : sum {i in BEAMS} X[i] * beam_values[i,k,h] == tumor_min - S[k,h];
+
+# total dosage at each critical location [k,h] should be = max critical dosage with slack variable
+subject to critical_limit {(k,h) in critical_area} : sum {i in BEAMS} X[i] * beam_values[i,k,h] == critical_max + T[k,h];
+
+
+

samples/AMPL/toy.ampl

@@ -0,0 +1,25 @@
+# A toy knapsack problem from the LocalSolver docs written in AMPL.
+
+set I;
+param Value{I};
+param Weight{I};
+param KnapsackBound;
+var Take{I} binary;
+
+maximize TotalValue: sum{i in I} Take[i] * Value[i];
+s.t. WeightLimit: sum{i in I} Take[i] * Weight[i] <= KnapsackBound;
+
+data;
+
+param:
+I: Weight Value :=
+0    10     1
+1    60    10
+2    30    15
+3    40    40
+4    30    60
+5    20    90
+6    20   100
+7     2    15;
+
+param KnapsackBound := 102;

samples/API Blueprint/actions.apib

@@ -0,0 +1,55 @@
+FORMAT: 1A
+
+# Advanced Action API
+A resource action is – in fact – a state transition. This API example demonstrates an action - state transition - to another resource.
+
+## API Blueprint
++ [Previous: Resource Model](11.%20Resource%20Model.md)
++ [This: Raw API Blueprint](https://raw.github.com/apiaryio/api-blueprint/master/examples/11.%20Advanced%20Action.md)
+
+# Tasks [/tasks/tasks{?status,priority}]
+
++ Parameters
+    + status (string)
+    + priority (number)
+
+## List All Tasks [GET]
+
++ Response 200 (application/json)
+
+        [
+            {
+                "id": 123,
+                "name": "Exercise in gym",
+                "done": false,
+                "type": "task"
+            },
+            {
+                "id": 124,
+                "name": "Shop for groceries",
+                "done": true,
+                "type": "task"
+            }
+        ]
+
+## Retrieve Task [GET /task/{id}]
+This is a state transition to another resource
+
++ Parameters
+    + id (string)
+
++ Response 200 (application/json)
+
+        {
+            "id": 123,
+            "name": "Go to gym",
+            "done": false,
+            "type": "task"
+        }
+
+## Delete Task [DELETE /task/{id}]
+
++ Parameters
+    + id (string)
+
++ Response 204

samples/API Blueprint/attributes.apib

@@ -0,0 +1,39 @@
+FORMAT: 1A
+
+# Attributes API
+This API example demonstrates how to describe body attributes of a request or response message.
+
+In this case, the description is complementary (and duplicate!) to the provided JSON example in the body section. The [Advanced Attributes](09.%20Advanced%20Attributes.md) API example will demonstrate how to avoid duplicates and how to reuse attributes descriptions.
+
+## API Blueprint
++ [Previous: Parameters](07.%20Parameters.md)
++ [This: Raw API Blueprint](https://raw.github.com/apiaryio/api-blueprint/master/examples/08.%20Attributes.md)
++ [Next: Advanced Attributes](09.%20Advanced%20Attributes.md)
+
+# Group Coupons
+
+## Coupon [/coupons/{id}]
+A coupon contains information about a percent-off or amount-off discount you might want to apply to a customer.
+
+### Retrieve a Coupon [GET]
+Retrieves the coupon with the given ID.
+
++ Response 200 (application/json)
+
+    + Attributes (object)
+        + id: 250FF (string)
+        + created: 1415203908 (number) - Time stamp
+        + percent_off: 25 (number)
+
+            A positive integer between 1 and 100 that represents the discount the coupon will apply.
+
+        + redeem_by (number) - Date after which the coupon can no longer be redeemed
+
+    + Body
+
+            {
+                "id": "250FF",
+                "created": 1415203908,
+                "percent_off": 25,
+                "redeem_by:" null
+            }

samples/API Blueprint/simple.apib

@@ -0,0 +1,18 @@
+FORMAT: 1A
+
+# The Simplest API
+This is one of the simplest APIs written in the **API Blueprint**.
+One plain resource combined with a method and that's it! We will explain what is going on in the next installment - [Resource and Actions](02.%20Resource%20and%20Actions.md).
+
+**Note:** As we progress through the examples, do not also forget to view the [Raw](https://raw.github.com/apiaryio/api-blueprint/master/examples/01.%20Simplest%20API.md) code to see what is really going on in the API Blueprint, as opposed to just seeing the output of the Github Markdown parser.
+
+Also please keep in mind that every single example in this course is a **real API Blueprint** and as such you can **parse** it with the [API Blueprint parser](https://github.com/apiaryio/drafter) or one of its [bindings](https://github.com/apiaryio/drafter#bindings).
+
+## API Blueprint
++ [This: Raw API Blueprint](https://raw.github.com/apiaryio/api-blueprint/master/examples/01.%20Simplest%20API.md)
++ [Next: Resource and Actions](02.%20Resource%20and%20Actions.md)
+
+# GET /message
++ Response 200 (text/plain)
+
+        Hello World!

samples/ATS/main.atxt

@@ -1,215 +0,0 @@
-%{
-#include "./../ATEXT/atextfun.hats"
-%}
-
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
-   "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml">
-
-<head>
-<meta http-equiv="content-type" content="text/html; charset=UTF-8" />
-<title>EFFECTIVATS-DiningPhil2</title>
-#patscode_style()
-</head>
-
-<body>
-
-<h1>
-Effective ATS: Dining Philosophers
-</h1>
-
-In this article, I present an implementation of a slight variant of the
-famous problem of 5-Dining-Philosophers by Dijkstra that makes simple but
-convincing use of linear types.
-
-<h2>
-The Original Problem 
-</h2>
-
-There are five philosophers sitting around a table and there are also 5
-forks placed on the table such that each fork is located between the left
-hand of a philosopher and the right hand of another philosopher. Each
-philosopher does the following routine repeatedly: thinking and dining.  In
-order to dine, a philosopher needs to first acquire two forks: one located
-on his left-hand side and the other on his right-hand side. After
-finishing dining, a philosopher puts the two acquired forks onto the table:
-one on his left-hand side and the other on his right-hand side.
-
-<h2>
-A Variant of the Original Problem 
-</h2>
-
-The following twist is added to the original version:
-
-<p>
-
-After a fork is used, it becomes a "dirty" fork and needs to be put in a
-tray for dirty forks. There is a cleaner who cleans dirty forks and then
-puts them back on the table.
-
-<h2>
-Channels for Communication
-</h2>
-
-A channel is just a shared queue of fixed capacity. The following two
-functions are for inserting an element into and taking an element out of a
-given channel:
-
-<pre
-class="patsyntax">
-#pats2xhtml_sats("\
-fun{a:vt0p} channel_insert (channel (a), a): void
-fun{a:vt0p} channel_takeout (chan: channel (a)): (a) 
-")</pre>
-
-If [channel_insert] is called on a channel that is full, then the caller is
-blocked until an element is taken out of the channel.  If [channel_takeout]
-is called on a channel that is empty, then the caller is blocked until an
-element is inserted into the channel.
-
-<h2>
-A Channel for Each Fork
-</h2>
-
-Forks are resources given a linear type. Each fork is initially stored in a
-channel, which can be obtained by calling the following function:
-
-<pre
-class="patsyntax">
-#pats2xhtml_sats("\
-fun fork_changet (n: nphil): channel(fork)
-")</pre>
-
-where the type [nphil] is defined to be [natLt(5)] (for natural numbers
-less than 5). The channels for storing forks are chosen to be of capacity
-2. The reason that channels of capacity 2 are chosen to store at most one
-element (in each of them) is to guarantee that these channels can never be
-full (so that there is no attempt made to send signals to awake callers
-supposedly being blocked due to channels being full).
-
-
-<h2>
-A Channel for the Fork Tray
-</h2>
-
-A tray for storing "dirty" forks is also a channel, which can be obtained
-by calling the following function:
-
-<pre
-class="patsyntax">
-#pats2xhtml_sats("\
-fun forktray_changet ((*void*)): channel(fork)
-")</pre>
-
-The capacity chosen for the channel is 6 (instead of 5) so that it can
-never become full (as there are only 5 forks in total).
-
-<h2>
-Philosopher Loop
-</h2>
-
-Each philosopher is implemented as a loop:
-
-<pre
-class="patsyntax">
-#pats2xhtml_dats('\
-implement
-phil_loop (n) = let
-//
-val () = phil_think (n)
-//
-val nl = phil_left (n) // = n
-val nr = phil_right (n) // = (n+1) % 5
-//
-val ch_lfork = fork_changet (nl)
-val ch_rfork = fork_changet (nr)
-//
-val lf = channel_takeout (ch_lfork)
-val () = println! ("phil_loop(", n, ") picks left fork")
-//
-val () = randsleep (2) // sleep up to 2 seconds
-//
-val rf = channel_takeout (ch_rfork)
-val () = println! ("phil_loop(", n, ") picks right fork")
-//
-val () = phil_dine (n, lf, rf)
-//
-val ch_forktray = forktray_changet ()
-val () = channel_insert (ch_forktray, lf) // left fork to dirty tray
-val () = channel_insert (ch_forktray, rf) // right fork to dirty tray
-//
-in
-  phil_loop (n)
-end // end of [phil_loop]
-')</pre>
-
-It should be straighforward to follow the code for [phil_loop].
-
-<h2>
-Fork Cleaner Loop
-</h2>
-
-A cleaner is implemented as a loop:
-
-<pre
-class="patsyntax">
-#pats2xhtml_dats('\
-implement
-cleaner_loop () = let
-//
-val ch = forktray_changet ()
-val f0 = channel_takeout (ch) // [f0] is dirty
-//
-val () = cleaner_wash (f0) // washes dirty [f0]
-val () = cleaner_return (f0) // puts back cleaned [f0]
-//
-in
-  cleaner_loop ()
-end // end of [cleaner_loop]
-')</pre>
-
-The function [cleaner_return] first finds out the number of a given fork
-and then uses the number to locate the channel for storing the fork. Its
-actual implementation is given as follows:
-
-<pre
-class="patsyntax">
-#pats2xhtml_dats('\
-implement
-cleaner_return (f) =
-{
-  val n = fork_get_num (f)
-  val ch = fork_changet (n)
-  val () = channel_insert (ch, f)
-}
-')</pre>
-
-It should now be straighforward to follow the code for [cleaner_loop].
-
-<h2>
-Testing
-</h2>
-
-The entire code of this implementation is stored in the following files:
-
-<pre>
-DiningPhil2.sats
-DiningPhil2.dats
-DiningPhil2_fork.dats
-DiningPhil2_thread.dats
-</pre>
-
-There is also a Makefile available for compiling the ATS source code into
-an excutable for testing. One should be able to encounter a deadlock after
-running the simulation for a while.
-
-<hr size="2">
-
-This article is written by <a href="http://www.cs.bu.edu/~hwxi/">Hongwei Xi</a>.
-
-</body>
-</html>
-
-%{
-implement main () = fprint_filsub (stdout_ref, "main_atxt.txt")
-%}

samples/ApacheConf/apache.vhost

@@ -0,0 +1,17 @@
+#######################
+# HOSTNAME
+######################
+
+<VirtualHost 127.0.0.1:PORT>
+    ServerAdmin patrick@heysparkbox.com
+    DocumentRoot "/var/www/HOSTNAME"
+    ServerName HOSTNAME
+
+    <Directory "/var/www/HOSTNAME">
+       Options Indexes MultiViews FollowSymLinks
+       AllowOverride All
+       Order allow,deny
+       Allow from all
+       DirectoryIndex index.php
+   </Directory>
+</VirtualHost>

samples/AsciiDoc/list.asc

@@ -1,6 +1,9 @@
 AsciiDoc Home Page
 ==================
 
+Title
+-----
+
 Example Articles
 ~~~~~~~~~~~~~~~~
 - Item 1

samples/Assembly/fp_sqr32_160_comba.inc

@@ -0,0 +1,245 @@
+
+    push r2
+    dint
+    nop
+    bis #MPYDLYWRTEN,&MPY32CTL0
+    bic #MPYDLY32,&MPY32CTL0
+    mov #SUMEXT,r13
+    clr r12
+
+    mov @r15+,r4
+    mov @r15+,r5
+
+    mov @r15+,r6
+    mov @r15+,r7
+
+    mov @r15+,r8
+    mov @r15+,r9
+
+    mov @r15+,r10
+    mov @r15+,r11
+
+    sub #2*8,r15
+
+    /* SELF_STEP_FIRST */
+    mov r4,&MPY32L
+    mov r5,&MPY32H
+    mov r4,&OP2L
+    mov r5,&OP2H
+
+    /* COLUMN_END */
+    mov &RES0,2*0(r14)
+    mov &RES1,2*(0+1)(r14)
+    mov &RES2,&RES0
+    mov &RES3,&RES1
+    mov r12,&RES2
+    clr &RES3
+
+    /* STEP_1 */
+    mov r4,&MAC32L
+    mov r5,&MAC32H
+    mov r6,&OP2L
+    mov r7,&OP2H
+    add &SUMEXT,r12
+    mov r6,&OP2L
+    mov r7,&OP2H
+
+    /* COLUMN_END */
+    mov &RES0,2*2(r14)
+    add @r13,r12
+    mov &RES1,2*(2+1)(r14)
+    mov &RES2,&RES0
+    mov &RES3,&RES1
+    mov r12,&RES2
+    clr &RES3
+    clr r12
+
+    /* STEP_1 */
+    mov r4,&MAC32L
+    mov r5,&MAC32H
+    mov r8,&OP2L
+    mov r9,&OP2H
+    add &SUMEXT,r12
+    mov r8,&OP2L
+    mov r9,&OP2H
+
+    /* SELF_STEP */
+    mov r6,&MAC32L
+    mov r7,&MAC32H
+    add @r13,r12
+    mov r6,&OP2L
+    mov r7,&OP2H
+
+    /* COLUMN_END */
+    mov &RES0,2*4(r14)
+    add @r13,r12
+    mov &RES1,2*(4+1)(r14)
+    mov &RES2,&RES0
+    mov &RES3,&RES1
+    mov r12,&RES2
+    clr &RES3
+    clr r12
+
+    /* STEP_1 */
+    mov r4,&MAC32L
+    mov r5,&MAC32H
+    mov r10,&OP2L
+    mov r11,&OP2H
+    add &SUMEXT,r12
+    mov r10,&OP2L
+    mov r11,&OP2H
+
+    /* STEP_2MORE */
+    mov r6,&MAC32L
+    mov r7,&MAC32H
+    add @r13,r12
+    mov r8,&OP2L
+    mov r9,&OP2H
+    add &SUMEXT,r12
+    mov r8,&OP2L
+    mov r9,&OP2H
+
+    /* COLUMN_END */
+    mov &RES0,2*6(r14)
+    add @r13,r12
+    mov &RES1,2*(6+1)(r14)
+    mov &RES2,&RES0
+    mov &RES3,&RES1
+    mov r12,&RES2
+    clr &RES3
+    clr r12
+
+    /* STEP_1 */
+    mov r4,&MAC32L
+    mov r5,&MAC32H
+    mov 2*8(r15),&OP2L
+    mov 2*9(r15),&OP2H
+    add &SUMEXT,r12
+    mov 2*8(r15),&OP2L
+    mov 2*9(r15),&OP2H
+
+    /* STEP_2MORE */
+    mov r6,&MAC32L
+    mov r7,&MAC32H
+    add @r13,r12
+    mov r10,&OP2L
+    mov r11,&OP2H
+    add &SUMEXT,r12
+    mov r10,&OP2L
+    mov r11,&OP2H
+
+    /* SELF_STEP */
+    mov r8,&MAC32L
+    mov r9,&MAC32H
+    add @r13,r12
+    mov r8,&OP2L
+    mov r9,&OP2H
+
+    /* COLUMN_END */
+    mov &RES0,2*8(r14)
+    add @r13,r12
+    mov &RES1,2*(8+1)(r14)
+    mov &RES2,&RES0
+    mov &RES3,&RES1
+    mov r12,&RES2
+    clr &RES3
+    clr r12
+
+    mov 2*8(r15),r4
+    mov 2*(8+1)(r15),r5
+
+    /* STEP_1 */
+    mov r6,&MAC32L
+    mov r7,&MAC32H
+    mov r4,&OP2L
+    mov r5,&OP2H
+    add &SUMEXT,r12
+    mov r4,&OP2L
+    mov r5,&OP2H
+
+    /* STEP_2MORE */
+    mov r8,&MAC32L
+    mov r9,&MAC32H
+    add @r13,r12
+    mov r10,&OP2L
+    mov r11,&OP2H
+    add &SUMEXT,r12
+    mov r10,&OP2L
+    mov r11,&OP2H
+
+    /* COLUMN_END */
+    mov &RES0,2*10(r14)
+    add @r13,r12
+    mov &RES1,2*(10+1)(r14)
+    mov &RES2,&RES0
+    mov &RES3,&RES1
+    mov r12,&RES2
+    clr &RES3
+    clr r12
+
+    /* STEP_1 */
+    mov r8,&MAC32L
+    mov r9,&MAC32H
+    mov r4,&OP2L
+    mov r5,&OP2H
+    add &SUMEXT,r12
+    mov r4,&OP2L
+    mov r5,&OP2H
+
+    /* SELF_STEP */
+    mov r10,&MAC32L
+    mov r11,&MAC32H
+    add @r13,r12
+    mov r10,&OP2L
+    mov r11,&OP2H
+
+    /* COLUMN_END */
+    mov &RES0,2*12(r14)
+    add @r13,r12
+    mov &RES1,2*(12+1)(r14)
+    mov &RES2,&RES0
+    mov &RES3,&RES1
+    mov r12,&RES2
+    clr &RES3
+    clr r12
+
+    /* STEP_1 */
+    mov r10,&MAC32L
+    mov r11,&MAC32H
+    mov r4,&OP2L
+    mov r5,&OP2H
+    add &SUMEXT,r12
+    mov r4,&OP2L
+    mov r5,&OP2H
+
+    /* COLUMN_END */
+    mov &RES0,2*14(r14)
+    add @r13,r12
+    mov &RES1,2*(14+1)(r14)
+    mov &RES2,&RES0
+    mov &RES3,&RES1
+    mov r12,&RES2
+    clr &RES3
+    clr r12
+
+    /* SELF_STEP_1 */
+    mov r4,&MAC32L
+    mov r5,&MAC32H
+    mov r4,&OP2L
+    mov r5,&OP2H
+
+    /* COLUMN_END */
+    mov &RES0,2*16(r14)
+    add @r13,r12
+    mov &RES1,2*(16+1)(r14)
+    mov &RES2,&RES0
+    mov &RES3,&RES1
+    mov r12,&RES2
+    clr &RES3
+    clr r12
+
+    /* END */
+    mov &RES0,2*18(r14)
+    mov &RES1,2*(18+1)(r14)
+    pop r2
+    eint

samples/Assembly/lib.inc

@@ -0,0 +1,170 @@
+
+; ------------------------------------------------------------------------
+; 显示 AL 中的数字
+; ------------------------------------------------------------------------
+DispAL:
+	push	ecx
+	push	edx
+	push	edi
+
+	mov	edi, [dwDispPos]
+
+	mov	ah, 0Fh			; 0000b: 黑底    1111b: 白字
+	mov	dl, al
+	shr	al, 4
+	mov	ecx, 2
+.begin:
+	and	al, 01111b
+	cmp	al, 9
+	ja	.1
+	add	al, '0'
+	jmp	.2
+.1:
+	sub	al, 0Ah
+	add	al, 'A'
+.2:
+	mov	[gs:edi], ax
+	add	edi, 2
+
+	mov	al, dl
+	loop	.begin
+	;add	edi, 2
+
+	mov	[dwDispPos], edi
+
+	pop	edi
+	pop	edx
+	pop	ecx
+
+	ret
+; DispAL 结束-------------------------------------------------------------
+
+
+; ------------------------------------------------------------------------
+; 显示一个整形数
+; ------------------------------------------------------------------------
+DispInt:
+	mov	eax, [esp + 4]
+	shr	eax, 24
+	call	DispAL
+
+	mov	eax, [esp + 4]
+	shr	eax, 16
+	call	DispAL
+
+	mov	eax, [esp + 4]
+	shr	eax, 8
+	call	DispAL
+
+	mov	eax, [esp + 4]
+	call	DispAL
+
+	mov	ah, 07h			; 0000b: 黑底    0111b: 灰字
+	mov	al, 'h'
+	push	edi
+	mov	edi, [dwDispPos]
+	mov	[gs:edi], ax
+	add	edi, 4
+	mov	[dwDispPos], edi
+	pop	edi
+
+	ret
+; DispInt 结束------------------------------------------------------------
+
+; ------------------------------------------------------------------------
+; 显示一个字符串
+; ------------------------------------------------------------------------
+DispStr:
+	push	ebp
+	mov	ebp, esp
+	push	ebx
+	push	esi
+	push	edi
+
+	mov	esi, [ebp + 8]	; pszInfo
+	mov	edi, [dwDispPos]
+	mov	ah, 0Fh
+.1:
+	lodsb
+	test	al, al
+	jz	.2
+	cmp	al, 0Ah	; 是回车吗?
+	jnz	.3
+	push	eax
+	mov	eax, edi
+	mov	bl, 160
+	div	bl
+	and	eax, 0FFh
+	inc	eax
+	mov	bl, 160
+	mul	bl
+	mov	edi, eax
+	pop	eax
+	jmp	.1
+.3:
+	mov	[gs:edi], ax
+	add	edi, 2
+	jmp	.1
+
+.2:
+	mov	[dwDispPos], edi
+
+	pop	edi
+	pop	esi
+	pop	ebx
+	pop	ebp
+	ret
+; DispStr 结束------------------------------------------------------------
+
+; ------------------------------------------------------------------------
+; 换行
+; ------------------------------------------------------------------------
+DispReturn:
+	push	szReturn
+	call	DispStr			;printf("\n");
+	add	esp, 4
+
+	ret
+; DispReturn 结束---------------------------------------------------------
+
+
+; ------------------------------------------------------------------------
+; 内存拷贝，仿 memcpy
+; ------------------------------------------------------------------------
+; void* MemCpy(void* es:pDest, void* ds:pSrc, int iSize);
+; ------------------------------------------------------------------------
+MemCpy:
+	push	ebp
+	mov	ebp, esp
+
+	push	esi
+	push	edi
+	push	ecx
+
+	mov	edi, [ebp + 8]	; Destination
+	mov	esi, [ebp + 12]	; Source
+	mov	ecx, [ebp + 16]	; Counter
+.1:
+	cmp	ecx, 0		; 判断计数器
+	jz	.2		; 计数器为零时跳出
+
+	mov	al, [ds:esi]		; ┓
+	inc	esi			; ┃
+					; ┣ 逐字节移动
+	mov	byte [es:edi], al	; ┃
+	inc	edi			; ┛
+
+	dec	ecx		; 计数器减一
+	jmp	.1		; 循环
+.2:
+	mov	eax, [ebp + 8]	; 返回值
+
+	pop	ecx
+	pop	edi
+	pop	esi
+	mov	esp, ebp
+	pop	ebp
+
+	ret			; 函数结束，返回
+; MemCpy 结束-------------------------------------------------------------
+

samples/Assembly/macros.inc

@@ -0,0 +1,321 @@
+BLARGG_MACROS_INCLUDED = 1
+
+; Allows extra error checking with modified version
+; of ca65. Otherwise acts like a constant of 0.
+ADDR = 0
+
+; Switches to Segment and places Line there.
+; Line can be an .align directive, .res, .byte, etc.
+; Examples:
+; seg_data BSS, .align 256
+; seg_data RODATA, {message: .byte "Test",0}
+.macro seg_data Segment, Line
+	.pushseg
+	.segment .string(Segment)
+		Line
+	.popseg
+.endmacro
+
+; Reserves Size bytes in Segment for Name.
+; If Size is omitted, reserves one byte.
+.macro seg_res Segment, Name, Size
+	.ifblank Size
+		seg_data Segment, Name: .res 1
+	.else
+		seg_data Segment, Name: .res Size
+	.endif
+.endmacro
+
+; Shortcuts for zeropage, bss, and stack
+.define zp_res  seg_res ZEROPAGE,
+.define nv_res  seg_res NVRAM,
+.define bss_res seg_res BSS,
+.define sp_res  seg_res STACK,
+.define zp_byte zp_res
+
+; Copies byte from Src to Addr. If Src begins with #,
+; it sets Addr to the immediate value.
+; Out: A = byte copied
+; Preserved: X, Y
+.macro mov Addr, Src
+	lda Src
+	sta Addr
+.endmacro
+
+; Copies word from Src to Addr. If Src begins with #,
+; it sets Addr the immediate value.
+; Out: A = high byte of word
+; Preserved: X, Y
+.macro movw Addr, Src
+	.if .match( .left( 1, {Src} ), # )
+		lda #<(.right( .tcount( {Src} )-1, {Src} ))
+		sta Addr
+		lda #>(.right( .tcount( {Src} )-1, {Src} ))
+		sta 1+(Addr)
+	.else
+		lda Src
+		sta Addr
+		lda 1+(Src)
+		sta 1+(Addr)
+	.endif
+.endmacro
+
+; Increments 16-bit value at Addr.
+; Out: EQ/NE based on resulting 16-bit value
+; Preserved: A, X, Y
+.macro incw Addr
+	.local @Skip
+	inc Addr
+	bne @Skip
+	inc 1+(Addr)
+@Skip:
+.endmacro
+
+; Adds Src to word at Addr.
+; Out: A = high byte of result, carry set appropriately
+; Preserved: X, Y
+.macro addw Addr, Src
+	.if .match( .left( 1, {Src} ), # )
+		addw_ Addr,(.right( .tcount( {Src} )-1, {Src} ))
+	.else
+		lda Addr
+		clc
+		adc Src
+		sta Addr
+		
+		lda 1+(Addr)
+		adc 1+(Src)
+		sta 1+(Addr)
+	.endif
+.endmacro
+.macro addw_ Addr, Imm
+	lda Addr
+	clc
+	adc #<Imm
+	sta Addr
+	
+	;.if (Imm >> 8) <> 0
+		lda 1+(Addr)
+		adc #>Imm
+		sta 1+(Addr)
+	;.else
+	;   .local @Skip
+	;   bcc @Skip
+	;   inc 1+(Addr)
+	;@Skip:
+	;.endif
+.endmacro
+
+; Splits list of words into tables of low and high bytes
+; Example: split_words foo, {$1234, $5678}
+; expands to:
+; foo_l: $34, $78
+; foo_h: $12, $56
+; foo_count = 2
+.macro split_words Label, Words
+	.ident (.concat (.string(Label), "_l")): .lobytes Words
+	.ident (.concat (.string(Label), "_h")): .hibytes Words
+	.ident (.concat (.string(Label), "_count")) = * - .ident (.concat (.string(Label), "_h"))
+.endmacro
+
+.macro SELECT Bool, True, False, Extra
+	.ifndef Bool
+		False Extra
+	.elseif Bool <> 0
+		True Extra
+	.else
+		False Extra
+	.endif
+.endmacro
+
+.macro DEFAULT Name, Value
+	.ifndef Name
+		Name = Value
+	.endif
+.endmacro
+
+.ifp02
+	; 6502 doesn't define these alternate names
+	.define blt bcc
+	.define bge bcs
+.endif
+.define jlt jcc
+.define jge jcs
+
+; Jxx Target = Bxx Target, except it can go farther than
+; 128 bytes. Implemented via branch around a JMP.
+
+; Don't use ca65's longbranch, because they fail for @labels
+;.macpack longbranch
+
+.macro jeq Target
+	bne *+5
+	jmp Target
+.endmacro
+
+.macro jne Target
+	beq *+5
+	jmp Target
+.endmacro
+
+.macro jmi Target
+	bpl *+5
+	jmp Target
+.endmacro
+
+.macro jpl Target
+	bmi *+5
+	jmp Target
+.endmacro
+
+.macro jcs Target
+	bcc *+5
+	jmp Target
+.endmacro
+
+.macro jcc Target
+	bcs *+5
+	jmp Target
+.endmacro
+
+.macro jvs Target
+	bvc *+5
+	jmp Target
+.endmacro
+
+.macro jvc Target
+	bvs *+5
+	jmp Target
+.endmacro
+
+
+; Passes constant data to routine in addr
+; Preserved: A, X, Y
+.macro jsr_with_addr routine,data
+	.local Addr
+	pha
+	lda #<Addr
+	sta addr
+	lda #>Addr
+	sta addr+1
+	pla
+	jsr routine
+	seg_data RODATA,{Addr: data}
+.endmacro
+
+; Calls routine multiple times, with A having the
+; value 'start' the first time, 'start+step' the
+; second time, up to 'end' for the last time.
+.macro for_loop routine,start,end,step
+	.local @for_loop
+	lda #start
+@for_loop:
+	pha
+	jsr routine
+	pla
+	clc
+	adc #step
+	cmp #<((end)+(step))
+	bne @for_loop
+.endmacro
+
+; Calls routine n times. The value of A in the routine
+; counts from 0 to n-1.
+.macro loop_n_times routine,n
+	for_loop routine,0,n-1,+1
+.endmacro
+
+; Same as for_loop, except uses 16-bit value in YX.
+; -256 <= step <= 255
+.macro for_loop16 routine,start,end,step
+.if (step) < -256 || (step) > 255
+	.error "Step must be within -256 to 255"
+.endif
+	.local @for_loop_skip
+	.local @for_loop
+	ldy #>(start)
+	lda #<(start)
+@for_loop:
+	tax
+	pha
+	tya
+	pha
+	jsr routine
+	pla
+	tay
+	pla
+	clc
+	adc #step
+.if (step) > 0
+	bcc @for_loop_skip
+	iny
+.else
+	bcs @for_loop_skip
+	dey
+.endif
+@for_loop_skip:
+	cmp #<((end)+(step))
+	bne @for_loop
+	cpy #>((end)+(step))
+	bne @for_loop
+.endmacro
+
+; Stores byte at addr
+; Preserved: X, Y
+.macro setb addr, byte
+	lda #byte
+	sta addr
+.endmacro
+
+; Stores word at addr
+; Preserved: X, Y
+.macro setw addr, word
+	lda #<(word)
+	sta addr
+	lda #>(word)
+	sta addr+1
+.endmacro
+
+; Loads XY with 16-bit immediate or value at address
+.macro ldxy Arg
+	.if .match( .left( 1, {Arg} ), # )
+		ldy #<(.right( .tcount( {Arg} )-1, {Arg} ))
+		ldx #>(.right( .tcount( {Arg} )-1, {Arg} ))
+	.else
+		ldy (Arg)
+		ldx (Arg)+1
+	.endif
+.endmacro
+
+; Increments XY as 16-bit register, in CONSTANT time.
+; Z flag set based on entire result.
+; Preserved: A
+; Time: 7 clocks
+.macro inxy
+	iny ; 2
+	beq *+4 ; 3
+		; -1
+	bne *+3 ; 3
+		; -1
+	inx ; 2
+.endmacro
+
+; Negates A and adds it to operand
+.macro subaf Operand
+	eor #$FF
+	sec
+	adc Operand
+.endmacro
+
+; Initializes CPU registers to reasonable values
+; Preserved: A, Y
+.macro init_cpu_regs
+	sei
+	cld     ; unnecessary on NES, but might help on clone
+	ldx #$FF
+	txs
+	.ifndef BUILD_NSF
+		inx
+		stx PPUCTRL
+	.endif
+.endmacro

samples/BitBake/gstreamer-libav.bb

@@ -0,0 +1,25 @@
+include gstreamer1.0-libav.inc
+
+LIC_FILES_CHKSUM = "file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263 \
+                    file://COPYING.LIB;md5=6762ed442b3822387a51c92d928ead0d \
+                    file://ext/libav/gstav.h;beginline=1;endline=18;md5=a752c35267d8276fd9ca3db6994fca9c \
+                    file://gst-libs/ext/libav/LICENSE;md5=23a54f59b82572c203a559346e89ed57 \
+                    file://gst-libs/ext/libav/COPYING.GPLv2;md5=b234ee4d69f5fce4486a80fdaf4a4263 \
+                    file://gst-libs/ext/libav/COPYING.GPLv3;md5=d32239bcb673463ab874e80d47fae504 \
+                    file://gst-libs/ext/libav/COPYING.LGPLv2.1;md5=bd7a443320af8c812e4c18d1b79df004 \
+                    file://gst-libs/ext/libav/COPYING.LGPLv3;md5=e6a600fd5e1d9cbde2d983680233ad02"
+
+SRC_URI = " \
+    http://gstreamer.freedesktop.org/src/gst-libav/gst-libav-${PV}.tar.xz \
+    file://0001-Disable-yasm-for-libav-when-disable-yasm.patch \
+    "
+SRC_URI[md5sum] = "86540dee14d31daf976eb2713f2294f3"
+SRC_URI[sha256sum] = "585eb7971006100ad771a852e07bd2f3e23bcc6eb0b1253a40b5a0e40e4e7418"
+
+LIBAV_EXTRA_CONFIGURE_COMMON_ARG = "--target-os=linux \
+  --cc='${CC}' --as='${CC}' --ld='${CC}' --nm='${NM}' --ar='${AR}' \
+  --ranlib='${RANLIB}' \
+  ${GSTREAMER_1_0_DEBUG}"
+
+S = "${WORKDIR}/gst-libav-${PV}"
+

samples/BitBake/qtbase-native.bb

@@ -0,0 +1,13 @@
+require qt5-git.inc
+require ${PN}.inc
+
+do_install_append() {
+    # for modules which are still using syncqt and call qtPrepareTool(QMAKE_SYNCQT, syncqt)
+    # e.g. qt3d, qtwayland
+    ln -sf syncqt.pl ${D}${OE_QMAKE_PATH_QT_BINS}/syncqt
+}
+
+QT_MODULE_BRANCH = "release"
+# v5.2.1 + 168 commits
+SRCREV = "08cbbde61778276ccdda73d89fd64d02c623779f"
+

samples/Brainfuck/factor.b

@@ -0,0 +1,195 @@
+* factor an arbitrarily large positive integer
+*
+* Copyright (C) 1999 by Brian Raiter
+* under the GNU General Public License
+
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>-
+
+*
+* read in the number
+*
+
+<<<<<<<<<+
+[-[>>>>>>>>>>][-]<<<<<<<<<<[[->>>>>>>>>>+<<<<<<<<<<]<<<<<<<<<<]
+  >>>>>>>>>>,----------]
+>>>>>>>>>>[------------------------------------->>>>>>>>>->]
+<[+>[>>>>>>>>>+>]<-<<<<<<<<<<]-
+
+*
+* display the number and initialize the loop variable to two
+*
+
+[>++++++++++++++++++++++++++++++++++++++++++++++++.
+  ------------------------------------------------<<<<<<<<<<<]
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.
+--------------------------.[-]
+>>>>>>>>>>>>++<<<<+
+
+*
+* the main loop
+*
+
+[ [-]>>
+
+  *
+  * make copies of the number and the loop variable
+  *
+
+  [>>>>[-]>[-]>[-]>[-]
+    >[-]>[-]
+    <<<<<<<[->>>+>+<<<<]>>>>>>>>]
+  <<<<<<<<<<[>>>>>>[-<<<<+>>>>]<<<<<<<<<<<<<<<<]>>>>>>>>>>
+  [>[->>>+>>+<<<<<]>>>>>>>>>]
+  <<<<<<<<<<[>>>>>>[-<<<<<+>>>>>]<<<<<<<<<<<<<<<<]>>>>>>>>>>
+
+  *
+  * divide the number by the loop variable
+  *
+
+  [>>>[-]>>>[-]>[-]>>>]                                  initialize
+  <<<<<<<<<<[<<<<<<<<<<]
+  >>>>>>>>>[-]>>>>>>>+<<<<<<<<[+]+
+  [ ->>                               double divisor until above dividend
+    [>>>>>>[->++<]>>>>]<<<<<<<<<<
+    [>>>>>>>>[-]>[-]
+       <<<<[->>>++<<<]<<<<<<<<<<<<<<<]>>>>>>>>>>
+    [>>>>>>>>[->+<[->+<[->+<[->+<[->+<[->+<[->+<[->+<[->+<
+            [->--------->>>>>>>>>+<<<<<<<<<<[->+<]]]]]]]]]]]>>]
+    <<<<<<<<<<[>>>>>>>>>[-<+<<<+>>>>]<<<<<<<<<<<<<<<<<<<]>>>>>>>>>>
+    [>>>>>>>[-<+>[-<+>[-<+>[-<+>[-<+>[-<+>[-<+>[-<+>[-<+>
+            [-<--------->>>>>>>>>>>+<<<<<<<<<<[-<+>]]]]]]]]]]]>>>]
+    <<<<<<<<<<
+    [>>>>[->>>+>>+<<<<<]<<<<<<<<<<<<<<]
+    >>>>>>>>>>[>>>>>>>[-<<<+>>>]>>>]<<<<<<<<<<
+    [>>>>>>>>[->-<]>
+      [<<<<<<<<<[<[-]>>>>>>>>>>[-<<<<<<<<<<+>>>>>>>>>>]<<<<<<<<<<<<<<<<<<<]
+        >>>>>>>>>>>>>>>>>>>]
+      <<<<<<<<<<<<<<<<<<<]
+    >>>>>>>>>[+[+[+[+[+[+[+[+[+[+[[-]<+>]]]]]]]]]]]<
+  ]
+  >>>>>>>>
+  [                                   subtract divisor from dividend
+    <<<<<<
+    [>>>>>>>>[-]>[-]<<<<<[->>>+>+<<<<]>>>>>>]<<<<<<<<<<
+    [>>>>>>>>[-<<<<+>>>>]<<<[->>>+>+<<<<]<<<<<<<<<<<<<<<]>>>>>>>>>>
+    [>>>>>>>>>[-<<<<+>>>>]>]<<<<<<<<<<
+    [>>>>>>>>[-<->]<<<<<<<<<<<<<<<<<<]>>>>>>>>>>
+    [>>>>>>>[->+<[->+<[->+<[->+<[->+<[->+<[->+<[->+<[->+<[->+<
+            [++++++++++[+>-<]>>>>>>>>>>-<<<<<<<<<<]]]]]]]]]]]>>>]
+    >>>>>>>+
+    [                                 if difference is nonnegative then
+      [-]<<<<<<<<<<<<<<<<<            replace dividend and increment quotient
+      [>>>>[-]>>>>[-<<<<+>>>>]<<[->>+<<]<<<<<<<<<<<<<<<<]>>>>>>>>>>
+      [>>>>>>>>[->+<<<+>>]>>]<<<<<<<<<<
+      [>>>[->>>>>>+<<<<<<]<<<<<<<<<<<<<]>>>>>>>>>>
+      [>>>>>>>>>[-<<<<<<+>>>>>>[-<<<<<<+>>>>>>
+                [-<<<<<<+>>>>>>[-<<<<<<+>>>>>>
+                [-<<<<<<+>>>>>>[-<<<<<<+>>>>>>
+                [-<<<<<<+>>>>>>[-<<<<<<+>>>>>>
+                [-<<<<<<+>>>>>>[-<<<<<<--------->>>>>>>>>>>>>>>>+<<<<<<<<<<
+                [-<<<<<<+>>>>>>]]]]]]]]]]]>]
+      >>>>>>>
+    ]                                 halve divisor and loop until zero
+    <<<<<<<<<<<<<<<<<[<<<<<<<<<<]>>>>>>>>>>
+    [>>>>>>>>[-]<<[->+<]<[->>>+<<<]>>>>>]<<<<<<<<<<
+    [+>>>>>>>[-<<<<<<<+>>>>>>>[-<<<<<<<->>>>>>+>
+             [-<<<<<<<+>>>>>>>[-<<<<<<<->>>>>>+>
+             [-<<<<<<<+>>>>>>>[-<<<<<<<->>>>>>+>
+             [-<<<<<<<+>>>>>>>[-<<<<<<<->>>>>>+>
+             [-<<<<<<<+>>>>>>>]]]]]]]]]<<<<<<<
+             [->>>>>>>+<<<<<<<]-<<<<<<<<<<]
+    >>>>>>>
+    [-<<<<<<<<<<<+>>>>>>>>>>>]
+      >>>[>>>>>>>[-<<<<<<<<<<<+++++>>>>>>>>>>>]>>>]<<<<<<<<<<
+    [+>>>>>>>>[-<<<<<<<<+>>>>>>>>[-<<<<<<<<->>>>>+>>>
+              [-<<<<<<<<+>>>>>>>>[-<<<<<<<<->>>>>+>>>
+              [-<<<<<<<<+>>>>>>>>[-<<<<<<<<->>>>>+>>>
+              [-<<<<<<<<+>>>>>>>>[-<<<<<<<<->>>>>+>>>
+              [-<<<<<<<<+>>>>>>>>]]]]]]]]]<<<<<<<<
+              [->>>>>>>>+<<<<<<<<]-<<<<<<<<<<]
+    >>>>>>>>[-<<<<<<<<<<<<<+>>>>>>>>>>>>>]>>
+    [>>>>>>>>[-<<<<<<<<<<<<<+++++>>>>>>>>>>>>>]>>]<<<<<<<<<<
+    [<<<<<<<<<<]>>>>>>>>>>
+    >>>>>>
+  ]
+  <<<<<<
+
+  *
+  * make copies of the loop variable and the quotient
+  *
+
+  [>>>[->>>>+>+<<<<<]>>>>>>>]
+  <<<<<<<<<<
+  [>>>>>>>[-<<<<+>>>>]<<<<<[->>>>>+>>+<<<<<<<]<<<<<<<<<<<<]
+  >>>>>>>>>>[>>>>>>>[-<<<<<+>>>>>]>>>]<<<<<<<<<<
+
+  *
+  * break out of the loop if the quotient is larger than the loop variable
+  *
+
+  [>>>>>>>>>[-<->]<
+    [<<<<<<<<
+      [<<[-]>>>>>>>>>>[-<<<<<<<<<<+>>>>>>>>>>]<<<<<<<<<<<<<<<<<<]
+    >>>>>>>>>>>>>>>>>>]<<<<<<<<<<<<<<<<<<]
+  >>>>>>>>[>-<[+[+[+[+[+[+[+[+[+[[-]>+<]]]]]]]]]]]>+
+
+  [ [-]
+
+    *
+    * partially increment the loop variable
+    *
+
+    <[-]+>>>>+>>>>>>>>[>>>>>>>>>>]<<<<<<<<<<
+
+    *
+    * examine the remainder for nonzero digits
+    *
+
+    [<<<<<<[<<<<[<<<<<<<<<<]>>>>+<<<<<<<<<<]<<<<]
+    >>>>>>>>>>>>>>>>>>>>[>>>>>>>>>>]<<<<<<<<<<[<<<<<<<<<<]
+    >>>>-
+
+    [ [+]
+
+      *
+      * decrement the loop variable and replace the number with the quotient
+      *
+
+      >>>>>>>>-<<[>[-]>>[-<<+>>]>>>>>>>]<<<<<<<<<<
+
+      *
+      * display the loop variable
+      *
+
+      [+>>[>>>>>>>>+>>]<<-<<<<<<<<<<]-
+      [>>++++++++++++++++++++++++++++++++++++++++++++++++.
+         ------------------------------------------------<<<<<<<<<<<<]
+      ++++++++++++++++++++++++++++++++.[-]>>>>
+
+    ]
+
+    *
+    * normalize the loop variable
+    *
+
+    >>>>>>
+    [>>[->>>>>+<<<<<[->>>>>+<<<<<
+       [->>>>>+<<<<<[->>>>>+<<<<<
+       [->>>>>+<<<<<[->>>>>+<<<<<
+       [->>>>>+<<<<<[->>>>>+<<<<<
+       [->>>>>+<<<<<[->>>>>--------->>>>>+<<<<<<<<<<
+       [->>>>>+<<<<<]]]]]]]]]]]>>>>>>>>]
+    <<<<<<<<<<[>>>>>>>[-<<<<<+>>>>>]<<<<<<<<<<<<<<<<<]
+    >>>>>>>>>
+
+  ]<
+
+]>>
+
+*
+* display the number and end
+*
+
+[>>>>>>>>>>]<<<<<<<<<<[+>[>>>>>>>>>+>]<-<<<<<<<<<<]-
+[>++++++++++++++++++++++++++++++++++++++++++++++++.<<<<<<<<<<<]
+++++++++++.

samples/Brainfuck/fib100.bf

@@ -0,0 +1,13 @@
+# Calculate and output all fibonacci numbers under 100
+
++++++++++++
+>+>>>>++++++++++++++++++++++++++++++++++++++++++++
+>++++++++++++++++++++++++++++++++<<<<<<[>[>>>>>>+>
++<<<<<<<-]>>>>>>>[<<<<<<<+>>>>>>>-]<[>++++++++++[-
+<-[>>+>+<<<-]>>>[<<<+>>>-]+<[>[-]<[-]]>[<<[>>>+<<<
+-]>>[-]]<<]>>>[>>+>+<<<-]>>>[<<<+>>>-]+<[>[-]<[-]]
+>[<<+>>[-]]<<<<<<<]>>>>>[+++++++++++++++++++++++++
++++++++++++++++++++++++.[-]]++++++++++<[->-<]>++++
+++++++++++++++++++++++++++++++++++++++++++++.[-]<<
+<<<<<<<<<<[>>>+>+<<<<-]>>>>[<<<<+>>>>-]<-[>>.>.<<<
+[-]]<<[>>+>+<<<-]>>>[<<<+>>>-]<<[<+>-]>[<+>-]<<<-]

samples/Brainfuck/hello.bf

@@ -0,0 +1,4 @@
+// More complex version of hello world
+
+>++++++++[<+++++++++>-]<.>>+>+>++>[-]+<[>[->+<<++++>]<<]>.+++++++..+++.>
+>+++++++.<<<[[-]<[-]>]<+++++++++++++++.>>.+++.------.--------.>>+.>++++.

samples/Brainfuck/helloworld.bf

@@ -0,0 +1,3 @@
+// Hello World
+
+++++++++[>++++[>++>+++>+++>+<<<<-]>+>+>->>+[<]<-]>>.>---.+++++++..+++.>>.<-.<.+++.------.--------.>>+.>++.

samples/Brainfuck/rot13.bf

@@ -0,0 +1,30 @@
+# ROT13 cipher
+
+-,+[                         Read first character and start outer character reading loop
+    -[                       Skip forward if character is 0
+        >>++++[>++++++++<-]  Set up divisor (32) for division loop
+                               (MEMORY LAYOUT: dividend copy remainder divisor quotient zero zero)
+        <+<-[                Set up dividend (x minus 1) and enter division loop
+            >+>+>-[>>>]      Increase copy and remainder / reduce divisor / Normal case: skip forward
+            <[[>+<-]>>+>]    Special case: move remainder back to divisor and increase quotient
+            <<<<<-           Decrement dividend
+        ]                    End division loop
+    ]>>>[-]+                 End skip loop; zero former divisor and reuse space for a flag
+    >--[-[<->+++[-]]]<[         Zero that flag unless quotient was 2 or 3; zero quotient; check flag
+        ++++++++++++<[       If flag then set up divisor (13) for second division loop
+                               (MEMORY LAYOUT: zero copy dividend divisor remainder quotient zero zero)
+            >-[>+>>]         Reduce divisor; Normal case: increase remainder
+            >[+[<+>-]>+>>]   Special case: increase remainder / move it back to divisor / increase quotient
+            <<<<<-           Decrease dividend
+        ]                    End division loop
+        >>[<+>-]             Add remainder back to divisor to get a useful 13
+        >[                   Skip forward if quotient was 0
+            -[               Decrement quotient and skip forward if quotient was 1
+                -<<[-]>>     Zero quotient and divisor if quotient was 2
+            ]<<[<<->>-]>>    Zero divisor and subtract 13 from copy if quotient was 1
+        ]<<[<<+>>-]          Zero divisor and add 13 to copy if quotient was 0
+    ]                        End outer skip loop (jump to here if ((character minus 1)/32) was not 2 or 3)
+    <[-]                     Clear remainder from first division if second division was skipped
+    <.[-]                    Output ROT13ed character from copy and clear it
+    <-,+                     Read next character
+]                            End character reading loop

samples/C#/AssemblyInfo.cs

@@ -0,0 +1,26 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+
+// Information about this assembly is defined by the following attributes. 
+// Change them to the values specific to your project.
+
+[assembly: AssemblyTitle("Simple")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("")]
+[assembly: AssemblyCopyright("")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}".
+// The form "{Major}.{Minor}.*" will automatically update the build and revision,
+// and "{Major}.{Minor}.{Build}.*" will update just the revision.
+
+[assembly: AssemblyVersion("1.0.*")]
+
+// The following attributes are used to specify the signing key for the assembly, 
+// if desired. See the Mono documentation for more information about signing.
+
+//[assembly: AssemblyDelaySign(false)]
+//[assembly: AssemblyKeyFile("")]

samples/C#/BsonPropertyValue.cs

@@ -0,0 +1,20 @@
+using System;
+
+namespace MongoDB.Serialization.Descriptors
+{
+    internal class BsonPropertyValue
+    {
+        public bool IsDictionary { get; private set; }
+
+        public Type Type { get; private set; }
+
+        public object Value { get; private set; }
+
+        public BsonPropertyValue(Type type, object value, bool isDictionary)
+        {
+            Type = type;
+            Value = value;
+            IsDictionary = isDictionary;
+        }
+    }
+}

samples/C#/MongoExpressionVisitor.cs

@@ -0,0 +1,153 @@
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Linq;
+using System.Linq.Expressions;
+
+namespace MongoDB.Linq.Expressions
+{
+    internal class MongoExpressionVisitor : ExpressionVisitor
+    {
+        protected override Expression Visit(Expression exp)
+        {
+            if (exp == null)
+                return null;
+            switch ((MongoExpressionType)exp.NodeType)
+            {
+                case MongoExpressionType.Collection:
+                    return VisitCollection((CollectionExpression)exp);
+                case MongoExpressionType.Field:
+                    return VisitField((FieldExpression)exp);
+                case MongoExpressionType.Projection:
+                    return VisitProjection((ProjectionExpression)exp);
+                case MongoExpressionType.Select:
+                    return VisitSelect((SelectExpression)exp);
+                case MongoExpressionType.Aggregate:
+                    return VisitAggregate((AggregateExpression)exp);
+                case MongoExpressionType.AggregateSubquery:
+                    return VisitAggregateSubquery((AggregateSubqueryExpression)exp);
+                case MongoExpressionType.Scalar:
+                    return VisitScalar((ScalarExpression)exp);
+                default:
+                    return base.Visit(exp);
+            }
+        }
+
+        protected virtual Expression VisitAggregate(AggregateExpression aggregate)
+        {
+            var exp = Visit(aggregate.Argument);
+            if (exp != aggregate.Argument)
+                return new AggregateExpression(aggregate.Type, aggregate.AggregateType, exp, aggregate.Distinct);
+
+            return aggregate;
+        }
+
+        protected virtual Expression VisitAggregateSubquery(AggregateSubqueryExpression aggregateSubquery)
+        {
+            Expression e = Visit(aggregateSubquery.AggregateAsSubquery);
+            ScalarExpression subquery = (ScalarExpression)e;
+            if (subquery != aggregateSubquery.AggregateAsSubquery)
+                return new AggregateSubqueryExpression(aggregateSubquery.GroupByAlias, aggregateSubquery.AggregateInGroupSelect, subquery);
+            return aggregateSubquery;
+        }
+
+        protected virtual Expression VisitCollection(CollectionExpression collection)
+        {
+            return collection;
+        }
+
+        protected virtual Expression VisitField(FieldExpression field)
+        {
+            var e = Visit(field.Expression);
+            if (field.Expression != e)
+                field = new FieldExpression(e, field.Alias, field.Name);
+
+            return field;
+        }
+
+        protected virtual Expression VisitProjection(ProjectionExpression projection)
+        {
+            var source = (SelectExpression)Visit(projection.Source);
+            var projector = Visit(projection.Projector);
+            if (source != projection.Source || projector != projection.Projector)
+                return new ProjectionExpression(source, projector, projection.Aggregator);
+            return projection;
+        }
+
+        protected ReadOnlyCollection<OrderExpression> VisitOrderBy(ReadOnlyCollection<OrderExpression> orderBys)
+        {
+            if (orderBys != null) 
+            {
+                List<OrderExpression> alternate = null;
+                for (int i = 0, n = orderBys.Count; i < n; i++) 
+                {
+                    OrderExpression expr = orderBys[i];
+                    Expression e = this.Visit(expr.Expression);
+                    if (alternate == null && e != expr.Expression) 
+                        alternate = orderBys.Take(i).ToList();
+                    if (alternate != null) 
+                        alternate.Add(new OrderExpression(expr.OrderType, e));
+                }
+                if (alternate != null) 
+                    return alternate.AsReadOnly();
+            }
+            return orderBys;
+        }
+
+        protected virtual Expression VisitScalar(ScalarExpression scalar)
+        {
+            SelectExpression select = (SelectExpression)Visit(scalar.Select);
+            if (select != scalar.Select)
+                return new ScalarExpression(scalar.Type, select);
+            return scalar;
+        }
+
+        protected virtual Expression VisitSelect(SelectExpression select)
+        {
+            var from = VisitSource(select.From);
+            var where = Visit(select.Where);
+            var groupBy = Visit(select.GroupBy);
+            var orderBy = VisitOrderBy(select.OrderBy);
+            var skip = Visit(select.Skip);
+            var take = Visit(select.Take);
+            var fields = VisitFieldDeclarationList(select.Fields);
+            if (from != select.From || where != select.Where || orderBy != select.OrderBy || groupBy != select.GroupBy || skip != select.Skip || take != select.Take || fields != select.Fields)
+                return new SelectExpression(select.Alias, fields, from, where, orderBy, groupBy, select.IsDistinct, skip, take);
+            return select;
+        }
+
+        protected virtual Expression VisitSource(Expression source)
+        {
+            return Visit(source);
+        }
+
+        protected virtual Expression VisitSubquery(SubqueryExpression subquery)
+        {
+            switch ((MongoExpressionType)subquery.NodeType)
+            {
+                case MongoExpressionType.Scalar:
+                    return VisitScalar((ScalarExpression)subquery);
+            }
+            return subquery;
+        }
+
+        protected virtual ReadOnlyCollection<FieldDeclaration> VisitFieldDeclarationList(ReadOnlyCollection<FieldDeclaration> fields)
+        {
+            if (fields == null)
+                return fields;
+
+            List<FieldDeclaration> alternate = null;
+            for (int i = 0, n = fields.Count; i < n; i++)
+            {
+                var f = fields[i];
+                var e = Visit(f.Expression);
+                if (f.Expression != e && alternate == null)
+                    alternate = fields.Take(i).ToList();
+                if (alternate != null)
+                    alternate.Add(new FieldDeclaration(f.Name, e));
+            }
+            if (alternate != null)
+                return alternate.AsReadOnly();
+            return fields;
+        }
+    }
+}

samples/C#/build.cake

@@ -0,0 +1,86 @@
+///////////////////////////////////////////////////////////////////////////////
+// ARGUMENTS
+///////////////////////////////////////////////////////////////////////////////
+
+var target = Argument<string>("target", "Default");
+var configuration = Argument<string>("configuration", "Release");
+
+///////////////////////////////////////////////////////////////////////////////
+// GLOBAL VARIABLES
+///////////////////////////////////////////////////////////////////////////////
+
+var solutions = GetFiles("./**/*.sln");
+var solutionPaths = solutions.Select(solution => solution.GetDirectory());
+
+///////////////////////////////////////////////////////////////////////////////
+// SETUP / TEARDOWN
+///////////////////////////////////////////////////////////////////////////////
+
+Setup(() =>
+{
+    // Executed BEFORE the first task.
+    Information("Running tasks...");
+});
+
+Teardown(() =>
+{
+    // Executed AFTER the last task.
+    Information("Finished running tasks.");
+});
+
+///////////////////////////////////////////////////////////////////////////////
+// TASK DEFINITIONS
+///////////////////////////////////////////////////////////////////////////////
+
+Task("Clean")
+    .Does(() =>
+{
+    // Clean solution directories.
+    foreach(var path in solutionPaths)
+    {
+        Information("Cleaning {0}", path);
+        CleanDirectories(path + "/**/bin/" + configuration);
+        CleanDirectories(path + "/**/obj/" + configuration);
+    }
+});
+
+Task("Restore")
+    .Does(() =>
+{
+    // Restore all NuGet packages.
+    foreach(var solution in solutions)
+    {
+        Information("Restoring {0}...", solution);
+        NuGetRestore(solution);
+    }
+});
+
+Task("Build")
+    .IsDependentOn("Clean")
+    .IsDependentOn("Restore")
+    .Does(() =>
+{
+    // Build all solutions.
+    foreach(var solution in solutions)
+    {
+        Information("Building {0}", solution);
+        MSBuild(solution, settings => 
+            settings.SetPlatformTarget(PlatformTarget.MSIL)
+                .WithProperty("TreatWarningsAsErrors","true")
+                .WithTarget("Build")
+                .SetConfiguration(configuration));
+    }
+});
+
+///////////////////////////////////////////////////////////////////////////////
+// TARGETS
+///////////////////////////////////////////////////////////////////////////////
+
+Task("Default")
+    .IsDependentOn("Build");
+
+///////////////////////////////////////////////////////////////////////////////
+// EXECUTION
+///////////////////////////////////////////////////////////////////////////////
+
+RunTarget(target);

samples/C++/ClasspathVMSystemProperties.inc

@@ -0,0 +1,57 @@
+//===- ClasspathVMSystem/Properties.cpp -----------------------------------===//
+//===--------------------- GNU classpath gnu/classpath/VMSystemProperties -===//
+//
+//                            The VMKit project
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include <sys/utsname.h>
+
+#include "types.h"
+
+#include "Classpath.h"
+#include "JavaArray.h"
+#include "JavaClass.h"
+#include "JavaObject.h"
+#include "JavaThread.h"
+#include "JavaUpcalls.h"
+#include "Jnjvm.h"
+
+#include "SetProperties.inc"
+
+using namespace j3;
+
+extern "C" {
+
+
+JNIEXPORT void JNICALL Java_gnu_classpath_VMSystemProperties_preInit(
+#ifdef NATIVE_JNI
+JNIEnv *env,
+jclass clazz,
+#endif
+JavaObject* prop) {
+  
+  llvm_gcroot(prop, 0);
+
+  BEGIN_NATIVE_EXCEPTION(0)
+
+  setProperties(prop);
+
+  END_NATIVE_EXCEPTION
+}
+
+extern "C" void Java_gnu_classpath_VMSystemProperties_postInit__Ljava_util_Properties_2(JavaObject* prop) {
+
+  llvm_gcroot(prop, 0);
+
+  BEGIN_NATIVE_EXCEPTION(0)
+
+  setCommandLineProperties(prop);
+
+  END_NATIVE_EXCEPTION
+}
+
+}

samples/C++/Entity.h

@@ -0,0 +1,98 @@
+/** 
+* @file Entity.h
+* @page EntityPage Entity
+* @brief represent an entity in the game
+* @author vinz243
+* @version 0.1.0
+* This file represents an Entity in the game system
+* This parent type is a static entity which is shown and loaded into the Physics engine but never updated
+*/
+
+#ifndef ENTITY_H
+#define ENTITY_H
+
+#include "base.h"
+/// @namespace Whitedrop
+namespace Whitedrop {
+	/** @class Entity
+	* This parent type is a static entity which is shown and loaded into the Physics engine but never updated
+	*/
+	class Entity {
+	public:
+		/**
+		 * @brief Create static entity
+		 * @details creates a static entity instance according to the mesh and the id, the position
+		 * This needs to be attached to a World after!
+		 * The material name is not the file name but the material name!
+		 * @ref WorldPage
+		 * @param mesh the name of the mesh for the object, file must be in media/meshes
+		 * @param id an unique identifier for the object, shortest as possible
+		 * @param dimensions an Ogre::Vector3 which contains the dimensions in meter
+		 * @param position the Vector3 which contains it position 
+		 * @param material the material name
+		 */
+		Entity(std::string mesh, std::string id, Ogre::Vector3 dimensions, Ogre::Vector3 position, std::string material);
+		/**
+		 * @brief The copy constructor
+		 * @details A copy constr
+		 * 
+		 * @param ref the Entity to be copied from
+		 */
+		Entity(const Entity &ref);
+
+		/**
+		 * @brief The assignement operator
+		 * @details 
+		 * 
+		 * @param ent the entity to be copied
+		 */
+		Entity& operator=(const Entity ent);
+
+		/**
+		 * @brief destrctor
+		 * @details
+		 */
+		virtual ~Entity(void);
+
+		/**
+		 * @brief a constance type of the entity
+		 * @details depends of the class. 
+		 * May contain STATIC, DYNAMIC or ETHERAL
+		 */
+		const std::string type = "STATIC";
+
+		/**
+		 * @brief Attach the entity to specified sceneManager
+		 * @details This creates the OgreEntity using sceneMgr,
+		 * set material, create a Node with name as `<id>_n`,
+		 * scale it to match dimensions and translate the node to pos
+		 * @param sceneMgr the scene manager to use
+		 */
+		virtual void setup(Ogre::SceneManager* sceneMgr);
+
+		/**
+		 * @brief the update method
+		 * @details this method should be called on each world update.
+		 * Even though the method is necessary declared, the main impl of 
+		 * a static entity should be empty since it is not updated by physics
+		 * However, a Dynamic entity should implement this function in order to:
+		 * 1) Get from the physics engine the actor position in the physic world
+		 * 2) Update the OgreEntity position and rotation from the previous actor
+		 * @return whether it was successful or not, if falsey engine should stop
+		 */
+		virtual bool update(void);
+
+	protected:
+		std::string 			mMesh = "cube.mesh";
+		std::string 			mId;
+		std::string 			mMaterial;
+		Ogre::Vector3 			mDimensions;
+		Ogre::Vector3 			mPosition;
+		Ogre::Entity* 			mEntity;
+		Ogre::SceneNode* 		mNode;
+
+	};
+}
+
+
+#endif

samples/C++/instances.inc

@@ -0,0 +1,34 @@
+#include "QPBO.h"
+
+#ifdef _MSC_VER
+#pragma warning(disable: 4661)
+#endif
+
+// Instantiations
+
+template class QPBO<int>;
+template class QPBO<float>;
+template class QPBO<double>;
+
+template <> 
+	inline void QPBO<int>::get_type_information(char*& type_name, char*& type_format)
+{
+	type_name = "int";
+	type_format = "d";
+}
+
+template <> 
+	inline void QPBO<float>::get_type_information(char*& type_name, char*& type_format)
+{
+	type_name = "float";
+	type_format = "f";
+}
+
+template <> 
+	inline void QPBO<double>::get_type_information(char*& type_name, char*& type_format)
+{
+	type_name = "double";
+	type_format = "Lf";
+}
+
+

samples/C++/qsciprinter.cp

@@ -0,0 +1,116 @@
+// This module defines interface to the QsciPrinter class.
+//
+// Copyright (c) 2011 Riverbank Computing Limited <info@riverbankcomputing.com>
+// 
+// This file is part of QScintilla.
+// 
+// This file may be used under the terms of the GNU General Public
+// License versions 2.0 or 3.0 as published by the Free Software
+// Foundation and appearing in the files LICENSE.GPL2 and LICENSE.GPL3
+// included in the packaging of this file.  Alternatively you may (at
+// your option) use any later version of the GNU General Public
+// License if such license has been publicly approved by Riverbank
+// Computing Limited (or its successors, if any) and the KDE Free Qt
+// Foundation. In addition, as a special exception, Riverbank gives you
+// certain additional rights. These rights are described in the Riverbank
+// GPL Exception version 1.1, which can be found in the file
+// GPL_EXCEPTION.txt in this package.
+// 
+// If you are unsure which license is appropriate for your use, please
+// contact the sales department at sales@riverbankcomputing.com.
+// 
+// This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
+// WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+
+
+#ifndef QSCIPRINTER_H
+#define QSCIPRINTER_H
+
+#ifdef __APPLE__
+extern "C++" {
+#endif
+
+#include <qprinter.h>
+
+#include <Qsci/qsciglobal.h>
+#include <Qsci/qsciscintilla.h>
+
+
+QT_BEGIN_NAMESPACE
+class QRect;
+class QPainter;
+QT_END_NAMESPACE
+
+class QsciScintillaBase;
+
+
+//! \brief The QsciPrinter class is a sub-class of the Qt QPrinter class that
+//! is able to print the text of a Scintilla document.
+//!
+//! The class can be further sub-classed to alter to layout of the text, adding
+//! headers and footers for example.
+class QSCINTILLA_EXPORT QsciPrinter : public QPrinter
+{
+public:
+    //! Constructs a printer paint device with mode \a mode.
+    QsciPrinter(PrinterMode mode = ScreenResolution);
+
+    //! Destroys the QsciPrinter instance.
+    virtual ~QsciPrinter();
+
+    //! Format a page, by adding headers and footers for example, before the
+    //! document text is drawn on it.  \a painter is the painter to be used to
+    //! add customised text and graphics.  \a drawing is true if the page is
+    //! actually being drawn rather than being sized.  \a painter drawing
+    //! methods must only be called when \a drawing is true.  \a area is the
+    //! area of the page that will be used to draw the text.  This should be
+    //! modified if it is necessary to reserve space for any customised text or
+    //! graphics.  By default the area is relative to the printable area of the
+    //! page.  Use QPrinter::setFullPage() because calling printRange() if you
+    //! want to try and print over the whole page.  \a pagenr is the number of
+    //! the page.  The first page is numbered 1.
+    virtual void formatPage(QPainter &painter, bool drawing, QRect &area,
+            int pagenr);
+
+    //! Return the number of points to add to each font when printing.
+    //!
+    //! \sa setMagnification()
+    int magnification() const {return mag;}
+
+    //! Sets the number of points to add to each font when printing to \a
+    //! magnification.
+    //!
+    //! \sa magnification()
+    virtual void setMagnification(int magnification);
+
+    //! Print a range of lines from the Scintilla instance \a qsb.  \a from is
+    //! the first line to print and a negative value signifies the first line
+    //! of text.  \a to is the last line to print and a negative value
+    //! signifies the last line of text.  true is returned if there was no
+    //! error.
+    virtual int printRange(QsciScintillaBase *qsb, int from = -1, int to = -1);
+
+    //! Return the line wrap mode used when printing.  The default is
+    //! QsciScintilla::WrapWord.
+    //!
+    //! \sa setWrapMode()
+    QsciScintilla::WrapMode wrapMode() const {return wrap;}
+
+    //! Sets the line wrap mode used when printing to \a wmode.
+    //!
+    //! \sa wrapMode()
+    virtual void setWrapMode(QsciScintilla::WrapMode wmode);
+
+private:
+    int mag;
+    QsciScintilla::WrapMode wrap;
+
+    QsciPrinter(const QsciPrinter &);
+    QsciPrinter &operator=(const QsciPrinter &);
+};
+
+#ifdef __APPLE__
+}
+#endif
+
+#endif

samples/C/bitmap.h

@@ -0,0 +1,47 @@
+#pragma once
+
+/* Copyright © 2010 Christoph Sünderhauf
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "generic.h"
+
+typedef struct {
+	uint32_t numbits;
+	/* an array large enough for numbits to fit in. Might 
+	 * (if numbits%8!=0) have some spare bits at the end
+	 */
+	uint32_t* bits;
+} bitmap_t;
+
+
+// creates a new bitmap.
+// CONTENT IS RANDOM!  - use bitmap_clearall() to clear the bitmap.
+bitmap_t bitmap_init(uint32_t numbits);
+
+// returns 1 or 0
+uint8_t bitmap_get(bitmap_t bitmap, uint32_t bitnum);
+// sets a bit (to 1)
+void bitmap_set(bitmap_t bitmap, uint32_t bitnum);
+// clears a bit (to 0)
+void bitmap_clear(bitmap_t bitmap, uint32_t bitnum);
+
+// clears every bit to 0
+void bitmap_clearAll(bitmap_t bitmap);
+
+// finds the first bit set to 0    returns 0 if no cleared bit found (0 is also returned if the first bit is cleared)
+uint32_t bitmap_findFirstClear(bitmap_t bitmap);

samples/C/color.h

@@ -0,0 +1,44 @@
+#pragma once
+
+/* Copyright © 2011 Fritz Grimpen
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <lib/generic.h>
+
+typedef struct {
+	uint32_t background;
+	uint32_t foreground;
+} console_color_t;
+
+#define CONSOLE_COLOR_BLACK    0x0
+#define CONSOLE_COLOR_BLUE     0x1
+#define CONSOLE_COLOR_GREEN    0x2
+#define CONSOLE_COLOR_CYAN     0x3
+#define CONSOLE_COLOR_RED      0x4
+#define CONSOLE_COLOR_MAGENTA  0x5
+#define CONSOLE_COLOR_BROWN    0x6
+#define CONSOLE_COLOR_LGREY    0x7
+#define CONSOLE_COLOR_DGREY    0x8
+#define CONSOLE_COLOR_LBLUE    0x9
+#define CONSOLE_COLOR_LGREEN   0xa
+#define CONSOLE_COLOR_LCYAN    0xb
+#define CONSOLE_COLOR_LRED     0xc
+#define CONSOLE_COLOR_LMAGENTA 0xd
+#define CONSOLE_COLOR_YELLOW   0xe
+#define CONSOLE_COLOR_WHITE    0xf
+

samples/C/driver.h

@@ -0,0 +1,52 @@
+#pragma once
+
+/* Copyright © 2011 Fritz Grimpen
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <lib/generic.h>
+#include <console/info.h>
+
+#define CONSOLE_DRV_CAP_CLEAR 0x01
+#define CONSOLE_DRV_CAP_SCROLL  0x02
+#define CONSOLE_DRV_CAP_SET_CURSOR 0x04
+
+// Input modifier keys
+typedef struct {
+	bool shift_left:1;
+	bool shift_right:1;
+	bool control_left:1;
+	bool control_right:1;
+	bool alt:1;
+	bool super:1;
+} console_modifiers_t;
+
+typedef struct {
+	char character;
+	console_modifiers_t* modifiers;
+} console_read_t;
+
+typedef struct {
+	int (*write)(console_info_t*, char);
+	console_read_t* (*read)(console_info_t*);
+
+	int capabilities;
+
+	int (*_clear)(console_info_t*);
+	int (*scroll)(console_info_t*, int32_t);
+	void (*setCursor)(console_info_t*, uint32_t, uint32_t);
+} console_driver_t;

samples/C/elf.h

@@ -0,0 +1,70 @@
+#pragma once
+
+/* Copyright © 2011 Lukas Martini
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <lib/generic.h>
+#include <tasks/scheduler.h>
+
+#define ELF_TYPE_NONE 0
+#define ELF_TYPE_REL 1
+#define ELF_TYPE_EXEC 2
+#define ELF_TYPE_DYN 3
+#define ELF_TYPE_CORE 4
+
+#define ELF_ARCH_NONE 0
+#define ELF_ARCH_386 3
+
+#define ELF_VERSION_CURRENT 1
+
+typedef struct {
+	unsigned char magic[4];
+	/* Note: There _is_ other stuff in here, but we don't need it */
+	unsigned char pad[12]; 
+} __attribute__((packed)) elf_ident_t;
+
+typedef struct {
+	uint32_t    type;
+	uint32_t    offset;
+	void*		virtaddr;
+	void*		physaddr;
+	uint32_t    filesize;
+	uint32_t    memsize;
+	uint32_t    flags;
+	uint32_t    alignment;
+} __attribute__((packed)) elf_program_t;
+
+typedef struct {
+	elf_ident_t ident;
+	uint16_t	type;		/* Object file type */
+	uint16_t	machine;	/* Architecture */
+	uint32_t	version;	/* Object file version */
+	void*		entry;		/* Entry point virtual address */
+	uint32_t	phoff;		/* Program header table file offset */
+	uint32_t	shoff;		/* Section header table file offset */
+	uint32_t	flags;		/* Processor-specific flags */
+	uint16_t	ehsize;		/* ELF header size in bytes */
+	uint16_t	phentsize;	/* Program header table entry size */
+	uint16_t	phnum;		/* Program header table entry count */
+	uint16_t	shentsize;	/* Section header table entry size */
+	uint16_t	shnum;		/* Section header table entry count */
+	uint16_t	shstrndx;	/* Section header string table index */
+} __attribute__((packed)) elf_t;
+
+task_t* elf_load(elf_t* bin, char* name, char** environ, char** argv, int argc);
+task_t* elf_load_file(char* path, char** environ, char** argv, int argc);

samples/C/filter.h

@@ -0,0 +1,45 @@
+#pragma once
+
+/* Copyright © 2011 Fritz Grimpen
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <lib/generic.h>
+#include <console/info.h>
+#include <console/driver.h>
+
+struct console_filter {
+	// General callback for all actions etc.
+	// Preferred prototype:
+	// char <name>(char c, console_info_t *info, console_driver_t *input, console_driver_t *output);
+	char (*callback)(char, console_info_t*, console_driver_t*, console_driver_t*);
+
+	// Specific callbacks for read and write
+	// Preferred prototype:
+	// char <name>(char c, console_info_t *info, console_driver_t *input);
+	char (*read_callback)(char, console_info_t*, console_driver_t*);
+
+	// Preferred prototype:
+	// char <name>(char c, console_info_t *info, console_driver_t *output);
+	char (*write_callback)(char, console_info_t*, console_driver_t*);
+
+	// The next filter in the filter chain
+	struct console_filter* next;
+};
+
+typedef struct console_filter console_filter_t;
+

samples/C/info.h

@@ -0,0 +1,44 @@
+#pragma once
+
+/* Copyright © 2011 Fritz Grimpen
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <lib/generic.h>
+#include <console/color.h>
+
+typedef struct {
+	uint32_t cursor_x;
+	uint32_t cursor_y;
+
+	uint32_t rows;
+	uint32_t columns;
+
+	uint32_t tabstop;
+
+	console_color_t default_color;
+	console_color_t current_color;
+
+	uint8_t nonblocking;
+	uint8_t reverse_video;
+	uint8_t bold;
+	uint8_t blink;
+	uint8_t underline;
+	uint8_t newline_mode;
+	uint8_t auto_echo;
+	uint8_t handle_backspace;
+} console_info_t;

samples/C/interface.h

@@ -0,0 +1,47 @@
+#pragma once
+
+/* Copyright © 2011 Fritz Grimpen
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <lib/generic.h>
+#include <console/info.h>
+#include <console/filter.h>
+#include <console/driver.h>
+
+typedef struct {
+	console_info_t info;
+
+	console_filter_t* input_filter;
+	console_filter_t* output_filter;
+
+	console_driver_t* input_driver;
+	console_driver_t* output_driver;
+} console_t;
+
+console_t* default_console;
+
+// Generate raw console, connected to the Display, Keyboard and the
+// ECMA-48-Filter
+void console_init();
+
+size_t console_write(console_t* console, const char* buffer, int32_t length);
+#define console_write2(console, buffer) console_write(console, buffer, strlen(buffer))
+size_t console_read(console_t* console, char* buffer, size_t length);
+size_t console_scroll(console_t* console, int32_t pages);
+
+void console_clear(console_t* console);

samples/C/ip4.h

@@ -0,0 +1,50 @@
+#pragma once
+
+/* Copyright © 2011 Lukas Martini
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <lib/generic.h>
+#include <net/net.h>
+
+#define IP4_TOS_ICMP 0
+
+typedef uint32_t ip4_addr_t;
+
+typedef struct {
+	unsigned int hl:4; /* both fields are 4 bits */
+	unsigned int version:4;
+	uint8_t	tos;
+	uint16_t len;
+	uint16_t id;
+	uint16_t off;
+	uint8_t ttl;
+	uint8_t p;
+	uint16_t checksum;
+	ip4_addr_t src;
+	ip4_addr_t dst;
+} ip4_header_t;
+
+typedef struct {
+	uint8_t type;
+	uint8_t code;
+	uint16_t checksum;
+	uint16_t id;
+	uint16_t sequence;
+} ip4_icmp_header_t;
+
+void ip4_receive(net_device_t* origin, net_l2proto_t proto, size_t size, void* raw);

samples/C/multiboot.h

@@ -0,0 +1,110 @@
+#pragma once
+
+/* Copyright © 2010, 2011 Lukas Martini
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <lib/generic.h>
+
+#define MULTIBOOT_KERNELMAGIC 0x2BADB002
+
+#define MULTIBOOT_FLAG_MEM	 0x001
+#define MULTIBOOT_FLAG_DEVICE  0x002
+#define MULTIBOOT_FLAG_CMDLINE 0x004
+#define MULTIBOOT_FLAG_MODS	0x008
+#define MULTIBOOT_FLAG_AOUT	0x010
+#define MULTIBOOT_FLAG_ELF	 0x020
+#define MULTIBOOT_FLAG_MMAP	0x040
+#define MULTIBOOT_FLAG_CONFIG  0x080
+#define MULTIBOOT_FLAG_LOADER  0x100
+#define MULTIBOOT_FLAG_APM	 0x200
+#define MULTIBOOT_FLAG_VBE	 0x400
+
+// The symbol table for a.out.
+typedef struct
+{
+	uint32_t	 tabSize;
+	uint32_t 	strSize;
+	uint32_t 	addr;
+	uint32_t 	reserved;
+} __attribute__((packed)) multiboot_aoutSymbolTable_t;
+     
+// The section header table for ELF.
+typedef struct
+{
+	uint32_t	 num;
+	uint32_t	 size;
+	uint32_t 	addr;
+	uint32_t	 shndx;
+} __attribute__((packed)) multiboot_elfSectionHeaderTable_t;
+
+typedef struct
+{
+	uint32_t	size;
+	uint64_t	addr;
+	uint64_t	length;
+	uint32_t	type;
+} __attribute__((packed)) multiboot_memoryMap_t;
+
+typedef struct
+{
+	uint32_t	start;
+	uint32_t	end;
+	char*		cmdLine;
+	uint32_t	reserved;
+} __attribute__((packed)) multiboot_module_t;
+
+typedef struct
+{
+	uint32_t	flags;
+	uint32_t	memLower;
+	uint32_t	memUpper;
+	uint32_t	bootDevice;
+	char*	cmdLine;
+	uint32_t	modsCount;
+	multiboot_module_t*	modsAddr;
+
+	union
+	{
+		multiboot_aoutSymbolTable_t aoutSym;
+		multiboot_elfSectionHeaderTable_t elfSec;
+	} u;
+	
+	uint32_t	mmapLength;
+	uint32_t	mmapAddr;
+	
+	uint32_t drivesLength;
+	uint32_t drivesAddr;
+	
+	// ROM configuration table
+	uint32_t configTable;
+	
+	char* bootLoaderName;
+	uint32_t apmTable;
+	
+	// Video
+	uint32_t vbeControlInfo;
+	uint32_t vbeModeInfo;
+	uint16_t vbeMode;
+	uint16_t vbeInterfaceSeg;
+	uint16_t vbeInterfaceOff;
+	uint16_t vbeInterfaceLen;
+} __attribute__((packed)) multiboot_info_t;
+
+multiboot_info_t* multiboot_info;
+
+void arch_multiboot_printInfo();

samples/C/portio.h

@@ -0,0 +1,43 @@
+#pragma once
+
+/* Copyright © 2011 Lukas Martini
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <lib/generic.h>
+#include <lib/stdint.h>
+
+// Legacy
+#define outb(args...) portio_out8(args)
+#define outw(args...) portio_out16(args)
+#define outl(args...) portio_out32(args)
+#define outq(args...) portio_out64(args)
+
+#define inb(args...) portio_in8(args)
+#define inw(args...) portio_in16(args)
+#define inl(args...) portio_in32(args)
+#define inq(args...) portio_in64(args)
+
+void portio_out8(uint16_t port, uint8_t value);
+void portio_out16(uint16_t port, uint16_t value);
+void portio_out32(uint16_t port, uint32_t value);
+void portio_out64(uint16_t port, uint64_t value);
+
+uint8_t portio_in8(uint16_t port);
+uint16_t portio_in16(uint16_t port);
+uint32_t portio_in32(uint16_t port);
+uint64_t portio_in64(uint16_t port);

samples/C/pqiv.h

@@ -0,0 +1,166 @@
+/**
+ * pqiv
+ *
+ * Copyright (c) 2013-2014, Phillip Berndt
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+// This file contains the definition of files, image types and
+// the plugin infrastructure. It should be included in file type
+// handlers.
+
+#ifndef _PQIV_H_INCLUDED
+#define _PQIV_H_INCLUDED
+
+#include <glib.h>
+#include <gtk/gtk.h>
+#include <gio/gio.h>
+#include "lib/bostree.h"
+
+#ifndef PQIV_VERSION
+#define PQIV_VERSION "2.3"
+#endif
+
+#define FILE_FLAGS_ANIMATION      (guint)(1)
+#define FILE_FLAGS_MEMORY_IMAGE   (guint)(1<<1)
+
+// The structure for images {{{
+typedef struct file_type_handler_struct_t file_type_handler_t;
+typedef struct {
+	// File type
+	const file_type_handler_t *file_type;
+
+	// Special flags
+	// FILE_FLAGS_ANIMATION        -> Animation functions are invoked
+	//                                Set by file type handlers
+	// FILE_FLAGS_MEMORY_IMAGE     -> File lives in memory
+	guint file_flags;
+
+	// The file name to display and to sort by
+	gchar *display_name;
+
+	// The URI or file name of the file
+	gchar *file_name;
+
+	// If the file is a memory image, the actual image data
+	GBytes *file_data;
+
+	// The file monitor structure is used for inotify-watching of
+	// the files
+	GFileMonitor *file_monitor;
+
+	// This flag stores whether this image is currently loaded
+	// and valid. i.e. if it is set, you can assume that
+	// private_data contains a representation of the image;
+	// if not, you can NOT assume that it does not.
+	gboolean is_loaded;
+
+	// Cached image size
+	guint width;
+	guint height;
+
+	// File-type specific data, allocated and freed by the file type handlers
+	void *private;
+} file_t;
+// }}}
+// Definition of the built-in file types {{{
+
+// If you want to implement your own file type, you'll have to implement the
+// following functions and a non-static initialization function named
+// file_type_NAME_initializer that fills a file_type_handler_t with pointers to
+// the functions. Store the file in backends/NAME.c and adjust the Makefile to
+// add the required libraries if your backend is listed in the $(BACKENDS)
+// variable.
+
+typedef enum { PARAMETER, RECURSION, INOTIFY, BROWSE_ORIGINAL_PARAMETER, FILTER_OUTPUT } load_images_state_t;
+// Allocation function: Allocate the ->private structure within a file and add the
+// image(s) to the list of available images via load_images_handle_parameter_add_file()
+// If an image is not to be loaded for any reason, the file structure should be
+// deallocated using file_free()
+// Returns a pointer to the first added image
+// Optional, you can also set the pointer to this function to NULL.
+typedef BOSNode *(*file_type_alloc_fn_t)(load_images_state_t state, file_t *file);
+
+// Deallocation, if a file is removed from the images list. Free the ->private structure.
+// Only called if ->private is non-NULL.
+typedef void (*file_type_free_fn_t)(file_t *file);
+
+// Actually load a file into memory
+typedef void (*file_type_load_fn_t)(file_t *file, GInputStream *data, GError **error_pointer);
+
+// Unload a file
+typedef void (*file_type_unload_fn_t)(file_t *file);
+
+// Animation support: Initialize memory for animations, return ms until first frame
+// Optional, you can also set the pointer to this function to NULL.
+typedef double (*file_type_animation_initialize_fn_t)(file_t *file);
+
+// Animation support: Advance to the next frame, return ms until next frame
+// Optional, you can also set the pointer to this function to NULL.
+typedef double (*file_type_animation_next_frame_fn_t)(file_t *file);
+
+// Draw the current view to a cairo context
+typedef void (*file_type_draw_fn_t)(file_t *file, cairo_t *cr);
+
+struct file_type_handler_struct_t {
+	// All files will be filtered with this filter. If it lets it pass,
+	// a handler is assigned to a file. If none do, the file is
+	// discarded if it was found during directory traversal or
+	// loaded using the first image backend if it was an explicit
+	// parameter.
+	GtkFileFilter *file_types_handled;
+
+	// Pointers to the functions defined above
+	file_type_alloc_fn_t alloc_fn;
+	file_type_free_fn_t free_fn;
+	file_type_load_fn_t load_fn;
+	file_type_unload_fn_t unload_fn;
+	file_type_animation_initialize_fn_t animation_initialize_fn;
+	file_type_animation_next_frame_fn_t animation_next_frame_fn;
+	file_type_draw_fn_t draw_fn;
+};
+
+// Initialization function: Tell pqiv about a backend
+typedef void (*file_type_initializer_fn_t)(file_type_handler_t *info);
+
+// pqiv symbols available to plugins {{{
+
+// Global cancellable that should be used for every i/o operation
+extern GCancellable *image_loader_cancellable;
+
+// Current scale level. For backends that don't support cairo natively.
+extern gdouble current_scale_level;
+
+// Load a file from disc/memory/network
+GInputStream *image_loader_stream_file(file_t *file, GError **error_pointer);
+
+// Add a file to the list of loaded files
+// Should be called at least once in a file_type_alloc_fn_t, with the state being
+// forwarded unaltered.
+BOSNode *load_images_handle_parameter_add_file(load_images_state_t state, file_t *file);
+
+// Load all data from an input stream into memory, conveinience function
+GBytes *g_input_stream_read_completely(GInputStream *input_stream, GCancellable *cancellable, GError **error_pointer);
+
+// Free a file
+void file_free(file_t *file);
+
+// }}}
+
+// File type handlers, used in the initializer and file type guessing
+extern file_type_handler_t file_type_handlers[];
+
+/* }}} */
+
+#endif

samples/C/scheduler.h

@@ -0,0 +1,69 @@
+#pragma once
+
+/* Copyright © 2011 Lukas Martini
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <lib/generic.h>
+#include <hw/cpu.h>
+#include <memory/vmem.h>
+
+#define SCHEDULER_MAXNAME 256
+#define SCHEDULER_TASK_PATH_MAX 256
+
+// Single linked list
+typedef struct task {
+	uint32_t pid;
+	char name[SCHEDULER_MAXNAME];
+	struct task *parent;
+	cpu_state_t* state;
+	struct task* next;
+	struct task* previous;
+
+	void* stack;
+	void* entry;
+	struct vmem_context *memory_context;
+
+	// Current task state
+	enum {
+		TASK_STATE_KILLED,
+		TASK_STATE_TERMINATED,
+		TASK_STATE_BLOCKING,
+		TASK_STATE_STOPPED,
+		TASK_STATE_RUNNING
+	} task_state;
+
+	char** environ;
+	char** argv;
+	int argc;
+
+	// TODO Is this actually the same as PATH_MAX in our toolchain?
+	char cwd[SCHEDULER_TASK_PATH_MAX + 1];
+} task_t;
+
+int scheduler_state;
+
+task_t* scheduler_new(void* entry, task_t* parent, char name[SCHEDULER_MAXNAME],
+	char** environ, char** argv, int argc, struct vmem_context* memory_context, bool map_structs);
+void scheduler_add(task_t *task);
+void scheduler_terminate_current();
+task_t* scheduler_get_current();
+task_t* scheduler_select(cpu_state_t* lastRegs);
+void scheduler_init();
+void scheduler_yield();
+void scheduler_remove(task_t *t);
+task_t* scheduler_fork(task_t* to_fork, cpu_state_t* state);

samples/C/syscalls.h

@@ -0,0 +1,95 @@
+#pragma once
+
+/* Copyright © 2011 Fritz Grimpen
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <lib/generic.h>
+#include <tasks/syscall.h>
+
+#include "syscalls/write.h"
+#include "syscalls/exit.h"
+#include "syscalls/getpid.h"
+#include "syscalls/getppid.h"
+#include "syscalls/read.h"
+#include "syscalls/brk.h"
+#include "syscalls/mmap.h"
+#include "syscalls/munmap.h"
+#include "syscalls/test.h"
+#include "syscalls/hostname.h"
+#include "syscalls/uname.h"
+#include "syscalls/open.h"
+#include "syscalls/execve.h"
+#include "syscalls/seek.h"
+#include "syscalls/opendir.h"
+#include "syscalls/readdir.h"
+#include "syscalls/kill.h"
+#include "syscalls/getexecdata.h"
+#include "syscalls/cwd.h"
+#include "syscalls/fork.h"
+
+syscall_t syscall_table[] = {
+	NULL,
+	sys_exit,			// 1
+	sys_read,			// 2
+	sys_write,			// 3
+	sys_getpid,			// 4
+	sys_brk,			// 5
+	sys_getppid,		// 6
+	sys_mmap,			// 7
+	sys_munmap,			// 8
+	sys_test,			// 9
+	sys_get_hostname,	// 10
+	sys_set_hostname,	// 11
+	sys_uname,			// 12
+	sys_open,			// 13
+	sys_execve,			// 14
+	sys_seek,			// 15
+	sys_opendir,		// 16
+	sys_readdir,		// 17
+	sys_kill,			// 18
+	sys_getexecdata,	// 19
+	sys_chdir,			// 20
+	sys_getcwd,			// 21
+	sys_fork,			// 22
+};
+
+char* syscall_name_table[] = {
+	NULL,
+	"exit",			// 1
+	"read",			// 2
+	"write",		// 3
+	"getpid",		// 4
+	"brk",			// 5
+	"getppid",		// 6
+	"mmap",			// 7
+	"munmap",		// 8
+	"test",			// 9
+	"get_hostname",	// 10
+	"set_hostname",	// 11
+	"uname",		// 12
+	"open",			// 13
+	"execve",		// 14
+	"seek",			// 15
+	"opendir",		// 16
+	"readdir",		// 17
+	"kill",			// 18
+	"getexecdata",	// 19
+	"chdir",		// 20
+	"getcwd",		// 21
+	"fork",			// 22
+};

samples/C/vfs.h

@@ -0,0 +1,56 @@
+#pragma once
+
+/* Copyright © 2010, 2011 Lukas Martini
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <lib/generic.h>
+
+#define VFS_SEEK_SET 0
+#define VFS_SEEK_CUR 1
+#define VFS_SEEK_END 2
+
+typedef struct {
+   uint64_t num;
+   char path[512];
+   char mount_path[512];
+   uint32_t offset;
+   uint32_t mountpoint;
+} vfs_file_t;
+
+typedef struct {
+   uint64_t num;
+   char path[512];
+   char mount_path[512];
+   uint32_t mountpoint;
+} vfs_dir_t;
+
+typedef void* (*vfs_read_callback_t)(char* path, uint32_t offset, uint32_t size);
+typedef char* (*vfs_read_dir_callback_t)(char* path, uint32_t offset);
+
+
+// Used to always store the last read/write attempt (used for kernel panic debugging)
+char vfs_last_read_attempt[512];
+
+vfs_file_t* vfs_get_from_id(uint32_t id);
+vfs_dir_t* vfs_get_dir_from_id(uint32_t id);
+void* vfs_read(vfs_file_t* fp, uint32_t size);
+char* vfs_dir_read(vfs_dir_t* dir, uint32_t offset);
+void vfs_seek(vfs_file_t* fp, uint32_t offset, int origin);
+vfs_file_t* vfs_open(char* path);
+vfs_dir_t* vfs_dir_open(char* path);
+int vfs_mount(char* path, vfs_read_callback_t read_callback, vfs_read_dir_callback_t read_dir_callback);

samples/C/vmem.h

@@ -0,0 +1,94 @@
+#pragma once
+
+/* Copyright © 2011 Fritz Grimpen
+ * Copyright © 2013 Lukas Martini
+ *
+ * This file is part of Xelix.
+ *
+ * Xelix is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Xelix is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Xelix. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <lib/generic.h>
+
+struct vmem_context;
+
+struct vmem_page
+{
+	enum
+	{
+		VMEM_SECTION_STACK,   /* Initial stack */
+		VMEM_SECTION_CODE,    /* Contains program code and is read-only */
+		VMEM_SECTION_DATA,    /* Contains static data */
+		VMEM_SECTION_HEAP,    /* Allocated by brk(2) at runtime */
+		VMEM_SECTION_MMAP,    /* Allocated by mmap(2) at runtime */
+		VMEM_SECTION_KERNEL,  /* Contains kernel-internal data */
+		VMEM_SECTION_UNMAPPED /* Unmapped */
+	} section;
+
+	bool readonly:1;
+	bool cow:1; /* Copy-on-Write mechanism */
+	bool allocated:1;
+
+	void *cow_src_addr;
+	void *virt_addr;
+	void *phys_addr;
+};
+
+typedef void (*vmem_iterator_t)(struct vmem_context *, struct vmem_page *, uint32_t);
+
+/* Initialize vmem_kernelContext for paging_init() */
+void vmem_init();
+struct vmem_context *vmem_kernelContext;
+struct vmem_context *vmem_currentContext;
+struct vmem_context *vmem_processContext;
+void *vmem_faultAddress;
+
+/* Some callbacks for magic functions */
+void (*vmem_applyPage)(struct vmem_context *, struct vmem_page *);
+
+/* Generate new page context */
+struct vmem_context *vmem_new();
+struct vmem_page *vmem_new_page();
+
+int vmem_add_page(struct vmem_context *ctx, struct vmem_page *pg);
+
+struct vmem_page *vmem_get_page_phys(struct vmem_context *ctx, void *phys_addr);
+struct vmem_page *vmem_get_page_virt(struct vmem_context *ctx, void *virt_addr);
+struct vmem_page *vmem_get_page(struct vmem_context *ctx, uint32_t offset);
+
+/* Remove pages in a specific context by physical or virtual address */
+struct vmem_page *vmem_rm_page_phys(struct vmem_context *ctx, void *phys_addr);
+struct vmem_page *vmem_rm_page_virt(struct vmem_context *ctx, void *virt_addr);
+
+/* Iterator */
+int vmem_iterate(struct vmem_context *ctx, vmem_iterator_t callback);
+
+uint32_t vmem_count_pages(struct vmem_context *ctx);
+void vmem_dump_page(struct vmem_page *pg);
+void vmem_dump(struct vmem_context *ctx);
+void vmem_handle_fault(uint32_t code, void *addr, void *instruction);
+
+/* Get/Set cached paging context */
+void vmem_set_cache(struct vmem_context *ctx, void *cache);
+void *vmem_get_cache(struct vmem_context *ctx);
+
+#ifdef __i386__
+	#define PAGE_SIZE 4096
+	#define VMEM_ALIGN(x) (typeof(x))(((intptr_t)(x) & 0xFFFFF000) + 0x1000)
+	#define VMEM_ALIGN_DOWN(x) (typeof(x))( \
+		((intptr_t)(x) - ((intptr_t)(x) % PAGE_SIZE)))
+#else
+	#define PAGE_SIZE 0
+	#define VMEM_ALIGN(x) (x)
+#endif

samples/CLIPS/demo.clp

@@ -0,0 +1,343 @@
+;;;***************************
+;;;* DEFFACTS KNOWLEDGE BASE *
+;;;***************************
+
+(deffacts MAIN::knowledge-base
+   (welcome (message WelcomeMessage))
+   (goal (variable type.animal))
+   (legalanswers (values yes no))
+   (displayanswers (values "Yes" "No"))
+   (rule (if backbone is yes) 
+         (then superphylum is backbone))
+   (rule (if backbone is no) 
+         (then superphylum is jellyback))
+   (question (variable backbone)
+             (query backbone.query))
+   (rule (if superphylum is backbone and
+          warm.blooded is yes) 
+         (then phylum is warm))
+   (rule (if superphylum is backbone and
+          warm.blooded is no) 
+         (then phylum is cold))
+   (question (variable warm.blooded)
+             (query warm.blooded.query))
+   (rule (if superphylum is jellyback and
+          live.prime.in.soil is yes) 
+         (then phylum is soil))
+   (rule (if superphylum is jellyback and
+          live.prime.in.soil is no) 
+         (then phylum is elsewhere))
+   (question (variable live.prime.in.soil)
+             (query live.prime.in.soil.query))
+   (rule (if phylum is warm and
+          has.breasts is yes) 
+         (then class is breasts))
+   (rule (if phylum is warm and
+          has.breasts is no) 
+         (then type.animal is bird))
+   (question (variable has.breasts)
+             (query has.breasts.query))
+   (rule (if phylum is cold and
+          always.in.water is yes) 
+         (then class is water))
+   (rule (if phylum is cold and
+          always.in.water is no) 
+         (then class is dry))
+   (question (variable always.in.water)
+             (query always.in.water.query))
+   (rule (if phylum is soil and
+          flat.bodied is yes) 
+         (then type.animal is flatworm))
+   (rule (if phylum is soil and
+          flat.bodied is no) 
+         (then type.animal is worm.leech))
+   (question (variable flat.bodied)
+             (query flat.bodied.query))
+   (rule (if phylum is elsewhere and
+          body.in.segments is yes) 
+         (then class is segments))
+   (rule (if phylum is elsewhere and
+          body.in.segments is no) 
+         (then class is unified))
+   (question (variable body.in.segments)
+             (query body.in.segments.query))
+   (rule (if class is breasts and
+          can.eat.meat is yes) 
+         (then order is meat))
+   (rule (if class is breasts and
+          can.eat.meat is no) 
+         (then order is vegy))
+   (question (variable can.eat.meat)
+             (query can.eat.meat.query))
+   (rule (if class is water and
+          boney is yes) 
+         (then type.animal is fish))
+   (rule (if class is water and
+          boney is no) 
+         (then type.animal is shark.ray))
+   (question (variable boney)
+             (query boney.query))
+   (rule (if class is dry and
+          scaly is yes) 
+         (then order is scales))
+   (rule (if class is dry and
+          scaly is no) 
+         (then order is soft))
+   (question (variable scaly)
+             (query scaly.query))
+   (rule (if class is segments and
+          shell is yes) 
+         (then order is shell))
+   (rule (if class is segments and
+          shell is no) 
+         (then type.animal is centipede.millipede.insect))
+   (question (variable shell)
+             (query shell.query))
+   (rule (if class is unified and
+          digest.cells is yes) 
+         (then order is cells))
+   (rule (if class is unified and
+          digest.cells is no) 
+         (then order is stomach))
+   (question (variable digest.cells)
+             (query digest.cells.query))
+   (rule (if order is meat and
+          fly is yes) 
+         (then type.animal is bat))
+   (rule (if order is meat and
+          fly is no) 
+         (then family is nowings))
+   (question (variable fly)
+             (query fly.query))
+   (rule (if order is vegy and
+          hooves is yes) 
+         (then family is hooves))
+   (rule (if order is vegy and
+          hooves is no) 
+         (then family is feet))
+   (question (variable hooves)
+             (query hooves.query))
+   (rule (if order is scales and
+          rounded.shell is yes) 
+         (then type.animal is turtle))
+   (rule (if order is scales and
+          rounded.shell is no) 
+         (then family is noshell))
+   (question (variable rounded.shell)
+             (query rounded.shell.query))
+   (rule (if order is soft and
+          jump is yes) 
+         (then type.animal is frog))
+   (rule (if order is soft and
+          jump is no) 
+         (then type.animal is salamander))
+   (question (variable jump)
+             (query jump.query))
+   (rule (if order is shell and
+          tail is yes) 
+         (then type.animal is lobster))
+   (rule (if order is shell and
+          tail is no) 
+         (then type.animal is crab))
+   (question (variable tail)
+             (query tail.query))
+   (rule (if order is cells and
+          stationary is yes) 
+         (then family is stationary))
+   (rule (if order is cells and
+          stationary is no) 
+         (then type.animal is jellyfish))
+   (question (variable stationary)
+             (query stationary.query))
+   (rule (if order is stomach and
+          multicelled is yes) 
+         (then family is multicelled))
+   (rule (if order is stomach and
+          multicelled is no) 
+         (then type.animal is protozoa))
+   (question (variable multicelled)
+             (query multicelled.query))
+   (rule (if family is nowings and
+          opposing.thumb is yes) 
+         (then genus is thumb))
+   (rule (if family is nowings and
+          opposing.thumb is no) 
+         (then genus is nothumb))
+   (question (variable opposing.thumb)
+             (query opposing.thumb.query))
+   (rule (if family is hooves and
+          two.toes is yes) 
+         (then genus is twotoes))
+   (rule (if family is hooves and
+          two.toes is no) 
+         (then genus is onetoe))
+   (question (variable two.toes)
+             (query two.toes.query))
+   (rule (if family is feet and
+          live.in.water is yes) 
+         (then genus is water))
+   (rule (if family is feet and
+          live.in.water is no) 
+         (then genus is dry))
+   (question (variable live.in.water)
+             (query live.in.water.query))
+   (rule (if family is noshell and
+          limbs is yes) 
+         (then type.animal is crocodile.alligator))
+   (rule (if family is noshell and
+          limbs is no) 
+         (then type.animal is snake))
+   (question (variable limbs)
+             (query limbs.query))
+   (rule (if family is stationary and
+          spikes is yes) 
+         (then type.animal is sea.anemone))
+   (rule (if family is stationary and
+          spikes is no) 
+         (then type.animal is coral.sponge))
+   (question (variable spikes)
+             (query spikes.query))
+   (rule (if family is multicelled and
+          spiral.shell is yes) 
+         (then type.animal is snail))
+   (rule (if family is multicelled and
+          spiral.shell is no) 
+         (then genus is noshell))
+   (question (variable spiral.shell)
+             (query spiral.shell.query))
+   (rule (if genus is thumb and
+          prehensile.tail is yes) 
+         (then type.animal is monkey))
+   (rule (if genus is thumb and
+          prehensile.tail is no) 
+         (then species is notail))
+   (question (variable prehensile.tail)
+             (query prehensile.tail.query))
+   (rule (if genus is nothumb and
+          over.400 is yes) 
+         (then species is 400))
+   (rule (if genus is nothumb and
+          over.400 is no) 
+         (then species is under400))
+   (question (variable over.400)
+             (query over.400.query))
+   (rule (if genus is twotoes and
+          horns is yes) 
+         (then species is horns))
+   (rule (if genus is twotoes and
+          horns is no) 
+         (then species is nohorns))
+   (question (variable horns)
+             (query horns.query))
+   (rule (if genus is onetoe and
+          plating is yes) 
+         (then type.animal is rhinoceros))
+   (rule (if genus is onetoe and
+          plating is no) 
+         (then type.animal is horse.zebra))
+   (question (variable plating)
+             (query plating.query))
+   (rule (if genus is water and
+          hunted is yes) 
+         (then type.animal is whale))
+   (rule (if genus is water and
+          hunted is no) 
+         (then type.animal is dolphin.porpoise))
+   (question (variable hunted)
+             (query hunted.query))
+   (rule (if genus is dry and
+          front.teeth is yes) 
+         (then species is teeth))
+   (rule (if genus is dry and
+          front.teeth is no) 
+         (then species is noteeth))
+   (question (variable front.teeth)
+             (query front.teeth.query))
+   (rule (if genus is noshell and
+          bivalve is yes) 
+         (then type.animal is clam.oyster))
+   (rule (if genus is noshell and
+          bivalve is no) 
+         (then type.animal is squid.octopus))
+   (question (variable bivalve)
+             (query bivalve.query))
+   (rule (if species is notail and
+          nearly.hairless is yes) 
+         (then type.animal is man))
+   (rule (if species is notail and
+          nearly.hairless is no) 
+         (then subspecies is hair))
+   (question (variable nearly.hairless)
+             (query nearly.hairless.query))
+   (rule (if species is 400 and
+          land.based is yes) 
+         (then type.animal is bear.tiger.lion))
+   (rule (if species is 400 and
+          land.based is no) 
+         (then type.animal is walrus))
+   (question (variable land.based)
+             (query land.based.query))
+   (rule (if species is under400 and
+          thintail is yes) 
+         (then type.animal is cat))
+   (rule (if species is under400 and
+          thintail is no) 
+         (then type.animal is coyote.wolf.fox.dog))
+   (question (variable thintail)
+             (query thintail.query))
+   (rule (if species is nohorns and
+          lives.in.desert is yes) 
+         (then type.animal is camel))
+   (rule (if species is nohorns and
+          lives.in.desert is no and
+          semi.aquatic is no) 
+         (then type.animal is giraffe))
+   (rule (if species is nohorns and
+          lives.in.desert is no and
+          semi.aquatic is yes) 
+         (then type.animal is hippopotamus))
+   (question (variable lives.in.desert)
+             (query lives.in.desert.query))
+   (question (variable semi.aquatic)
+             (query semi.aquatic.query))
+   (rule (if species is teeth and
+          large.ears is yes) 
+         (then type.animal is rabbit))
+   (rule (if species is teeth and
+          large.ears is no)
+         (then type.animal is rat.mouse.squirrel.beaver.porcupine))
+   (question (variable large.ears)
+             (query large.ears.query))
+   (rule (if species is noteeth and
+          pouch is yes) 
+         (then type.animal is kangaroo.koala.bear))
+   (rule (if species is noteeth and
+          pouch is no) 
+         (then type.animal is mole.shrew.elephant))
+   (question (variable pouch)
+             (query pouch.query))
+   (rule (if subspecies is hair and
+          long.powerful.arms is yes) 
+         (then type.animal is orangutan.gorilla.chimpanzee))
+   (rule (if subspecies is hair and
+          long.powerful.arms is no) 
+         (then type.animal is baboon))
+   (question (variable long.powerful.arms)
+             (query long.powerful.arms.query))
+   (rule (if species is horns and
+          fleece is yes) 
+         (then type.animal is sheep.goat))
+   (rule (if species is horns and
+          fleece is no) 
+         (then subsubspecies is nofleece))
+   (question (variable fleece)
+             (query fleece.query))
+   (rule (if subsubspecies is nofleece and
+          domesticated is yes) 
+         (then type.animal is cow))
+   (rule (if subsubspecies is nofleece and
+          domesticated is no) 
+         (then type.animal is deer.moose.antelope))
+   (question (variable domesticated)
+             (query domesticated.query))
+   (answer (prefix "I think your animal is a ") (variable type.animal) (postfix ".")))

samples/CLIPS/sudoku.clp

@@ -0,0 +1,281 @@
+;;; http://www.angusj.com/sudoku/hints
+;;; http://www.scanraid.com/BasicStrategies.htm
+;;; http://www.sudokuoftheday.com/pages/techniques-overview
+;;; http://www.sudokuonline.us/sudoku_solving_techniques
+;;; http://www.sadmansoftware.com/sudoku/techniques.htm
+;;; http://www.krazydad.com/blog/2005/09/29/an-index-of-sudoku-strategies/
+
+;;; #######################
+;;; DEFTEMPLATES & DEFFACTS
+;;; #######################
+
+(deftemplate possible
+   (slot row)
+   (slot column)
+   (slot value)
+   (slot group)
+   (slot id))
+   
+(deftemplate impossible
+   (slot id)
+   (slot value)
+   (slot priority)
+   (slot reason))
+   
+(deftemplate technique-employed
+   (slot reason)
+   (slot priority))
+
+(deftemplate technique
+   (slot name)
+   (slot priority))
+   
+(deffacts startup
+   (phase grid-values))
+
+(deftemplate size-value
+   (slot size)
+   (slot value))
+   
+(deffacts values
+   (size-value (size 1) (value 1))
+   (size-value (size 2) (value 2))
+   (size-value (size 2) (value 3))
+   (size-value (size 2) (value 4))
+   (size-value (size 3) (value 5))
+   (size-value (size 3) (value 6))
+   (size-value (size 3) (value 7))
+   (size-value (size 3) (value 8))
+   (size-value (size 3) (value 9))
+   (size-value (size 4) (value 10))
+   (size-value (size 4) (value 11))
+   (size-value (size 4) (value 12))
+   (size-value (size 4) (value 13))
+   (size-value (size 4) (value 14))
+   (size-value (size 4) (value 15))
+   (size-value (size 4) (value 16))
+   (size-value (size 5) (value 17))
+   (size-value (size 5) (value 18))
+   (size-value (size 5) (value 19))
+   (size-value (size 5) (value 20))
+   (size-value (size 5) (value 21))
+   (size-value (size 5) (value 22))
+   (size-value (size 5) (value 23))
+   (size-value (size 5) (value 24))
+   (size-value (size 5) (value 25)))
+   
+;;; ###########
+;;; SETUP RULES
+;;; ###########
+
+;;; ***********
+;;; stress-test
+;;; ***********
+
+(defrule stress-test
+   
+   (declare (salience 10))
+   
+   (phase match)
+   
+   (stress-test)
+   
+   (priority ?last)
+   
+   (not (priority ?p&:(> ?p ?last)))
+   
+   (technique (priority ?next&:(> ?next ?last)))
+   
+   (not (technique (priority ?p&:(> ?p ?last)&:(< ?p ?next))))
+   
+   =>
+   
+   (assert (priority ?next)))
+   
+;;; *****************
+;;; enable-techniques
+;;; *****************
+
+(defrule enable-techniques
+
+   (declare (salience 10))
+   
+   (phase match)
+   
+   (size ?)
+   
+   (not (possible (value any)))
+   
+   =>
+   
+   (assert (priority 1)))
+
+;;; **********
+;;; expand-any
+;;; **********
+
+(defrule expand-any
+
+   (declare (salience 10))
+
+   (phase expand-any)
+   
+   ?f <- (possible (row ?r) (column ?c) (value any) (group ?g) (id ?id))
+  
+   (not (possible (value any) (id ?id2&:(< ?id2 ?id))))
+   
+   (size ?s)
+   
+   (size-value (size ?as&:(<= ?as ?s)) (value ?v))
+   
+   (not (possible (row ?r) (column ?c) (value ?v)))
+  
+   (not (and (size-value (value ?v2&:(< ?v2 ?v)))
+               
+             (not (possible (row ?r) (column ?c) (value ?v2)))))
+   
+   =>
+   
+   (assert (possible (row ?r) (column ?c) (value ?v) (group ?g) (id ?id))))
+   
+;;; *****************
+;;; position-expanded
+;;; *****************
+
+(defrule position-expanded
+
+   (declare (salience 10))
+
+   (phase expand-any)
+   
+   ?f <- (possible (row ?r) (column ?c) (value any) (group ?g) (id ?id))
+     
+   (size ?s)
+   
+   (not (and (size-value (size ?as&:(<= ?as ?s)) (value ?v))
+   
+             (not (possible (row ?r) (column ?c) (value ?v)))))
+
+   =>
+   
+   (retract ?f))
+   
+;;; ###########
+;;; PHASE RULES
+;;; ###########
+
+;;; ***************
+;;; expand-any-done
+;;; ***************
+
+(defrule expand-any-done
+
+   (declare (salience 10))
+
+   ?f <- (phase expand-any)
+
+   (not (possible (value any)))
+   
+   =>
+   
+   (retract ?f)
+   
+   (assert (phase initial-output))
+   (assert (print-position 1 1)))
+   
+;;; ***********
+;;; begin-match
+;;; ***********
+
+(defrule begin-match
+
+   (declare (salience -20))
+   
+   ?f <- (phase initial-output)
+      
+   =>
+   
+   (retract ?f)
+   
+   (assert (phase match)))
+
+;;; *****************
+;;; begin-elimination
+;;; *****************
+
+(defrule begin-elimination
+
+   (declare (salience -20))
+   
+   ?f <- (phase match)
+   
+   (not (not (impossible)))
+   
+   =>
+   
+   (retract ?f)
+   
+   (assert (phase elimination)))
+
+;;; *************
+;;; next-priority
+;;; *************
+
+(defrule next-priority
+
+   (declare (salience -20))
+   
+   (phase match)
+   
+   (not (impossible))
+   
+   (priority ?last)
+   
+   (not (priority ?p&:(> ?p ?last)))
+   
+   (technique (priority ?next&:(> ?next ?last)))
+   
+   (not (technique (priority ?p&:(> ?p ?last)&:(< ?p ?next))))
+   
+   =>
+   
+   (assert (priority ?next)))
+
+;;; ************
+;;; begin-output
+;;; ************
+
+(defrule begin-output
+
+   (declare (salience -20))
+   
+   ?f <- (phase match)
+   
+   (not (impossible))
+   
+   (priority ?last)
+   
+   (not (priority ?p&:(> ?p ?last)))
+
+   (not (technique (priority ?next&:(> ?next ?last))))
+   
+   =>
+   
+   (retract ?f)
+   
+   (assert (phase final-output))
+   (assert (print-position 1 1)))
+
+   
+
+  
+    
+   
+   
+   
+   
+   
+   
+   
+   
+   

samples/CMake/filenames/CMakeLists.txt

@@ -0,0 +1,12 @@
+cmake_minimum_required(VERSION 2.8)
+
+project(Foo)
+
+set(CMAKE_SKIP_RPATH TRUE)
+set(CMAKE_INSTALL_PREFIX "/usr/local")
+
+add_subdirectory(bar)
+
+add_executable(foo foo.c)
+target_link_libraries(foo pthread)
+install(TARGETS foo DESTINATION bin)

samples/CMake/sample1.cmake

@@ -0,0 +1,15 @@
+cmake_minimum_required(VERSION 2.6)
+
+enable_testing()
+
+set(CMAKE_BUILD_TYPE debug)
+
+include_directories("/usr/local/include")
+
+find_library(ssl_LIBRARY NAMES ssl PATHS "/usr/local/lib")
+
+add_custom_command(OUTPUT "ver.c" "ver.h" COMMAND ./ver.sh)
+
+add_executable(foo foo.c bar.c baz.c ver.c)
+
+target_link_libraries(foo ${ssl_LIBRARY})

samples/CMake/sample2.cmake

@@ -0,0 +1,25 @@
+cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
+ 
+project(PCLVisualizer)
+target_link_libraries (PCLVisualizer ${PCL_LIBRARIES})
+
+#it seems it's needed only on OS X 10.9
+find_package(GLEW REQUIRED)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/include -v")
+
+find_package(PCL 1.7 REQUIRED)
+ 
+include_directories(${PCL_INCLUDE_DIRS})
+link_directories(${PCL_LIBRARY_DIRS})
+add_definitions(${PCL_DEFINITIONS})
+ 
+set(PCL_BUILD_TYPE Release)
+ 
+file(GLOB PCL_openni_viewer_SRC
+    "src/*.h"
+    "src/*.cpp"
+)
+add_executable(PCLVisualizer ${PCL_openni_viewer_SRC})
+
+#add this line to solve probem in mac os x 10.9
+target_link_libraries(PCLVisualizer ${PCL_COMMON_LIBRARIES} ${PCL_IO_LIBRARIES} ${PCL_VISUALIZATION_LIBRARIES} ${PCL_FEATURES_LIBRARIES}) 

samples/CMake/sample3.cmake

@@ -0,0 +1,33 @@
+# Specifications for building user and development documentation.
+#
+# ====================================================================
+# Copyright (c) 2009 Ian Blumel.  All rights reserved.
+# 
+# This software is licensed as described in the file LICENSE, which
+# you should have received as part of this distribution.  
+# ====================================================================
+
+CMAKE_MINIMUM_REQUIRED(VERSION 2.6)
+
+FIND_FILE( SPHINX sphinx-build.exe) 
+
+# If we are windows call to the make.bat file, otherwise rely on the Makefile
+# to handle the processing.
+IF(WIN32)
+   SET(SPHINX_MAKE make.bat)
+ELSE(WIN32)
+   SET(SPHINX_MAKE make)
+ENDIF(WIN32)
+
+ADD_CUSTOM_TARGET(
+   doc_usr
+   COMMAND ${SPHINX_MAKE} html
+   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/usr
+)
+
+ADD_CUSTOM_TARGET(
+   doc_dev
+   COMMAND ${SPHINX_MAKE} html
+   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/dev
+)
+

samples/CMake/sample4.cmake

@@ -0,0 +1,33 @@
+cmake_minimum_required (VERSION 2.6)
+
+set (CMAKE_RUNTIME_OUTPUT_DIRECTORY "bin")
+
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/vala)
+find_package(Vala REQUIRED)
+include(ValaPrecompile)
+include(ValaVersion)
+ensure_vala_version("0.11.0" MINIMUM)
+
+project (template C)
+
+find_package(PkgConfig)
+
+pkg_check_modules(GOBJECT REQUIRED gobject-2.0)
+add_definitions(${GOBJECT_CFLAGS} ${GOBJECT_CFLAGS_OTHER})
+link_libraries(${GOBJECT_LIBRARIES})
+link_directories(${GOBJECT_LIBRARY_DIRS})
+
+
+vala_precompile(VALA_C
+    src/template.vala
+PACKAGES
+OPTIONS
+    --thread
+CUSTOM_VAPIS
+GENERATE_VAPI
+GENERATE_HEADER
+DIRECTORY
+	gen
+)
+
+add_executable("template" ${VALA_C})

samples/CMake/sample5.cmake

@@ -0,0 +1,89 @@
+# - Check if the STDCALL function exists.
+#   This works for non-cdecl functions (kernel32 functions, for example)
+# CHECK_STDCALL_FUNCTION_EXISTS(FUNCTION FUNCTION_DUMMY_ARGS VARIABLE)
+# - macro which checks if the stdcall function exists
+#  FUNCTION_DECLARATION - the definition of the function ( e.g.: Sleep(500) )
+#  VARIABLE - variable to store the result
+#
+# The following variables may be set before calling this macro to
+# modify the way the check is run:
+#
+#  CMAKE_REQUIRED_FLAGS = string of compile command line flags
+#  CMAKE_REQUIRED_DEFINITIONS = list of macros to define (-DFOO=bar)
+#  CMAKE_REQUIRED_INCLUDES = list of include directories
+#  CMAKE_REQUIRED_LIBRARIES = list of libraries to link
+#  CMAKE_EXTRA_INCLUDE_FILES = list of extra includes to check in
+
+MACRO(CHECK_STDCALL_FUNCTION_EXISTS FUNCTION_DECLARATION VARIABLE)
+  IF("${VARIABLE}" MATCHES "^${VARIABLE}$")
+    #get includes  
+    SET(CHECK_STDCALL_FUNCTION_PREMAIN)
+    FOREACH(def ${CMAKE_EXTRA_INCLUDE_FILES})
+      SET(CHECK_STDCALL_FUNCTION_PREMAIN "${CHECK_STDCALL_FUNCTION_PREMAIN}#include \"${def}\"\n")
+    ENDFOREACH(def)
+    
+    #add some default includes
+    IF ( HAVE_WINDOWS_H )
+        SET(CHECK_STDCALL_FUNCTION_PREMAIN "${CHECK_STDCALL_FUNCTION_PREMAIN}#include \"windows.h\"\n")
+    ENDIF ( HAVE_WINDOWS_H )
+    IF ( HAVE_UNISTD_H )
+        SET(CHECK_STDCALL_FUNCTION_PREMAIN "${CHECK_STDCALL_FUNCTION_PREMAIN}#include \"unistd.h\"\n")
+    ENDIF ( HAVE_UNISTD_H )
+    IF ( HAVE_DIRECT_H )
+        SET(CHECK_STDCALL_FUNCTION_PREMAIN "${CHECK_STDCALL_FUNCTION_PREMAIN}#include \"direct.h\"\n")
+    ENDIF ( HAVE_DIRECT_H )
+    IF ( HAVE_IO_H )
+        SET(CHECK_STDCALL_FUNCTION_PREMAIN "${CHECK_STDCALL_FUNCTION_PREMAIN}#include \"io.h\"\n")
+    ENDIF ( HAVE_IO_H )
+    IF ( HAVE_SYS_TIMEB_H )
+        SET(CHECK_STDCALL_FUNCTION_PREMAIN "${CHECK_STDCALL_FUNCTION_PREMAIN}#include \"sys/timeb.h\"\n")
+    ENDIF ( HAVE_SYS_TIMEB_H )
+    
+    STRING(REGEX REPLACE "(\\(.*\\))" "" CHECK_STDCALL_FUNCTION_EXISTS_FUNCTION ${FUNCTION_DECLARATION} )
+    
+    SET(MACRO_CHECK_STDCALL_FUNCTION_DEFINITIONS "${CMAKE_REQUIRED_FLAGS}")
+    MESSAGE(STATUS "Looking for ${CHECK_STDCALL_FUNCTION_EXISTS_FUNCTION}")
+    
+    IF(CMAKE_REQUIRED_LIBRARIES)
+      SET(CHECK_STDCALL_FUNCTION_EXISTS_ADD_LIBRARIES
+        "-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}")
+    ELSE(CMAKE_REQUIRED_LIBRARIES)
+      SET(CHECK_STDCALL_FUNCTION_EXISTS_ADD_LIBRARIES)
+    ENDIF(CMAKE_REQUIRED_LIBRARIES)
+    
+    IF(CMAKE_REQUIRED_INCLUDES)
+      SET(CHECK_STDCALL_FUNCTION_EXISTS_ADD_INCLUDES
+        "-DINCLUDE_DIRECTORIES:STRING=${CMAKE_REQUIRED_INCLUDES}")
+    ELSE(CMAKE_REQUIRED_INCLUDES)
+      SET(CHECK_STDCALL_FUNCTION_EXISTS_ADD_INCLUDES)
+    ENDIF(CMAKE_REQUIRED_INCLUDES)
+    
+    SET(CHECK_STDCALL_FUNCTION_DECLARATION ${FUNCTION_DECLARATION})
+    CONFIGURE_FILE("${clucene-shared_SOURCE_DIR}/cmake/CheckStdCallFunctionExists.cpp.in"
+      "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/CheckStdCallFunctionExists.cpp" IMMEDIATE @ONLY)
+    FILE(READ "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/CheckStdCallFunctionExists.cpp"
+      CHECK_STDCALL_FUNCTION_CONTENT)
+
+    TRY_COMPILE(${VARIABLE}
+      ${CMAKE_BINARY_DIR}
+      "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/CheckStdCallFunctionExists.cpp"
+      COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS}
+      CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_STDCALL_FUNCTION_DEFINITIONS}
+      "${CHECK_STDCALL_FUNCTION_EXISTS_ADD_LIBRARIES}"
+      "${CHECK_STDCALL_FUNCTION_EXISTS_ADD_INCLUDES}"
+      OUTPUT_VARIABLE OUTPUT)
+    IF(${VARIABLE})
+      SET(${VARIABLE} 1 CACHE INTERNAL "Have function ${FUNCTION_DECLARATION}")
+      MESSAGE(STATUS "Looking for ${FUNCTION_DECLARATION} - found")
+      FILE(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log 
+        "Determining if the stdcall function ${FUNCTION_DECLARATION} exists passed with the following output:\n"
+        "${OUTPUT}\nCheckStdCallFunctionExists.cpp:\n${CHECK_STDCALL_FUNCTION_CONTENT}\n\n")
+    ELSE(${VARIABLE})
+      MESSAGE(STATUS "Looking for ${FUNCTION_DECLARATION} - not found")
+      SET(${VARIABLE} "" CACHE INTERNAL "Have function ${FUNCTION_DECLARATION}")
+      FILE(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log 
+        "Determining if the stdcall function ${FUNCTION_DECLARATION} exists failed with the following output:\n"
+        "${OUTPUT}\nCheckStdCallFunctionExists.cpp:\n${CHECK_STDCALL_FUNCTION_CONTENT}\n\n")
+    ENDIF(${VARIABLE})
+  ENDIF("${VARIABLE}" MATCHES "^${VARIABLE}$")
+ENDMACRO(CHECK_STDCALL_FUNCTION_EXISTS)

samples/CMake/uninstall.cmake.in

@@ -0,0 +1,22 @@
+IF (NOT EXISTS "@PROJECT_BINARY_DIR@/install_manifest.txt")
+    MESSAGE (FATAL_ERROR "Cannot find install manifest: \"@PROJECT_BINARY_DIR@/install_manifest.txt\"")
+ENDIF (NOT EXISTS "@PROJECT_BINARY_DIR@/install_manifest.txt")
+
+FILE (READ "@PROJECT_BINARY_DIR@/install_manifest.txt" files)
+STRING (REGEX REPLACE "\n" ";" files "${files}")
+FOREACH (file ${files})
+    MESSAGE (STATUS "Uninstalling \"$ENV{DESTDIR}${file}\"")
+    IF (EXISTS "$ENV{DESTDIR}${file}")
+        EXEC_PROGRAM (
+            "@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\""
+            OUTPUT_VARIABLE rm_out
+            RETURN_VALUE rm_retval
+            )
+        IF (NOT "${rm_retval}" STREQUAL 0)
+            MESSAGE (FATAL_ERROR "Problem when removing \"$ENV{DESTDIR}${file}\"")
+        ENDIF (NOT "${rm_retval}" STREQUAL 0)
+    ELSE (EXISTS "$ENV{DESTDIR}${file}")
+        MESSAGE (STATUS "File \"$ENV{DESTDIR}${file}\" does not exist.")
+    ENDIF (EXISTS "$ENV{DESTDIR}${file}")
+ENDFOREACH (file)
+

samples/CSV/cars.csv

@@ -1,3 +1,3 @@
 Year,Make,Model,Length
 1997,Ford,E350,2.34
-2000,Mercury,Cougar,2.38
+2000,Mercury,Cougar,2.38

samples/Chapel/distributions.chpl

@@ -58,6 +58,34 @@ var BA: [BlockSpace] int;
 forall ba in BA do
   ba = here.id;
 
+//
+// The 'hasSingleLocalSubdomain' method on arrays will return true if the 
+// index set for a locale can be represented by a single domain.
+//
+if !BA.hasSingleLocalSubdomain() then
+  halt("For a Block distribution, the index set per locale should be \
+      represented by a single domain");
+
+//
+// If the distribution's subdomains can be represented as single subdomain,
+// we can use the 'localSubdomain' method to get the index set for the 
+// current locale.
+//
+// Below, we'll use the index set to confirm that the array elements have the 
+// correct locale id.
+//
+
+for L in Locales {
+  on L {
+    const indices = BA.localSubdomain();
+    for i in indices {
+      if BA[i] != L.id then
+        halt("Error: incorrect locale id");
+    }
+  }
+}
+
+
 //
 // Output the Block-distributed array to visually see how the elements
 // are partitioned across the locales.
@@ -104,6 +132,14 @@ writeln("Block Array Index Map");
 writeln(BA2);
 writeln();
 
+//
+// We can use the 'targetLocales' method available on an array to get the
+// locales array used as targets.
+//
+for (L, ML) in zip(BA2.targetLocales(), MyLocales) do
+  if L != ML then
+    halt("Error: BA2.targetLocales() should equal MyLocales");
+
 
 
 //
@@ -126,6 +162,18 @@ writeln("Cyclic Array Index Map");
 writeln(CA);
 writeln();
 
+//
+// The domain returned by 'localSubdomain' need not be a dense block, as is
+// the case for the Cyclic Distribution.
+//
+on Locales[0] {
+  const indices = CA.localSubdomain();
+  for i in indices {
+    if CA[i] != 0 then
+      halt("Error: Cyclic array values on Locale 0 should be zero");
+  }
+}
+
 
 //
 // Next, we'll declare a Block-Cyclic distribution.  These
@@ -134,7 +182,7 @@ writeln();
 // of indices.  Thus, the BlockCyclic distribution is parameterized
 // by a starting index (as with Cyclic) and a block size (per
 // dimension) specifying how large the chunks to be dealt out are.
-// 
+//
 const BlkCycSpace = Space dmapped BlockCyclic(startIdx=Space.low, 
                                               blocksize=(2, 3));
 var BCA: [BlkCycSpace] int;
@@ -146,6 +194,46 @@ writeln("Block-Cyclic Array Index Map");
 writeln(BCA);
 writeln();
 
+//
+// A locale's index set for a Block-Cyclic distribution cannot be represented
+// by a single subdomain.
+//
+if BCA.hasSingleLocalSubdomain() then
+  halt("A Block-Cyclic index set cannot be represented by a single subdomain");
+
+//
+// If the local index set cannot be represented by a single subdomain, 
+// we can use the 'localSubdomains' iterator to yield a number of domains
+// that represent the whole index set.
+//
+// Let's write a function that will use 'localSubdomains' to verify the 
+// correctness of the array values.
+//
+
+proc verifyID(Data: []) {
+  for L in Locales {
+    on L {
+      for indices in Data.localSubdomains() {
+        for i in indices {
+          if Data[i] != L.id then
+            halt("Error: incorrect locale id");
+        }
+      }
+    }
+  }
+}
+verifyID(BCA);
+
+//
+// The 'localSubdomains' iterator is also available on distributions that
+// can represent a locale's index set with a single domain. This allows us to
+// write more general code that will work for all distributions.
+//
+// This means that we can call the 'verifyID' function on any array, like the
+// 'BA' array from earlier.
+//
+verifyID(BA);
+
 
 //
 // The ReplicatedDist distribution is different: each of the

samples/Chapel/lulesh.chpl

@@ -292,14 +292,14 @@ proc main() {
     LagrangeLeapFrog();
 
     if debug {
-      //      deprint("[[ Forces ]]", fx, fy, fz);
+      deprintatomic("[[ Forces ]]", fx, fy, fz);
       deprint("[[ Positions ]]", x, y, z);
       deprint("[[ p, e, q ]]", p, e, q);
     }
     if showProgress then
-      writeln("time = ", format("%e", time), ", dt=", format("%e", deltatime),
-              if doTiming then ", elapsed = " + (getCurrentTime()-iterTime) 
-                          else "");
+      writef("time = %er, dt=%er, %s", time, deltatime,
+             if doTiming then ", elapsed = " + (getCurrentTime()-iterTime) +"\n"
+                         else "\n");
   }
   if (cycle == maxcycles) {
     writeln("Stopped early due to reaching maxnumsteps");
@@ -314,12 +314,10 @@ proc main() {
   if printCoords {
     var outfile = open("coords.out", iomode.cw);
     var writer = outfile.writer();
-    var fmtstr = if debug then "%1.9e" else "%1.4e";
-    for i in Nodes {
-      writer.writeln(format(fmtstr, x[i]), " ", 
-                     format(fmtstr, y[i]), " ", 
-                     format(fmtstr, z[i]));
-    }
+    var fmtstr = if debug then "%1.9re %1.9er %1.9er\n" 
+                          else "%1.4er %1.4er %1.4er\n";
+    for i in Nodes do
+      writer.writef(fmtstr, x[i], y[i], z[i]);
     writer.close();
     outfile.close();
   }
@@ -479,7 +477,7 @@ inline proc localizeNeighborNodes(eli: index(Elems),
                                   y: [] real, ref y_local: 8*real,
                                   z: [] real, ref z_local: 8*real) {
 
-  for param i in 1..nodesPerElem {
+  for i in 1..nodesPerElem {
     const noi = elemToNode[eli][i];
 
     x_local[i] = x[noi];
@@ -670,7 +668,7 @@ proc SumElemStressesToNodeForces(b_x: 8*real, b_y: 8*real, b_z: 8*real,
                                  ref fx: 8*real,
                                  ref fy: 8*real,
                                  ref fz: 8*real) {
-  for param i in 1..8 {
+  for i in 1..8 {
     fx[i] = -(stress_xx * b_x[i]);
     fy[i] = -(stress_yy * b_y[i]);
     fz[i] = -(stress_zz * b_z[i]);
@@ -725,17 +723,17 @@ inline proc CalcElemFBHourglassForce(xd: 8*real, yd: 8*real, zd: 8*real,
   var hx, hy, hz: 4*real;
 
   // reduction
-  for param i in 1..4 {
-    for param j in 1..8 {
+  for i in 1..4 {
+    for j in 1..8 {
       hx[i] += hourgam[j][i] * xd[j];
       hy[i] += hourgam[j][i] * yd[j];
       hz[i] += hourgam[j][i] * zd[j];
     }
   }
 
-  for param i in 1..8 {
+  for i in 1..8 {
     var shx, shy, shz: real;
-    for param j in 1..4 {
+    for j in 1..4 {
       shx += hourgam[i][j] * hx[j];
       shy += hourgam[i][j] * hy[j];
       shz += hourgam[i][j] * hz[j];
@@ -1088,16 +1086,16 @@ proc CalcFBHourglassForceForElems(determ, x8n, y8n, z8n, dvdx, dvdy, dvdz) {
 
     /* TODO: Can we enable this local block? */
     // local {
-      for param i in 1..4 {
+      for i in 1..4 {
         var hourmodx, hourmody, hourmodz: real;
         // reduction
-        for param j in 1..8 {
+        for j in 1..8 {
           hourmodx += x8n[eli][j] * gammaCoef[i][j];
           hourmody += y8n[eli][j] * gammaCoef[i][j];
           hourmodz += z8n[eli][j] * gammaCoef[i][j];
         }
 
-        for param j in 1..8 {
+        for j in 1..8 {
           hourgam[j][i] = gammaCoef[i][j] - volinv * 
             (dvdx[eli][j] * hourmodx +
              dvdy[eli][j] * hourmody +
@@ -1221,7 +1219,7 @@ proc CalcKinematicsForElems(dxx, dyy, dzz, const dt: real) {
       arealg[k] = CalcElemCharacteristicLength(x_local, y_local, z_local,
                                                volume);
 
-      for param i in 1..8 {
+      for i in 1..8 {
         x_local[i] -= dt2 * xd_local[i];
         y_local[i] -= dt2 * yd_local[i];
         z_local[i] -= dt2 * zd_local[i];
@@ -1669,7 +1667,7 @@ proc CalcSoundSpeedForElems(vnewc, rho0:real, enewc, pnewc, pbvc, bvc) {
 
 
 iter elemToNodes(elem) {
-  for param i in 1..nodesPerElem do
+  for i in 1..nodesPerElem do
     yield elemToNode[elem][i];
 }
                                  
@@ -1679,14 +1677,19 @@ iter elemToNodesTuple(e) {
 }
 
 
-proc deprint(title:string, x:[?D] real, y:[D]real, z:[D]real) {
+proc deprint(title:string, x:[?D] real, y:[D] real, z:[D] real) {
   writeln(title);
-  for i in D {
-    writeln(format("%3d", i), ": ", 
-            format("%3.4e", x[i]), " ", 
-            format("%3.4e", y[i]), " ", 
-            format("%3.4e", z[i]));
-  }
+  for i in D do
+    writef("%3i: %3.4er %3.4er %3.4er\n", 
+           if use3DRepresentation then idx3DTo1D(i, D.dim(1).size) else i, 
+           x[i], y[i], z[i]);
 }
 
 
+proc deprintatomic(title:string, x:[?D] atomic real, y:[] atomic real, z:[] atomic real) {
+  writeln(title);
+  for i in D do
+    writef("%3i: %3.4er %3.4er %3.4er\n", 
+           if use3DRepresentation then idx3DTo1D(i, D.dim(1).size) else i, 
+           x[i].peek(), y[i].peek(), z[i].peek());
+}

samples/Charity/example.ch

@@ -0,0 +1,6 @@
+%
+%   Some very badly written Charity
+%
+
+data LA(A) -> D = ss: A -> D
+                | ff: -> D.

samples/Clarion/CStringClass.clw

@@ -0,0 +1,172 @@
+      Member()
+         omit('***$***',_VER_C55)
+_ABCDllMode_  EQUATE(0)
+_ABCLinkMode_ EQUATE(1)
+         ***$***
+      Include('Equates.CLW'),ONCE
+      Include('Keycodes.CLW'),ONCE
+      Include('Errors.CLW'),ONCE
+      Map
+      End ! map
+      Include('CStringClass.inc'),ONCE
+CStringClass.Construct PROCEDURE                       ! Declare Procedure
+  CODE
+  SELF.bufferSize  = DEFAULT_CS_BUFFER_SIZE
+  SELF.CS         &= New(CSTRING(SELF.bufferSize))
+CStringClass.Destruct PROCEDURE                        ! Declare Procedure
+  CODE
+  Dispose(SELF.cs)
+CStringClass.Cat PROCEDURE  (STRING pStr) !,*CSTRING,PROC ! Declare Procedure
+newLen                       LONG,AUTO
+oldCS                        &CSTRING
+  CODE
+  newLen = Len(pStr)
+  IF (newLen+SELF.strLength+2) > SELF.newStrSize
+    ! Only grow the internal string if the result of the cat will be larger than the string currently is.
+    ! The reason for the "+2" is because this is used in the string slicing outside this IF. Without this matching +2 there is potential for an out of bounds slice which would be bad!
+
+    ! Save a temporary copy of the old string so we can us it in the concatination after we have grown it!
+    oldCS &= New(CSTRING(SELF.strLength+1))
+    oldCS = SELF.CS
+    Dispose(SELF.CS)
+
+    SELF.newStrSize = newLen + SELF.strLength + 1 + SELF.bufferSize
+    SELF.CS &= New(CSTRING(SELF.newStrSize))
+    SELF.CS = oldCS
+    Dispose(oldCS)
+  END
+
+  ! Append the new string directly to the end of the old one.
+  SELF.CS[SELF.strLength+1 : SELF.strLength+newLen] = pStr
+  ! And terminate the CSTRING manually
+  SELF.CS[SELF.strLength+newLen+1] = '<0>'
+
+  ! This is the same as doing "SELF.strLength = Len(SELF.CS)" but the Len() is _really_ slow on large strings. This is much faster!
+  SELF.strLength += newLen
+
+  ! This is what it used to be:
+  ! SELF.Str(SELF.Str() & s)
+  ! It is a nice and neat solution but performance, especially on large strings was terrible!
+
+  RETURN SELF.Str()
+CStringClass.Str PROCEDURE  (STRING pStr) !,*CSTRING, PROC   ! Declare Procedure
+  CODE
+  IF Len(pStr) > SELF.newStrSize
+    ! Only Dispose/New the internal string if the new one requires it.
+    ! This might be slightly innefficient in terms of memory usage when the string gets smaller
+    ! But it is _vasty_ better for performance when the string gets added to a lot.
+    Dispose(SELF.CS)
+    SELF.newStrSize = Len(pStr) + 1 + SELF.bufferSize
+    SELF.CS &= New(CSTRING(SELF.newStrSize))
+  END
+
+  SELF.CS        = pStr
+  SELF.strLength = Len(SELF.CS)
+
+  RETURN SELF.CS
+CStringClass.Len PROCEDURE  !,LONG                     ! Declare Procedure
+  CODE
+  RETURN SELF.strLength
+CStringClass.Replace PROCEDURE  (STRING pFind, STRING pReplace) !,*CSTRING,PROC ! Declare Procedure
+! FindString , ReplaceWith
+locate                       LONG,AUTO
+lastLocate                   LONG
+  CODE
+  LOOP
+    locate = InString(Upper(pFind), Upper(SELF.Str()), 1, lastLocate+1)
+    IF ~locate
+      BREAK
+    END
+
+    ! So we dont end up having recursive replacement.
+    lastLocate = locate + Len(pReplace)-1
+
+    SELF.Str(Sub(SELF.Str(), 1, locate-1)                  & |
+             pReplace                                      & |
+             Sub(SELF.Str(), locate+Len(pFind), SELF.Len())    |
+             )
+  END
+
+  RETURN SELF.Str()
+CStringClass.Str PROCEDURE  () !,*CSTRING              ! Declare Procedure 3
+  CODE
+  RETURN SELF.CS
+!------------------------------------------------------------------------------
+CStringClass.Contains PROCEDURE  (STRING pFind, BYTE pCaseSensitive=TRUE) !,BYTE ! Declare Procedure
+! Returns a value (TRUE) indicating whether the specified String occurs within this string.
+! Second parameter defaults to a case sensitive search.
+  CODE
+  IF pCaseSensitive = TRUE
+    IF InString(pFind, SELF.Str(), 1 , 1) > 0
+      RETURN TRUE
+    END
+  ELSE
+    IF InString(Lower(pFind), SELF.Lower(), 1 , 1) > 0
+      RETURN TRUE
+    END
+  END
+
+  RETURN FALSE
+CStringClass.Lower PROCEDURE  () !,STRING              ! Declare Procedure
+! Returns a "Lowered" version of the self.cs doesnt change the self.cs
+  CODE
+  RETURN Lower(SELF.CS)
+CStringClass.SubString PROCEDURE  (LONG pPosition, LONG pLength) !,STRING,PROC ! Declare Procedure
+  CODE
+  RETURN Sub(SELF.Str(), pPosition, pLength)
+CStringClass.ToLower PROCEDURE  () !,*CSTRING,PROC     ! Declare Procedure
+! Converts this string to lowercase and returns the converted string
+
+  CODE
+  RETURN SELF.Str(SELF.Lower())
+CStringClass.ToUpper PROCEDURE  () !,*CSTRING,PROC     ! Declare Procedure
+! Converts this string to uppercase and returns the converted string
+
+  CODE
+  RETURN SELF.Str(SELF.Upper())
+CStringClass.Trim PROCEDURE  () !,*CSTRING,PROC        ! Declare Procedure
+  CODE
+  SELF.Str(Left(SELF.Str()))
+  SELF.Str(Clip(SELF.Str()))
+  RETURN SELF.Str()
+CStringClass.Upper PROCEDURE  () !,STRING              ! Declare Procedure
+  CODE
+  RETURN Upper(SELF.Str())
+CStringClass.IndexOf PROCEDURE  (STRING pLookIn, BYTE pCaseSensitive=TRUE) !,LONG ! Declare Procedure
+! Returns the index of the first parameter (pLookIn) is found within the SELF.CS
+! zero if it is not found
+  CODE
+  IF pCaseSensitive = TRUE
+    RETURN InString(SELF.Str(), pLookIn, 1 , 1)
+  ELSE
+    RETURN InString(SELF.Lower(), Lower(pLookIn), 1 , 1)
+  END
+CStringClass.FoundIn PROCEDURE  (STRING pLookIn, BYTE pCaseSensitive=TRUE) !,BYTE ! Declare Procedure
+! Returns TRUE if the first parameter (pLookIn) is found within the SELF.CS
+! FALSE if it is no
+  CODE
+  IF SELF.IndexOf(pLookIn, pCaseSensitive) > 0
+    RETURN TRUE
+  ELSE
+    RETURN FALSE
+  END
+CStringClass.SetBuffer PROCEDURE  (LONG pNewBuffer)    ! Declare Procedure
+  CODE
+  SELF.bufferSize = pNewBuffer
+CStringClass.EscapeXml PROCEDURE  (<STRING pStr>) !,STRING ! Declare Procedure
+CS CStringClass
+  CODE
+  IF Omitted(pStr)=FALSE
+    CS.Str(pStr)
+  ELSE
+    ! Make a copy so we don't alter the original
+    CS.Str(SELF.Str())
+  END
+  CS.Replace('&', '&amp;')
+  CS.Replace('<', '&lt;')
+  CS.Replace('>', '&gt;')
+  CS.Replace('"', '&quot;')
+  CS.Replace('''', '&apos;')
+
+  RETURN CS.Str()
+

samples/Clarion/ConsoleSupport.clw

@@ -0,0 +1,68 @@
+      Member()
+      Include('ConsoleSupport.inc'),ONCE
+	  Map
+        MODULE('32-bit Windows API')
+            ! General functions
+            GetLastError(),DWORD,PASCAL
+ 
+            ! Console functions
+            GetStdHandle(DWORD),HANDLE,PASCAL,PROC,RAW
+            WriteConsole(Handle,Long,Dword,long,long),bool,Raw,Pascal,name('WriteConsoleA')
+            ReadConsole(Handle,Long,Dword,long,long),bool,Raw,Pascal,name('ReadConsoleA')
+            SetConsoleTitle(Long),Bool,Raw,Pascal,name('SetConsoleTitleA')
+            GetConsoleTitle(Long,dword),Bool,Raw,Pascal,name('GetConsoleTitleA')
+            SetConsoleMode(Handle,dWord),BOOL,RAW,PASCAL
+            GetConsoleMode(Handle,Long),BOOL,RAW,PASCAL
+        End
+      End 
+      
+ConsoleSupport.Construct PROCEDURE
+
+  CODE
+
+ConsoleSupport.Destruct PROCEDURE
+
+  CODE
+
+ConsoleSupport.Init				   PROCEDURE () !,BYTE,VIRTUAL 
+  CODE
+
+    SELF.OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE)
+    If SELF.OutputHandle = INVALID_HANDLE_VALUE
+        Halt(1,'Unable to get output handle (' & GetLastError() & ')')
+        RETURN INVALID_HANDLE_VALUE
+    End
+ 
+    SELF.InputHandle = GetStdHandle(STD_INPUT_HANDLE)
+    if SELF.InputHandle = INVALID_HANDLE_VALUE
+        Halt(2,'Unable to get console input handle (' & GetLastError() & ')')
+        RETURN INVALID_HANDLE_VALUE
+    End
+ 
+    If ~SetConsoleMode(SELF.InputHandle,ENABLE_PROCESSED_INPUT )
+        Halt(3,'Unable to set console mode (' & GetLastError() & ')')
+        RETURN INVALID_OTHER
+    End
+
+    RETURN FALSE
+
+ConsoleSupport.WriteLine			   PROCEDURE (STRING pText) !,BYTE,PROC,VIRTUAL 
+  CODE
+    SELF.TextBuffer = SELF.Prefix & pText & '<13,10>'
+    If WriteConsole(SELF.OutputHandle, ADDRESS(SELF.TextBuffer), LEN(SELF.TextBuffer),ADDRESS(SELF.BytesWritten), NULL) = 0
+        Halt(4,'WriteConsoleError (' & GetLastError() & ')')
+        RETURN -1
+    End
+    RETURN FALSE
+
+Consolesupport.ReadKey  			   PROCEDURE () !,STRING,PROC,VIRTUAL 
+  CODE
+  SELF.WriteLine('Press any key to continue...')
+  Clear(SELF.InBuffer)
+  Loop
+    IF ReadConsole(SELF.InputHandle,Address(SELF.InBuffer),100,Address(SELF.BytesRead),NULL) = 0 THEN
+      Halt(5,'Error on read console (' & GetLastError() & ')')
+      Break
+    End
+  Until SELF.BytesRead > 0
+  RETURN SELF.InBuffer

samples/Clarion/HelloWorld.clw

@@ -0,0 +1,10 @@
+  PROGRAM
+
+  MAP
+  END
+
+  CODE
+
+  MESSAGE('Hello World!')
+
+  RETURN

samples/Clarion/hello.clw

@@ -0,0 +1,12 @@
+  MEMBER()
+  INCLUDE('HelloClass.inc'),ONCE
+  MAP
+  END
+
+HelloClass.Construct              PROCEDURE
+  CODE
+HelloClass.Destruct               PROCEDURE() !,VIRTUAL
+  CODE
+HelloClass.SayHello               PROCEDURE
+  CODE
+  MESSAGE('Hello World!')

samples/Click/sr2.click

@@ -0,0 +1,133 @@
+rates :: AvailableRates
+elementclass sr2 {
+  $sr2_ip, $sr2_nm, $wireless_mac, $gateway, $probes|
+
+
+arp :: ARPTable();
+lt :: LinkTable(IP $sr2_ip);
+
+
+gw :: SR2GatewaySelector(ETHTYPE 0x062c,
+		      IP $sr2_ip,
+		      ETH $wireless_mac,
+		      LT lt,
+		      ARP arp,
+		      PERIOD 15,
+		      GW $gateway);
+
+
+gw -> SR2SetChecksum -> [0] output;
+
+set_gw :: SR2SetGateway(SEL gw);
+
+
+es :: SR2ETTStat(ETHTYPE 0x0641, 
+	      ETH $wireless_mac, 
+	      IP $sr2_ip, 
+	      PERIOD 30000,
+	      TAU 300000,
+	      ARP arp,
+	      PROBES $probes,
+	      ETT metric,
+	      RT rates);
+
+
+metric :: SR2ETTMetric(LT lt);
+
+
+forwarder :: SR2Forwarder(ETHTYPE 0x0643, 
+			      IP $sr2_ip, 
+			      ETH $wireless_mac, 
+			      ARP arp, 
+			      LT lt);
+
+
+querier :: SR2Querier(ETH $wireless_mac, 
+		     SR forwarder,
+		     LT lt, 
+		     ROUTE_DAMPENING true,
+		     TIME_BEFORE_SWITCH 5,
+		     DEBUG true);
+
+
+query_forwarder :: SR2MetricFlood(ETHTYPE 0x0644,
+			       IP $sr2_ip, 
+			       ETH $wireless_mac, 
+			       LT lt, 
+			       ARP arp,
+			       DEBUG false);
+
+query_responder :: SR2QueryResponder(ETHTYPE 0x0645,
+				    IP $sr2_ip, 
+				    ETH $wireless_mac, 
+				    LT lt, 
+				    ARP arp,
+				    DEBUG true);
+
+
+query_responder -> SR2SetChecksum -> [0] output;
+query_forwarder -> SR2SetChecksum -> SR2Print(forwarding) -> [0] output;
+query_forwarder [1] -> query_responder;
+
+data_ck :: SR2SetChecksum() 
+
+input [1] 
+-> host_cl :: IPClassifier(dst net $sr2_ip mask $sr2_nm,
+				-)
+-> querier
+-> data_ck;
+
+
+host_cl [1] -> [0] set_gw [0] -> querier;
+
+forwarder[0] 
+  -> dt ::DecIPTTL
+  -> data_ck
+  -> [2] output;
+
+
+dt[1] 
+-> Print(ttl-error) 
+-> ICMPError($sr2_ip, timeexceeded, 0) 
+-> querier;
+
+
+// queries
+querier [1] -> [1] query_forwarder;
+es -> SetTimestamp() -> [1] output;
+
+
+forwarder[1] //ip packets to me
+  -> SR2StripHeader()
+  -> CheckIPHeader()
+  -> from_gw_cl :: IPClassifier(src net $sr2_ip mask $sr2_nm,
+				-)
+  -> [3] output;
+
+from_gw_cl [1] -> [1] set_gw [1] -> [3] output;
+
+ input [0]
+   -> ncl :: Classifier(
+			12/0643 , //sr2_forwarder
+			12/0644 , //sr2
+			12/0645 , //replies
+			12/0641 , //sr2_es
+			12/062c , //sr2_gw
+			);
+ 
+ 
+ ncl[0] -> SR2CheckHeader() -> [0] forwarder;
+ ncl[1] -> SR2CheckHeader() -> PrintSR(query) -> query_forwarder
+ ncl[2] -> SR2CheckHeader() -> query_responder;
+ ncl[3] -> es;
+ ncl[4] -> SR2CheckHeader() -> gw;
+ 
+}
+
+
+
+Idle -> s :: sr2(2.0.0.1, 255.0.0.0, 00:00:00:00:00:01, false, "12 60 12 1500") -> Discard;
+Idle -> [1] s;
+s[1] -> Discard;
+s[2] -> Discard;
+s[3] -> Discard;

samples/Click/thomer-nat.click

@@ -0,0 +1,142 @@
+// This Click configuration implements a firewall and NAT, roughly based on the
+// mazu-nat.click example.
+//
+// This example assumes there is one interface that is IP-aliased.  In this
+// example, eth0 and eth0:0 have IP addresses 66.68.65.90 and 192.168.1.1,
+// respectively.  There is a local network, 192.168.1.0/24, and an upstream
+// gateway, 66.58.65.89.  Traffic from the local network is NATed.
+//
+// Connections can be initiated from the NAT box itself, also.
+// 
+// For bugs, suggestions, and, corrections, please email me.
+//
+// Author: Thomer M. Gil (click@thomer.com)
+
+AddressInfo(
+    eth0-in     192.168.1.1     192.168.1.0/24  00:0d:87:9d:1c:e9,
+    eth0-ex     66.58.65.90                     00:0d:87:9d:1c:e9,
+    gw-addr     66.58.65.89                     00:20:6f:14:54:c2
+);
+
+
+elementclass SniffGatewayDevice {
+  $device |
+  from :: FromDevice($device)
+    -> t1 :: Tee
+    -> output;
+  input -> q :: Queue(1024)
+    -> t2 :: PullTee
+    -> to :: ToDevice($device);
+  t1[1] -> ToHostSniffers;
+  t2[1] -> ToHostSniffers($device);
+  ScheduleInfo(from .1, to 1);
+}
+
+
+device :: SniffGatewayDevice(eth0);
+arpq_in :: ARPQuerier(eth0-in) -> device;
+ip_to_extern :: GetIPAddress(16)
+        -> CheckIPHeader
+        -> EtherEncap(0x800, eth0-ex, gw-addr)
+        -> device;
+ip_to_host :: EtherEncap(0x800, gw-addr, eth0-ex)
+        -> ToHost;
+ip_to_intern :: GetIPAddress(16)
+        -> CheckIPHeader
+        -> arpq_in;
+
+                 
+arp_class :: Classifier(
+        12/0806 20/0001,        // [0] ARP requests
+        12/0806 20/0002,        // [1] ARP replies to host
+        12/0800);               // [2] IP packets
+
+device -> arp_class;
+
+// ARP crap
+arp_class[0] -> ARPResponder(eth0-in, eth0-ex) -> device;
+arp_class[1] -> arp_t :: Tee;
+                arp_t[0] -> ToHost;
+                arp_t[1] -> [1]arpq_in;
+
+
+// IP packets
+arp_class[2] -> Strip(14)
+   -> CheckIPHeader
+   -> ipclass :: IPClassifier(dst host eth0-ex,
+                              dst host eth0-in,
+                              src net eth0-in);
+
+// Define pattern NAT
+iprw :: IPRewriterPatterns(NAT eth0-ex 50000-65535 - -);
+
+// Rewriting rules for UDP/TCP packets
+// output[0] rewritten to go into the wild
+// output[1] rewritten to come back from the wild or no match
+rw :: IPRewriter(pattern NAT 0 1,
+                 pass 1);
+
+// Rewriting rules for ICMP packets
+irw :: ICMPPingRewriter(eth0-ex, -);
+irw[0] -> ip_to_extern;
+irw[1] -> icmp_me_or_intern :: IPClassifier(dst host eth0-ex, -);
+          icmp_me_or_intern[0] -> ip_to_host;
+          icmp_me_or_intern[1] -> ip_to_intern;
+
+// Rewriting rules for ICMP error packets
+ierw :: ICMPRewriter(rw irw);
+ierw[0] -> icmp_me_or_intern;
+ierw[1] -> icmp_me_or_intern;
+
+
+// Packets directed at eth0-ex.
+// Send it through IPRewriter(pass).  If there was a mapping, it will be
+// rewritten such that dst is eth0-in:net, otherwise dst will still be for
+// eth0-ex.
+ipclass[0] -> [1]rw;
+
+// packets that were rewritten, heading into the wild world.
+rw[0] -> ip_to_extern;
+
+// packets that come back from the wild or are not part of an established
+// connection.
+rw[1] -> established_class :: IPClassifier(dst host eth0-ex,
+                                           dst net eth0-in);
+
+         // not established yet or returning packets for a connection that was
+         // established from this host itself.
+         established_class[0] ->
+           firewall :: IPClassifier(dst tcp port ssh,
+                                    dst tcp port smtp,
+                                    dst tcp port domain,
+                                    dst udp port domain,
+                                    icmp type echo-reply,
+                                    proto icmp,
+                                    port > 4095,
+                                    -);
+
+                                    firewall[0] -> ip_to_host; // ssh
+                                    firewall[1] -> ip_to_host; // smtp
+                                    firewall[2] -> ip_to_host; // domain (t)
+                                    firewall[3] -> ip_to_host; // domain (u)
+                                    firewall[4] -> [0]irw;     // icmp reply
+                                    firewall[5] -> [0]ierw;    // other icmp
+                                    firewall[6] -> ip_to_host; // port > 4095, probably for connection
+                                                               // originating from host itself
+                                    firewall[7] -> Discard;    // don't allow incoming for port <= 4095
+
+        // established connection
+        established_class[1] -> ip_to_intern;
+
+// To eth0-in.  Only accept from inside network.
+ipclass[1] -> IPClassifier(src net eth0-in) -> ip_to_host;
+
+// Packets from eth0-in:net either stay on local network or go to the wild.
+// Those that go into the wild need to go through the appropriate rewriting
+// element.  (Either UDP/TCP rewriter or ICMP rewriter.)
+ipclass[2] -> inter_class :: IPClassifier(dst net eth0-in, -);
+              inter_class[0] -> ip_to_intern;
+              inter_class[1] -> ip_udp_class :: IPClassifier(tcp or udp,
+                                                             icmp type echo);
+                                ip_udp_class[0] -> [0]rw;
+                                ip_udp_class[1] -> [0]irw;