Merge pull request #2017 from github/revert-2014-revert-1976-path-for-fileblob

Cut 4.5.0 release
v4.5.1
2025-10-29 17:50:22 +00:00 · 2015-03-14 20:42:20 -05:00 · 2015-03-14 20:32:18 -05:00 · 2015-03-14 20:19:11 -05:00 · 2015-03-14 17:18:36 -05:00 · 2015-03-14 17:17:52 -05:00
145 changed files with 14144 additions and 378 deletions
--- a/.gitmodules
+++ b/.gitmodules
@@ -249,7 +249,7 @@
 	url = https://github.com/shellderp/sublime-robot-plugin
 [submodule "vendor/grammars/actionscript3-tmbundle"]
 	path = vendor/grammars/actionscript3-tmbundle
-	url = https://github.com/simongregory/actionscript3-tmbundle
+	url = https://github.com/honzabrecka/actionscript3-tmbundle
 [submodule "vendor/grammars/Sublime-QML"]
 	path = vendor/grammars/Sublime-QML
 	url = https://github.com/skozlovf/Sublime-QML
@@ -397,9 +397,6 @@
 [submodule "vendor/grammars/pascal.tmbundle"]
 	path = vendor/grammars/pascal.tmbundle
 	url = https://github.com/textmate/pascal.tmbundle
-[submodule "vendor/grammars/perl.tmbundle"]
-	path = vendor/grammars/perl.tmbundle
-	url = https://github.com/textmate/perl.tmbundle
 [submodule "vendor/grammars/php-smarty.tmbundle"]
 	path = vendor/grammars/php-smarty.tmbundle
 	url = https://github.com/textmate/php-smarty.tmbundle
@@ -552,9 +549,6 @@
 [submodule "vendor/grammars/liquid.tmbundle"]
 	path = vendor/grammars/liquid.tmbundle
 	url = https://github.com/bastilian/validcode-textmate-bundles
-[submodule "vendor/grammars/AutoHotkey"]
-	path = vendor/grammars/AutoHotkey
-	url = https://github.com/ahkscript/AutoHotkey
 [submodule "vendor/grammars/ats.sublime"]
 	path = vendor/grammars/ats.sublime
 	url = https://github.com/steinwaywhw/ats-mode-sublimetext
@@ -579,3 +573,75 @@
 [submodule "vendor/grammars/JSyntax"]
 	path = vendor/grammars/JSyntax
 	url = https://github.com/bcj/JSyntax
+[submodule "vendor/grammars/TXL"]
+	path = vendor/grammars/TXL
+	url = https://github.com/MikeHoffert/Sublime-Text-TXL-syntax
+[submodule "vendor/grammars/G-Code"]
+	path = vendor/grammars/G-Code
+	url = https://github.com/robotmaster/sublime-text-syntax-highlighting
+[submodule "vendor/grammars/grace-tmbundle"]
+	path = vendor/grammars/grace-tmbundle
+	url = https://github.com/zmthy/grace-tmbundle
+[submodule "vendor/grammars/sublime-text-ox"]
+	path = vendor/grammars/sublime-text-ox
+	url = https://github.com/andreashetland/sublime-text-ox
+[submodule "vendor/grammars/AutoHotkey"]
+	path = vendor/grammars/AutoHotkey
+	url = https://github.com/ahkscript/SublimeAutoHotkey
+[submodule "vendor/grammars/ec.tmbundle"]
+	path = vendor/grammars/ec.tmbundle
+	url = https://github.com/ecere/ec.tmbundle
+[submodule "vendor/grammars/InnoSetup"]
+	path = vendor/grammars/InnoSetup
+	url = https://github.com/idleberg/InnoSetup-Sublime-Text
+[submodule "vendor/grammars/gap-tmbundle"]
+	path = vendor/grammars/gap-tmbundle
+	url = https://github.com/dhowden/gap-tmbundle
+[submodule "vendor/grammars/SublimePapyrus"]
+	path = vendor/grammars/SublimePapyrus
+	url = https://github.com/Kapiainen/SublimePapyrus
+[submodule "vendor/grammars/sublime-spintools"]
+	path = vendor/grammars/sublime-spintools
+	url = https://github.com/bitbased/sublime-spintools
+[submodule "vendor/grammars/PogoScript.tmbundle"]
+	path = vendor/grammars/PogoScript.tmbundle
+	url = https://github.com/featurist/PogoScript.tmbundle
+[submodule "vendor/grammars/sublime-opal"]
+	path = vendor/grammars/sublime-opal
+	url = https://github.com/artifactz/sublime-opal
+[submodule "vendor/grammars/mediawiki.tmbundle"]
+	path = vendor/grammars/mediawiki.tmbundle
+	url = https://github.com/textmate/mediawiki.tmbundle
+[submodule "vendor/grammars/oracle.tmbundle"]
+	path = vendor/grammars/oracle.tmbundle
+	url = https://github.com/mulander/oracle.tmbundle.git
+[submodule "vendor/grammars/BrightScript.tmbundle"]
+	path = vendor/grammars/BrightScript.tmbundle
+	url = https://github.com/cmink/BrightScript.tmbundle
+[submodule "vendor/grammars/Stylus"]
+	path = vendor/grammars/Stylus
+	url = https://github.com/billymoon/Stylus
+[submodule "vendor/grammars/asciidoc.tmbundle"]
+	path = vendor/grammars/asciidoc.tmbundle
+	url = https://github.com/zuckschwerdt/asciidoc.tmbundle
+[submodule "vendor/grammars/sublime-text-pig-latin"]
+	path = vendor/grammars/sublime-text-pig-latin
+	url = https://github.com/goblindegook/sublime-text-pig-latin
+[submodule "vendor/grammars/Lean.tmbundle"]
+	path = vendor/grammars/Lean.tmbundle
+	url = https://github.com/leanprover/Lean.tmbundle
+[submodule "vendor/grammars/ampl"]
+	path = vendor/grammars/ampl
+	url = https://github.com/ampl/sublime-ampl
+[submodule "vendor/grammars/openscad.tmbundle"]
+	path = vendor/grammars/openscad.tmbundle
+	url = https://github.com/tbuser/openscad.tmbundle
+[submodule "vendor/grammars/sublime-varnish"]
+	path = vendor/grammars/sublime-varnish
+	url = https://github.com/brandonwamboldt/sublime-varnish
+[submodule "vendor/grammars/xc.tmbundle"]
+	path = vendor/grammars/xc.tmbundle
+	url = https://github.com/graymalkin/xc.tmbundle
+[submodule "vendor/grammars/perl.tmbundle"]
+	path = vendor/grammars/perl.tmbundle
+	url = https://github.com/textmate/perl.tmbundle
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,3 @@
-sudo: false
 before_install: script/travis/before_install
 rvm:
  - 1.9.3
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,41 +1,81 @@
-## Contributing
+# Contributing

-The majority of contributions won't need to touch any Ruby code at all. The [master language list][languages] is just a YAML configuration file.
+Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great. The majority of contributions won't need to touch any Ruby code at all.

-Almost all bug fixes or new language additions should come with some additional code samples. Just drop them under [`samples/`][samples] in the correct subdirectory and our test suite will automatically test them. In most cases you shouldn't need to add any new assertions.
+## Adding a language

-### My code is detected as the wrong language
+We try only to add languages once they have some usage on GitHub. In most cases we prefer that languages be in use in hundreds of repositories before supporting them in Linguist.

-This can usually be solved either by adding a new filename or file name extension to the language's entry in [`languages.yml`][languages] or adding more [samples][samples] for your language to the repository to make Linguist's classifier smarter.
-
-### Syntax highlighting looks wrong
-
-Assuming your code is being detected as the right language (see above), in most cases this is due to a bug in the language grammar rather than a bug in Linguist. [`grammars.yml`][grammars] lists all the grammars we use for syntax highlighting on github.com. Find the one corresponding to your code's programming language and submit a bug report upstream. If you can, try to reproduce the highlighting problem in the text editor that the grammar is designed for (TextMate, Sublime Text, or Atom) and include that information in your bug report.
-
-You can also try to fix the bug yourself and submit a Pull Request. [This piece from TextMate's documentation](http://manual.macromates.com/en/language_grammars) offers a good introduction on how to work with TextMate-compatible grammars. You can test grammars using [Lightshow](https://lightshow.githubapp.com).
-
-Once the bug has been fixed upstream, please let us know and we'll pick it up for GitHub.
-
-### I want to add support for the `X` programming language
-
-Great! You'll need to:
+To add support for a new language:

 0. Add an entry for your language to [`languages.yml`][languages].
 0. Add a grammar for your language. Please only add grammars that have a license that permits redistribution.
-    0. Add your grammar as a submodule: `git submodule add https://github.com/JaneSmith/MyGrammar vendor/grammars/MyGrammar`.
-    0. Add your grammar to [`grammars.yml`][grammars] by running `script/convert-grammars --add vendor/grammars/MyGrammar`.
-0. Add samples for your language to the [samples directory][samples].
+  0. Add your grammar as a submodule: `git submodule add https://github.com/JaneSmith/MyGrammar vendor/grammars/MyGrammar`.
+  0. Add your grammar to [`grammars.yml`][grammars] by running `script/convert-grammars --add vendor/grammars/MyGrammar`.
+0. Add samples for your language to the [samples directory][samples] in the correct subdirectory.
+0. Open a pull request, linking to a [GitHub search result](https://github.com/search?utf8=%E2%9C%93&q=extension%3Aboot+NOT+nothack&type=Code&ref=searchresults) showing in-the-wild usage.

 In addition, if your new language defines an extension that's already listed in [`languages.yml`][languages] (such as `.foo`) then sometimes a few more steps will need to be taken:

-0. Make sure that example `.foo` files are present in the [samples directory][samples] for each language that uses `.foo`. 
+0. Make sure that example `.foo` files are present in the [samples directory][samples] for each language that uses `.foo`.
 0. Test the performance of the Bayesian classifier with a relatively large number (1000s) of sample `.foo` files. (ping @arfon or @bkeepers to help with this) to ensure we're not misclassifying files.
 0. If the Bayesian classifier does a bad job with the sample `.foo` files then a [heuristic](https://github.com/github/linguist/blob/master/lib/linguist/heuristics.rb) may need to be written to help.

 Remember, the goal here is to try and avoid false positives!

-We try only to add languages once they have some usage on GitHub, so please note in-the-wild usage examples in your pull request. In most cases we prefer that languages already be in use in hundreds of repositories before supporting them in Linguist.
+## Fixing a misclassified language
+
+Most languages are detected by their file extension defined in [languages.yml][languages].  For disambiguating between files with common extensions, linguist applies some [heuristics](/lib/linguist/heuristics.rb) and a [statistical classifier](lib/linguist/classifier.rb). This process can help differentiate between, for example, `.h` files which could be either C, C++, or Obj-C.
+
+Misclassifications can often be solved by either adding a new filename or extension for the language or adding more [samples][samples] to make the classifier smarter.
+
+## Fixing syntax highlighting
+
+Syntax highlighting in GitHub is performed using TextMate-compatible grammars. These are the same grammars that TextMate, Sublime Text and Atom use. Every language in [languages.yml][languages] is mapped to its corresponding TM `scope`. This scope will be used when picking up a grammar for highlighting.
+
+Assuming your code is being detected as the right language, in most cases this is due to a bug in the language grammar rather than a bug in Linguist. [`grammars.yml`][grammars] lists all the grammars we use for syntax highlighting on github.com. Find the one corresponding to your code's programming language and submit a bug report upstream. If you can, try to reproduce the highlighting problem in the text editor that the grammar is designed for (TextMate, Sublime Text, or Atom) and include that information in your bug report.
+
+You can also try to fix the bug yourself and submit a Pull Request. [TextMate's documentation](http://manual.macromates.com/en/language_grammars) offers a good introduction on how to work with TextMate-compatible grammars. You can test grammars using [Lightshow](https://github-lightshow.herokuapp.com).
+
+Once the bug has been fixed upstream, please let us know and we'll pick it up for GitHub.
+
+## Testing
+
+For development you are going to want to checkout out the source. To get it, clone the repo and run [Bundler](http://gembundler.com/) to install its dependencies.
+
+    git clone https://github.com/github/linguist.git
+    cd linguist/
+    script/bootstrap
+
+To run the tests:
+
+    bundle exec rake test
+
+Sometimes getting the tests running can be too much work, especially if you don't have much Ruby experience. It's okay: be lazy and let our build bot [Travis](http://travis-ci.org/#!/github/linguist) run the tests for you. Just open a pull request and the bot will start cranking away.
+
+Here's our current build status: [![Build Status](https://secure.travis-ci.org/github/linguist.png?branch=master)](http://travis-ci.org/github/linguist)
+
+## Releasing
+
+If you are the current maintainer of this gem:
+
+0. Create a branch for the release: `git checkout -b cut-release-vxx.xx.xx`
+0. Make sure your local dependencies are up to date: `script/bootstrap`
+0. If grammar submodules have not been updated recently, update them: `git submodule update --remote && git commit -a`
+0. Ensure that samples are updated: `bundle exec rake samples`
+0. Ensure that tests are green: `bundle exec rake test`
+0. Bump gem version in `lib/linguist/version.rb`, [like this](https://github.com/github/linguist/commit/8d2ea90a5ba3b2fe6e1508b7155aa4632eea2985).
+0. Make a PR to github/linguist, [like this](https://github.com/github/linguist/pull/1238).
+0. Build a local gem: `bundle exec rake build_gem`
+0. Test the gem:
+  0. Bump the Gemfile and Gemfile.lock versions for an app which relies on this gem
+  0. Install the new gem locally
+  0. Test behavior locally, branch deploy, whatever needs to happen
+0. Merge github/linguist PR
+0. Tag and push: `git tag vx.xx.xx; git push --tags`
+0. Push to rubygems.org -- `gem push github-linguist-3.0.0.gem`

 [grammars]: /grammars.yml
 [languages]: /lib/linguist/languages.yml
 [samples]: /samples
+[new-issue]: https://github.com/github/linguist/issues/new
--- a/README.md
+++ b/README.md
@@ -1,45 +1,80 @@
 # Linguist

-We use this library at GitHub to detect blob languages, ignore binary files, suppress generated files in diffs, and generate language breakdown graphs.
+[issues]: https://github.com/github/linguist/issues
+[new-issue]: https://github.com/github/linguist/issues/new

-Tips for filing issues and creating pull requests can be found in [`CONTRIBUTING.md`](/CONTRIBUTING.md).
+This library is used on GitHub.com to detect blob languages, ignore binary or vendored files, suppress generated files in diffs, and generate language breakdown graphs.

-## Features
+See [Troubleshooting](#troubleshooting) and [`CONTRIBUTING.md`](/CONTRIBUTING.md) before filing an issue or creating a pull request.

-### Language detection
+## Troubleshooting

-Linguist defines a list of all languages known to GitHub in a [yaml file](https://github.com/github/linguist/blob/master/lib/linguist/languages.yml).
+### My repository is detected as the wrong language

-Most languages are detected by their file extension. For disambiguating between files with common extensions, we first apply some common-sense heuristics to pick out obvious languages. After that, we use a
-[statistical
-classifier](https://github.com/github/linguist/blob/master/lib/linguist/classifier.rb).
-This process can help us tell the difference between, for example, `.h` files which could be either C, C++, or Obj-C.
+![language stats bar](https://cloud.githubusercontent.com/assets/173/5562290/48e24654-8ddf-11e4-8fe7-735b0ce3a0d3.png)

-```ruby
+The Language stats bar is built by aggregating the languages of each file in that repository. If it is reporting a language that you don't expect:

-Linguist::FileBlob.new("lib/linguist.rb").language.name #=> "Ruby"
+0. Click on the name of the language in the stats bar to see a list of the files that are identified as that language.
+0. If you see files that you didn't write, consider moving the files into one of the [paths for vendored  code](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml), or use the [manual overrides](#overrides) feature to ignore them.
+0. If the files are being misclassified, search for [open issues][issues] to see if anyone else has already reported the issue. Any information you an add, especially links to public repositories, is helpful.
+0. If there are no reported issues of this misclassification, [open an issue][new-issue] and include a link to the repository or a sample of the code that is being misclassified.

-Linguist::FileBlob.new("bin/linguist").language.name #=> "Ruby"
+## Overrides
+
+Linguist supports a number of different custom overrides strategies for language definitions and vendored paths.
+
+### Using gitattributes
+
+Add a `.gitattributes` file to your project and use standard git-style path matchers for the files you want to override to set `linguist-documentation`, `linguist-language`, and `linguist-vendored`. `.gitattributes` will be used to determine language statistics, but will not be used to syntax highlight files. To manually set syntax highlighting, use [Vim or Emacs modelines](#using-emacs-and-vim-modelines).
+
+```
+$ cat .gitattributes
+*.rb linguist-language=Java
 ```

-See [lib/linguist/language.rb](https://github.com/github/linguist/blob/master/lib/linguist/language.rb) and [lib/linguist/languages.yml](https://github.com/github/linguist/blob/master/lib/linguist/languages.yml).
+Checking code you didn't write, such as JavaScript libraries, into your git repo is a common practice, but this often inflates your project's language stats and may even cause your project to be labeled as another language. By default, Linguist treats all of the paths defined in [lib/linguist/vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml) as vendored and therefore doesn't include them in the language statistics for a repository. Vendored files are also hidden by default in diffs on github.com.

-### Syntax Highlighting
+Use the `linguist-vendored` attribute to vendor or un-vendor paths.

-Syntax highlighting in GitHub is performed using TextMate-compatible grammars. These are the same grammars that TextMate, Sublime Text and Atom use.
+```
+$ cat .gitattributes
+special-vendored-path/* linguist-vendored
+jquery.js linguist-vendored=false
+```

-Every language in `languages.yml` is mapped to its corresponding TM `scope`. This scope will be used when picking up a grammar for highlighting. **When adding a new language to Linguist, please add its corresponding scope too (assuming there's an existing TextMate bundle, Sublime Text package, or Atom package) so syntax highlighting works for it**.
+Similar to vendored files, Linguist excludes documentation files from your project's language stats. (Unlike vendored files, documentation files are displayed in diffs on github.com.) [lib/linguist/documentation.yml](lib/linguist/documentation.yml) lists common documentation paths and excludes them from the language statistics for your repository.

-### Stats
+Use the `linguist-documentation` attribute to mark or unmark paths as documentation.

-The Language stats bar that you see on every repository is built by aggregating the languages of each file in that repository. The top language in the graph determines the project's primary language.
+```
+$ cat .gitattributes
+project-docs/* linguist-documentation
+docs/formatter.rb linguist-documentation=false
+```

-The repository stats API, accessed through `#languages`, can be used on a directory:
+### Using Emacs and Vim modelines

-***API UPDATE***
+Alternatively, you can use Vim and Emacs style modelines to set the language for a single file. Modelines can be placed anywhere within a file and are respected when determining how to syntax-highlight a file on GitHub.com

-Since [Version 3.0.0](https://github.com/github/linguist/releases/tag/v3.0.0) Linguist expects a git repository (in the form of a [Rugged::Repository](https://github.com/libgit2/rugged#repositories)) to be passed when initializing `Linguist::Repository`.
+```
+Vim
+vim: set filetype=prolog:
+vim: set ft=cpp:

+Emacs
+-*- mode: php;-*-
+```
+
+## Usage
+
+Install the gem:
+
+```
+$ gem install github-linguist
+```
+
+Then use it in your application:

 ```ruby
 require 'rugged'
@@ -51,147 +86,27 @@ project.language       #=> "Ruby"
 project.languages      #=> { "Ruby" => 119387 }
 ```

-These stats are also printed out by the `linguist` binary. You can use the
+These stats are also printed out by the `linguist` executable. You can use the
 `--breakdown` flag, and the binary will also output the breakdown of files by language.

 You can try running `linguist` on the root directory in this repository itself:

-    $ bundle exec linguist --breakdown
+```
+$ bundle exec linguist --breakdown

-    100.00% Ruby
+100.00% Ruby

-    Ruby:
-    Gemfile
-    Rakefile
-    bin/linguist
-    github-linguist.gemspec
-    lib/linguist.rb
-    lib/linguist/blob_helper.rb
-    lib/linguist/classifier.rb
-    lib/linguist/file_blob.rb
-    lib/linguist/generated.rb
-    lib/linguist/heuristics.rb
-    lib/linguist/language.rb
-    lib/linguist/lazy_blob.rb
-    lib/linguist/md5.rb
-    lib/linguist/repository.rb
-    lib/linguist/samples.rb
-    lib/linguist/tokenizer.rb
-    lib/linguist/version.rb
-    test/test_blob.rb
-    test/test_classifier.rb
-    test/test_heuristics.rb
-    test/test_language.rb
-    test/test_md5.rb
-    test/test_pedantic.rb
-    test/test_repository.rb
-    test/test_samples.rb
-    test/test_tokenizer.rb
-
-#### Ignore vendored files
-
-Checking other code into your git repo is a common practice. But this often inflates your project's language stats and may even cause your project to be labeled as another language. We are able to identify some of these files and directories and exclude them.
-
-```ruby
-Linguist::FileBlob.new("vendor/plugins/foo.rb").vendored? # => true
+Ruby:
+Gemfile
+Rakefile
+bin/linguist
+github-linguist.gemspec
+lib/linguist.rb
+…
 ```

-See [Linguist::BlobHelper#vendored?](https://github.com/github/linguist/blob/master/lib/linguist/blob_helper.rb) and [lib/linguist/vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml).
+## Contributing

-#### Generated file detection
+Please check out our [contributing guidelines](CONTRIBUTING.md).

-Not all plain text files are true source files. Generated files like minified js and compiled CoffeeScript can be detected and excluded from language stats. As an extra bonus, these files are suppressed in diffs.
-
-```ruby
-Linguist::FileBlob.new("underscore.min.js").generated? # => true
-```
-
-See [Linguist::Generated#generated?](https://github.com/github/linguist/blob/master/lib/linguist/generated.rb).
-
-## Overrides
-
-Linguist supports custom overrides for language definitions and vendored paths. Add a `.gitattributes` file to your project using the keys `linguist-language` and `linguist-vendored` with the standard git-style path matchers for the files you want to override.
-
-Please note that the overrides currently only affect the language statistics for a repository and not the syntax-highlighting of files.
-
-```
-$ cat .gitattributes
-*.rb linguist-language=Java
-
-$ linguist --breakdown
-100.00% Java
-
-Java:
-ruby_file.rb
-```
-
-By default, Linguist treats all of the paths defined in [lib/linguist/vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml) as vendored and therefore doesn't include them in the language statistics for a repository. Use the `linguist-vendored` attribute to vendor or un-vendor paths.
-
-```
-$ cat .gitattributes
-special-vendored-path/* linguist-vendored
-jquery.js linguist-vendored=false
-```
-
-## Installation
-
-Github.com is usually running the latest version of the `github-linguist` gem that is released on [RubyGems.org](http://rubygems.org/gems/github-linguist).
-
-But for development you are going to want to checkout out the source. To get it, clone the repo and run [Bundler](http://gembundler.com/) to install its dependencies.
-
-    git clone https://github.com/github/linguist.git
-    cd linguist/
-    script/bootstrap
-
-To run the tests:
-
-    bundle exec rake test
-
-### A note on language extensions
-
-Linguist has a number of methods available to it for identifying the language of a particular file. The initial lookup is based upon the extension of the file, possible file extensions are defined in an array called `extensions`. Take a look at this example for example for `Perl`:
-
-```
-Perl:
-  type: programming
-  ace_mode: perl
-  color: "#0298c3"
-  extensions:
-  - .pl
-  - .PL
-  - .perl
-  - .ph
-  - .plx
-  - .pm
-  - .pod
-  - .psgi
-  interpreters:
-  - perl
-```
-Any of the extensions defined are valid but the first in this array should be the most popular.
-
-### Testing
-
-Sometimes getting the tests running can be too much work, especially if you don't have much Ruby experience. It's okay: be lazy and let our build bot [Travis](http://travis-ci.org/#!/github/linguist) run the tests for you. Just open a pull request and the bot will start cranking away.
-
-Here's our current build status, which is hopefully green: [![Build Status](https://secure.travis-ci.org/github/linguist.png?branch=master)](http://travis-ci.org/github/linguist)
-
-### Releasing
-
-If you are the current maintainer of this gem:
-
- 0. Create a branch for the release: `git checkout -b cut-release-vxx.xx.xx`
- 0. Make sure your local dependencies are up to date: `script/bootstrap`
- 0. If grammar submodules have not been updated recently, update them: `git submodule update --remote && git commit -a`
- 0. Ensure that samples are updated: `bundle exec rake samples`
- 0. Ensure that tests are green: `bundle exec rake test`
- 0. Bump gem version in `lib/linguist/version.rb`.  For example, [like this](https://github.com/github/linguist/commit/8d2ea90a5ba3b2fe6e1508b7155aa4632eea2985).
- 0. Make a PR to github/linguist.  For example, [#1238](https://github.com/github/linguist/pull/1238).
- 0. Build a local gem: `bundle exec rake build_gem`
- 0. Testing:
-   0. Bump the Gemfile and Gemfile.lock versions for an app which relies on this gem
-   0. Install the new gem locally
-   0. Test behavior locally, branch deploy, whatever needs to happen
- 0. Merge github/linguist PR
- 0. Tag and push: `git tag vx.xx.xx; git push --tags`
- 0. Push to rubygems.org -- `gem push github-linguist-3.0.0.gem`
+##
--- a/bin/linguist
+++ b/bin/linguist
@@ -2,10 +2,8 @@

 # linguist — detect language type for a file, or, given a directory, determine language breakdown
 #     usage: linguist <path> [<--breakdown>]
-
-require 'linguist/file_blob'
-require 'linguist/language'
-require 'linguist/repository'
+#
+require 'linguist'
 require 'rugged'

 path = ARGV[0] || Dir.pwd
--- a/grammars.yml
+++ b/grammars.yml
@@ -24,8 +24,11 @@ vendor/grammars/Agda.tmbundle:
 - source.agda
 vendor/grammars/Alloy.tmbundle:
 - source.alloy
-vendor/grammars/AutoHotkey:
+vendor/grammars/AutoHotkey/:
 - source.ahk
+vendor/grammars/BrightScript.tmbundle/:
+- source.brightauthorproject
+- source.brightscript
 vendor/grammars/CLIPS-sublime:
 - source.clips
 vendor/grammars/ColdFusion:
@@ -39,12 +42,20 @@ vendor/grammars/Docker.tmbundle:
 - source.dockerfile
 vendor/grammars/Elm.tmLanguage:
 - source.elm
+vendor/grammars/G-Code/:
+- source.LS
+- source.MCPOST
+- source.MOD
+- source.apt
+- source.gcode
 vendor/grammars/GDScript-sublime/:
 - source.gdscript
 vendor/grammars/Handlebars:
 - text.html.handlebars
 vendor/grammars/IDL-Syntax:
 - source.webidl
+vendor/grammars/InnoSetup/:
+- source.inno
 vendor/grammars/Isabelle.tmbundle:
 - source.isabelle.root
 - source.isabelle.theory
@@ -52,6 +63,8 @@ vendor/grammars/JSyntax/:
 - source.j
 vendor/grammars/Julia.tmbundle:
 - source.julia
+vendor/grammars/Lean.tmbundle:
+- source.lean
 vendor/grammars/LiveScript.tmbundle:
 - source.livescript
 vendor/grammars/Modelica/:
@@ -64,6 +77,8 @@ vendor/grammars/NimLime:
 - source.nimcfg
 vendor/grammars/PHP-Twig.tmbundle:
 - text.html.twig
+vendor/grammars/PogoScript.tmbundle/:
+- source.pogoscript
 vendor/grammars/RDoc.tmbundle:
 - text.rdoc
 vendor/grammars/Racket:
@@ -78,6 +93,8 @@ vendor/grammars/Slash.tmbundle:
 vendor/grammars/Stata.tmbundle:
 - source.mata
 - source.stata
+vendor/grammars/Stylus/:
+- source.stylus
 vendor/grammars/Sublime-Coq:
 - source.coq
 vendor/grammars/Sublime-HTTP:
@@ -104,8 +121,14 @@ vendor/grammars/Sublime-VimL:
 - source.viml
 vendor/grammars/SublimeBrainfuck:
 - source.bf
+vendor/grammars/SublimePapyrus/:
+- source.compiled-papyrus
+- source.papyrus
+- source.papyrus-assembly
 vendor/grammars/SublimeXtend:
 - source.xtend
+vendor/grammars/TXL/:
+- source.txl
 vendor/grammars/Textmate-Gosu-Bundle:
 - source.gosu.2
 vendor/grammars/VBDotNetSyntax:
@@ -120,6 +143,8 @@ vendor/grammars/actionscript3-tmbundle:
 - text.xml.flex-config
 vendor/grammars/ada.tmbundle:
 - source.ada
+vendor/grammars/ampl:
+- source.ampl
 vendor/grammars/ant.tmbundle:
 - text.xml.ant
 vendor/grammars/antlr.tmbundle:
@@ -129,6 +154,8 @@ vendor/grammars/apache.tmbundle:
 - source.apache-config.mod_perl
 vendor/grammars/applescript.tmbundle:
 - source.applescript
+vendor/grammars/asciidoc.tmbundle/:
+- text.html.asciidoc
 vendor/grammars/asp.tmbundle:
 - source.asp
 - text.html.asp
@@ -192,6 +219,8 @@ vendor/grammars/dylan.tmbundle:
 - source.makegen
 vendor/grammars/ebundles/Bundles/MSDOS batch file.tmbundle:
 - source.dosbatch
+vendor/grammars/ec.tmbundle/:
+- source.c.ec
 vendor/grammars/eiffel.tmbundle:
 - source.eiffel
 vendor/grammars/elixir-tmbundle:
@@ -213,12 +242,16 @@ vendor/grammars/fortran.tmbundle:
 - source.fortran.modern
 vendor/grammars/fsharpbinding:
 - source.fsharp
+vendor/grammars/gap-tmbundle/:
+- source.gap
 vendor/grammars/gettext.tmbundle:
 - source.po
 vendor/grammars/gnuplot-tmbundle:
 - source.gnuplot
 vendor/grammars/go-tmbundle:
 - source.go
+vendor/grammars/grace-tmbundle/:
+- source.grace
 vendor/grammars/gradle.tmbundle:
 - source.groovy.gradle
 vendor/grammars/graphviz.tmbundle:
@@ -324,6 +357,8 @@ vendor/grammars/matlab.tmbundle:
 - source.octave
 vendor/grammars/maven.tmbundle:
 - text.xml.pom
+vendor/grammars/mediawiki.tmbundle/:
+- text.html.mediawiki
 vendor/grammars/mercury-tmlanguage:
 - source.mercury
 vendor/grammars/monkey.tmbundle:
@@ -350,12 +385,17 @@ vendor/grammars/ooc.tmbundle:
 - source.ooc
 vendor/grammars/opa.tmbundle:
 - source.opa
+vendor/grammars/openscad.tmbundle/:
+- source.scad
+vendor/grammars/oracle.tmbundle:
+- source.plsql.oracle
 vendor/grammars/oz-tmbundle/Syntaxes/Oz.tmLanguage:
 - source.oz
 vendor/grammars/pascal.tmbundle:
 - source.pascal
-vendor/grammars/perl.tmbundle:
+vendor/grammars/perl.tmbundle/:
 - source.perl
+- source.perl.6
 vendor/grammars/php-smarty.tmbundle:
 - source.smarty
 vendor/grammars/php.tmbundle:
@@ -439,17 +479,30 @@ vendor/grammars/sublime-nginx:
 - source.nginx
 vendor/grammars/sublime-nix:
 - source.nix
+vendor/grammars/sublime-opal/:
+- source.opal
+- source.opalsysdefs
 vendor/grammars/sublime-robot-plugin:
 - text.robot
 vendor/grammars/sublime-rust:
 - source.rust
 vendor/grammars/sublime-sourcepawn:
 - source.sp
+vendor/grammars/sublime-spintools/:
+- source.regexp.spin
+- source.spin
 vendor/grammars/sublime-tea:
 - source.tea
+vendor/grammars/sublime-text-ox/:
+- source.ox
+vendor/grammars/sublime-text-pig-latin/:
+- source.pig_latin
+vendor/grammars/sublime-varnish:
+- source.varnish.vcl
 vendor/grammars/sublime_cobol:
 - source.acucobol
 - source.cobol
+- source.jcl
 - source.opencobol
 vendor/grammars/sublime_man_page_support:
 - source.man
@@ -479,6 +532,8 @@ vendor/grammars/verilog.tmbundle:
 - source.verilog
 vendor/grammars/x86-assembly-textmate-bundle:
 - source.asm.x86
+vendor/grammars/xc.tmbundle/:
+- source.xc
 vendor/grammars/xml.tmbundle:
 - text.xml
 - text.xml.xsl
--- a/lib/linguist.rb
+++ b/lib/linguist.rb
@@ -6,3 +6,15 @@ require 'linguist/repository'
 require 'linguist/samples'
 require 'linguist/shebang'
 require 'linguist/version'
+
+class << Linguist
+  attr_accessor :instrumenter
+
+  def instrument(*args, &bk)
+    if instrumenter
+      instrumenter.instrument(*args, &bk)
+    else
+      yield if block_given?
+    end
+  end
+end
--- a/lib/linguist/blob_helper.rb
+++ b/lib/linguist/blob_helper.rb
@@ -99,7 +99,7 @@ module Linguist
      elsif name.nil?
        "attachment"
      else
-        "attachment; filename=#{EscapeUtils.escape_url(File.basename(name))}"
+        "attachment; filename=#{EscapeUtils.escape_url(name)}"
      end
    end

@@ -233,7 +233,22 @@ module Linguist
    #
    # Return true or false
    def vendored?
-      name =~ VendoredRegexp ? true : false
+      path =~ VendoredRegexp ? true : false
+    end
+
+    documentation_paths = YAML.load_file(File.expand_path("../documentation.yml", __FILE__))
+    DocumentationRegexp = Regexp.new(documentation_paths.join('|'))
+
+    # Public: Is the blob in a documentation directory?
+    #
+    # Documentation files are ignored by language statistics.
+    #
+    # See "documentation.yml" for a list of documentation conventions that match
+    # this pattern.
+    #
+    # Return true or false
+    def documentation?
+      path =~ DocumentationRegexp ? true : false
    end

    # Public: Get each line of data
@@ -301,7 +316,7 @@ module Linguist
    #
    # Return true or false
    def generated?
-      @_generated ||= Generated.generated?(name, lambda { data })
+      @_generated ||= Generated.generated?(path, lambda { data })
    end

    # Public: Detects the Language of the blob.
@@ -317,5 +332,15 @@ module Linguist
    def tm_scope
      language && language.tm_scope
    end
+
+    DETECTABLE_TYPES = [:programming, :markup].freeze
+
+    # Internal: Should this blob be included in repository language statistics?
+    def include_in_language_stats?
+      !vendored? &&
+      !documentation? &&
+      !generated? &&
+      language && DETECTABLE_TYPES.include?(language.type)
+    end
  end
 end
--- a/lib/linguist/documentation.yml
+++ b/lib/linguist/documentation.yml
@@ -0,0 +1,23 @@
+# Documentation files and directories are excluded from language
+# statistics.
+#
+# Lines in this file are Regexps that are matched against the file
+# pathname.
+#
+# Please add additional test coverage to
+# `test/test_blob.rb#test_documentation` if you make any changes.
+
+## Documentation directories ##
+
+- ^docs?/
+- (^|/)[Dd]ocumentation/
+- (^|/)javadoc/
+- ^man/
+
+## Documentation files ##
+
+- (^|/)CONTRIBUTING(\.|$)
+- (^|/)COPYING(\.|$)
+- (^|/)INSTALL(\.|$)
+- (^|/)LICEN[CS]E(\.|$)
+- (^|/)README(\.|$)
--- a/lib/linguist/file_blob.rb
+++ b/lib/linguist/file_blob.rb
@@ -3,7 +3,7 @@ require 'linguist/blob_helper'
 module Linguist
  # A FileBlob is a wrapper around a File object to make it quack
  # like a Grit::Blob. It provides the basic interface: `name`,
-  # `data`, and `size`.
+  # `data`, `path` and `size`.
  class FileBlob
    include BlobHelper

@@ -14,43 +14,50 @@ module Linguist
    #
    # Returns a FileBlob.
    def initialize(path, base_path = nil)
-      @path = path
-      @name = base_path ? path.sub("#{base_path}/", '') : path
+      @fullpath = path
+      @path = base_path ? path.sub("#{base_path}/", '') : path
    end

    # Public: Filename
    #
    # Examples
    #
-    #   FileBlob.new("/path/to/linguist/lib/linguist.rb").name
+    #   FileBlob.new("/path/to/linguist/lib/linguist.rb").path
    #   # =>  "/path/to/linguist/lib/linguist.rb"
    #
    #   FileBlob.new("/path/to/linguist/lib/linguist.rb",
-    #                "/path/to/linguist").name
+    #                "/path/to/linguist").path
    #   # =>  "lib/linguist.rb"
    #
    # Returns a String
-    attr_reader :name
+    attr_reader :path

    # Public: Read file permissions
    #
    # Returns a String like '100644'
    def mode
-      File.stat(@path).mode.to_s(8)
+      File.stat(@fullpath).mode.to_s(8)
+    end
+
+    # Public: File name
+    #
+    # Returns a String
+    def name
+      File.basename(@fullpath)
    end

    # Public: Read file contents.
    #
    # Returns a String.
    def data
-      File.read(@path)
+      File.read(@fullpath)
    end

    # Public: Get byte size
    #
    # Returns an Integer.
    def size
-      File.size(@path)
+      File.size(@fullpath)
    end

    # Public: Get file extension.
@@ -67,7 +74,7 @@ module Linguist
    #
    # Returns an Array
    def extensions
-      basename, *segments = File.basename(name).split(".")
+      basename, *segments = name.downcase.split(".")

      segments.map.with_index do |segment, index|
        "." + segments[index..-1].join(".")
--- a/lib/linguist/generated.rb
+++ b/lib/linguist/generated.rb
@@ -62,6 +62,8 @@ module Linguist
      generated_parser? ||
      generated_net_docfile? ||
      generated_postscript? ||
+      compiled_cython_file? ||
+      generated_protocol_buffer_go? ||
      generated_protocol_buffer? ||
      generated_jni_header? ||
      vcr_cassette?
@@ -202,6 +204,13 @@ module Linguist
        creator.include?("ImageMagick")
    end

+    def generated_protocol_buffer_go?
+      return false unless extname == '.go'
+      return false unless lines.count > 1
+
+      return lines[0].include?("Code generated by protoc-gen-go")
+    end
+
    # Internal: Is the blob a C++, Java or Python source file generated by the
    # Protocol Buffer compiler?
    #
@@ -262,5 +271,18 @@ module Linguist
      # VCR Cassettes have "recorded_with: VCR" in the second last line.
      return lines[-2].include?("recorded_with: VCR")
    end
+
+    # Internal: Is this a compiled C/C++ file from Cython?
+    #
+    # Cython-compiled C/C++ files typically contain:
+    # /* Generated by Cython x.x.x on ... */
+    # on the first line.
+    #
+    # Return true or false
+    def compiled_cython_file?
+      return false unless ['.c', '.cpp'].include? extname
+      return false unless lines.count > 1
+      return lines[0].include?("Generated by Cython")
+    end
  end
 end
--- a/lib/linguist/heuristics.rb
+++ b/lib/linguist/heuristics.rb
@@ -33,7 +33,7 @@ module Linguist
    #     disambiguate "Perl", "Prolog" do |data|
    #       if data.include?("use strict")
    #         Language["Perl"]
-    #       elsif data.include?(":-")
+    #       elsif /^[^#]+:-/.match(data)
    #         Language["Prolog"]
    #       end
    #     end
@@ -92,25 +92,29 @@ module Linguist
    disambiguate "Perl", "Perl6", "Prolog" do |data|
      if data.include?("use v6")
        Language["Perl6"]
-      elsif data.include?("use strict")
+      elsif data.match(/use strict|use\s+v?5\./)
        Language["Perl"]
-      elsif data.include?(":-")
+      elsif /^[^#]+:-/.match(data)
        Language["Prolog"]
      end
    end

    disambiguate "ECL", "Prolog" do |data|
-      if data.include?(":-")
+      if /^[^#]+:-/.match(data)
        Language["Prolog"]
      elsif data.include?(":=")
        Language["ECL"]
      end
    end

-    disambiguate "IDL", "Prolog" do |data|
-      if data.include?(":-")
+    disambiguate "IDL", "Prolog", "INI", "QMake" do |data|
+      if /^[^#]+:-/.match(data)
        Language["Prolog"]
-      else
+      elsif data.include?("last_client=")
+        Language["INI"]
+      elsif data.include?("HEADERS") && data.include?("SOURCES")
+        Language["QMake"]
+      elsif /^\s*function[ \w,]+$/.match(data)
        Language["IDL"]
      end
    end
@@ -150,14 +154,20 @@ module Linguist
      end
    end

-    disambiguate "AsciiDoc", "AGS Script" do |data|
-      Language["AsciiDoc"] if /^=+(\s|\n)/.match(data)
+    disambiguate "AsciiDoc", "AGS Script", "Public Key" do |data|
+      if /^[=-]+(\s|\n)|{{[A-Za-z]/.match(data)
+        Language["AsciiDoc"]
+      elsif /^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])/.match(data)
+        Language["AGS Script"]
+      elsif /^-----BEGIN/.match(data)
+        Language["Public Key"]
+      end
    end

    disambiguate "FORTRAN", "Forth" do |data|
      if /^: /.match(data)
        Language["Forth"]
-      elsif /^([c*][^a-z]|      (subroutine|program)\s|!)/i.match(data)
+      elsif /^([c*][^a-z]|      (subroutine|program)\s|\s*!)/i.match(data)
        Language["FORTRAN"]
      end
    end
@@ -172,11 +182,13 @@ module Linguist
      end
    end

-    disambiguate "M", "Mathematica", "Matlab", "Mercury", "Objective-C" do |data|
+    disambiguate "M", "MUF", "Mathematica", "Matlab", "Mercury", "Objective-C" do |data|
      if ObjectiveCRegex.match(data)
        Language["Objective-C"]
      elsif data.include?(":- module")
        Language["Mercury"]
+      elsif /^: /.match(data)
+        Language["MUF"]
      elsif /^\s*;/.match(data)
        Language["M"]
      elsif /^\s*\(\*/.match(data)
@@ -223,5 +235,31 @@ module Linguist
        Language["Text"]
      end
    end
+
+    disambiguate "PLSQL", "SQLPL", "PLpgSQL", "SQL" do |data|
+      if /^\\i\b|AS \$\$|LANGUAGE '+plpgsql'+/i.match(data) || /SECURITY (DEFINER|INVOKER)/i.match(data) || /BEGIN( WORK| TRANSACTION)?;/i.match(data)
+        #Postgres
+        Language["PLpgSQL"]
+      elsif /(alter module)|(language sql)|(begin( NOT)+ atomic)/i.match(data)  || /signal SQLSTATE '[0-9]+'/i.match(data)
+        #IBM db2
+        Language["SQLPL"]
+      elsif /pragma|\$\$PLSQL_|XMLTYPE|sysdate|systimestamp|\.nextval|connect by|AUTHID (DEFINER|CURRENT_USER)/i.match(data) || /constructor\W+function/i.match(data)
+        #Oracle
+        Language["PLSQL"]
+      elsif ! /begin|boolean|package|exception/i.match(data)
+        #Generic SQL
+        Language["SQL"]
+      end
+    end
+
+    disambiguate "D", "DTrace", "Makefile" do |data|
+      if /^module /.match(data)
+        Language["D"]
+      elsif /^((dtrace:::)?BEGIN|provider |#pragma (D (option|attributes)|ident)\s)/.match(data)
+        Language["DTrace"]
+      elsif /(\/.*:( .* \\)$| : \\$|^ : |: \\$)/.match(data)
+        Language["Makefile"]
+      end
+    end
  end
 end
--- a/lib/linguist/language.rb
+++ b/lib/linguist/language.rb
@@ -32,13 +32,6 @@ module Linguist
    # Valid Languages types
    TYPES = [:data, :markup, :programming, :prose]

-    # Names of non-programming languages that we will still detect
-    #
-    # Returns an array
-    def self.detectable_markup
-      ["CSS", "Less", "Sass", "SCSS", "Stylus", "TeX"]
-    end
-
    # Detect languages by a specific type
    #
    # type - A symbol that exists within TYPES
@@ -80,7 +73,7 @@ module Linguist
          raise ArgumentError, "Extension is missing a '.': #{extension.inspect}"
        end

-        @extension_index[extension] << language
+        @extension_index[extension.downcase] << language
      end

      language.interpreters.each do |interpreter|
@@ -96,8 +89,8 @@ module Linguist

    STRATEGIES = [
      Linguist::Strategy::Modeline,
-      Linguist::Strategy::Filename,
      Linguist::Shebang,
+      Linguist::Strategy::Filename,
      Linguist::Heuristics,
      Linguist::Classifier
    ]
@@ -112,19 +105,31 @@ module Linguist
      # Bail early if the blob is binary or empty.
      return nil if blob.likely_binary? || blob.binary? || blob.empty?

-      # Call each strategy until one candidate is returned.
-      STRATEGIES.reduce([]) do |languages, strategy|
-        candidates = strategy.call(blob, languages)
-        if candidates.size == 1
-          return candidates.first
-        elsif candidates.size > 1
-          # More than one candidate was found, pass them to the next strategy.
-          candidates
-        else
-          # No candiates were found, pass on languages from the previous strategy.
-          languages
+      Linguist.instrument("linguist.detection", :blob => blob) do
+        # Call each strategy until one candidate is returned.
+        languages = []
+        returning_strategy = nil
+
+        STRATEGIES.each do |strategy|
+          returning_strategy = strategy
+          candidates = Linguist.instrument("linguist.strategy", :blob => blob, :strategy => strategy, :candidates => languages) do
+            strategy.call(blob, languages)
+          end
+          if candidates.size == 1
+            languages = candidates
+            break
+          elsif candidates.size > 1
+            # More than one candidate was found, pass them to the next strategy.
+            languages = candidates
+          else
+            # No candidates, try the next strategy
+          end
        end
-      end.first
+
+        Linguist.instrument("linguist.detected", :blob => blob, :strategy => returning_strategy, :language => languages.first)
+
+        languages.first
+      end
    end

    # Public: Get all Languages
@@ -198,7 +203,7 @@ module Linguist
    # Returns all matching Languages or [] if none were found.
    def self.find_by_extension(extname)
      extname = ".#{extname}" unless extname.start_with?(".")
-      @extension_index[extname]
+      @extension_index[extname.downcase]
    end

    # DEPRECATED
@@ -535,8 +540,8 @@ module Linguist

    if extnames = extensions[name]
      extnames.each do |extname|
-        if !options['extensions'].index { |x| x.end_with? extname }
-          warn "#{name} has a sample with extension (#{extname}) that isn't explicitly defined in languages.yml" unless extname == '.script!'
+        if !options['extensions'].index { |x| x.downcase.end_with? extname.downcase }
+          warn "#{name} has a sample with extension (#{extname.downcase}) that isn't explicitly defined in languages.yml" unless extname == '.script!'
          options['extensions'] << extname
        end
      end
--- a/lib/linguist/languages.yml
+++ b/lib/linguist/languages.yml
@@ -18,6 +18,8 @@
 #                     language. This should match one of the scopes listed in
 #                     the grammars.yml file. Use "none" if there is no grammar
 #                     for this language.
+# group             - Name of the parent language. Languages in a group are counted
+#                     in the statistics as the parent language.
 #
 # Any additions or modifications (even trivial) should have corresponding
 # test change in `test/test_blob.rb`.
@@ -41,6 +43,14 @@ AGS Script:
  tm_scope: source.c++
  ace_mode: c_cpp

+AMPL:
+  type: programming
+  color: "#00008B"
+  extensions:
+  - .ampl
+  tm_scope: source.ampl
+  ace_mode: text
+
 ANTLR:
  type: programming
  color: "#9DC3FF"
@@ -186,7 +196,7 @@ AsciiDoc:
  - .asciidoc
  - .adoc
  - .asc
-  tm_scope: none
+  tm_scope: text.html.asciidoc

 AspectJ:
  type: programming
@@ -198,14 +208,14 @@ AspectJ:

 Assembly:
  type: programming
-  color: "#a67219"
+  color: "#6E4C13"
  search_term: nasm
  aliases:
  - nasm
  extensions:
  - .asm
-  - .ASM
  - .a51
+  - .nasm
  tm_scope: source.asm.x86
  ace_mode: assembly_x86

@@ -270,6 +280,7 @@ Batchfile:
  ace_mode: batchfile

 Befunge:
+  type: programming
  extensions:
  - .befunge
  ace_mode: text
@@ -325,6 +336,7 @@ Boo:
  ace_mode: text

 Brainfuck:
+  type: programming
  extensions:
  - .b
  - .bf
@@ -335,7 +347,7 @@ Brightscript:
  type: programming
  extensions:
  - .brs
-  tm_scope: none
+  tm_scope: source.brightscript
  ace_mode: text

 Bro:
@@ -346,15 +358,15 @@ Bro:

 C:
  type: programming
-  color: "#555"
+  color: "#555555"
  extensions:
  - .c
-  - .C
-  - .H
  - .cats
  - .h
  - .idc
  - .w
+  interpreters:
+  - tcc
  ace_mode: c_cpp

 C#:
@@ -418,6 +430,7 @@ CLIPS:
  ace_mode: text

 CMake:
+  type: programming
  extensions:
  - .cmake
  - .cmake.in
@@ -429,8 +442,6 @@ COBOL:
  type: programming
  extensions:
  - .cob
-  - .COB
-  - .CPY
  - .cbl
  - .ccp
  - .cobol
@@ -438,6 +449,8 @@ COBOL:
  ace_mode: cobol

 CSS:
+  type: markup
+  tm_scope: source.css
  ace_mode: css
  color: "#563d7c"
  extensions:
@@ -475,6 +488,7 @@ Chapel:
  ace_mode: text

 ChucK:
+  type: programming
  extensions:
  - .ck
  tm_scope: source.java
@@ -636,6 +650,7 @@ Crystal:
  - crystal

 Cucumber:
+  type: programming
  extensions:
  - .feature
  tm_scope: text.gherkin.feature
@@ -694,7 +709,17 @@ DM:
  tm_scope: source.c++
  ace_mode: c_cpp

+DTrace:
+  type: programming
+  extensions:
+  - .d
+  interpreters:
+  - dtrace
+  tm_scope: source.c
+  ace_mode: c_cpp
+
 Darcs Patch:
+  type: programming
  search_term: dpatch
  aliases:
  - dpatch
@@ -712,6 +737,7 @@ Dart:
  ace_mode: dart

 Diff:
+  type: programming
  extensions:
  - .diff
  - .patch
@@ -867,14 +893,6 @@ FORTRAN:
  color: "#4d41b1"
  extensions:
  - .f90
-  - .F
-  - .F03
-  - .F08
-  - .F77
-  - .F90
-  - .F95
-  - .FOR
-  - .FPP
  - .f
  - .f03
  - .f08
@@ -918,9 +936,7 @@ Forth:
  color: "#341708"
  extensions:
  - .fth
-  - .4TH
  - .4th
-  - .F
  - .f
  - .for
  - .forth
@@ -943,7 +959,7 @@ G-code:
  - .g
  - .gco
  - .gcode
-  tm_scope: none
+  tm_scope: source.gcode
  ace_mode: gcode

 GAMS:
@@ -961,7 +977,7 @@ GAP:
  - .gd
  - .gi
  - .tst
-  tm_scope: none
+  tm_scope: source.gap
  ace_mode: text

 GAS:
@@ -969,7 +985,6 @@ GAS:
  group: Assembly
  extensions:
  - .s
-  - .S
  tm_scope: source.asm.x86
  ace_mode: assembly_x86

@@ -981,7 +996,6 @@ GDScript:
  ace_mode: text

 GLSL:
-  group: C
  type: programming
  extensions:
  - .glsl
@@ -1009,6 +1023,7 @@ Game Maker Language:
  ace_mode: c_cpp

 Genshi:
+  type: programming
  extensions:
  - .kid
  tm_scope: text.xml.genshi
@@ -1018,6 +1033,7 @@ Genshi:
  ace_mode: xml

 Gentoo Ebuild:
+  type: programming
  group: Shell
  extensions:
  - .ebuild
@@ -1025,6 +1041,7 @@ Gentoo Ebuild:
  ace_mode: sh

 Gentoo Eclass:
+  type: programming
  group: Shell
  extensions:
  - .eclass
@@ -1032,6 +1049,7 @@ Gentoo Eclass:
  ace_mode: sh

 Gettext Catalog:
+  type: prose
  search_term: pot
  searchable: false
  aliases:
@@ -1093,7 +1111,7 @@ Grace:
  type: programming
  extensions:
  - .grace
-  tm_scope: none
+  tm_scope: source.grace
  ace_mode: text

 Gradle:
@@ -1127,11 +1145,11 @@ Graphviz (DOT):
  tm_scope: source.dot
  extensions:
  - .dot
-  - .DOT
  - .gv
  ace_mode: text

 Groff:
+  type: programming
  extensions:
  - .man
  - '.1'
@@ -1159,6 +1177,7 @@ Groovy:
  - groovy

 Groovy Server Pages:
+  type: programming
  group: Groovy
  aliases:
  - gsp
@@ -1172,6 +1191,7 @@ HTML:
  type: markup
  tm_scope: text.html.basic
  ace_mode: html
+  color: "#e44b23"
  aliases:
  - xhtml
  extensions:
@@ -1307,6 +1327,7 @@ INI:
  - .ini
  - .cfg
  - .prefs
+  - .pro
  - .properties
  tm_scope: source.ini
  aliases:
@@ -1314,6 +1335,7 @@ INI:
  ace_mode: ini

 IRC log:
+  type: data
  search_term: irc
  aliases:
  - irc
@@ -1344,9 +1366,10 @@ Inform 7:
  ace_mode: text

 Inno Setup:
+  type: programming
  extensions:
  - .iss
-  tm_scope: none
+  tm_scope: source.inno
  ace_mode: text

 Io:
@@ -1375,6 +1398,7 @@ Isabelle:

 J:
  type: programming
+  color: "#2d8abd"
  extensions:
  - .ijs
  tm_scope: source.j
@@ -1438,6 +1462,7 @@ Java:
  - .java

 Java Server Pages:
+  type: programming
  group: Java
  search_term: jsp
  aliases:
@@ -1533,6 +1558,7 @@ LFE:
  ace_mode: lisp

 LLVM:
+  type: programming
  extensions:
  - .ll
  ace_mode: text
@@ -1584,6 +1610,13 @@ Latte:
  tm_scope: source.smarty
  ace_mode: smarty

+Lean:
+  type: programming
+  extensions:
+  - .lean
+  - .hlean
+  ace_mode: lean
+
 Less:
  type: markup
  group: CSS
@@ -1593,6 +1626,7 @@ Less:
  ace_mode: less

 LilyPond:
+  type: programming
  extensions:
  - .ly
  - .ily
@@ -1713,6 +1747,15 @@ MTML:
  tm_scope: text.html.basic
  ace_mode: html

+MUF:
+  type: programming
+  group: Forth
+  extensions:
+  - .muf
+  - .m
+  tm_scope: none
+  ace_mode: forth
+
 Makefile:
  type: programming
  aliases:
@@ -1721,6 +1764,7 @@ Makefile:
  - mf
  extensions:
  - .mak
+  - .d
  - .mk
  filenames:
  - GNUmakefile
@@ -1731,6 +1775,7 @@ Makefile:
  ace_mode: makefile

 Mako:
+  type: programming
  extensions:
  - .mako
  - .mao
@@ -1767,6 +1812,8 @@ Mathematica:
  - .ma
  - .nb
  - .nbp
+  - .wl
+  - .wlt
  aliases:
  - mma
  ace_mode: text
@@ -1807,7 +1854,7 @@ MediaWiki:
  wrap: true
  extensions:
  - .mediawiki
-  tm_scope: none
+  tm_scope: text.html.mediawiki
  ace_mode: text

 Mercury:
@@ -1823,6 +1870,7 @@ Mercury:
  ace_mode: prolog

 MiniD: # Legacy
+  type: programming
  searchable: false
  extensions:
  - .minid # Dummy extension
@@ -1870,12 +1918,21 @@ MoonScript:
  ace_mode: text

 Myghty:
+  type: programming
  extensions:
  - .myt
  tm_scope: none
  ace_mode: text

+NL:
+  type: data
+  extensions:
+  - .nl
+  tm_scope: none
+  ace_mode: text
+
 NSIS:
+  type: programming
  extensions:
  - .nsi
  - .nsh
@@ -1967,6 +2024,7 @@ Nu:
  - nush

 NumPy:
+  type: programming
  group: Python
  extensions:
  - .numpy
@@ -2052,7 +2110,7 @@ Opal:
  color: "#f7ede0"
  extensions:
  - .opal
-  tm_scope: none
+  tm_scope: source.opal
  ace_mode: text

 OpenCL:
@@ -2080,7 +2138,7 @@ OpenSCAD:
  type: programming
  extensions:
  - .scad
-  tm_scope: none
+  tm_scope: source.scad
  ace_mode: scad

 Org:
@@ -2097,7 +2155,7 @@ Ox:
  - .ox
  - .oxh
  - .oxo
-  tm_scope: none
+  tm_scope: source.ox
  ace_mode: text

 Oxygene:
@@ -2145,6 +2203,27 @@ PHP:
  aliases:
  - inc

+#Oracle 
+PLSQL:
+  type: programming
+  ace_mode: sql
+  tm_scope: source.plsql.oracle
+  extensions:
+  - .pls
+  - .pkb
+  - .pks
+  - .plb
+  - .plsql
+  - .sql
+
+#Postgres 
+PLpgSQL:
+  type: programming
+  ace_mode: pgsql
+  tm_scope: source.sql
+  extensions:
+  - .sql
+
 Pan:
  type: programming
  color: '#cc0000'
@@ -2158,7 +2237,7 @@ Papyrus:
  color: "#6600cc"
  extensions:
  - .psc
-  tm_scope: none
+  tm_scope: source.papyrus
  ace_mode: text

 Parrot:
@@ -2206,11 +2285,11 @@ Pascal:

 Perl:
  type: programming
+  tm_scope: source.perl
  ace_mode: perl
  color: "#0298c3"
  extensions:
  - .pl
-  - .PL
  - .cgi
  - .fcgi
  - .perl
@@ -2242,7 +2321,7 @@ Perl6:
  - Rexfile
  interpreters:
  - perl6
-  tm_scope: none
+  tm_scope: source.perl.6
  ace_mode: perl

 PigLatin:
@@ -2250,7 +2329,7 @@ PigLatin:
  color: "#fcd7de"
  extensions:
  - .pig
-  tm_scope: none
+  tm_scope: source.pig_latin
  ace_mode: text

 Pike:
@@ -2276,7 +2355,7 @@ PogoScript:
  color: "#d80074"
  extensions:
  - .pogo
-  tm_scope: none
+  tm_scope: source.pogoscript
  ace_mode: text

 PostScript:
@@ -2323,7 +2402,7 @@ Propeller Spin:
  color: "#2b446d"
  extensions:
  - .spin
-  tm_scope: none
+  tm_scope: source.spin
  ace_mode: text

 Protocol Buffer:
@@ -2418,7 +2497,7 @@ Python traceback:
  ace_mode: text

 QML:
-  type: markup
+  type: programming
  color: "#44a51c"
  extensions:
  - .qml
@@ -2426,6 +2505,7 @@ QML:
  ace_mode: text

 QMake:
+  type: programming
  extensions:
  - .pro
  - .pri
@@ -2442,8 +2522,6 @@ R:
  - splus
  extensions:
  - .r
-  - .R
-  - .Rd
  - .rd
  - .rsx
  filenames:
@@ -2496,7 +2574,6 @@ RMarkdown:
  ace_mode: markdown
  extensions:
  - .rmd
-  - .Rmd
  tm_scope: none

 Racket:
@@ -2522,6 +2599,7 @@ Ragel in Ruby Host:
  ace_mode: text

 Raw token data:
+  type: data
  search_term: raw
  aliases:
  - raw
@@ -2553,6 +2631,7 @@ Red:
  ace_mode: text

 Redcode:
+  type: programming
  extensions:
  - .cw
  tm_scope: none
@@ -2591,6 +2670,7 @@ Ruby:
  - .gemspec
  - .god
  - .irbrc
+  - .jbuilder
  - .mspec
  - .pluginspec
  - .podspec
@@ -2675,6 +2755,15 @@ SQL:
  - .udf
  - .viw

+#IBM DB2
+SQLPL:
+  type: programming
+  ace_mode: sql
+  tm_scope: source.sql
+  extensions:
+  - .sql
+  - .db2
+
 STON:
  type: data
  group: Smalltalk
@@ -2683,6 +2772,13 @@ STON:
  tm_scope: source.smalltalk
  ace_mode: text

+SVG:
+  type: data
+  extensions:
+  - .svg
+  tm_scope: text.xml
+  ace_mode: xml
+
 Sage:
  type: programming
  group: Python
@@ -2832,6 +2928,7 @@ Smalltalk:
  ace_mode: text

 Smarty:
+  type: programming
  extensions:
  - .tpl
  ace_mode: smarty
@@ -2843,6 +2940,7 @@ SourcePawn:
  - sourcemod
  extensions:
  - .sp
+  - .sma
  tm_scope: source.sp
  ace_mode: text

@@ -2883,7 +2981,7 @@ Stylus:
  group: CSS
  extensions:
  - .styl
-  tm_scope: none
+  tm_scope: source.stylus
  ace_mode: stylus

 SuperCollider:
@@ -2922,7 +3020,7 @@ TXL:
  type: programming
  extensions:
  - .txl
-  tm_scope: none
+  tm_scope: source.txl
  ace_mode: text

 Tcl:
@@ -3054,11 +3152,11 @@ UnrealScript:

 VCL:
  type: programming
-  ace_mode: perl
  color: "#0298c3"
  extensions:
  - .vcl
-  tm_scope: source.perl
+  tm_scope: source.varnish.vcl
+  ace_mode: text

 VHDL:
  type: programming
@@ -3148,13 +3246,14 @@ WebIDL:

 XC:
  type: programming
+  color: "#99DA07"
  extensions:
  - .xc
-  tm_scope: source.c
+  tm_scope: source.xc
  ace_mode: c_cpp

 XML:
-  type: markup
+  type: data
  ace_mode: xml
  aliases:
  - rss
@@ -3198,14 +3297,13 @@ XML:
  - .srdf
  - .stTheme
  - .sublime-snippet
-  - .svg
  - .targets
  - .tmCommand
+  - .tml
  - .tmLanguage
  - .tmPreferences
  - .tmSnippet
  - .tmTheme
-  - .tml
  - .ts
  - .ui
  - .urdf
@@ -3256,6 +3354,7 @@ XQuery:
  ace_mode: xquery

 XS:
+  type: programming
  extensions:
  - .xs
  tm_scope: source.c
@@ -3328,11 +3427,12 @@ desktop:

 eC:
  type: programming
+  color: "#4A4773"
  search_term: ec
  extensions:
  - .ec
  - .eh
-  tm_scope: none
+  tm_scope: source.c.ec
  ace_mode: text

 edn:
@@ -3352,6 +3452,7 @@ fish:
  ace_mode: text

 mupad:
+  type: programming
  extensions:
  - .mu
  ace_mode: text
--- a/lib/linguist/lazy_blob.rb
+++ b/lib/linguist/lazy_blob.rb
@@ -4,7 +4,7 @@ require 'rugged'

 module Linguist
  class LazyBlob
-    GIT_ATTR = ['linguist-language', 'linguist-vendored']
+    GIT_ATTR = ['linguist-documentation', 'linguist-language', 'linguist-vendored']
    GIT_ATTR_OPTS = { :priority => [:index], :skip_system => true }
    GIT_ATTR_FLAGS = Rugged::Repository::Attributes.parse_opts(GIT_ATTR_OPTS)

@@ -14,13 +14,15 @@ module Linguist

    attr_reader :repository
    attr_reader :oid
-    attr_reader :name
+    attr_reader :path
    attr_reader :mode

-    def initialize(repo, oid, name, mode = nil)
+    alias :name :path
+
+    def initialize(repo, oid, path, mode = nil)
      @repository = repo
      @oid = oid
-      @name = name
+      @path = path
      @mode = mode
    end

@@ -37,11 +39,19 @@ module Linguist
      end
    end

+    def documentation?
+      if attr = git_attributes['linguist-documentation']
+        boolean_attribute(attr)
+      else
+        super
+      end
+    end
+
    def language
      return @language if defined?(@language)

      @language = if lang = git_attributes['linguist-language']
-        Language.find_by_name(lang)
+        Language.find_by_alias(lang)
      else
        super
      end
--- a/lib/linguist/popular.yml
+++ b/lib/linguist/popular.yml
@@ -9,21 +9,21 @@
 - CSS
 - Clojure
 - CoffeeScript
- Common Lisp
- Diff
- Emacs Lisp
- Erlang
+- Go
 - HTML
 - Haskell
 - Java
 - JavaScript
 - Lua
+- Matlab
 - Objective-C
 - PHP
 - Perl
 - Python
+- R
 - Ruby
- SQL
 - Scala
- Scheme
 - Shell
+- Swift
+- TeX
+- VimL
--- a/lib/linguist/repository.rb
+++ b/lib/linguist/repository.rb
@@ -156,13 +156,8 @@ module Linguist

          blob = Linguist::LazyBlob.new(repository, delta.new_file[:oid], new, mode.to_s(8))

-          # Skip vendored or generated blobs
-          next if blob.vendored? || blob.generated? || blob.language.nil?
-
-          # Only include programming languages and acceptable markup languages
-          if blob.language.type == :programming || Language.detectable_markup.include?(blob.language.name)
-            file_map[new] = [blob.language.group.name, blob.size]
-          end
+          next unless blob.include_in_language_stats?
+          file_map[new] = [blob.language.group.name, blob.size]
        end
      end

--- a/lib/linguist/strategy/modeline.rb
+++ b/lib/linguist/strategy/modeline.rb
@@ -1,8 +1,8 @@
 module Linguist
  module Strategy
    class Modeline
-      EmacsModeline = /-\*-\s*mode:\s*(\w+);?\s*-\*-/
-      VimModeline = /\/\*\s*vim:\s*set\s*(?:ft|filetype)=(\w+):\s*\*\//
+      EmacsModeline = /-\*-\s*mode:\s*(\w+);?\s*-\*-/i
+      VimModeline = /\/\*\s*vim:\s*set\s*(?:ft|filetype)=(\w+):\s*\*\//i

      # Public: Detects language based on Vim and Emacs modelines
      #
--- a/lib/linguist/vendor.yml
+++ b/lib/linguist/vendor.yml
@@ -40,7 +40,7 @@
 # Minified JavaScript and CSS
 - (\.|-)min\.(js|css)$

-#Stylesheets imported from packages
+# Stylesheets imported from packages
 - ([^\s]*)import\.(css|less|scss|styl)$

 # Bootstrap css and js
@@ -251,3 +251,6 @@
 # ProGuard
 - proguard.pro
 - proguard-rules.pro
+
+# Android Google APIs
+- (^|/)\.google_apis/
--- a/lib/linguist/version.rb
+++ b/lib/linguist/version.rb
@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "4.3.1"
+  VERSION = "4.5.1"
 end
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
  "repository": "https://github.com/github/linguist",
  "dependencies": {
-    "season": "~>3.0"
+    "season": "~>5.0"
  }
 }
--- a/samples/AMPL/toy.ampl
+++ b/samples/AMPL/toy.ampl
@@ -0,0 +1,25 @@
+# A toy knapsack problem from the LocalSolver docs written in AMPL.
+
+set I;
+param Value{I};
+param Weight{I};
+param KnapsackBound;
+var Take{I} binary;
+
+maximize TotalValue: sum{i in I} Take[i] * Value[i];
+s.t. WeightLimit: sum{i in I} Take[i] * Weight[i] <= KnapsackBound;
+
+data;
+
+param:
+I: Weight Value :=
+0    10     1
+1    60    10
+2    30    15
+3    40    40
+4    30    60
+5    20    90
+6    20   100
+7     2    15;
+
+param KnapsackBound := 102;
--- a/samples/Assembly/forth.nasm
+++ b/samples/Assembly/forth.nasm
--- a/samples/Brainfuck/fib100.bf
+++ b/samples/Brainfuck/fib100.bf
@@ -0,0 +1,13 @@
+# Calculate and output all fibonacci numbers under 100
+
+++++++++++
+>+>>>>++++++++++++++++++++++++++++++++++++++++++++
+>++++++++++++++++++++++++++++++++<<<<<<[>[>>>>>>+>
+<<<<<<<-]>>>>>>>[<<<<<<<+>>>>>>>-]<[>++++++++++[-
+<-[>>+>+<<<-]>>>[<<<+>>>-]+<[>[-]<[-]]>[<<[>>>+<<<
+-]>>[-]]<<]>>>[>>+>+<<<-]>>>[<<<+>>>-]+<[>[-]<[-]]
+>[<<+>>[-]]<<<<<<<]>>>>>[+++++++++++++++++++++++++
+++++++++++++++++++++++.[-]]++++++++++<[->-<]>++++
++++++++++++++++++++++++++++++++++++++++++++.[-]<<
+<<<<<<<<<<[>>>+>+<<<<-]>>>>[<<<<+>>>>-]<-[>>.>.<<<
+[-]]<<[>>+>+<<<-]>>>[<<<+>>>-]<<[<+>-]>[<+>-]<<<-]
--- a/samples/Brainfuck/hello.bf
+++ b/samples/Brainfuck/hello.bf
@@ -0,0 +1,4 @@
+// More complex version of hello world
+
+>++++++++[<+++++++++>-]<.>>+>+>++>[-]+<[>[->+<<++++>]<<]>.+++++++..+++.>
+>+++++++.<<<[[-]<[-]>]<+++++++++++++++.>>.+++.------.--------.>>+.>++++.
--- a/samples/Brainfuck/helloworld.bf
+++ b/samples/Brainfuck/helloworld.bf
@@ -0,0 +1,3 @@
+// Hello World
+
++++++++[>++++[>++>+++>+++>+<<<<-]>+>+>->>+[<]<-]>>.>---.+++++++..+++.>>.<-.<.+++.------.--------.>>+.>++.
--- a/samples/Brainfuck/rot13.bf
+++ b/samples/Brainfuck/rot13.bf
@@ -0,0 +1,30 @@
+# ROT13 cipher
+
+-,+[                         Read first character and start outer character reading loop
+    -[                       Skip forward if character is 0
+        >>++++[>++++++++<-]  Set up divisor (32) for division loop
+                               (MEMORY LAYOUT: dividend copy remainder divisor quotient zero zero)
+        <+<-[                Set up dividend (x minus 1) and enter division loop
+            >+>+>-[>>>]      Increase copy and remainder / reduce divisor / Normal case: skip forward
+            <[[>+<-]>>+>]    Special case: move remainder back to divisor and increase quotient
+            <<<<<-           Decrement dividend
+        ]                    End division loop
+    ]>>>[-]+                 End skip loop; zero former divisor and reuse space for a flag
+    >--[-[<->+++[-]]]<[         Zero that flag unless quotient was 2 or 3; zero quotient; check flag
+        ++++++++++++<[       If flag then set up divisor (13) for second division loop
+                               (MEMORY LAYOUT: zero copy dividend divisor remainder quotient zero zero)
+            >-[>+>>]         Reduce divisor; Normal case: increase remainder
+            >[+[<+>-]>+>>]   Special case: increase remainder / move it back to divisor / increase quotient
+            <<<<<-           Decrease dividend
+        ]                    End division loop
+        >>[<+>-]             Add remainder back to divisor to get a useful 13
+        >[                   Skip forward if quotient was 0
+            -[               Decrement quotient and skip forward if quotient was 1
+                -<<[-]>>     Zero quotient and divisor if quotient was 2
+            ]<<[<<->>-]>>    Zero divisor and subtract 13 from copy if quotient was 1
+        ]<<[<<+>>-]          Zero divisor and add 13 to copy if quotient was 0
+    ]                        End outer skip loop (jump to here if ((character minus 1)/32) was not 2 or 3)
+    <[-]                     Clear remainder from first division if second division was skipped
+    <.[-]                    Output ROT13ed character from copy and clear it
+    <-,+                     Read next character
+]                            End character reading loop
--- a/samples/C/filenames/script
+++ b/samples/C/filenames/script
--- a/samples/C/pqiv.h
+++ b/samples/C/pqiv.h
@@ -0,0 +1,166 @@
+/**
+ * pqiv
+ *
+ * Copyright (c) 2013-2014, Phillip Berndt
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+// This file contains the definition of files, image types and
+// the plugin infrastructure. It should be included in file type
+// handlers.
+
+#ifndef _PQIV_H_INCLUDED
+#define _PQIV_H_INCLUDED
+
+#include <glib.h>
+#include <gtk/gtk.h>
+#include <gio/gio.h>
+#include "lib/bostree.h"
+
+#ifndef PQIV_VERSION
+#define PQIV_VERSION "2.3"
+#endif
+
+#define FILE_FLAGS_ANIMATION      (guint)(1)
+#define FILE_FLAGS_MEMORY_IMAGE   (guint)(1<<1)
+
+// The structure for images {{{
+typedef struct file_type_handler_struct_t file_type_handler_t;
+typedef struct {
+	// File type
+	const file_type_handler_t *file_type;
+
+	// Special flags
+	// FILE_FLAGS_ANIMATION        -> Animation functions are invoked
+	//                                Set by file type handlers
+	// FILE_FLAGS_MEMORY_IMAGE     -> File lives in memory
+	guint file_flags;
+
+	// The file name to display and to sort by
+	gchar *display_name;
+
+	// The URI or file name of the file
+	gchar *file_name;
+
+	// If the file is a memory image, the actual image data
+	GBytes *file_data;
+
+	// The file monitor structure is used for inotify-watching of
+	// the files
+	GFileMonitor *file_monitor;
+
+	// This flag stores whether this image is currently loaded
+	// and valid. i.e. if it is set, you can assume that
+	// private_data contains a representation of the image;
+	// if not, you can NOT assume that it does not.
+	gboolean is_loaded;
+
+	// Cached image size
+	guint width;
+	guint height;
+
+	// File-type specific data, allocated and freed by the file type handlers
+	void *private;
+} file_t;
+// }}}
+// Definition of the built-in file types {{{
+
+// If you want to implement your own file type, you'll have to implement the
+// following functions and a non-static initialization function named
+// file_type_NAME_initializer that fills a file_type_handler_t with pointers to
+// the functions. Store the file in backends/NAME.c and adjust the Makefile to
+// add the required libraries if your backend is listed in the $(BACKENDS)
+// variable.
+
+typedef enum { PARAMETER, RECURSION, INOTIFY, BROWSE_ORIGINAL_PARAMETER, FILTER_OUTPUT } load_images_state_t;
+// Allocation function: Allocate the ->private structure within a file and add the
+// image(s) to the list of available images via load_images_handle_parameter_add_file()
+// If an image is not to be loaded for any reason, the file structure should be
+// deallocated using file_free()
+// Returns a pointer to the first added image
+// Optional, you can also set the pointer to this function to NULL.
+typedef BOSNode *(*file_type_alloc_fn_t)(load_images_state_t state, file_t *file);
+
+// Deallocation, if a file is removed from the images list. Free the ->private structure.
+// Only called if ->private is non-NULL.
+typedef void (*file_type_free_fn_t)(file_t *file);
+
+// Actually load a file into memory
+typedef void (*file_type_load_fn_t)(file_t *file, GInputStream *data, GError **error_pointer);
+
+// Unload a file
+typedef void (*file_type_unload_fn_t)(file_t *file);
+
+// Animation support: Initialize memory for animations, return ms until first frame
+// Optional, you can also set the pointer to this function to NULL.
+typedef double (*file_type_animation_initialize_fn_t)(file_t *file);
+
+// Animation support: Advance to the next frame, return ms until next frame
+// Optional, you can also set the pointer to this function to NULL.
+typedef double (*file_type_animation_next_frame_fn_t)(file_t *file);
+
+// Draw the current view to a cairo context
+typedef void (*file_type_draw_fn_t)(file_t *file, cairo_t *cr);
+
+struct file_type_handler_struct_t {
+	// All files will be filtered with this filter. If it lets it pass,
+	// a handler is assigned to a file. If none do, the file is
+	// discarded if it was found during directory traversal or
+	// loaded using the first image backend if it was an explicit
+	// parameter.
+	GtkFileFilter *file_types_handled;
+
+	// Pointers to the functions defined above
+	file_type_alloc_fn_t alloc_fn;
+	file_type_free_fn_t free_fn;
+	file_type_load_fn_t load_fn;
+	file_type_unload_fn_t unload_fn;
+	file_type_animation_initialize_fn_t animation_initialize_fn;
+	file_type_animation_next_frame_fn_t animation_next_frame_fn;
+	file_type_draw_fn_t draw_fn;
+};
+
+// Initialization function: Tell pqiv about a backend
+typedef void (*file_type_initializer_fn_t)(file_type_handler_t *info);
+
+// pqiv symbols available to plugins {{{
+
+// Global cancellable that should be used for every i/o operation
+extern GCancellable *image_loader_cancellable;
+
+// Current scale level. For backends that don't support cairo natively.
+extern gdouble current_scale_level;
+
+// Load a file from disc/memory/network
+GInputStream *image_loader_stream_file(file_t *file, GError **error_pointer);
+
+// Add a file to the list of loaded files
+// Should be called at least once in a file_type_alloc_fn_t, with the state being
+// forwarded unaltered.
+BOSNode *load_images_handle_parameter_add_file(load_images_state_t state, file_t *file);
+
+// Load all data from an input stream into memory, conveinience function
+GBytes *g_input_stream_read_completely(GInputStream *input_stream, GCancellable *cancellable, GError **error_pointer);
+
+// Free a file
+void file_free(file_t *file);
+
+// }}}
+
+// File type handlers, used in the initializer and file type guessing
+extern file_type_handler_t file_type_handlers[];
+
+/* }}} */
+
+#endif
--- a/samples/D/mpq.d
+++ b/samples/D/mpq.d
@@ -0,0 +1,318 @@
+/*
+ *  mpq.d -- D programming language module for libmpq
+ *
+ *  Copyright (c) 2008 Georg Lukas <georg@op-co.de>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ *  This module is written to support Phobos. Patches to allow binding to
+ *  Tango are welcome.
+ */
+
+module mpq;
+
+/* the following pragma does not work on DMD/Linux, generates a warning on
+ * GDC/Linux and has not been tested on Windows. Commented out for now. */
+// pragma(lib, "libmpq");
+
+import std.string; // for format() and toStringz()
+import std.traits; // for ParameterTypeTuple!()
+
+/* XXX: this assumes that libmpq is compiled with Large File Support on */
+alias long off_t;
+
+/* libmpq error return values */
+const LIBMPQ_ERROR_OPEN			= -1;	/* open error on file. */
+const LIBMPQ_ERROR_CLOSE		= -2;	/* close error on file. */
+const LIBMPQ_ERROR_SEEK			= -3;	/* lseek error on file. */
+const LIBMPQ_ERROR_READ			= -4;	/* read error on file. */
+const LIBMPQ_ERROR_WRITE		= -5;	/* write error on file. */
+const LIBMPQ_ERROR_MALLOC		= -6;	/* memory allocation error. */
+const LIBMPQ_ERROR_FORMAT		= -7;	/* format errror. */
+const LIBMPQ_ERROR_NOT_INITIALIZED	= -8;	/* init() wasn't called. */
+const LIBMPQ_ERROR_SIZE			= -9;	/* buffer size is to small. */
+const LIBMPQ_ERROR_EXIST		= -10;	/* file or block does not exist in archive. */
+const LIBMPQ_ERROR_DECRYPT		= -11;	/* we don't know the decryption seed. */
+const LIBMPQ_ERROR_UNPACK		= -12;	/* error on unpacking file. */
+
+/** libmpq internal meta-data for an archive */
+extern struct mpq_archive_s;
+
+extern(C) {
+
+/* libmpq__generic information about library. */
+char *libmpq__version();
+
+/* libmpq__generic mpq archive information. */
+int libmpq__archive_open(mpq_archive_s **mpq_archive, char *mpq_filename, off_t archive_offset);
+int libmpq__archive_close(mpq_archive_s *mpq_archive);
+int libmpq__archive_packed_size(mpq_archive_s *mpq_archive, off_t *packed_size);
+int libmpq__archive_unpacked_size(mpq_archive_s *mpq_archive, off_t *unpacked_size);
+int libmpq__archive_offset(mpq_archive_s *mpq_archive, off_t *offset);
+int libmpq__archive_version(mpq_archive_s *mpq_archive, uint *version_);
+int libmpq__archive_files(mpq_archive_s *mpq_archive, uint *files);
+
+/* libmpq__generic file processing functions. */
+int libmpq__file_packed_size(mpq_archive_s *mpq_archive, uint file_number, off_t *packed_size);
+int libmpq__file_unpacked_size(mpq_archive_s *mpq_archive, uint file_number, off_t *unpacked_size);
+int libmpq__file_offset(mpq_archive_s *mpq_archive, uint file_number, off_t *offset);
+int libmpq__file_blocks(mpq_archive_s *mpq_archive, uint file_number, uint *blocks);
+int libmpq__file_encrypted(mpq_archive_s *mpq_archive, uint file_number, uint *encrypted);
+int libmpq__file_compressed(mpq_archive_s *mpq_archive, uint file_number, uint *compressed);
+int libmpq__file_imploded(mpq_archive_s *mpq_archive, uint file_number, uint *imploded);
+int libmpq__file_number(mpq_archive_s *mpq_archive, char *filename, uint *number);
+int libmpq__file_read(mpq_archive_s *mpq_archive, uint file_number, ubyte *out_buf, off_t out_size, off_t *transferred);
+
+/* libmpq__generic block processing functions. */
+int libmpq__block_open_offset(mpq_archive_s *mpq_archive, uint file_number);
+int libmpq__block_close_offset(mpq_archive_s *mpq_archive, uint file_number);
+int libmpq__block_unpacked_size(mpq_archive_s *mpq_archive, uint file_number, uint block_number, off_t *unpacked_size);
+int libmpq__block_read(mpq_archive_s *mpq_archive, uint file_number, uint block_number, ubyte *out_buf, off_t out_size, off_t *transferred);
+
+}
+
+
+/** exception class for failed libmpq calls */
+class MPQException : Exception {
+	const string[] Errors = [
+		"unknown error",
+		"open error on file",
+		"close error on file",
+		"lseek error on file",
+		"read error on file",
+		"write error on file",
+		"memory allocation error",
+		"format errror",
+		"init() wasn't called",
+		"buffer size is to small",
+		"file or block does not exist in archive",
+		"we don't know the decryption seed",
+		"error on unpacking file"];
+
+	public int errno;
+	this(char[] fnname = "unknown_function", int errno = 0) {
+
+		this.errno = errno;
+		if (-errno >= Errors.length)
+			errno = 0;
+		super(std.string.format("Error in %s(): %s (%d)",
+					fnname, Errors[-errno], errno));
+	}
+}
+
+
+/** template to wrap function calls and throw exceptions in case of error
+ *
+ * thanks for the idea to while(nan) blog,
+ * http://while-nan.blogspot.com/2007/06/wrapping-functions-for-fun-and-profit.html
+ *
+ * use: MPQ_CHECKERR(libmpq__archive_open)(&m, "foo.mpq", -1);
+ *   returns the retval of archive_open on success;
+ *   throws an MPQException on failure.
+ *
+ * @param Fn libmpq__function reference
+ * @param args libmpq__function parameters
+ * @return return value of libmpq__function on success
+ * @throw MPQException on error
+ */
+int MPQ_CHECKERR(alias Fn)(ParameterTypeTuple!(Fn) args)
+{
+	int result = Fn(args);
+	if (result < 0) {
+		/* XXX: relying on non-specified stringof() behaviour */
+		throw new MPQException((&Fn).stringof[2..$], result);
+	}
+        return result;
+}
+
+
+/** mixin alias to wrap library functions into MPQ_CHECKERR.
+ *
+ * alias mpq.func_name(...) to MPQ_CHECKERR(libmpq__func_name)(...)
+ * @param func_name name of the function to be wrapped
+ */
+template MPQ_FUNC(char[] func_name) {
+	const char[] MPQ_FUNC = "alias MPQ_CHECKERR!(libmpq__" ~ func_name ~ ") " ~ func_name ~ ";";
+}
+
+alias libmpq__version libversion; /* must be direct alias because it returns char*, not error int */
+mixin(MPQ_FUNC!("archive_open"));
+mixin(MPQ_FUNC!("archive_close"));
+mixin(MPQ_FUNC!("archive_packed_size"));
+mixin(MPQ_FUNC!("archive_unpacked_size"));
+mixin(MPQ_FUNC!("archive_offset"));
+mixin(MPQ_FUNC!("archive_version"));
+mixin(MPQ_FUNC!("archive_files"));
+mixin(MPQ_FUNC!("file_packed_size"));
+mixin(MPQ_FUNC!("file_unpacked_size"));
+mixin(MPQ_FUNC!("file_offset"));
+mixin(MPQ_FUNC!("file_blocks"));
+mixin(MPQ_FUNC!("file_encrypted"));
+mixin(MPQ_FUNC!("file_compressed"));
+mixin(MPQ_FUNC!("file_imploded"));
+mixin(MPQ_FUNC!("file_number"));
+mixin(MPQ_FUNC!("file_read"));
+mixin(MPQ_FUNC!("block_open_offset"));
+mixin(MPQ_FUNC!("block_close_offset"));
+mixin(MPQ_FUNC!("block_unpacked_size"));
+mixin(MPQ_FUNC!("block_read"));
+
+/** getter function named name for returning archive_* single values:
+ *
+ *   <type> Archive.<name>() { return libmpq__archive_<name>() }
+ *
+ * @param type return type for the original function reference
+ * @param name name of the original function
+ * @param name2 name for the prototype (defaults to name, used for "version")
+ * @return getter function mixin
+ */
+template MPQ_A_GET(char[] type, char[] name, char[] name2 = name) {
+	const char[] MPQ_A_GET = type ~ " " ~ name2 ~ "() { " ~
+			type ~ " ret; " ~
+			"archive_" ~ name ~ "(m, &ret); return ret;" ~
+		"}";
+}
+
+/** wrapper class for an MPQ Archive
+ *
+ * syntax: auto a = new mpq.Archive("somefile.mpq");
+ */
+class Archive {
+	mpq_archive_s *m;
+	File listfile;
+	char[][] listfiledata;
+
+	this(char[] archivename, off_t offset = -1) {
+		archive_open(&m, toStringz(archivename), offset);
+	}
+
+	mixin(MPQ_A_GET!("off_t", "packed_size"));
+	mixin(MPQ_A_GET!("off_t", "unpacked_size"));
+	mixin(MPQ_A_GET!("off_t", "offset"));
+	mixin(MPQ_A_GET!("uint", "version", "version_"));
+	mixin(MPQ_A_GET!("uint", "files"));
+
+	~this() {
+		archive_close(m);
+	}
+
+	mpq_archive_s* archive() {
+		return m;
+	}
+
+	File opIndex(char[] fname) {
+		return new File(this, fname);
+	}
+	File opIndex(int fno) {
+		return new File(this, fno);
+	}
+
+	char[][] filelist() {
+		try {
+			if (!listfile) {
+				listfile = this["(listfile)"];
+				listfiledata = (cast(char[])listfile.read()).splitlines();
+			}
+			return listfiledata;
+		} catch (MPQException e) {
+			return [];
+		}
+	}
+
+	/+uint filenumber(char[] filename) {
+		try {
+			if (!listfile) {
+				listfile = this["(listfile)"];
+				listfiledata = (cast(char[])listfile.read()).splitlines();
+			}
+			return listfiledata;
+		} catch (MPQException e) {
+			return [];
+		}
+	}+/
+
+}
+
+
+/** getter function named name for returning file_* single values:
+ *
+ *   <type> File.<name>() { return libmpq__file_<name>() }
+ *
+ * @param type return type for the original function reference
+ * @param name name of the original function
+ * @param name2 name for the prototype (defaults to name, used for "version")
+ * @return getter function mixin
+ */
+template MPQ_F_GET(char[] type, char[] name, char[] name2 = name) {
+	const char[] MPQ_F_GET = type ~ " " ~ name2 ~ "() { " ~
+			type ~ " ret; " ~
+			"file_" ~ name ~ "(am, fileno, &ret); " ~
+			"return ret;" ~
+		"}";
+}
+
+/** wrapper class for a single file in an MPQ Archive
+ *
+ * syntax:
+ *    auto a = new mpq.Archive("somefile.mpq");
+ *    auto f = a["(listfile)"];
+ *    auto f2 = a[0];
+ *    auto f3 = new File(a, "(listfile)");
+ */
+class File {
+	Archive a;
+	mpq_archive_s* am;
+	char[] filename;
+	uint fileno;
+
+	this(Archive a, int fileno) {
+		this.a = a;
+		this.am = a.archive();
+		if (fileno >= a.files) {
+			throw new MPQException(format("File(%d)", fileno),
+				LIBMPQ_ERROR_EXIST);
+		}
+		this.filename = format("file%04d.xxx", fileno);
+		this.fileno = fileno;
+	}
+
+	this(Archive a, char[] filename) {
+		this.a = a;
+		this.am = a.archive();
+		this.filename = filename;
+		/* this line will throw an exception when the file is not there */
+		mpq.file_number(am, toStringz(filename), &this.fileno);
+	}
+
+	mixin(MPQ_F_GET!("off_t", "packed_size"));
+	mixin(MPQ_F_GET!("off_t", "unpacked_size"));
+	mixin(MPQ_F_GET!("off_t", "offset"));
+	mixin(MPQ_F_GET!("uint", "blocks"));
+	mixin(MPQ_F_GET!("uint", "encrypted"));
+	mixin(MPQ_F_GET!("uint", "compressed"));
+	mixin(MPQ_F_GET!("uint", "imploded"));
+
+	uint no() {	return fileno; }
+	char[] name() {	return filename; }
+
+	ubyte[] read() {
+		ubyte[] content;
+		content.length = this.unpacked_size();
+		off_t trans;
+		mpq.file_read(am, fileno, content.ptr, content.length, &trans);
+		content.length = trans;
+		return content;
+	}
+}
--- a/samples/DTrace/counts.d
+++ b/samples/DTrace/counts.d
@@ -0,0 +1,23 @@
+/*
+ * This software is in the public domain.
+ *
+ * $Id: counts.d 10510 2005-08-15 01:46:19Z kateturner $
+ */
+
+#pragma D option quiet
+
+self int tottime;
+BEGIN {
+	tottime = timestamp;
+}
+
+php$target:::function-entry
+	@counts[copyinstr(arg0)] = count();
+}
+
+END {
+	printf("Total time: %dus\n", (timestamp - tottime) / 1000);
+	printf("# calls by function:\n");
+	printa("%-40s %@d\n", @counts);
+}
+
--- a/samples/DTrace/javascript-trace.d
+++ b/samples/DTrace/javascript-trace.d
@@ -0,0 +1,73 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * Copyright (C) 2007  Sun Microsystems, Inc. All Rights Reserved.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+/*
+ * javascript provider probes
+ *
+ * function-entry       (filename, classname, funcname)
+ * function-info        (filename, classname, funcname, lineno,
+ *                      runfilename, runlineno)
+ * function-args        (filename, classname, funcname, argc, argv, argv0,
+ *                      argv1, argv2, argv3, argv4)
+ * function-rval        (filename, classname, funcname, lineno, rval, rval0)
+ * function-return      (filename, classname, funcname)
+ * object-create-start  (filename, classname)
+ * object-create        (filename, classname, *object, rlineno)
+ * object-create-done   (filename, classname)
+ * object-finalize      (NULL, classname, *object)
+ * execute-start        (filename, lineno)
+ * execute-done         (filename, lineno)
+ */
+
+provider javascript {
+ probe function__entry(char *, char *, char *);
+ probe function__info(char *, char *, char *, int, char *, int);
+ probe function__args(char *, char *, char *, int, void *, void *, void *,
+     void *, void *, void *);
+ probe function__rval(char *, char *, char *, int, void *, void *);
+ probe function__return(char *, char *, char *);
+ probe object__create__start(char *, char *);
+ probe object__create__done(char *, char *);
+ /* XXX must use unsigned longs here instead of uintptr_t for OS X 
+    (Apple radar: 5194316 & 5565198) */
+ probe object__create(char *, char *, unsigned long, int);
+ probe object__finalize(char *, char *, unsigned long);
+ probe execute__start(char *, int);
+ probe execute__done(char *, int);
+};
+
+/*
+#pragma D attributes Unstable/Unstable/Common provider mozilla provider
+#pragma D attributes Private/Private/Unknown provider mozilla module
+#pragma D attributes Private/Private/Unknown provider mozilla function
+#pragma D attributes Unstable/Unstable/Common provider mozilla name
+#pragma D attributes Unstable/Unstable/Common provider mozilla args
+*/
+
--- a/samples/DTrace/probes.d
+++ b/samples/DTrace/probes.d
@@ -0,0 +1,93 @@
+/* ----------
+ *	DTrace probes for PostgreSQL backend
+ *
+ *	Copyright (c) 2006-2009, PostgreSQL Global Development Group
+ *
+ *	$PostgreSQL: pgsql/src/backend/utils/probes.d,v 1.11 2009/04/02 20:59:10 momjian Exp $
+ * ----------
+ */
+
+
+/*
+ * Typedefs used in PostgreSQL.
+ *
+ * NOTE: Do not use system-provided typedefs (e.g. uintptr_t, uint32_t, etc)
+ * in probe definitions, as they cause compilation errors on Mac OS X 10.5.
+ */
+#define LocalTransactionId unsigned int
+#define LWLockId int
+#define LWLockMode int
+#define LOCKMODE int
+#define BlockNumber unsigned int
+#define Oid unsigned int
+#define ForkNumber int
+#define bool char
+
+provider postgresql {
+
+	probe transaction__start(LocalTransactionId);
+	probe transaction__commit(LocalTransactionId);
+	probe transaction__abort(LocalTransactionId);
+
+	probe lwlock__acquire(LWLockId, LWLockMode);
+	probe lwlock__release(LWLockId);
+	probe lwlock__wait__start(LWLockId, LWLockMode);
+	probe lwlock__wait__done(LWLockId, LWLockMode);
+	probe lwlock__condacquire(LWLockId, LWLockMode);
+	probe lwlock__condacquire__fail(LWLockId, LWLockMode);
+
+	probe lock__wait__start(unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, LOCKMODE);
+	probe lock__wait__done(unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, LOCKMODE);
+
+	probe query__parse__start(const char *);
+	probe query__parse__done(const char *);
+	probe query__rewrite__start(const char *);
+	probe query__rewrite__done(const char *);
+	probe query__plan__start();
+	probe query__plan__done();
+	probe query__execute__start();
+	probe query__execute__done();
+	probe query__start(const char *);
+	probe query__done(const char *);
+	probe statement__status(const char *);
+
+	probe sort__start(int, bool, int, int, bool);
+	probe sort__done(bool, long);
+
+	probe buffer__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, bool, bool);
+	probe buffer__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, bool, bool, bool);
+	probe buffer__flush__start(ForkNumber, BlockNumber, Oid, Oid, Oid);
+	probe buffer__flush__done(ForkNumber, BlockNumber, Oid, Oid, Oid);
+
+	probe buffer__checkpoint__start(int);
+	probe buffer__checkpoint__sync__start();
+	probe buffer__checkpoint__done();
+	probe buffer__sync__start(int, int);
+	probe buffer__sync__written(int);
+	probe buffer__sync__done(int, int, int);
+	probe buffer__write__dirty__start(ForkNumber, BlockNumber, Oid, Oid, Oid);
+	probe buffer__write__dirty__done(ForkNumber, BlockNumber, Oid, Oid, Oid);
+
+	probe deadlock__found();
+
+	probe checkpoint__start(int);
+	probe checkpoint__done(int, int, int, int, int);
+	probe clog__checkpoint__start(bool);
+	probe clog__checkpoint__done(bool);
+	probe subtrans__checkpoint__start(bool);
+	probe subtrans__checkpoint__done(bool);
+	probe multixact__checkpoint__start(bool);
+	probe multixact__checkpoint__done(bool);
+	probe twophase__checkpoint__start();
+	probe twophase__checkpoint__done();
+
+	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid);
+	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int);
+	probe smgr__md__write__start(ForkNumber, BlockNumber, Oid, Oid, Oid);
+	probe smgr__md__write__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int);
+
+	probe xlog__insert(unsigned char, unsigned char);
+	probe xlog__switch();
+	probe wal__buffer__write__dirty__start();
+	probe wal__buffer__write__dirty__done();
+};
--- a/samples/Eiffel/application.e
+++ b/samples/Eiffel/application.e
@@ -0,0 +1,44 @@
+note
+	description : "nino application root class"
+	date        : "$Date$"
+	revision    : "$Revision$"
+
+class
+	APPLICATION
+
+inherit
+	ARGUMENTS
+
+	HTTP_SERVER_SHARED_CONFIGURATION
+
+create
+	make
+
+feature {NONE} -- Initialization
+
+	make
+			-- Run application.
+		local
+			l_server : HTTP_SERVER
+			l_cfg: HTTP_SERVER_CONFIGURATION
+			l_http_handler : HTTP_HANDLER
+		do
+			create l_cfg.make
+			l_cfg.http_server_port := 9_000
+			l_cfg.document_root := default_document_root
+			set_server_configuration (l_cfg)
+			debug ("nino")
+				l_cfg.set_is_verbose (True)
+			end
+
+			create l_server.make (l_cfg)
+			create {APPLICATION_CONNECTION_HANDLER} l_http_handler.make (l_server)
+			l_server.setup (l_http_handler)
+		end
+
+feature -- Access
+
+	default_document_root: STRING = "webroot"
+
+end
+
--- a/samples/Eiffel/book_collection.e
+++ b/samples/Eiffel/book_collection.e
@@ -0,0 +1,82 @@
+class
+	BOOK_COLLECTION
+
+create
+	make
+
+feature {NONE} -- Initialization
+
+	make (a_name: STRING_32)
+			-- Create a book collection with `a_name' as `name'.
+		do
+			set_name (a_name)
+			create book_index.make (10)
+		ensure
+			name_set: name = a_name
+		end
+
+feature -- Access
+
+	name: STRING_32
+			-- Name.
+
+	books: LIST [BOOK]
+			-- collection of book.
+		do
+			create {LINKED_LIST [BOOK]} Result.make
+			across
+				book_index as it
+			loop
+				Result.append (it.item)
+			end
+		end
+
+	books_by_author (a_author: STRING_32): LIST [BOOK]
+			-- Books wrote by `a_author' in this collection.
+		do
+			if attached book_index [a_author] as l_result then
+				Result := l_result
+			else
+				create {LINKED_LIST [BOOK]} Result.make
+			end
+		end
+
+feature -- Change
+
+	set_name (a_name: STRING_32)
+			-- Set `name' with `a_name'.
+		do
+			name := a_name
+		ensure
+			name_set: name = a_name
+		end
+
+	add_book (a_book: BOOK)
+			-- Extend collection with `a_book'.
+		local
+			l: detachable LIST [BOOK]
+		do
+			l := book_index.at (a_book.author.name)
+			if l = Void then
+				create {LINKED_LIST [BOOK]} l.make
+				book_index.put (l, a_book.author.name)
+			end
+			l.force (a_book)
+		end
+
+	add_books (book_list: like books)
+			-- Append collection with `book_list'.
+		do
+			across
+				book_list as it
+			loop
+				add_book (it.item)
+			end
+		end
+
+feature {NONE} -- Implementation
+
+	book_index: HASH_TABLE [LIST [BOOK], STRING_32]
+			-- Association of author name and its books.
+
+end -- class BOOK_COLLECTION
--- a/samples/Eiffel/git_checkout_command.e
+++ b/samples/Eiffel/git_checkout_command.e
@@ -0,0 +1,41 @@
+note
+	description: "Git checkout command."
+	author: "Olivier Ligot"
+
+class
+	GIT_CHECKOUT_COMMAND
+
+inherit
+	GIT_COMMAND
+
+create
+	make,
+	make_master
+
+feature {NONE} -- Initialization
+
+	make (a_branch: STRING)
+			-- Checkout the branch `a_branch'.
+		do
+			initialize
+			arguments.force_last (a_branch)
+			branch := a_branch
+		ensure
+			branch_set: branch = a_branch
+		end
+
+	make_master
+			-- Checkout the master branch.
+		do
+			make ("master")
+		end
+
+feature -- Access
+
+	branch: STRING
+			-- Branch to checkout
+
+	name: STRING = "checkout"
+			-- Git subcommand name
+
+end
--- a/samples/Go/api.pb.go
+++ b/samples/Go/api.pb.go
--- a/samples/INI/MouseKeyboard.pro
+++ b/samples/INI/MouseKeyboard.pro
@@ -0,0 +1,71 @@
+update=Sun 15 Feb 2015 01:10:10 PM EST
+last_client=eeschema
+[pcbnew]
+version=1
+PageLayoutDescrFile=
+LastNetListRead=
+UseCmpFile=1
+PadDrill=0.6
+PadDrillOvalY=0.6
+PadSizeH=1.5
+PadSizeV=1.5
+PcbTextSizeV=1.5
+PcbTextSizeH=1.5
+PcbTextThickness=0.3
+ModuleTextSizeV=1
+ModuleTextSizeH=1
+ModuleTextSizeThickness=0.15
+SolderMaskClearance=0
+SolderMaskMinWidth=0
+DrawSegmentWidth=0.2
+BoardOutlineThickness=0.09999999999999999
+ModuleOutlineThickness=0.15
+[pcbnew/libraries]
+LibDir=
+[general]
+version=1
+[eeschema]
+version=1
+PageLayoutDescrFile=
+SubpartIdSeparator=0
+SubpartFirstId=65
+LibDir=/home/hschmale/KiCad/LibMods-3rdParty
+NetFmtName=
+RptD_X=0
+RptD_Y=100
+RptLab=1
+LabSize=60
+[eeschema/libraries]
+LibName1=power
+LibName2=device
+LibName3=transistors
+LibName4=conn
+LibName5=linear
+LibName6=regul
+LibName7=74xx
+LibName8=cmos4000
+LibName9=adc-dac
+LibName10=memory
+LibName11=xilinx
+LibName12=special
+LibName13=microcontrollers
+LibName14=dsp
+LibName15=microchip
+LibName16=analog_switches
+LibName17=motorola
+LibName18=texas
+LibName19=intel
+LibName20=audio
+LibName21=interface
+LibName22=digital-audio
+LibName23=philips
+LibName24=display
+LibName25=cypress
+LibName26=siliconi
+LibName27=opto
+LibName28=atmel
+LibName29=contrib
+LibName30=valves
+LibName31=arduino_shieldsNCL
+LibName32=con-usb-2
+LibName33=2axispotwselect
--- a/Setup/expat.iss
+++ b/Setup/expat.iss
@@ -0,0 +1,69 @@
+; Basic setup script for the Inno Setup installer builder.  For more
+; information on the free installer builder, see www.jrsoftware.org.
+;
+; This script was contributed by Tim Peters.
+; It was designed for Inno Setup 2.0.19 but works with later versions as well.
+
+[Setup]
+AppName=Expat
+AppId=expat
+AppVersion=2.1.0
+AppVerName=Expat 2.1.0
+AppCopyright=Copyright 1998-2012 Thai Open Source Software Center, Clark Cooper, and the Expat maintainers
+AppPublisher=The Expat Developers
+AppPublisherURL=http://www.libexpat.org/
+AppSupportURL=http://www.libexpat.org/
+AppUpdatesURL=http://www.libexpat.org/
+UninstallDisplayName=Expat XML Parser 2.1.0
+VersionInfoVersion=2.1.0
+
+DefaultDirName={pf}\Expat 2.1.0
+UninstallFilesDir={app}\Uninstall
+
+Compression=lzma
+SolidCompression=yes
+SourceDir=..
+OutputDir=win32
+DisableStartupPrompt=yes
+AllowNoIcons=yes
+DisableProgramGroupPage=yes
+DisableReadyPage=yes
+
+[Files]
+Flags: ignoreversion; Source: win32\bin\Release\xmlwf.exe;  DestDir: "{app}\Bin"
+Flags: ignoreversion; Source: win32\MANIFEST.txt;           DestDir: "{app}"
+Flags: ignoreversion; Source: Changes;                      DestDir: "{app}"; DestName: Changes.txt
+Flags: ignoreversion; Source: COPYING;                      DestDir: "{app}"; DestName: COPYING.txt
+Flags: ignoreversion; Source: README;                       DestDir: "{app}"; DestName: README.txt
+Flags: ignoreversion; Source: doc\*.html;                   DestDir: "{app}\Doc"
+Flags: ignoreversion; Source: doc\*.css;                    DestDir: "{app}\Doc"
+Flags: ignoreversion; Source: doc\*.png;                    DestDir: "{app}\Doc"
+Flags: ignoreversion; Source: win32\bin\Release\*.dll;      DestDir: "{app}\Bin"
+Flags: ignoreversion; Source: win32\bin\Release\*.lib;      DestDir: "{app}\Bin"
+Flags: ignoreversion; Source: expat.dsw;                    DestDir: "{app}\Source"
+Flags: ignoreversion; Source: win32\README.txt;             DestDir: "{app}\Source"
+Flags: ignoreversion; Source: bcb5\*.bp*;                   DestDir: "{app}\Source\bcb5"
+Flags: ignoreversion; Source: bcb5\*.mak;                   DestDir: "{app}\Source\bcb5"
+Flags: ignoreversion; Source: bcb5\*.def;                   DestDir: "{app}\Source\bcb5"
+Flags: ignoreversion; Source: bcb5\*.txt;                   DestDir: "{app}\Source\bcb5"
+Flags: ignoreversion; Source: bcb5\*.bat;                   DestDir: "{app}\Source\bcb5"
+Flags: ignoreversion; Source: lib\*.c;                      DestDir: "{app}\Source\lib"
+Flags: ignoreversion; Source: lib\*.h;                      DestDir: "{app}\Source\lib"
+Flags: ignoreversion; Source: lib\*.def;                    DestDir: "{app}\Source\lib"
+Flags: ignoreversion; Source: lib\*.dsp;                    DestDir: "{app}\Source\lib"
+Flags: ignoreversion; Source: examples\*.c;                 DestDir: "{app}\Source\examples"
+Flags: ignoreversion; Source: examples\*.dsp;               DestDir: "{app}\Source\examples"
+Flags: ignoreversion; Source: tests\*.c;                    DestDir: "{app}\Source\tests"
+Flags: ignoreversion; Source: tests\*.cpp;                  DestDir: "{app}\Source\tests"
+Flags: ignoreversion; Source: tests\*.h;                    DestDir: "{app}\Source\tests"
+Flags: ignoreversion; Source: tests\README.txt;             DestDir: "{app}\Source\tests"
+Flags: ignoreversion; Source: tests\benchmark\*.c;          DestDir: "{app}\Source\tests\benchmark"
+Flags: ignoreversion; Source: tests\benchmark\*.ds*;        DestDir: "{app}\Source\tests\benchmark"
+Flags: ignoreversion; Source: tests\benchmark\README.txt;   DestDir: "{app}\Source\tests\benchmark"
+Flags: ignoreversion; Source: xmlwf\*.c*;                   DestDir: "{app}\Source\xmlwf"
+Flags: ignoreversion; Source: xmlwf\*.h;                    DestDir: "{app}\Source\xmlwf"
+Flags: ignoreversion; Source: xmlwf\*.dsp;                  DestDir: "{app}\Source\xmlwf"
+
+[Messages]
+WelcomeLabel1=Welcome to the Expat XML Parser Setup Wizard
+WelcomeLabel2=This will install [name/ver] on your computer.%n%nExpat is an XML parser with a C-language API, and is primarily made available to allow developers to build applications which use XML using a portable API and fast implementation.%n%nIt is strongly recommended that you close all other applications you have running before continuing. This will help prevent any conflicts during the installation process.
--- a/samples/Lean/binary.lean
+++ b/samples/Lean/binary.lean
@@ -0,0 +1,75 @@
+/-
+Copyright (c) 2014 Microsoft Corporation. All rights reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+
+Module: algebra.binary
+Authors: Leonardo de Moura, Jeremy Avigad
+
+General properties of binary operations.
+-/
+
+import logic.eq
+open eq.ops
+
+namespace binary
+  section
+    variable {A : Type}
+    variables (op₁ : A → A → A) (inv : A → A) (one : A)
+
+    local notation a * b := op₁ a b
+    local notation a ⁻¹  := inv a
+    local notation 1     := one
+
+    definition commutative := ∀a b, a * b = b * a
+    definition associative := ∀a b c, (a * b) * c = a * (b * c)
+    definition left_identity := ∀a, 1 * a = a
+    definition right_identity := ∀a, a * 1 = a
+    definition left_inverse := ∀a, a⁻¹ * a = 1
+    definition right_inverse := ∀a, a * a⁻¹ = 1
+    definition left_cancelative := ∀a b c, a * b = a * c → b = c
+    definition right_cancelative := ∀a b c, a * b = c * b → a = c
+
+    definition inv_op_cancel_left := ∀a b, a⁻¹ * (a * b) = b
+    definition op_inv_cancel_left := ∀a b, a * (a⁻¹ * b) = b
+    definition inv_op_cancel_right := ∀a b, a * b⁻¹ * b =  a
+    definition op_inv_cancel_right := ∀a b, a * b * b⁻¹ = a
+
+    variable (op₂ : A → A → A)
+
+    local notation a + b := op₂ a b
+
+    definition left_distributive := ∀a b c, a * (b + c) = a * b + a * c
+    definition right_distributive := ∀a b c, (a + b) * c = a * c + b * c
+  end
+
+  context
+    variable {A : Type}
+    variable {f : A → A → A}
+    variable H_comm : commutative f
+    variable H_assoc : associative f
+    infixl `*` := f
+    theorem left_comm : ∀a b c, a*(b*c) = b*(a*c) :=
+    take a b c, calc
+      a*(b*c) = (a*b)*c  : H_assoc
+        ...   = (b*a)*c  : H_comm
+        ...   = b*(a*c)  : H_assoc
+
+    theorem right_comm : ∀a b c, (a*b)*c = (a*c)*b :=
+    take a b c, calc
+      (a*b)*c = a*(b*c) : H_assoc
+        ...   = a*(c*b) : H_comm
+        ...   = (a*c)*b : H_assoc
+  end
+
+  context
+    variable {A : Type}
+    variable {f : A → A → A}
+    variable H_assoc : associative f
+    infixl `*` := f
+    theorem assoc4helper (a b c d) : (a*b)*(c*d) = a*((b*c)*d) :=
+    calc
+      (a*b)*(c*d) = a*(b*(c*d)) : H_assoc
+              ... = a*((b*c)*d) : H_assoc
+  end
+
+end binary
--- a/samples/Lean/set.hlean
+++ b/samples/Lean/set.hlean
@@ -0,0 +1,70 @@
+-- Copyright (c) 2015 Jakob von Raumer. All rights reserved.
+-- Released under Apache 2.0 license as described in the file LICENSE.
+-- Authors: Jakob von Raumer
+-- Category of sets
+
+import .basic types.pi trunc
+
+open truncation sigma sigma.ops pi function eq morphism precategory
+open equiv
+
+namespace precategory
+
+  universe variable l
+
+  definition set_precategory : precategory.{l+1 l} (Σ (A : Type.{l}), is_hset A) :=
+  begin
+    fapply precategory.mk.{l+1 l},
+                  intros, apply (a.1 → a_1.1),
+                intros, apply trunc_pi, intros, apply b.2,
+              intros, intro x, exact (a_1 (a_2 x)),
+            intros, exact (λ (x : a.1), x),
+          intros, apply funext.path_pi, intro x, apply idp,
+        intros, apply funext.path_pi, intro x, apply idp,
+      intros, apply funext.path_pi, intro x, apply idp,
+  end
+
+end precategory
+
+namespace category
+
+  universe variable l
+  local attribute precategory.set_precategory.{l+1 l} [instance]
+
+  definition set_category_equiv_iso (a b : (Σ (A : Type.{l}), is_hset A))
+    : (a ≅ b) = (a.1 ≃ b.1) :=
+  /-begin
+    apply ua, fapply equiv.mk,
+      intro H,
+        apply (isomorphic.rec_on H), intros (H1, H2),
+        apply (is_iso.rec_on H2), intros (H3, H4, H5),
+        fapply equiv.mk,
+        apply (isomorphic.rec_on H), intros (H1, H2),
+        exact H1,
+      fapply is_equiv.adjointify, exact H3,
+          exact sorry,
+        exact sorry,
+  end-/ sorry
+
+  definition set_category : category.{l+1 l} (Σ (A : Type.{l}), is_hset A) :=
+  /-begin
+    assert (C : precategory.{l+1 l} (Σ (A : Type.{l}), is_hset A)),
+      apply precategory.set_precategory,
+    apply category.mk,
+    assert (p : (λ A B p, (set_category_equiv_iso A B) ▹ iso_of_path p) = (λ A B p, @equiv_path A.1 B.1 p)),
+    apply is_equiv.adjointify,
+        intros,
+        apply (isomorphic.rec_on a_1), intros (iso', is_iso'),
+        apply (is_iso.rec_on is_iso'), intros (f', f'sect, f'retr),
+        fapply sigma.path,
+          apply ua, fapply equiv.mk, exact iso',
+          fapply is_equiv.adjointify,
+              exact f',
+            intros, apply (f'retr ▹ _),
+          intros, apply (f'sect ▹ _),
+        apply (@is_hprop.elim),
+        apply is_trunc_is_hprop,
+      intros,
+  end -/ sorry
+
+end category
--- a/samples/MUF/39.m
+++ b/samples/MUF/39.m
@@ -0,0 +1,278 @@
+$include $lib/strings
+$include $lib/match
+lvar check-obj-addr
+   
+: check-next-loop (d -- )
+   dup not if pop exit then
+   dup exit? over thing? or
+   me @ 3 pick .controls and if
+      dup check-obj-addr @ execute
+   then
+   next check-next-loop
+;
+   
+: check-contents (d -- )
+   contents check-next-loop
+;
+   
+: check-exits (d -- )
+   exits check-next-loop
+;
+   
+: exec-err (d mtypestr warnstr -- )
+   "On " 4 rotate unparseobj strcat
+   ", in it's " strcat rot strcat
+   ", " strcat swap strcat .tell
+;
+   
+: can-linkto? (player object -- i)
+   dup "link_ok" flag? if pop pop 1 exit then
+   .controls
+;
+   
+: check-exec (d mtype execstr -- )
+   dup "@" 1 strncmp if pop pop pop exit then
+   1 strcut swap pop
+   " " .split pop
+   dup "$" 1 strncmp not if
+      dup match ok? not if
+         " is not a known registered program." strcat
+         exec-err exit
+      then
+      dup match program? not if
+         " is not a program." strcat
+         exec-err exit
+      then
+      3 pick owner over match can-linkto? not if
+         " is not Link_OK." strcat
+         exec-err exit
+      then
+   else
+      dup number? not if
+         " is not a program dbref." strcat
+         "@" swap strcat exec-err exit
+      then
+      dup atoi dbref ok? not if
+         " is not a valid program reference." strcat
+         "@" swap strcat exec-err exit
+      then
+      dup atoi dbref program? not if
+         " is not a valid program reference." strcat
+         "@" swap strcat exec-err exit
+      then
+      3 pick owner over atoi dbref can-linkto? not if
+         " is not Link_OK." strcat
+         "@" swap strcat exec-err exit
+      then
+   then
+   pop pop pop
+;
+   
+   
+: missing-err ( d s -- )
+   swap unparseobj
+   " is missing an "
+   strcat swap strcat
+   " message." strcat .tell
+;
+   
+: colon-err ( d s -- )
+   swap unparseobj
+   " has an unnecesary ':' at the start of its "
+   strcat swap strcat
+   " message." strcat .tell
+;
+   
+: check-desc (d -- )
+   dup desc not if
+      "@description" missing-err
+   else
+      "@description" over
+      desc check-exec
+   then
+;
+   
+: check-succ (d -- )
+   dup succ not if
+      "@success" missing-err
+   else
+      "@success" over
+      succ check-exec
+   then
+;
+   
+: check-fail (d -- )
+   dup fail not if
+      "@fail" missing-err
+   else
+      "@fail" over
+      fail check-exec
+   then
+;
+   
+: check-drop (d -- )
+   dup drop not if
+      "@drop" missing-err
+   else
+      "@drop" over
+      drop check-exec
+   then
+;
+   
+: check-osucc (d -- )
+   dup osucc not if
+      "@osuccess" missing-err
+   else
+      dup osucc ":" 1 strncmp not if
+         "@osuccess" colon-err
+      else pop
+      then
+   then
+;
+   
+: check-ofail (d -- )
+   dup ofail not if
+      "@ofail" missing-err
+   else
+      dup ofail ":" 1 strncmp not if
+         "@ofail" colon-err
+      else pop
+      then
+   then
+;
+   
+: check-odrop (d -- )
+   dup odrop not if
+      "@odrop" missing-err
+   else
+      dup odrop ":" 1 strncmp not if
+         "@odrop" colon-err
+      else pop
+      then
+   then
+;
+   
+   
+$define islocked? (d -- i) getlockstr "*UNLOCKED*" stringcmp $enddef
+   
+: islocked_always? (d -- i)
+   getlockstr dup "#0" stringcmp not if pop 1 exit then
+   dup "#" STRsplit swap pop atoi
+   "#" swap intostr strcat
+   (lockstr "#dbref")
+   dup "&!" over strcat strcat
+   3 pick stringcmp not if pop pop 1 exit then
+   "&" over strcat strcat "!" swap strcat
+   stringcmp not if 1 exit then
+   0
+;
+   
+: check-link ( d -- )
+   dup getlink not if
+      dup unparseobj " is unlinked." strcat .tell
+   else
+      dup getlink over location dbcmp if
+         dup islocked? not if
+            dup unparseobj
+            " is linked to it's location, but is unlocked."
+            strcat .tell
+         then
+      else (is not linked to it's location)
+         dup getlink program? if
+            dup dup owner swap getlink can-linkto? not if
+               dup unparseobj
+               " is linked to a program which is not Link_OK."
+               strcat .tell
+            then
+         then
+      then
+   then
+   pop
+;
+         
+: check-room (d -- )
+   dup check-desc
+   dup islocked? if
+      dup islocked_always? not if
+         dup check-succ
+      then
+      dup check-fail
+   then
+   dup getlink if
+      dup check-drop
+      dup check-odrop
+   then
+   dup check-contents
+   check-exits
+;
+   
+: check-exit ( d -- )
+   dup check-link
+   dup check-desc
+   dup getlink dup ok? if
+      program? not if
+         dup islocked_always? not if
+            dup check-succ
+            dup check-osucc
+            dup check-odrop
+         then
+         dup islocked? if
+            dup check-fail
+            dup check-ofail
+         then
+      then
+   else pop
+   then
+   pop
+;
+   
+: check-thing ( d -- )
+   dup check-desc
+   dup islocked_always? not if
+      dup check-succ
+      dup check-osucc
+   then
+   dup islocked? if
+      dup check-fail
+      dup check-ofail
+   then
+   dup check-drop
+   dup check-odrop
+   check-exits
+;
+   
+: check-player ( d -- )
+   dup check-desc
+   dup islocked_always? not if
+      dup check-succ
+      dup check-osucc
+   then
+   dup islocked? if
+      dup check-fail
+      dup check-ofail
+   then
+   dup check-contents
+   check-exits
+;
+   
+: check-program ( d -- )
+   check-desc
+;
+   
+: check-obj (d -- )
+   dup room?   if check-room   exit then
+   dup exit?   if check-exit   exit then
+   dup thing?  if check-thing  exit then
+   dup player? if check-player exit then
+   check-program
+;
+   
+: main
+   'check-obj check-obj-addr !
+   .strip dup not if pop "here" then
+   .match_controlled
+   dup #-3 dbcmp if pop me @ getlink then
+   dup ok? not if pop exit then
+   check-obj
+   me @ "Check done." notify
+;
--- a/samples/MUF/cmd-say.muf
+++ b/samples/MUF/cmd-say.muf
@@ -0,0 +1,275 @@
+@program cmd-say.muf
+1 1000 d
+i
+( cmd-say.muf by Natasha@HLM
+
+  Copyright 2002-2004 Natasha Snunkmeox. Copyright 2002-2004 Here Lie Monsters.
+  "@view $box/mit" for license information.
+)
+$author Natasha Snunkmeox <natmeox@neologasm.org>
+$note Say for Fuzzball 6.
+$version 1.0
+
+$include $lib/ignore
+$include $lib/strings
+$include $lib/match
+
+$def str_program "saypose"
+$def prop_third "_prefs/say/third"
+$def prop_quotes "_say/def/quotes"
+$def prop_overb "_say/def/osay"
+$def prop_verb "_say/def/say"
+$def prop_split "_prefs/say/split"
+$def prop_color "_prefs/say/color"
+$def prop_meow "_prefs/say/meow"
+
+lvar randomWord
+
+lvar verb
+lvar overb
+lvar lquo
+lvar rquo
+lvar splitsay
+
+: rtn-getThirdVerb[ var:overb -- ]
+    ( Get the third-person verb. )
+    me @ prop_overb getpropstr dup if  ( str strOverb )
+        strip dup dup "," instr not and if "," strcat then
+    else pop "says," then  ( str strOverb )
+    me @ "%D %s" fmtstring overb @ !  ( str )
+;
+
+: rtn-getFirstVerb[ var:verb var:overb -- ]
+    me @ prop_third getpropstr .yes? not if  ( str )
+        ( Get the first-person verb. )
+        me @ prop_verb getpropstr dup if  ( str strVerb )
+            strip dup dup "," instr not and if "," strcat then
+        else pop "say," then  ( str strVerb )
+        splitsay @ if "you %s" else "You %s" then fmtstring  ( str strVerb )
+    else overb @ @ then verb @ !  ( str )
+;
+
+: rtn-getQuotes[ var:lquo var:rquo -- ]
+    me @ prop_quotes getpropstr dup "%m" instr if  ( strQuotes )
+        "%m" split  ( strLquo strRquo )
+    else pop "\"" dup then  ( strLquo strRquo )
+    rquo @ ! lquo @ !  (  )
+;
+
+: do-say  ( str -- )
+    "" randomWord !
+
+    var who
+    var exclude
+
+    ( Ignoring? Get 'em outta here. )
+    loc @ contents_array  ( str arrHere )
+    dup me @ str_program array_get_ignorers  ( str arrHere arrIgnorers )
+    dup exclude !
+    swap array_diff who !
+
+
+    ( Anyone #meowing this player? Go ahead and notify before special formatting. )
+    who @ prop_meow me @ owner "*{%d}*" fmtstring array_filter_prop  ( str arrMeow )
+    dup if  ( str arrMeow )
+        dup who @ array_diff who !  ( str arrMeow )
+        dup exclude @ array_union exclude !  ( str arrMeow )
+
+        over ansi_strip  ( str arrMeow str )
+        "\\b[A-Z0-9_]+\\b" "MEOW" REG_ALL regsub  ( str arrMeow str' )
+        "\\b[A-Z0-9_][A-Za-z0-9_]*[a-z][A-Za-z0-9_]*\\b" "Meow" REG_ALL regsub  ( str arrMeow str' )
+        "\\b[a-z_][A-Za-z0-9_]*\\b" "meow" REG_ALL regsub  ( str arrMeow str' )
+        me @ "%D meows, \"%s\"" fmtstring  ( str arrMeow str" )
+        1 array_make swap array_notify  ( str )
+    else pop then  ( str )
+
+
+    var msg
+
+    dup ",," instr  ( str boolCommas )
+    me @ prop_split getpropstr .no? not  ( str boolCommas boolSplitOK )
+    and if  ( str )
+        ",," split  ( str- -str )
+
+        ( User-supplied verb? )
+        dup ",," instr if
+            ",," split  ( str- strVerb -str )
+            swap dup if  ( str- -str strVerb )
+                strip  ( str- -str strVerb )
+                dup me @ name instr over tolower "%n" instr or if  ( str- -str strVerb )
+                    "%n" "%N" subst me @ name "%n" subst  ( str- -str strVerb )
+                else
+                    me @ swap "%s %D," fmtstring  ( str- -str -str- )
+                then  ( str- -str -str- )
+                dup "*[-!.,:;]" smatch not if "," strcat then  ( str- -str -str- )
+                dup verb ! overb !  ( str- -str )
+            else pop then  ( str- -str )
+        then  ( str- -str )
+
+        2 array_make  ( arrMsg )
+        1
+    else 0 then splitsay ! msg !
+
+
+    verb @ string? not if
+        overb rtn-getThirdVerb
+        verb overb rtn-getFirstVerb
+    then
+    lquo rquo rtn-getQuotes  ( str )
+
+
+    ( Say. )
+    msg @ string? if
+        rquo @ msg @ lquo @  ( strRquo strMsg strLquo )
+        "%s %s%s%s"  ( strRquo strMsg strLquo strFormat )
+
+        4 dupn
+        verb  @ swap fmtstring .tell  ( strRquo strMsg strLquo strFormat )
+        overb @ swap fmtstring  ( strOsay )
+    else
+        rquo @ msg @ array_vals pop  ( strRquo strMsg strMsg2 )
+        swap dup "*[-!.,:;]" smatch not if "," strcat then swap  ( strRquo strMsg strMsg2 )
+        ( Only handle strMsg if there's no strMsg2. )
+        dup if  ( strRquo strMsg strMsg2 )
+            swap  ( strRquo strMsg2 strMsg )
+            lquo @ swap rquo @ swap lquo @  ( strRquo strMsg2 strLquo strRquo strMsg' strLquo )
+            "%s%s%s %s %s%s%s"  ( strRquo strMsg2 strLquo strRquo strMsg' strLquo strFormat )
+
+            7
+        else  ( strRquo strMsg strMsg2 )
+            pop lquo @  ( strRquo strMsg' strLquo )
+            "%s%s%s %s"  ( strRquo strMsg' strLquo strFormat )
+
+            verb @ ",$" "." 0 regsub verb !
+            overb @ ",$" "." 0 regsub overb !
+
+            4
+        then  ( ... strRquo strMsg strLquo strFormat intDepth )
+
+        dupn
+        verb  @ -5 rotate fmtstring .tell  ( ... strRquo strMsg strLquo strFormat )
+        overb @ -5 rotate fmtstring  ( strOsay )
+    then  ( strOsay )
+
+
+    ( Is there color to avoid? )
+    dup "\[[" instr if
+        who @ prop_color "{n*|0}" array_filter_prop  ( strOsay arrGreyed )
+        dup if  ( strOsay arrGreyed )
+            over ansi_strip 1 array_make  ( strOsay arrGreyed arrMsg )
+            over array_notify  ( strOsay arrGreyed )
+
+            exclude @ array_union exclude !  ( strOsay )
+        else pop then  ( strOsay )
+    then  ( strOsay )
+
+    loc @  ( strOsay db )
+    exclude @ array_vals  ( strOsay db dbExcludeN..dbExclude1 intN )
+    me @ swap ++  ( strOsay db dbGreyedN..dbGreyed1' intN' )
+    dup 3 + rotate  ( db dbGreyedN..dbGreyed1 intN strOsay )
+    notify_exclude  (  )
+;
+
+: do-help pop pop .showhelp ;
+: do-ignore pop str_program cmd-ignore-add ;
+: do-unignore pop str_program cmd-ignore-del ;
+
+: do-third  ( strY strZ -- )
+    pop pop  (  )
+    me @ prop_third "yes" setprop
+    me @ "You will see your own says in the third person (\"%D says\")." fmtstring .tellgood
+;
+: do-unthird  ( strY strZ -- )
+    pop pop  (  )
+    me @ prop_third remove_prop
+    "You will see your own says in the second person (\"You say\")." .tellgood
+;
+
+: do-grey  ( strY strZ -- )
+    pop pop  (  )
+    me @ prop_color "no" setprop
+    me @ "You will not see color in any says. Note you will see color in your own says." fmtstring .tellgood
+;
+: do-ungrey  ( strY strZ -- )
+    pop pop  (  )
+    me @ prop_color remove_prop
+    "You will see color in says." .tellgood
+;
+
+: do-meow  ( strY strZ -- )
+    pop  ( strY )
+    dup if
+        .noisy_pmatch dup ok? not if pop exit then  ( db )
+        me @ prop_meow 3 pick reflist_find if  ( db )
+            "%D is already in your #meow list." fmtstring .tellbad exit  (  )
+        then  ( db )
+        me @ prop_meow 3 pick reflist_add  ( db )
+        "%D added." fmtstring .tellgood
+    else
+        me @ prop_meow array_get_reflist  ( arr )
+        "" swap foreach swap pop "%D %s" fmtstring repeat
+        "Your meowlist: " swap strcat .tellgood
+    then
+;
+: do-unmeow  ( strY strZ -- )
+    pop  ( strY )
+    .noisy_pmatch dup ok? not if pop exit then  ( db )
+    me @ prop_meow 3 pick reflist_find not if  ( db )
+        "%D is not in your #meow list." fmtstring .tellbad exit  (  )
+    then  ( db )
+    me @ prop_meow 3 pick reflist_del  ( db )
+    "%D removed." fmtstring .tellgood
+;
+
+$define dict_commands {
+    "help"    'do-help
+    "ignore"  'do-ignore
+    "!ignore" 'do-unignore
+    "meow"    'do-meow
+    "!meow"   'do-unmeow
+    "third"   'do-third
+    "!third"  'do-unthird
+    "grey"    'do-grey
+    "gray"    'do-grey
+    "!grey"   'do-ungrey
+    "!gray"   'do-ungrey
+}dict $enddef
+
+: main  ( str -- )
+    dup STRparse  ( str strX strY strZ )
+    3 pick string? if 3 pick "#" stringpfx if  ( str strX strY strZ )
+        pop pop pop  ( str )
+        "#" split strcat  ( str' )
+        do-say exit  (  )
+    then then
+    3 pick int? if pop pop pop do-say exit then
+    4 rotate pop  ( strX strY strZ )
+
+    rot dict_commands over array_getitem  ( strY strZ strX ? )
+    dup address? if  ( strY strZ strX adr )
+        swap pop  ( strY strZ adr )
+        execute  (  )
+    else pop  ( strY strZ strX )
+        "I don't recognize the command '#%s'. Try 'say #help' for help, or using '##' to say something starting with '#'." fmtstring .tellbad  ( strY strZ )
+        pop pop  (  )
+    then  (  )
+;
+.
+c
+q
+
+lsedit #257=_help
+.del 1 $
+say <message>
+."<message>
+say #[!]ignore <names>
+say #[!]third
+say #[!]grey
+say #[!]meow <names>
+
+Speaks <message> to the room. Use #ignore <name> to not see <name>'s says, poses, and spoofs; use #meow <name> to see <name>'s says with all the words replaced with "meow." Use #third to see your own says in the third person (that is, "Puck says" instead of the normal "You say"). Use #grey to turn off color in others' says and poses.
+
+Say supports a "split" say if you put two consecutive commas in your message. For example, if CobaltBlue typed '"Hello,,how are you?' everyone would see '"Hello," says CobaltBlue, "how are you?"' You can also specify an "ad-hoc" verb by putting a message with your name or '%N' between pairs of commas: '"Hello,,%N welcomes Weiran,,how are you?' would display '"Hello," CobaltBlue welcomes Weiran, "how are you?"'
+.format 10=78
+.format 8=78
+.end
--- a/samples/Makefile/foo.o.d
+++ b/samples/Makefile/foo.o.d
@@ -0,0 +1,5 @@
+bar/foo.o: \
+ bar/foo.c \
+ bar/baz.h
+
+bar/baz.h:
--- a/samples/Mathematica/Predicates.wl
+++ b/samples/Mathematica/Predicates.wl
@@ -0,0 +1,150 @@
+(* ::Package:: *)
+
+BeginPackage["Predicates`"];
+
+
+(* ::Title:: *)
+(*Predicates*)
+
+
+(* ::Section::Closed:: *)
+(*Fuzzy Logic*)
+
+
+(* ::Subsection:: *)
+(*Documentation*)
+
+
+PossiblyTrueQ::usage="Returns True if the argument is not definitely False.";
+PossiblyFalseQ::usage="Returns True if the argument is not definitely True.";
+PossiblyNonzeroQ::usage="Returns True if and only if its argument is not definitely zero.";
+
+
+(* ::Subsection:: *)
+(*Implimentation*)
+
+
+Begin["`Private`"];
+
+
+PossiblyTrueQ[expr_]:=\[Not]TrueQ[\[Not]expr]
+
+
+PossiblyFalseQ[expr_]:=\[Not]TrueQ[expr]
+
+
+End[];
+
+
+(* ::Section::Closed:: *)
+(*Numbers and Lists*)
+
+
+(* ::Subsection:: *)
+(*Documentation*)
+
+
+AnyQ::usage="Given a predicate and a list, retuns True if and only if that predicate is True for at least one element of the list.";
+AnyElementQ::usage="Returns True if cond matches any element of L.";
+AllQ::usage="Given a predicate and a list, retuns True if and only if that predicate is True for all elements of the list.";
+AllElementQ::usage="Returns True if cond matches any element of L.";
+
+
+AnyNonzeroQ::usage="Returns True if L is a list such that at least one element is definitely not zero.";
+AnyPossiblyNonzeroQ::usage="Returns True if expr is a list such that at least one element is not definitely zero.";
+
+
+RealQ::usage="Returns True if and only if the argument is a real number";
+PositiveQ::usage="Returns True if and only if the argument is a positive real number";
+NonnegativeQ::usage="Returns True if and only if the argument is a non-negative real number";
+PositiveIntegerQ::usage="Returns True if and only if the argument is a positive integer";
+NonnegativeIntegerQ::usage="Returns True if and only if the argument is a non-negative integer";
+
+
+IntegerListQ::usage="Returns True if and only if the input is a list of integers.";
+PositiveIntegerListQ::usage="Returns True if and only if the input is a list of positive integers.";
+NonnegativeIntegerListQ::usage="Returns True if and only if the input is a list of non-negative integers.";
+IntegerOrListQ::usage="Returns True if and only if the input is a list of integers or an integer.";
+PositiveIntegerOrListQ::usage="Returns True if and only if the input is a list of positive integers or a positive integer.";
+NonnegativeIntegerOrListQ::usage="Returns True if and only if the input is a list of positive integers or a positive integer.";
+
+
+SymbolQ::usage="Returns True if argument is an unassigned symbol.";
+SymbolOrNumberQ::usage="Returns True if argument is a number of has head 'Symbol'";
+
+
+(* ::Subsection:: *)
+(*Implimentation*)
+
+
+Begin["`Private`"];
+
+
+AnyQ[cond_, L_] := Fold[Or, False, cond /@ L]
+
+
+AnyElementQ[cond_,L_]:=AnyQ[cond,Flatten[L]]
+
+
+AllQ[cond_, L_] := Fold[And, True, cond /@ L]
+
+
+AllElementQ[cond_, L_] := Fold[And, True, cond /@ L]
+
+
+AnyNonzeroQ[L_]:=AnyElementQ[#!=0&,L]
+
+
+PossiblyNonzeroQ[expr_]:=PossiblyTrueQ[expr!=0]
+
+
+AnyPossiblyNonzeroQ[expr_]:=AnyElementQ[PossiblyNonzeroQ,expr]
+
+
+RealQ[n_]:=TrueQ[Im[n]==0];
+
+
+PositiveQ[n_]:=Positive[n];
+
+
+PositiveIntegerQ[n_]:=PositiveQ[n]\[And]IntegerQ[n];
+
+
+NonnegativeQ[n_]:=TrueQ[RealQ[n]&&n>=0];
+
+
+NonnegativeIntegerQ[n_]:=NonnegativeQ[n]\[And]IntegerQ[n];
+
+
+IntegerListQ[input_]:=ListQ[input]&&Not[MemberQ[IntegerQ/@input,False]];
+
+
+IntegerOrListQ[input_]:=IntegerListQ[input]||IntegerQ[input];
+
+
+PositiveIntegerListQ[input_]:=IntegerListQ[input]&&Not[MemberQ[Positive[input],False]];
+
+
+PositiveIntegerOrListQ[input_]:=PositiveIntegerListQ[input]||PositiveIntegerQ[input];
+
+
+NonnegativeIntegerListQ[input_]:=IntegerListQ[input]&&Not[MemberQ[NonnegativeIntegerQ[input],False]];
+
+
+NonnegativeIntegerOrListQ[input_]:=NonnegativeIntegerListQ[input]||NonnegativeIntegerQ[input];
+
+
+SymbolQ[a_]:=Head[a]===Symbol;
+
+
+SymbolOrNumberQ[a_]:=NumericQ[a]||Head[a]===Symbol;
+
+
+End[];
+
+
+(* ::Section:: *)
+(*Epilogue*)
+
+
+EndPackage[];
--- a/samples/Mathematica/UnitTest.wlt
+++ b/samples/Mathematica/UnitTest.wlt
@@ -0,0 +1,17 @@
+BeginTestSection["Untitled-5"]
+
+VerificationTest[(* 1 *)
+	RotationMatrix[phi]
+	,
+	List[List[Cos[phi], Times[-1, Sin[phi]]], List[Sin[phi], Cos[phi]]]	
+]
+
+VerificationTest[(* 2 *)
+	Times[1, Power[Plus[a, Times[-1, a]], -1]]
+	,
+	ComplexInfinity
+	,
+	{Power::infy}
+]
+
+EndTestSection[]
--- a/samples/NL/assign0.nl
+++ b/samples/NL/assign0.nl
@@ -0,0 +1,84 @@
+g3 0 1 0	# problem assign0
+ 9 6 1 0 6	# vars, constraints, objectives, ranges, eqns
+ 0 0	# nonlinear constraints, objectives
+ 0 0	# network constraints: nonlinear, linear
+ 0 0 0	# nonlinear vars in constraints, objectives, both
+ 0 0 0 1	# linear network variables; functions; arith, flags
+ 9 0 0 0 0	# discrete variables: binary, integer, nonlinear (b,c,o)
+ 18 9	# nonzeros in Jacobian, gradients
+ 0 0	# max name lengths: constraints, variables
+ 0 0 0 0 0	# common exprs: b,c,o,c1,o1
+C0
+n0
+C1
+n0
+C2
+n0
+C3
+n0
+C4
+n0
+C5
+n0
+O0 0
+n0
+r
+4 1
+4 1
+4 1
+4 1
+4 1
+4 1
+b
+0 0 1
+0 0 1
+0 0 1
+0 0 1
+0 0 1
+0 0 1
+0 0 1
+0 0 1
+0 0 1
+k8
+2
+4
+6
+8
+10
+12
+14
+16
+J0 3
+0 1
+1 1
+2 1
+J1 3
+3 1
+4 1
+5 1
+J2 3
+6 1
+7 1
+8 1
+J3 3
+0 1
+3 1
+6 1
+J4 3
+1 1
+4 1
+7 1
+J5 3
+2 1
+5 1
+8 1
+G0 9
+0 1
+1 3
+2 3
+3 2
+4 3
+5 3
+6 3
+7 3
+8 2
--- a/samples/NL/balassign0.nl
+++ b/samples/NL/balassign0.nl
--- a/samples/NewLisp/queens.nl
+++ b/samples/NewLisp/queens.nl
@@ -0,0 +1,49 @@
+#!/usr/bin/env newlisp
+
+(constant 'NUM 8)
+
+(define (intersects? q1 q2)
+	(or 
+		(= (q1 0) (q2 0)) 
+		(= (q1 1) (q2 1))
+		(= (abs (- (q1 0) (q2 0))) (abs (- (q1 1) (q2 1))))))
+
+(define (variant? alist)
+	(set 'logic nil)
+	(cond
+		((= (length alist) 1) true)
+		((> (length alist) 1)
+			(while (> (length alist) 1)
+				(set 'q (pop alist -1))
+				(dolist (el alist)
+					(push 
+						(intersects? 
+							(list q (inc (length alist)))
+							(list el (+ 1 $idx)))
+					logic -1)))
+			(not (apply or logic)))))
+
+(define (fork-by-line alist)
+	(let (res '())
+		(dolist (i (sequence 1 NUM))
+			(set 'tmp alist)
+			(push i tmp -1)
+			(setf res (push tmp res -1)))
+		res))
+
+(define (find-variants num)
+	(let (res '())
+		(cond 
+			((< num 1) 
+				(begin (println "num < 1") (exit)))
+			((= num 1) 
+				(dolist (i (sequence 1 NUM)) (push (list i) res -1)))
+			((> num 1) 
+				(dolist (v (find-variants (dec num))) 
+					(set 'passed (filter variant? (fork-by-line v)))
+					(if (not (empty? passed)) (extend res passed)))))
+		res))
+		
+(set 'solutions (find-variants NUM))
+(println (length solutions))
+;;(exit)
--- a/samples/PLSQL/myobject.sql
+++ b/samples/PLSQL/myobject.sql
@@ -0,0 +1,15 @@
+create or replace type myobject
+AUTHID DEFINER
+AS OBJECT
+(
+        m_name varchar2(200),
+        member function toString RETURN VARCHAR2,
+        map member function Compare return varchar2
+
+)
+not instantiable not final;
+/
+
+prompt create type myarray
+create or replace type myarray as table of myobject;
+/
--- a/samples/PLSQL/packagebody.pkb
+++ b/samples/PLSQL/packagebody.pkb
@@ -0,0 +1,58 @@
+CREATE OR REPLACE PACKAGE BODY linguistpackage
+AS
+  /*
+  * Package:     linguist pacakage body
+  * Purpose:     a sample PLSQL file for linguist to work with
+  *
+  * Date:        03/03/2014
+  * Author:      david pyke le brun
+  * Comments:    initial version
+  */
+
+PROCEDURE proc_1
+IS
+BEGIN
+NULL;
+END;
+
+-- functions with 1 arg
+FUNCTION function1( param1 VARCHAR2 ) RETURN VARCHAR2
+IS
+CURSOR c IS
+select * from dual;
+v c%ROWTYPE;
+BEGIN
+ open c;
+ fetch c into v;
+ close c;
+
+ return v;
+end;
+
+FUNCTION function2( param1 NUMBER ) RETURN DATE
+IS
+BEGIN
+ return SYSDATE;
+end;
+
+--a few more to use all basic SQL types
+FUNCTION function3( param1 TIMESTAMP ) RETURN CHAR
+IS
+BEGIN
+IF 1 = 2 THEN
+return 'Y';
+ELSE
+return 'N';
+END IF;
+return NULL;
+END;
+
+
+FUNCTION function4( param1 CLOB ) RETURN BLOB
+IS
+BEGIN
+	return null;
+END;
+
+END linguistpackage;
+/
--- a/samples/PLSQL/packageheader.pks
+++ b/samples/PLSQL/packageheader.pks
@@ -0,0 +1,28 @@
+CREATE OR REPLACE PACKAGE linguistpackage
+AUTHID DEFINER
+AS
+  /*
+  * Package:     linguist pacakage
+  * Purpose:     a sample PLSQL file for linguist to work with
+  *
+  * Date:        03/03/2014
+  * Author:      david pyke le brun
+  * Comments:    initial version
+  */
+
+k_constant CONSTANT NUMBER(10,2) := 3.14;
+
+--basic procedure
+PROCEDURE proc_1;
+
+-- functions with 1 arg
+FUNCTION function1( param1 VARCHAR2 ) RETURN VARCHAR2;
+FUNCTION function2( param1 NUMBER ) RETURN DATE;
+
+--a few more to use all basic SQL types
+FUNCTION function3( param1 TIMESTAMP ) RETURN CHAR;
+FUNCTION function4( param1 CLOB ) RETURN BLOB;
+  
+END linguistpackage;
+/
+
--- a/samples/PLSQL/prime#.plsql
+++ b/samples/PLSQL/prime#.plsql
@@ -0,0 +1,93 @@
+create or replace package prime#
+is
+  invalid_argument_error                          exception;
+
+  function nth (
+    i_num                                         pls_integer
+  ) return number;
+end prime#;
+/
+
+create or replace package body prime#
+is
+  type t_primes is table of number index by pls_integer;
+  b_primes                                       t_primes;
+
+  function is_prime(
+    i_candidate                                   number
+  ) return boolean
+  is
+    l_num                           number := 1;
+    l_prime                         number;
+    l_result                        number;
+  begin
+  	if i_candidate < 2 then
+  	  return false;
+  	end if;
+
+    loop
+      l_prime := nth(l_num);
+      if l_prime = i_candidate then
+        return true;
+      end if;
+
+      l_result := i_candidate / l_prime;
+    	if l_result = ceil(l_result) then
+    	  return false;
+    	end if;
+
+    	l_num := l_num + 1;
+    	exit when l_result <= l_prime;
+    end loop;
+
+    return true;
+  end is_prime;
+
+  function next (
+    i_prime                                       pls_integer
+  ) return number
+  is
+    l_next                          number;
+  begin
+    l_next := i_prime + case mod(i_prime, 2) when 0 then 1 else 2 end;
+
+    while not is_prime(l_next) loop
+      l_next := l_next + 2;
+    end loop;
+    return l_next;
+  end next;
+	
+  function nth (
+    i_num                                         pls_integer
+  ) return number
+  is
+    l_index                         number := 2;
+    l_prime                         number := 3;
+  begin
+    if   i_num < 1
+      or ceil(i_num) != i_num
+    then
+      raise invalid_argument_error;
+    end if;
+
+    case i_num
+      when 1 then return 2;
+      else
+        if b_primes.exists(i_num) then
+          return b_primes(i_num);
+        end if;
+        while l_index < i_num loop
+          l_index := l_index + 1;
+          if b_primes.exists(l_index) then
+            l_prime := b_primes(l_index);
+          else
+            l_prime := next(l_prime);
+            b_primes(l_index) := l_prime;
+          end if;
+        end loop;
+        return l_prime;
+    end case;
+  end nth;
+
+end prime#;
+/
--- a/samples/PLSQL/who_called_me.sql
+++ b/samples/PLSQL/who_called_me.sql
@@ -0,0 +1,65 @@
+CREATE OR REPLACE PROCEDURE who_called_me
+( owner      OUT VARCHAR2,
+  name       OUT VARCHAR2,
+  lineno     OUT NUMBER,
+  caller_t   OUT VARCHAR2 ,
+  depth          NUMBER DEFAULT 1
+)
+AUTHID DEFINER
+AS
+--depth based version of who_called_me from asktom
+   call_stack  VARCHAR2(4096) default dbms_utility.format_call_stack;
+   n           NUMBER;
+   found_stack BOOLEAN DEFAULT FALSE;
+   line        VARCHAR2(255);
+   cnt         NUMBER := 0;
+BEGIN
+   LOOP
+       n := instr( call_stack, chr(10) );
+       exit when ( n is NULL or n = 0 );
+--
+       line := substr( call_stack, 1, n-1 );
+       call_stack := substr( call_stack, n+1 );
+--
+       if ( NOT found_stack ) then
+           if ( line like '%handle%number%name%' ) then
+               found_stack := TRUE;
+           end if;
+       else
+           cnt := cnt + 1;
+           -- cnt = 1 is ME
+           -- cnt = 2 is MY Caller
+           -- cnt = 3 is Their Caller
+           if ( cnt = (2+depth) ) then
+               lineno := to_number(substr( line, 13, 8 ));
+               line   := substr( line, 23 ); --set to rest of line .. change from 21 to 23
+               if ( line like 'pr%' ) then
+                   n := length( 'procedure ' );
+               elsif ( line like 'fun%' ) then
+                   n := length( 'function ' );
+               elsif ( line like 'package body%' ) then
+                   n := length( 'package body ' );
+               elsif ( line like 'pack%' ) then
+                   n := length( 'package ' );
+               elsif ( line like 'anonymous%' ) then
+                   n := length( 'anonymous block ' );
+               else
+                   n := null;
+               end if;
+               if ( n is not null ) then
+                  caller_t := ltrim(rtrim(upper(substr( line, 1, n-1 ))));
+               else
+                  caller_t := 'TRIGGER';
+               end if;
+
+               line := substr( line, nvl(n,1) );
+               n := instr( line, '.' );
+               owner := ltrim(rtrim(substr( line, 1, n-1 )));
+               name  := LTRIM(RTRIM(SUBSTR( LINE, N+1 )));
+               exit;
+           END IF;
+       END IF;
+   END LOOP;
+END;
+/
+
--- a/samples/PLpgSQL/plpgsql_lint-8.4.sql
+++ b/samples/PLpgSQL/plpgsql_lint-8.4.sql
@@ -0,0 +1,165 @@
+load 'plpgsql';
+load 'plpgsql_lint';
+
+create table t1(a int, b int);
+
+create function f1()
+returns void as $$
+begin
+  if false then
+    update t1 set c = 30;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create function g1(out a int, out b int)
+as $$
+  select 10,20;
+$$ language sql;
+
+create function f1()
+returns void as $$
+declare r record;
+begin
+  r := g1();
+  if false then 
+    raise notice '%', r.c;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+drop function g1();
+
+create function g1(out a int, out b int)
+returns setof record as $$
+select * from t1;
+$$ language sql;
+
+create function f1()
+returns void as $$
+declare r record;
+begin
+  for r in select * from g1()
+  loop
+    raise notice '%', r.c;
+  end loop;
+end;
+$$ language plpgsql;
+
+select f1();
+
+create or replace function f1()
+returns void as $$
+declare r record;
+begin
+  for r in select * from g1()
+  loop
+    r.c := 20;
+  end loop;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+drop function g1();
+
+create function f1()
+returns int as $$
+declare r int;
+begin
+  if false then
+    r := a + b;
+  end if;
+  return r;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+begin
+  if false then
+    raise notice '%', 1, 2;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+begin
+  if false then
+    raise notice '% %';
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+declare r int[];
+begin
+  if false then
+    r[c+10] := 20;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+
+create or replace function f1()
+returns void as $$
+declare r int;
+begin
+  if false then
+    r[10] := 20;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create type diagnostic_info_type as (
+  status text,
+  message text,
+  detail text,
+  row_count int);
+
+create or replace function f1()
+returns void as $$
+declare
+  dg record;
+begin
+  dg := NULL::diagnostic_info_type;
+  if false then
+    dg.status := '00000';
+    dg.mistake := 'hello';
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
--- a/samples/PLpgSQL/plpgsql_lint-9.0.sql
+++ b/samples/PLpgSQL/plpgsql_lint-9.0.sql
@@ -0,0 +1,165 @@
+load 'plpgsql';
+load 'plpgsql_lint';
+
+create table t1(a int, b int);
+
+create function f1()
+returns void as $$
+begin
+  if false then
+    update t1 set c = 30;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create function g1(out a int, out b int)
+as $$
+  select 10,20;
+$$ language sql;
+
+create function f1()
+returns void as $$
+declare r record;
+begin
+  r := g1();
+  if false then 
+    raise notice '%', r.c;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+drop function g1();
+
+create function g1(out a int, out b int)
+returns setof record as $$
+select * from t1;
+$$ language sql;
+
+create function f1()
+returns void as $$
+declare r record;
+begin
+  for r in select * from g1()
+  loop
+    raise notice '%', r.c;
+  end loop;
+end;
+$$ language plpgsql;
+
+select f1();
+
+create or replace function f1()
+returns void as $$
+declare r record;
+begin
+  for r in select * from g1()
+  loop
+    r.c := 20;
+  end loop;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+drop function g1();
+
+create function f1()
+returns int as $$
+declare r int;
+begin
+  if false then
+    r := a + b;
+  end if;
+  return r;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+begin
+  if false then
+    raise notice '%', 1, 2;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+begin
+  if false then
+    raise notice '% %';
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+declare r int[];
+begin
+  if false then
+    r[c+10] := 20;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+
+create or replace function f1()
+returns void as $$
+declare r int;
+begin
+  if false then
+    r[10] := 20;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create type diagnostic_info_type as (
+  status text,
+  message text,
+  detail text,
+  row_count int);
+
+create or replace function f1()
+returns void as $$
+declare
+  dg record;
+begin
+  dg := NULL::diagnostic_info_type;
+  if false then
+    dg.status := '00000';
+    dg.mistake := 'hello';
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
--- a/samples/PLpgSQL/plpgsql_lint-9.1.sql
+++ b/samples/PLpgSQL/plpgsql_lint-9.1.sql
@@ -0,0 +1,179 @@
+load 'plpgsql';
+load 'plpgsql_lint';
+
+create table t1(a int, b int);
+
+create function f1()
+returns void as $$
+begin
+  if false then
+    update t1 set c = 30;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create function g1(out a int, out b int)
+as $$
+  select 10,20;
+$$ language sql;
+
+create function f1()
+returns void as $$
+declare r record;
+begin
+  r := g1();
+  if false then 
+    raise notice '%', r.c;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+drop function g1();
+
+create function g1(out a int, out b int)
+returns setof record as $$
+select * from t1;
+$$ language sql;
+
+create function f1()
+returns void as $$
+declare r record;
+begin
+  for r in select * from g1()
+  loop
+    raise notice '%', r.c;
+  end loop;
+end;
+$$ language plpgsql;
+
+select f1();
+
+create or replace function f1()
+returns void as $$
+declare r record;
+begin
+  for r in select * from g1()
+  loop
+    r.c := 20;
+  end loop;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+drop function g1();
+
+create function f1()
+returns int as $$
+declare r int;
+begin
+  if false then
+    r := a + b;
+  end if;
+  return r;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+begin
+  if false then
+    raise notice '%', 1, 2;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+begin
+  if false then
+    raise notice '% %';
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+declare r int[];
+begin
+  if false then
+    r[c+10] := 20;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+
+create or replace function f1()
+returns void as $$
+declare r int;
+begin
+  if false then
+    r[10] := 20;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create type diagnostic_info_type as (
+  status text,
+  message text,
+  detail text,
+  row_count int);
+
+create or replace function f1()
+returns void as $$
+declare  dg record;
+begin
+  dg := NULL::diagnostic_info_type;
+  if false then
+    dg.status := '00000';
+    dg.mistake := 'hello';
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+declare  dg record;
+begin
+  if false then
+    dg := 10,20;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
--- a/samples/PLpgSQL/plpgsql_lint-9.2.sql
+++ b/samples/PLpgSQL/plpgsql_lint-9.2.sql
@@ -0,0 +1,166 @@
+load 'plpgsql';
+load 'plpgsql_lint';
+
+create table t1(a int, b int);
+
+create function f1()
+returns void as $$
+begin
+  if false then
+    update t1 set c = 30;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create function g1(out a int, out b int)
+as $$
+  select 10,20;
+$$ language sql;
+
+create function f1()
+returns void as $$
+declare r record;
+begin
+  r := g1();
+  if false then 
+    raise notice '%', r.c;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+drop function g1();
+
+create function g1(out a int, out b int)
+returns setof record as $$
+select * from t1;
+$$ language sql;
+
+create function f1()
+returns void as $$
+declare r record;
+begin
+  for r in select * from g1()
+  loop
+    raise notice '%', r.c;
+  end loop;
+end;
+$$ language plpgsql;
+
+select f1();
+
+create or replace function f1()
+returns void as $$
+declare r record;
+begin
+  for r in select * from g1()
+  loop
+    r.c := 20;
+  end loop;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+drop function g1();
+
+create function f1()
+returns int as $$
+declare r int;
+begin
+  if false then
+    r := a + b;
+  end if;
+  return r;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+begin
+  if false then
+    raise notice '%', 1, 2;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+begin
+  if false then
+    raise notice '% %';
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+declare r int[];
+begin
+  if false then
+    r[c+10] := 20;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+
+create or replace function f1()
+returns void as $$
+declare r int;
+begin
+  if false then
+    r[10] := 20;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create type _exception_type as (
+  state text,
+  message text,
+  detail text);
+
+create or replace function f1()
+returns void as $$
+declare
+  _exception record;
+begin
+  _exception := NULL::_exception_type;
+exception when others then
+  get stacked diagnostics
+        _exception.state = RETURNED_SQLSTATE,
+        _exception.message = MESSAGE_TEXT,
+        _exception.detail = PG_EXCEPTION_DETAIL,
+        _exception.hint = PG_EXCEPTION_HINT;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
--- a/samples/PLpgSQL/plpgsql_lint-9.3.sql
+++ b/samples/PLpgSQL/plpgsql_lint-9.3.sql
@@ -0,0 +1,166 @@
+load 'plpgsql';
+load 'plpgsql_lint';
+
+create table t1(a int, b int);
+
+create function f1()
+returns void as $$
+begin
+  if false then
+    update t1 set c = 30;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create function g1(out a int, out b int)
+as $$
+  select 10,20;
+$$ language sql;
+
+create function f1()
+returns void as $$
+declare r record;
+begin
+  r := g1();
+  if false then 
+    raise notice '%', r.c;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+drop function g1();
+
+create function g1(out a int, out b int)
+returns setof record as $$
+select * from t1;
+$$ language sql;
+
+create function f1()
+returns void as $$
+declare r record;
+begin
+  for r in select * from g1()
+  loop
+    raise notice '%', r.c;
+  end loop;
+end;
+$$ language plpgsql;
+
+select f1();
+
+create or replace function f1()
+returns void as $$
+declare r record;
+begin
+  for r in select * from g1()
+  loop
+    r.c := 20;
+  end loop;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+drop function g1();
+
+create function f1()
+returns int as $$
+declare r int;
+begin
+  if false then
+    r := a + b;
+  end if;
+  return r;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+begin
+  if false then
+    raise notice '%', 1, 2;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+begin
+  if false then
+    raise notice '% %';
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create or replace function f1()
+returns void as $$
+declare r int[];
+begin
+  if false then
+    r[c+10] := 20;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+
+create or replace function f1()
+returns void as $$
+declare r int;
+begin
+  if false then
+    r[10] := 20;
+  end if;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
+
+create type _exception_type as (
+  state text,
+  message text,
+  detail text);
+
+create or replace function f1()
+returns void as $$
+declare
+  _exception record;
+begin
+  _exception := NULL::_exception_type;
+exception when others then
+  get stacked diagnostics
+        _exception.state = RETURNED_SQLSTATE,
+        _exception.message = MESSAGE_TEXT,
+        _exception.detail = PG_EXCEPTION_DETAIL,
+        _exception.hint = PG_EXCEPTION_HINT;
+end;
+$$ language plpgsql;
+
+select f1();
+
+drop function f1();
--- a/samples/Perl/exception_handler.pl
+++ b/samples/Perl/exception_handler.pl
@@ -0,0 +1,117 @@
+package exception_handler;
+use sigtrap qw(die normal-signals);
+use IO::Handle;
+use Carp;
+use File::Spec;
+use File::Basename;
+use Data::Dumper;
+
+use sigtrap 'handler', \&tm_die;
+
+$Carp::CarpLevel = 1;     # How many extra package levels to skip on carp.
+
+BEGIN {
+    *CORE::GLOBAL::die = \&tm_die;
+    $main::SIG{__DIE__} = \&tm_die;
+    my $error_fd = $ENV{"TM_ERROR_FD"};
+	open (TM_ERROR_FD, ">&=$error_fd");
+	TM_ERROR_FD->autoflush(1);
+}
+
+sub realwarn { CORE::warn(@_); }
+sub realdie { CORE::die(@_); }
+
+sub longmess {
+    my ($arg, @rest) = shift;
+    {
+        local $@;
+        # XXX fix require to not clear $@?
+        # don't use require unless we need to (for Safe compartments)
+        require Carp::Heavy unless $INC{"Carp/Heavy.pm"};
+    }
+    # Icky backwards compatibility wrapper. :-(
+    my $call_pack = caller();
+    if ($Internal{$call_pack} or $Carp::CarpInternal{$call_pack}) {
+      return longmess_heavy($arg, @rest);
+    }
+    else {
+      local $Carp::CarpLevel = $Carp::CarpLevel + 1;
+      return longmess_heavy($arg, @rest);
+    }
+}
+
+sub longmess_heavy {
+  return @_ if ref($_[0]); # don't break references as exceptions
+  my $i = Carp::long_error_loc();
+  my ($arg, @rest) = @_;
+  return ret_backtrace($i, $arg, @rest);
+}
+
+sub quote {
+	my $str = shift;
+	$str =~ s/([^A-Za-z0-9\/_.-])/sprintf("%%%02X", ord($1))/seg;
+	return $str;
+}
+
+sub url_and_display_name {
+	my $file = shift;
+	my $url = "";
+	my $display_name = "";
+	$display_name = basename($file);
+	$url = 'url=file://' . quote($file);
+	return ($url, $display_name);
+}
+
+# Returns a full stack backtrace starting from where it is
+# told.
+sub ret_backtrace { 
+  my ($i, $arg, @rest) = @_;
+  my $mess;
+  $i++;
+
+  my $tid_msg = '';
+  if (defined &Thread::tid) {
+    my $tid = Thread->self->tid;
+    $tid_msg = " thread $tid" if $tid;
+  }
+
+  my %i = Carp::caller_info($i);
+  $arg =~ s/\n/\<br\>/g;
+  $i{sub} =~ s/tm_die/die/g;
+  $mess .= "<div id='exception_report' class='framed'>\n";
+  $mess .= "<p id='exception'><strong>$arg</strong></p>\n";
+  $mess .= "<blockquote><table border='0' cellspacing='0' cellpadding='0'>\n";
+  my ($url, $display_name) = url_and_display_name($i{file});
+  $mess .= "<tr><td><a href='txmt://open?line=$i{line}&" . $url . "'>$i{sub}</a></td><td>&nbsp;in $display_name at line $i{line}$tid_msg</td></tr>\n";
+  while (my %i = Carp::caller_info(++$i)) {
+      ($url, $display_name) = url_and_display_name($i{file});
+      $mess .= "<tr><td><a href='txmt://open?line=$i{line}&" . $url . "'>$i{sub}</a></td><td>&nbsp;in $display_name at line $i{line}$tid_msg</td></tr>\n";
+  }
+  $mess .= "</table></blockquote></div>";
+  return $mess;
+}
+
+sub ineval {
+  (exists $ENV{MOD_PERL} ? 0 : $^S) || Carp::longmess() =~ /eval [\{\']/m
+}
+
+sub htmlize {
+	my $l = shift;
+	$l =~ s/&/&amp;/g;
+	$l =~ s/</&lt;/g;
+	$l =~ s/>/&gt;/g;
+	return $l;
+}
+
+sub tm_die {
+  my ($arg,@rest) = @_;
+  if (ineval()) {
+      realdie ($arg,@rest) if ineval();
+  }
+  if (!ref($arg)) {
+    print TM_ERROR_FD longmess($arg,@rest);
+  }
+  exit($!);
+}
+
+1;
--- a/samples/Perl/use5.pl
+++ b/samples/Perl/use5.pl
@@ -0,0 +1,3 @@
+use Mojolicious::Lite;
+use 5.20.0;
+use experimental 'signatures';
--- a/samples/Ruby/index.json.jbuilder
+++ b/samples/Ruby/index.json.jbuilder
@@ -0,0 +1,4 @@
+json.array!(@courts) do |court|
+  json.extract! court, :id, :name_r, :region, :region_r, :email, :website
+  json.url court_url(court, format: :json)
+end
--- a/samples/SQL/create_stuff.sql
+++ b/samples/SQL/create_stuff.sql
@@ -0,0 +1,21 @@
+
+CREATE TABLE x AS SELECT * FROM DUAL;
+
+CREATE TABLE y (
+col1 NUMBER NOT NULL ,
+col2 VARCHAR2(200),
+col3 DATE,
+col4 TIMESTAMP WITH TIME ZONE NOT NULL
+);
+
+
+CREATE USER username IDENTIFIED BY password;
+
+GRANT CONNECT, RESOURCE TO username;
+
+
+GRANT CREATE TYPE TO username;
+GRANT CREATE PROCEDURE TO username;
+GRANT CREATE TABLE TO username;
+GRANT CREATE VIEW TO username;
+
--- a/samples/SQL/drop_stuff.sql
+++ b/samples/SQL/drop_stuff.sql
@@ -0,0 +1,13 @@
+drop procedure who_called_me;
+drop package body linguist_package;
+drop package linguist_package;
+drop function functionname1;
+
+drop table x;
+drop table y cascade;
+
+drop type typename1;
+drop type typename2;
+
+drop view viewname1;
+drop view viewname2;
--- a/samples/SQL/dual.sql
+++ b/samples/SQL/dual.sql
@@ -0,0 +1,3 @@
+--this is the most basic oracle sql command
+select * from dual;
+
--- a/samples/SQLPL/check_reorg.sql
+++ b/samples/SQLPL/check_reorg.sql
@@ -0,0 +1,39 @@
+create procedure check_reorg_tables (in v_schema varchar(128), out v_reorg_counter integer)
+begin
+
+  declare loc result_set_locator varying;
+
+  declare schema_out        varchar(128);
+  declare table_out         varchar(128);
+  declare card_out          integer;
+  declare overflow_out      integer;
+  declare npages_out        integer;
+  declare fpages_out        integer;
+  declare active_blocks_out integer;
+  declare tsize_out         integer;
+  declare f1_out            integer;
+  declare f2_out            integer;
+  declare f3_out            integer;
+  declare reorg_out         varchar(3);
+  declare cursor_end        smallint default 0;
+
+  declare continue handler for NOT FOUND
+ 
+  set cursor_end = 1;
+  set v_reorg_counter = 0;
+
+  call reorgchk_tb_stats('S', v_schema);
+  associate result set locator(loc) with procedure reorgchk_tb_stats;
+  allocate mycursor cursor for result set loc;
+
+  open mycursor;
+  repeat
+    fetch from mycursor into schema_out, table_out, card_out, overflow_out, npages_out, fpages_out, active_blocks_out, tsize_out, f1_out, f2_out, f3_out, reorg_out;
+     if reorg_out <> '---' then
+        set v_reorg_counter = v_reorg_counter + 1;
+     end if;
+     until cursor_end = 1
+  end repeat;
+  close mycursor;
+
+end!
--- a/samples/SQLPL/comm_amount.db2
+++ b/samples/SQLPL/comm_amount.db2
@@ -0,0 +1,30 @@
+DROP FUNCTION COMM_AMOUNT;
+CREATE FUNCTION COMM_AMOUNT(SALARY DEC(9,2))
+  RETURNS DEC(9,2)
+  LANGUAGE SQL READS SQL DATA
+  BEGIN ATOMIC
+    DECLARE REMAINDER DEC(9,2) DEFAULT 0.0;--
+    DECLARE COMM_PAID DEC(9,2) DEFAULT 0.0;--
+    DECLARE COMM_INCR INT      DEFAULT 1;--
+    DECLARE MAX_COMM  DEC(9,2) DEFAULT 0.0;--
+
+    IF (SALARY <= 0) THEN
+       SIGNAL SQLSTATE '75000'
+         SET MESSAGE_TEXT = 'Bad Salary';--
+    END IF;--
+
+    SET REMAINDER = SALARY;--
+
+L1: WHILE REMAINDER > 0.0 DO
+      SET COMM_PAID = COMM_PAID + (COMM_INCR * 500.00);--
+      SET REMAINDER = REMAINDER-(COMM_INCR * 5000.00);--
+      SET COMM_INCR = COMM_INCR + 1;--
+    END WHILE L1;--
+
+    SET MAX_COMM =
+       (SELECT SUM(SALARY)/100.00 FROM EMPLOYEE);--
+    IF (COMM_PAID > MAX_COMM) THEN
+       SET COMM_PAID = MAX_COMM;--
+    END IF;--
+    RETURN COMM_PAID;--
+END;
--- a/samples/SQLPL/drop_table.db2
+++ b/samples/SQLPL/drop_table.db2
@@ -0,0 +1,13 @@
+DROP TABLE TDEPT;
+CREATE TABLE TDEPT (DEPTNO CHAR(4));
+
+--#SET TERMINATOR @
+BEGIN ATOMIC
+  DECLARE COUNT INT DEFAULT 5;
+
+  WHILE COUNT > 0 DO
+    INSERT INTO TDEPT VALUES 'F'||
+       RTRIM(CHAR(COUNT));
+    SET COUNT = COUNT - 1;
+  END WHILE;
+END@
--- a/samples/SQLPL/runstats.sql
+++ b/samples/SQLPL/runstats.sql
@@ -0,0 +1,18 @@
+create procedure runstats (out nr_tables integer, out nr_ok integer)
+begin
+  declare SQLCODE integer;
+  declare stmt varchar(100);
+
+  set nr_tables = 0;
+  set nr_ok = 0;
+
+  for line as select tabschema, tabname from syscat.tables where type='T' and tabschema='SPODEN'
+  do
+    set nr_tables = nr_tables + 1;
+    set stmt = 'CALL SYSPROC.ADMIN_CMD (RUNSTATS ON TABLE ' concat rtrim(line.tabschema) concat '.' concat line.tabname concat ')';
+    execute immediate stmt;
+    if SQLCODE = 0 then
+       set nr_ok = nr_ok + 1;
+    end if;
+  end for;
+end!
--- a/samples/SQLPL/sleep.sql
+++ b/samples/SQLPL/sleep.sql
@@ -0,0 +1,9 @@
+create procedure sleep (in sleeptime integer)
+begin
+  declare wait_until timestamp;
+
+  set wait_until = (current timestamp + sleeptime seconds);
+  while (wait_until > current timestamp)
+    do
+    end while;
+end!
--- a/samples/SQLPL/trigger.sql
+++ b/samples/SQLPL/trigger.sql
@@ -0,0 +1,9 @@
+create trigger CHECK_HIREDATE
+no cascade before insert on EMPLOYEE
+referencing new as N
+for each row mode db2sql
+if n.hiredate > current date
+then
+  signal SQLSTATE '75000'
+  set MESSAGE_TEXT = 'Hire date must be in the past';
+end if!
--- a/samples/SourcePawn/foo.sma
+++ b/samples/SourcePawn/foo.sma
@@ -0,0 +1,272 @@
+// vim: set ts=4 sw=4 tw=99 noet:
+//
+// AMX Mod X, based on AMX Mod by Aleksander Naszko ("OLO").
+// Copyright (C) The AMX Mod X Development Team.
+//
+// This software is licensed under the GNU General Public License, version 3 or higher.
+// Additional exceptions apply. For full license details, see LICENSE.txt or visit:
+//     https://alliedmods.net/amxmodx-license
+
+//
+// TimeLeft Plugin
+//
+
+#include <amxmodx>
+
+const TASK_TIMEREMAIN_SHORT = 8648458	// 0.8s repeat task
+const TASK_TIMEREMAIN_LARGE = 34543		// 1.0s repeat task
+
+// time display flags
+const TD_BOTTOM_WHITE_TEXT = 1			// a - display white text on bottom
+const TD_USE_VOICE = 2					// b - use voice
+const TD_NO_REMAINING_VOICE = 4			// c - don't add "remaining" (only in voice)
+const TD_NO_HOURS_MINS_SECS_VOICE = 8		// d - don't add "hours/minutes/seconds" (only in voice)
+const TD_SHOW_SPEAK_VALUES_BELOW = 16	// e - show/speak if current time is less than this set in parameter
+
+new g_TimeSet[32][2]
+new g_LastTime
+new g_CountDown
+new g_Switch
+
+// pcvars
+new g_amx_time_voice, g_amx_timeleft
+new g_mp_timelimit
+
+public plugin_init()
+{
+	register_plugin("TimeLeft", AMXX_VERSION_STR, "AMXX Dev Team")
+	register_dictionary("timeleft.txt")
+	g_amx_time_voice = register_cvar("amx_time_voice", "1")
+	register_srvcmd("amx_time_display", "setDisplaying")
+	g_amx_timeleft = register_cvar("amx_timeleft", "00:00", FCVAR_SERVER|FCVAR_EXTDLL|FCVAR_UNLOGGED|FCVAR_SPONLY)
+	register_clcmd("say timeleft", "sayTimeLeft", 0, "- displays timeleft")
+	register_clcmd("say thetime", "sayTheTime", 0, "- displays current time")
+	
+	set_task(0.8, "timeRemain", TASK_TIMEREMAIN_SHORT, "", 0, "b")
+
+	g_mp_timelimit = get_cvar_pointer("mp_timelimit")
+}
+
+public sayTheTime(id)
+{
+	if (get_pcvar_num(g_amx_time_voice))
+	{
+		new mhours[6], mmins[6], whours[32], wmins[32], wpm[6]
+		
+		get_time("%H", mhours, charsmax(mhours))
+		get_time("%M", mmins, charsmax(mmins))
+		
+		new mins = str_to_num(mmins)
+		new hrs = str_to_num(mhours)
+		
+		if (mins)
+			num_to_word(mins, wmins, charsmax(wmins))
+		else
+			wmins[0] = EOS
+		
+		if (hrs < 12)
+			wpm = "am "
+		else
+		{
+			if (hrs > 12) hrs -= 12
+			wpm = "pm "
+		}
+
+		if (hrs) 
+			num_to_word(hrs, whours, charsmax(whours))
+		else
+			whours = "twelve "
+		
+		client_cmd(id, "spk ^"fvox/time_is_now %s_period %s%s^"", whours, wmins, wpm)
+	}
+	
+	new ctime[64]
+	
+	get_time("%m/%d/%Y - %H:%M:%S", ctime, charsmax(ctime))
+	client_print(0, print_chat, "%L:   %s", LANG_PLAYER, "THE_TIME", ctime)
+	
+	return PLUGIN_CONTINUE
+}
+
+public sayTimeLeft(id)
+{
+	if (get_pcvar_float(g_mp_timelimit))
+	{
+		new a = get_timeleft()
+		
+		if (get_pcvar_num(g_amx_time_voice))
+		{
+			new svoice[128]
+			setTimeVoice(svoice, charsmax(svoice), 0, a)
+			client_cmd(id, "%s", svoice)
+		}
+		client_print(0, print_chat, "%L:  %d:%02d", LANG_PLAYER, "TIME_LEFT", (a / 60), (a % 60))
+	}
+	else
+		client_print(0, print_chat, "%L", LANG_PLAYER, "NO_T_LIMIT")
+	
+	return PLUGIN_CONTINUE
+}
+
+setTimeText(text[], len, tmlf, id)
+{
+	new secs = tmlf % 60
+	new mins = tmlf / 60
+	
+	if (secs == 0)
+		formatex(text, len, "%d %L", mins, id, (mins > 1) ? "MINUTES" : "MINUTE")
+	else if (mins == 0)
+		formatex(text, len, "%d %L", secs, id, (secs > 1) ? "SECONDS" : "SECOND")
+	else
+		formatex(text, len, "%d %L %d %L", mins, id, (mins > 1) ? "MINUTES" : "MINUTE", secs, id, (secs > 1) ? "SECONDS" : "SECOND")
+}
+
+setTimeVoice(text[], len, flags, tmlf)
+{
+	new temp[7][32]
+	new secs = tmlf % 60
+	new mins = tmlf / 60
+	
+	// for (new a = 0;a < 7;++a) // we just created it, already null
+		// temp[a][0] = 0
+
+	if (secs > 0)
+	{
+		num_to_word(secs, temp[4], charsmax(temp[]))
+		
+		if ( ~flags & TD_NO_HOURS_MINS_SECS_VOICE ) 
+			temp[5] = "seconds "	/* there is no "second" in default hl */
+	}
+	
+	if (mins > 59)
+	{
+		new hours = mins / 60
+		
+		num_to_word(hours, temp[0], charsmax(temp[]))
+		
+		if ( ~flags & TD_NO_HOURS_MINS_SECS_VOICE )
+			temp[1] = "hours "
+		
+		mins = mins % 60
+	}
+	
+	if (mins > 0)
+	{
+		num_to_word(mins, temp[2], charsmax(temp[]))
+		
+		if ( ~flags & TD_NO_HOURS_MINS_SECS_VOICE )
+			temp[3] = "minutes "
+	}
+	
+	if ( ~flags & TD_NO_REMAINING_VOICE )
+		temp[6] = "remaining "
+	
+	return formatex(text, len, "spk ^"vox/%s%s%s%s%s%s%s^"", temp[0], temp[1], temp[2], temp[3], temp[4], temp[5], temp[6])
+}
+
+findDispFormat(_time)
+{
+	// it is important to check i<sizeof BEFORE g_TimeSet[i][0] to prevent out of bound error
+	for (new i = 0; i < sizeof(g_TimeSet) && g_TimeSet[i][0]; ++i)
+	{
+		if (g_TimeSet[i][1] & TD_SHOW_SPEAK_VALUES_BELOW)
+		{
+			if (g_TimeSet[i][0] > _time)
+			{
+				if (!g_Switch)
+				{
+					g_CountDown = g_Switch = _time
+					remove_task(TASK_TIMEREMAIN_SHORT)
+					set_task(1.0, "timeRemain", TASK_TIMEREMAIN_LARGE, "", 0, "b")
+				}
+				
+				return i
+			}
+		}
+		else if (g_TimeSet[i][0] == _time)
+		{
+			return i
+		}
+	}
+	
+	return -1
+}
+
+public setDisplaying()
+{
+	new arg[32], flags[32], num[32]
+	new argc = read_argc() - 1
+	new i = 0
+
+	while (i < argc && i < sizeof(g_TimeSet))
+	{
+		read_argv(i + 1, arg, charsmax(arg))
+		parse(arg, flags, charsmax(flags), num, charsmax(num))
+		
+		g_TimeSet[i][0] = str_to_num(num)
+		g_TimeSet[i][1] = read_flags(flags)
+		
+		i++
+	}
+
+	if( i < sizeof(g_TimeSet) )
+		g_TimeSet[i][0] = 0 // has to be zeroed in case command is sent twice
+	
+	return PLUGIN_HANDLED
+}
+
+public timeRemain(param[])
+{
+	new gmtm = get_timeleft()
+	new tmlf = g_Switch ? --g_CountDown : gmtm
+	new stimel[12]
+	
+	formatex(stimel, charsmax(stimel), "%02d:%02d", gmtm / 60, gmtm % 60)
+	set_pcvar_string(g_amx_timeleft, stimel)
+	
+	if (g_Switch && gmtm > g_Switch)
+	{
+		remove_task(TASK_TIMEREMAIN_LARGE)
+		g_Switch = 0
+		set_task(0.8, "timeRemain", TASK_TIMEREMAIN_SHORT, "", 0, "b")
+		
+		return
+	}
+
+	if (tmlf > 0 && g_LastTime != tmlf)
+	{
+		g_LastTime = tmlf
+		new tm_set = findDispFormat(tmlf)
+		
+		if (tm_set != -1)
+		{
+			new flags = g_TimeSet[tm_set][1]
+			new arg[128]
+			
+			if (flags & TD_BOTTOM_WHITE_TEXT)
+			{
+				new players[MAX_PLAYERS], pnum, plr
+				
+				get_players(players, pnum, "c")
+
+				if (flags & TD_SHOW_SPEAK_VALUES_BELOW) // yes this is correct flag, just because message should be shorter if it is shown every seconds
+					set_hudmessage(255, 255, 255, -1.0, 0.85, 0, 0.0, 1.1, 0.1, 0.5, -1)
+				else
+					set_hudmessage(255, 255, 255, -1.0, 0.85, 0, 0.0, 3.0, 0.0, 0.5, -1)
+					
+				for (new i = 0; i < pnum; i++)
+				{
+					plr = players[i]
+					setTimeText(arg, charsmax(arg), tmlf, plr)
+					show_hudmessage(plr, "%s", arg)
+				}
+			}
+
+			if (flags & TD_USE_VOICE)
+			{
+				setTimeVoice(arg, charsmax(arg), flags, tmlf)
+				client_cmd(0, "%s", arg)
+			}
+		}
+	}
+}
--- a/samples/XS/CommonMark.xs
+++ b/samples/XS/CommonMark.xs
@@ -0,0 +1,466 @@
+/*
+ * This software is copyright (C) by Nick Wellnhofer <wellnhofer@aevum.de>.
+ *
+ * This is free software; you can redistribute it and/or modify it under
+ * the same terms as the Perl 5 programming language system itself.
+ *
+ * Terms of the Perl programming language system itself
+ *
+ * a) the GNU General Public License as published by the Free
+ *    Software Foundation; either version 1, or (at your option) any
+ *    later version, or
+ * b) the "Artistic License"
+ */
+
+/*
+ * Notes on memory management
+ *
+ * - A pointer to the Perl SV representing a node is stored in the
+ *   user data slot of `struct cmark_node`, so there's a 1:1 mapping
+ *   between Perl and C objects.
+ * - Every node SV keeps a reference to the parent SV. This is done
+ *   indirectly by looking up the parent SV and increasing its refcount.
+ * - This makes sure that a document isn't freed if the last reference
+ *   from Perl to the root node is dropped, as references to child nodes
+ *   might still exist.
+ * - As a consequence, as long as a node is referenced from Perl, all its
+ *   ancestor nodes will also be associated with a Perl object.
+ */
+
+#define PERL_NO_GET_CONTEXT
+
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+
+#include <stdlib.h>
+#include <cmark.h>
+
+#if CMARK_VERSION < 0x001000
+    #error libcmark 0.16.0 is required.
+#endif
+
+/* Fix prefixes of render functions. */
+#define cmark_node_render_html cmark_render_html
+#define cmark_node_render_xml  cmark_render_xml
+#define cmark_node_render_man  cmark_render_man
+
+static SV*
+S_create_or_incref_node_sv(pTHX_ cmark_node *node) {
+    SV *new_obj = NULL;
+
+    while (node) {
+        SV *obj;
+        HV *stash;
+
+        /* Look for existing object. */
+        obj = (SV*)cmark_node_get_user_data(node);
+
+        if (obj) {
+            /* Incref if found. */
+            SvREFCNT_inc_simple_void_NN(obj);
+            if (!new_obj) {
+                new_obj = obj;
+            }
+            break;
+        }
+
+        /* Create a new SV. */
+        obj = newSViv(PTR2IV(node));
+        cmark_node_set_user_data(node, obj);
+        if (!new_obj) {
+            new_obj = obj;
+        }
+
+        /*
+         * Unfortunately, Perl doesn't offer an API function to bless an SV
+         * without a reference. The following code is mostly copied from
+         * sv_bless.
+         */
+        SvOBJECT_on(obj);
+#if (PERL_VERSION <= 16)
+        PL_sv_objcount++;
+#endif
+        SvUPGRADE(obj, SVt_PVMG);
+        stash = gv_stashpvn("CommonMark::Node", 16, GV_ADD);
+        SvSTASH_set(obj, (HV*)SvREFCNT_inc(stash));
+
+        /* Recurse into parent. */
+        node = cmark_node_parent(node);
+    }
+
+    return new_obj;
+}
+
+static void
+S_decref_node_sv(pTHX_ cmark_node *node) {
+    SV *obj;
+
+    if (!node) {
+        return;
+    }
+
+    obj = (SV*)cmark_node_get_user_data(node);
+    if (!obj) {
+        /* Should never happen. */
+        croak("Internal error: node SV not found");
+    }
+
+    SvREFCNT_dec_NN(obj);
+}
+
+/* Find or create an SV for a cmark_node. */
+static SV*
+S_node2sv(pTHX_ cmark_node *node) {
+    SV *obj;
+
+    if (!node) {
+        return &PL_sv_undef;
+    }
+
+    obj = S_create_or_incref_node_sv(aTHX_ node);
+
+    return newRV_noinc(obj);
+}
+
+/* Transfer refcount from a node to another. */
+static void
+S_transfer_refcount(pTHX_ cmark_node *from, cmark_node *to) {
+    if (from != to) {
+        S_create_or_incref_node_sv(aTHX_ to);
+        S_decref_node_sv(aTHX_ from);
+    }
+}
+
+/* Get C struct pointer from an SV argument. */
+static void*
+S_sv2c(pTHX_ SV *sv, const char *class_name, STRLEN len, CV *cv,
+       const char *var_name) {
+    if (!SvROK(sv) || !sv_derived_from_pvn(sv, class_name, len, 0)) {
+        const char *sub_name = GvNAME(CvGV(cv));
+        croak("%s: %s is not of type %s", sub_name, var_name, class_name);
+    }
+    return INT2PTR(void*, SvIV(SvRV(sv)));
+}
+
+
+MODULE = CommonMark  PACKAGE = CommonMark  PREFIX = cmark_
+
+PROTOTYPES: DISABLE
+
+BOOT:
+    if (cmark_version != CMARK_VERSION) {
+        warn("Compiled against libcmark %s, but runtime version is %s",
+             CMARK_VERSION_STRING, cmark_version_string);
+    }
+
+char*
+cmark_markdown_to_html(package, string)
+    SV *package = NO_INIT
+    SV *string
+PREINIT:
+    STRLEN len;
+    const char *buffer;
+CODE:
+    (void)package;
+    buffer = SvPVutf8(string, len);
+    RETVAL = cmark_markdown_to_html(buffer, len);
+OUTPUT:
+    RETVAL
+
+cmark_node*
+cmark_parse_document(package, string)
+    SV *package = NO_INIT
+    SV *string
+PREINIT:
+    STRLEN len;
+    const char *buffer;
+CODE:
+    (void)package;
+    buffer = SvPVutf8(string, len);
+    RETVAL = cmark_parse_document(buffer, len);
+OUTPUT:
+    RETVAL
+
+cmark_node*
+cmark_parse_file(package, file)
+    SV *package = NO_INIT
+    SV *file
+PREINIT:
+    PerlIO *perl_io;
+    FILE *stream = NULL;
+CODE:
+    (void)package;
+    perl_io = IoIFP(sv_2io(file));
+    if (perl_io) {
+        stream = PerlIO_findFILE(perl_io);
+    }
+    if (!stream) {
+        croak("parse_file: file is not a file handle");
+    }
+    RETVAL = cmark_parse_file(stream);
+OUTPUT:
+    RETVAL
+
+int
+cmark_version(package)
+    SV *package = NO_INIT
+CODE:
+    (void)package;
+    RETVAL = cmark_version;
+OUTPUT:
+    RETVAL
+
+const char*
+cmark_version_string(package)
+    SV *package = NO_INIT
+CODE:
+    (void)package;
+    RETVAL = cmark_version_string;
+OUTPUT:
+    RETVAL
+
+int
+cmark_compile_time_version(package)
+    SV *package = NO_INIT
+CODE:
+    (void)package;
+    RETVAL = CMARK_VERSION;
+OUTPUT:
+    RETVAL
+
+const char*
+cmark_compile_time_version_string(package)
+    SV *package = NO_INIT
+CODE:
+    (void)package;
+    RETVAL = CMARK_VERSION_STRING;
+OUTPUT:
+    RETVAL
+
+
+MODULE = CommonMark  PACKAGE = CommonMark::Node  PREFIX = cmark_node_
+
+cmark_node*
+new(package, type)
+    SV *package = NO_INIT
+    cmark_node_type type
+CODE:
+    (void)package;
+    RETVAL = cmark_node_new(type);
+OUTPUT:
+    RETVAL
+
+void
+DESTROY(cmark_node *node)
+CODE:
+    cmark_node *parent = cmark_node_parent(node);
+    if (parent) {
+        cmark_node_set_user_data(node, NULL);
+        S_decref_node_sv(aTHX_ parent);
+    }
+    else {
+        cmark_node_free(node);
+    }
+
+cmark_iter*
+iterator(cmark_node *node)
+CODE:
+    S_create_or_incref_node_sv(aTHX_ node);
+    RETVAL = cmark_iter_new(node);
+OUTPUT:
+    RETVAL
+
+cmark_node*
+interface_get_node(cmark_node *node)
+INTERFACE:
+    cmark_node_next
+    cmark_node_previous
+    cmark_node_parent
+    cmark_node_first_child
+    cmark_node_last_child
+
+int
+interface_get_int(cmark_node *node)
+INTERFACE:
+    cmark_node_get_type
+    cmark_node_get_header_level
+    cmark_node_get_list_type
+    cmark_node_get_list_delim
+    cmark_node_get_list_start
+    cmark_node_get_list_tight
+    cmark_node_get_start_line
+    cmark_node_get_start_column
+    cmark_node_get_end_line
+    cmark_node_get_end_column
+
+NO_OUTPUT int
+interface_set_int(cmark_node *node, int value)
+INTERFACE:
+    cmark_node_set_header_level
+    cmark_node_set_list_type
+    cmark_node_set_list_delim
+    cmark_node_set_list_start
+    cmark_node_set_list_tight
+POSTCALL:
+    if (!RETVAL) {
+        croak("%s: invalid operation", GvNAME(CvGV(cv)));
+    }
+
+const char*
+interface_get_utf8(cmark_node *node)
+INTERFACE:
+    cmark_node_get_type_string
+    cmark_node_get_literal
+    cmark_node_get_title
+    cmark_node_get_url
+    cmark_node_get_fence_info
+
+NO_OUTPUT int
+interface_set_utf8(cmark_node *node, const char *value)
+INTERFACE:
+    cmark_node_set_literal
+    cmark_node_set_title
+    cmark_node_set_url
+    cmark_node_set_fence_info
+POSTCALL:
+    if (!RETVAL) {
+        croak("%s: invalid operation", GvNAME(CvGV(cv)));
+    }
+
+void
+cmark_node_unlink(cmark_node *node)
+PREINIT:
+    cmark_node *old_parent;
+INIT:
+    old_parent = cmark_node_parent(node);
+POSTCALL:
+    S_decref_node_sv(aTHX_ old_parent);
+
+NO_OUTPUT int
+interface_move_node(cmark_node *node, cmark_node *other)
+PREINIT:
+    cmark_node *old_parent;
+    cmark_node *new_parent;
+INIT:
+    old_parent = cmark_node_parent(other);
+INTERFACE:
+    cmark_node_insert_before
+    cmark_node_insert_after
+    cmark_node_prepend_child
+    cmark_node_append_child
+POSTCALL:
+    if (!RETVAL) {
+        croak("%s: invalid operation", GvNAME(CvGV(cv)));
+    }
+    new_parent = cmark_node_parent(other);
+    S_transfer_refcount(aTHX_ old_parent, new_parent);
+
+char*
+interface_render(cmark_node *root, long options = 0)
+INTERFACE:
+    cmark_node_render_html
+    cmark_node_render_xml
+    cmark_node_render_man
+
+
+MODULE = CommonMark  PACKAGE = CommonMark::Iterator  PREFIX = cmark_iter_
+
+void
+DESTROY(cmark_iter *iter)
+CODE:
+    S_decref_node_sv(aTHX_ cmark_iter_get_node(iter));
+    S_decref_node_sv(aTHX_ cmark_iter_get_root(iter));
+    cmark_iter_free(iter);
+
+void
+cmark_iter_next(cmark_iter *iter)
+PREINIT:
+    I32 gimme;
+    cmark_node *old_node;
+    cmark_event_type ev_type;
+PPCODE:
+    gimme    = GIMME_V;
+    old_node = cmark_iter_get_node(iter);
+    ev_type  = cmark_iter_next(iter);
+
+    if (ev_type != CMARK_EVENT_DONE) {
+        cmark_node *node = cmark_iter_get_node(iter);
+
+        ST(0) = sv_2mortal(newSViv((IV)ev_type));
+
+        if (gimme == G_ARRAY) {
+            SV *obj = S_create_or_incref_node_sv(aTHX_ node);
+
+            /* A bit more efficient than S_transfer_refcount. */
+            if (old_node != node) {
+                S_decref_node_sv(aTHX_ old_node);
+                SvREFCNT_inc_simple_void_NN(obj);
+            }
+
+            ST(1) = sv_2mortal(newRV_noinc(obj));
+            XSRETURN(2);
+        }
+        else {
+            S_transfer_refcount(aTHX_ old_node, node);
+            XSRETURN(1);
+        }
+    }
+    else {
+        S_decref_node_sv(aTHX_ old_node);
+
+        if (gimme == G_ARRAY) {
+            XSRETURN_EMPTY;
+        }
+        else {
+            ST(0) = sv_2mortal(newSViv((IV)ev_type));
+            XSRETURN(1);
+        }
+    }
+
+cmark_node*
+cmark_iter_get_node(cmark_iter *iter)
+
+cmark_event_type
+cmark_iter_get_event_type(cmark_iter *iter)
+
+void
+cmark_iter_reset(iter, node, event_type)
+    cmark_iter *iter
+    cmark_node *node
+    cmark_event_type event_type
+PREINIT:
+    cmark_node *old_node;
+INIT:
+    old_node = cmark_iter_get_node(iter);
+    S_transfer_refcount(aTHX_ old_node, node);
+
+
+MODULE = CommonMark  PACKAGE = CommonMark::Parser  PREFIX = cmark_parser_
+
+cmark_parser*
+cmark_parser_new(package)
+    SV *package = NO_INIT
+CODE:
+    (void)package;
+    RETVAL = cmark_parser_new();
+OUTPUT:
+    RETVAL
+
+void
+DESTROY(cmark_parser *parser)
+CODE:
+    cmark_parser_free(parser);
+
+void
+cmark_parser_feed(cmark_parser *parser, SV *string)
+PREINIT:
+    STRLEN len;
+    const char *buffer;
+CODE:
+    buffer = SvPVutf8(string, len);
+    cmark_parser_feed(parser, buffer, len);
+
+cmark_node*
+cmark_parser_finish(cmark_parser *parser)
+
--- a/samples/eC/Designer.ec
+++ b/samples/eC/Designer.ec
@@ -0,0 +1,337 @@
+import "ide"
+
+class Designer : DesignerBase
+{
+   ~Designer()
+   {
+      if(GetActiveDesigner() == this)
+      {
+         SetActiveDesigner(null);
+      }
+      if(classDesigner)
+         delete classDesigner;
+   }
+
+   // *** DesignerBase Implementation ***
+
+   void ModifyCode()
+   {
+      codeEditor.ModifyCode();
+   }
+
+   void UpdateProperties()
+   {
+      codeEditor.DesignerModifiedObject();
+   }
+
+   void CodeAddObject(Instance instance, ObjectInfo * object)
+   {
+      codeEditor.AddObject(instance, object);
+   }
+
+   void SheetAddObject(ObjectInfo object)
+   {
+      codeEditor.sheet.AddObject(object, object.name, typeData, true); //className, true);
+   }
+
+   void AddToolBoxClass(Class _class)
+   {
+      ((IDEWorkSpace)master).toolBox.AddControl(_class);
+   }
+
+   void AddDefaultMethod(Instance instance, Instance classInstance)
+   {
+      Class _class = instance._class;
+      Method defaultMethod = null;
+
+      for( ; _class; _class = _class.base)
+      {
+         Method method;
+         int minID = MAXINT;
+         for(method = (Method)_class.methods.first; method; method = (Method)((BTNode)method).next)
+         {
+            if(method.type == virtualMethod)
+            {
+               if(!method.dataType)
+                  method.dataType = ProcessTypeString(method.dataTypeString, false);
+               if(method.vid < minID && (instance == classInstance || (method.dataType.thisClass && eClass_IsDerived(classInstance._class, method.dataType.thisClass.registered))))
+               {
+                  defaultMethod = method;
+                  minID = method.vid;
+               }
+            }
+         }
+         if(defaultMethod)
+            break;
+      }
+      codeEditor.AddMethod(defaultMethod);
+   }
+
+   bool ObjectContainsCode(ObjectInfo object)
+   {
+      // Confirmation if control contains code
+      if(object.instCode)
+      {
+         MembersInit members;
+         if(object.instCode.members)
+         {
+            for(members = object.instCode.members->first; members; members = members.next)
+            {
+               if(members.type == methodMembersInit)
+               {
+                  //if(!Code_IsFunctionEmpty(members.function))
+                  {
+                     return true;
+                  }
+               }
+            }
+         }
+      }
+      return false;
+   }
+
+   void DeleteObject(ObjectInfo object)
+   {
+      if(codeEditor)
+         codeEditor.DeleteObject(object);
+   }
+
+   void RenameObject(ObjectInfo object, const char * name)
+   {
+      if(object && (name || !object.classDefinition))
+         codeEditor.RenameObject(object, name);
+   }
+
+   bool FindObject(Instance * object, const char * string)
+   {
+      ObjectInfo classObject;
+      for(classObject = codeEditor.classes.first; classObject; classObject = classObject.next)
+      {
+         ObjectInfo check;
+         if(classObject.name && !strcmp(string, classObject.name))
+         {
+            *object = classObject.instance;
+            break;
+         }
+         for(check = classObject.instances.first; check; check = check.next)
+         {
+            if(check.name && !strcmp(string, check.name))
+            {
+               *object = check.instance;
+               break;
+            }
+         }
+         if(check)
+            return true;
+      }
+      return false;
+   }
+
+   void SelectObjectFromDesigner(ObjectInfo object)
+   {
+      codeEditor.SelectObjectFromDesigner(object);
+   }
+
+   borderStyle = sizable;
+   isActiveClient = true;
+   hasVertScroll = true;
+   hasHorzScroll = true;
+   hasClose = true;
+   hasMaximize = true;
+   hasMinimize = true;
+   text = $"Designer";
+   menu = Menu { };
+   anchor = Anchor { left = 300, right = 150, top = 0, bottom = 0 };
+
+   ToolBox toolBox;
+   CodeEditor codeEditor;
+
+   Menu fileMenu { menu, $"File", f };
+   MenuItem fileSaveItem
+   {
+      fileMenu, $"Save", s, ctrlS;
+      bool NotifySelect(MenuItem selection, Modifiers mods)
+      {
+         return codeEditor.MenuFileSave(selection, mods);
+      }
+   };
+   MenuItem fileSaveAsItem
+   {
+      fileMenu, $"Save As...", a;
+      bool NotifySelect(MenuItem selection, Modifiers mods)
+      {
+         return codeEditor.MenuFileSaveAs(selection, mods);
+      }
+   };
+   bool debugClosing;
+
+   bool OnClose(bool parentClosing)
+   {
+      if(!parentClosing)
+      {
+         if(codeEditor && codeEditor.inUseDebug && !debugClosing)
+         {
+            debugClosing = true;
+            closing = false;
+            if(CloseConfirmation(false))
+            {
+               visible = false;
+               if(modifiedDocument)
+                  OnFileModified({ modified = true }, null);
+            }
+            debugClosing = false;
+            return false;
+         }
+         if(codeEditor && !codeEditor.closing && !debugClosing)
+         {
+            if(!codeEditor.visible)
+            {
+               if(!codeEditor.Destroy(0))
+                  return false;
+               else
+                  codeEditor = null;
+            }
+            else
+            {
+               visible = false;
+               return false;
+            }
+         }
+      }
+      return true;
+   }
+
+   bool OnActivate(bool active, Window previous, bool * goOnWithActivation, bool direct)
+   {
+      if(active)
+      {
+         codeEditor.EnsureUpToDate();
+         codeEditor.fixCaret = true;
+         /*
+         if(classDesigner)
+            classDesigner.Activate();
+         */
+      }
+      return true;
+   }
+
+   bool OnKeyHit(Key key, unichar ch)
+   {
+      return codeEditor.sheet.OnKeyHit(key, ch);
+   }
+
+   watch(modifiedDocument)
+   {
+      fileSaveItem.disabled = !modifiedDocument && codeEditor.fileName;
+   };
+
+   // *** METHODS ACCESSED FROM PROPERTY SHEET/TOOLBOX/CODE EDITOR ***
+   void Reset()
+   {
+      if(classDesigner)
+      {
+         classDesigner.Reset();
+         classDesigner.SelectObject(null, null);
+         classDesigner.Destroy(0);
+         delete classDesigner;
+      }
+   }
+
+   void FillToolBox()
+   {
+      if(this && classDesigner)
+         classDesigner.ListToolBoxClasses(this);
+   }
+
+   void SelectObject(ObjectInfo object, Instance instance)
+   {
+      ClassDesignerBase classDesigner = this.classDesigner;
+#ifdef _DEBUG
+      if(instance && instance._class.module.application != codeEditor.privateModule)
+         printf("warning: SelectObject: instance._class.module.application != codeEditor.privateModule\n");
+#endif
+      if(!classDesigner || !instance || classDesigner._class != (Class)eInstance_GetDesigner(instance))
+      {
+         if(classDesigner)
+         {
+            classDesigner.SelectObject(null, null);
+            classDesigner.Destroy(0);
+            classDesigner = null;
+            delete this.classDesigner;
+         }
+         if(instance)
+         {
+            this.classDesigner = classDesigner = eInstance_New(eInstance_GetDesigner(instance));
+            incref classDesigner;
+            //if(!classDesigner.parent)
+            {
+               classDesigner.parent = this;
+               classDesigner.anchor = Anchor { left = 0, right = 0, top = 0, bottom = 0 };
+            }
+            classDesigner.Create();
+         }
+      }
+      // Call class editor SelectObject
+      if(classDesigner)
+         classDesigner.SelectObject(object, instance);
+   }
+
+   void AddObject()
+   {
+      // Call class editor AddObject
+      if(classDesigner)
+         classDesigner.AddObject();
+      if(visible)
+         Activate();
+      else
+         codeEditor.Activate();
+   }
+
+   void CreateObject(Instance instance, ObjectInfo object, bool isClass, Instance iclass)
+   {
+      subclass(ClassDesignerBase) designerClass = eInstance_GetDesigner(instance);
+
+      // Call class editor CreateObject
+      if(designerClass)
+         designerClass.CreateObject(this, instance, object, isClass, iclass);
+   }
+
+   void ::PostCreateObject(Instance instance, ObjectInfo object, bool isClass, Instance iclass)
+   {
+      subclass(ClassDesignerBase) designerClass = eInstance_GetDesigner(instance);
+
+      // Call class editor PostCreateObject
+      if(designerClass)
+         designerClass.PostCreateObject(instance, object, isClass, iclass);
+   }
+
+   void ::DroppedObject(Instance instance, ObjectInfo object, bool isClass, Instance iclass)
+   {
+      subclass(ClassDesignerBase) designerClass = eInstance_GetDesigner(instance);
+
+      // Call class editor PostCreateObject
+      if(designerClass)
+         designerClass.DroppedObject(instance, object, isClass, iclass);
+   }
+
+   void PrepareTestObject(Instance instance)
+   {
+      subclass(ClassDesignerBase) designerClass = eInstance_GetDesigner(instance);
+      if(designerClass)
+         designerClass.PrepareTestObject(this, instance);
+   }
+
+   void ::DestroyObject(Instance instance)
+   {
+      subclass(ClassDesignerBase) designerClass = eInstance_GetDesigner(instance);
+      if(designerClass)
+         designerClass.DestroyObject(instance);
+   }
+
+   void ::FixProperty(Property prop, Instance instance)
+   {
+      subclass(ClassDesignerBase) designerClass = eInstance_GetDesigner(instance);
+      if(designerClass)
+         designerClass.FixProperty(prop, instance);
+   }
+}
--- a/script/travis/before_install
+++ b/script/travis/before_install
@@ -5,6 +5,8 @@ set -ex
 # Fetch all commits/refs needed to run our tests.
 git fetch origin master:master v2.0.0:v2.0.0 test/attributes:test/attributes test/master:test/master

+sudo apt-get update
+
 script/vendor-deb libicu48 libicu-dev
 if ruby -e 'exit RUBY_VERSION >= "2.0" && RUBY_VERSION < "2.1"'; then
  # Workaround for https://bugs.ruby-lang.org/issues/8074. We can't use this
--- a/test/fixtures/Data/Modelines/iamphp.inc
+++ b/test/fixtures/Data/Modelines/iamphp.inc
@@ -1 +1 @@
-; -*- mode: php;-*-
+; -*- MoDe: PhP;-*-
--- a/test/fixtures/Data/Modelines/not_perl.pl
+++ b/test/fixtures/Data/Modelines/not_perl.pl
@@ -1,3 +1,3 @@
-/* vim: set filetype=prolog: */
+/* vim: set filEtype=pRoloG: */

 # I am Prolog
--- a/test/fixtures/Shell/crossbuild_liblua5.1
+++ b/test/fixtures/Shell/crossbuild_liblua5.1
@@ -0,0 +1,57 @@
+#! /bin/sh
+#
+# Builds and installs liblua5.1 for the cross toolchain.
+# Executed by build-uqm-dependencies.chroot
+
+# Include our common functions
+. /usr/lib/crossbuild/crossbuild.subr
+
+# envvar LIBLUA51_URL
+#
+# Specifies the URL of the liblua5.1 source tarball you want to use.
+export LIBLUA51_URL="http://www.lua.org/ftp/lua-5.1.5.tar.gz"
+
+# envvar INSTALL_TOP
+#
+# This determines where lua's makefiles install everything (we don't want to use
+# /usr/local!).
+export INSTALL_TOP="/usr/${HOST_TRIPLET}"
+
+# envvar TO_BIN
+#
+# Names of the binary files to install (that's right, lua's makefiles don't
+# determine this automatically, and since we end up with files named according
+# to Windows conventions the install chokes without these)
+export TO_BIN="lua.exe luac.exe"
+
+# envvar TO_LIB
+#
+# Names of the libraries to install, see TO_BIN
+export TO_LIB="liblua.a lua51.dll"
+
+
+# liblua5.1 uses custom makefiles and does not natively support cross-building.
+# However, with our cross toolchain in its PATH it successfully builds the mingw
+# target.
+export PATH=/usr/${HOST_TRIPLET}/bin:${PATH}
+
+echo "*************************************************************************"
+echo "--- BEGIN: crossbuild_liblua5.1 ---"
+
+get_tarball "liblua5.1" "${LIBLUA51_URL}" gz
+
+cd ${SRC_ROOT_DIR}/liblua5.1/*
+
+if [ -f Makefile ]; then
+  make clean
+  make --environment-overrides mingw install
+
+
+else
+  echo "crossbuild_liblua5.1 failed: Could not find Makefile"
+  echo "(is the liblua5.1 source tarball sane?)"
+  exit 1
+fi
+
+echo "--- END: crossbuild_liblua5.1 ---"
+echo "*************************************************************************"
--- a/test/fixtures/Shell/graylog2-server.init.d
+++ b/test/fixtures/Shell/graylog2-server.init.d
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+### BEGIN INIT INFO
+# Provides:        graylog2-server
+# Required-Start:  $network
+# Required-Stop:   $network
+# Default-Start:   2 3 4 5
+# Default-Stop:    1
+# Short-Description: Start Graylog2 server
+### END INIT INFO
+
+# Written by Lital Natan <litaln@gmail.com>
+
+PREFIX=/usr
+SHAREDIR=$PREFIX/share/graylog2-server
+SERVER_JAR=$SHAREDIR/graylog2-server.jar
+SYSLOG4J_JAR=$SHAREDIR/syslog4j-0.9.46-bin.jar
+SVCNAME="graylog2-server"
+
+CONFIG="/etc/graylog2.conf"
+LOGFILE="/var/log/graylog2.log"
+PIDFILE="/var/run/graylog2.pid"
+
+start() {
+    if [ ! -e $CONFIG ]; then
+        echo "Config file $CONFIG does not exist"
+        return 1
+    fi
+
+    echo "Starting ${SVCNAME}"
+        nohup `which java` -cp $SERVER_JAR:$SYSLOG4J_JAR org.graylog2.Main \
+      -p ${PIDFILE} -f ${CONFIG} > $LOGFILE 2>&1 &
+
+    # Sleep before testing the service
+    sleep 2
+
+    graylog2_test || return 1
+}
+
+stop() {
+    pid=`< $PIDFILE`
+    kill $pid
+    rm -f ${PIDFILE} # just in case
+}
+
+graylog2_test() {
+    # Graylog2 only deletes its PID file if it hits a config error
+    if [ ! -e ${PIDFILE} ]; then
+        echo "Configuration error, check ${CONFIG}"
+        return 1
+    fi
+
+    local pid=`cat ${PIDFILE}`
+
+    # Graylog2 isn't running, so that means there was a problem
+    if [ ! -e /proc/$pid ]; then
+        echo "Something went wrong, check ${LOGFILE}"
+        rm -f ${PIDFILE}
+        return 1
+    else
+        return 0
+    fi
+}
+
+
+status() {
+    graylog2_test > /dev/null 2>&1
+    if [ "$?" == "0" ]; then
+       echo "Graylog2 server is up"
+       return 0
+    else
+       echo "Graylog2 server is down"
+       return 1
+    fi
+}
+
+
+restart() {
+    stop
+    start
+}
+
+case "$1" in
+    start)
+  start
+  ;;
+    stop)
+  stop
+  ;;
+    status)
+        status
+        ;;
+    restart)
+  restart
+  ;;
+    *)
+  echo "Usage $0 {start|stop|restart|status}"
+  RETVAL=1
+esac
--- a/test/test_blob.rb
+++ b/test/test_blob.rb
@@ -216,6 +216,7 @@ class TestBlob < Minitest::Test
    assert sample_blob("C++/protocol-buffer.pb.cc").generated?
    assert sample_blob("Java/ProtocolBuffer.java").generated?
    assert sample_blob("Python/protocol_buffer_pb2.py").generated?
+    assert sample_blob("Go/api.pb.go").generated?

    # Generated JNI
    assert sample_blob("C/jni_layer.h").generated?
@@ -233,12 +234,15 @@ class TestBlob < Minitest::Test
    assert sample_blob("Zephir/filenames/exception.zep.php").generated?
    assert !sample_blob("Zephir/Router.zep").generated?

-
-    assert Linguist::Generated.generated?("node_modules/grunt/lib/grunt.js", nil)
+    assert sample_blob("node_modules/grunt/lib/grunt.js").generated?

    # Godep saved dependencies
    assert sample_blob("Godeps/Godeps.json").generated?
    assert sample_blob("Godeps/_workspace/src/github.com/kr/s3/sign.go").generated?
+
+    # Cython-generated C/C++
+    assert sample_blob("C/sgd_fast.c").generated?
+    assert sample_blob("C++/wrapper_inner.cpp").generated?
  end

  def test_vendored
@@ -274,6 +278,8 @@ class TestBlob < Minitest::Test
    assert sample_blob("deps/http_parser/http_parser.c").vendored?
    assert sample_blob("deps/v8/src/v8.h").vendored?

+    assert sample_blob("tools/something/else.c").vendored?
+
    # Chart.js
    assert sample_blob("some/vendored/path/Chart.js").vendored?
    assert !sample_blob("some/vendored/path/chart.js").vendored?
@@ -284,6 +290,9 @@ class TestBlob < Minitest::Test
    # Debian packaging
    assert sample_blob("debian/cron.d").vendored?

+    # Erlang
+    assert sample_blob("rebar").vendored?
+
    # Minified JavaScript and CSS
    assert sample_blob("foo.min.js").vendored?
    assert sample_blob("foo.min.css").vendored?
@@ -292,6 +301,9 @@ class TestBlob < Minitest::Test
    assert !sample_blob("foomin.css").vendored?
    assert !sample_blob("foo.min.txt").vendored?

+    #.osx
+    assert sample_blob(".osx").vendored?
+
    # Prototype
    assert !sample_blob("public/javascripts/application.js").vendored?
    assert sample_blob("public/javascripts/prototype.js").vendored?
@@ -299,6 +311,9 @@ class TestBlob < Minitest::Test
    assert sample_blob("public/javascripts/controls.js").vendored?
    assert sample_blob("public/javascripts/dragdrop.js").vendored?

+    # Samples
+    assert sample_blob("Samples/Ruby/foo.rb").vendored?
+
    # jQuery
    assert sample_blob("jquery.js").vendored?
    assert sample_blob("public/javascripts/jquery.js").vendored?
@@ -439,6 +454,54 @@ class TestBlob < Minitest::Test
    assert sample_blob("activator.bat").vendored?
    assert sample_blob("subproject/activator").vendored?
    assert sample_blob("subproject/activator.bat").vendored?
+
+    assert_predicate fixture_blob(".google_apis/bar.jar"), :vendored?
+    assert_predicate fixture_blob("foo/.google_apis/bar.jar"), :vendored?
+  end
+
+  def test_documentation
+    assert_predicate fixture_blob("doc/foo.html"), :documentation?
+    assert_predicate fixture_blob("docs/foo.html"), :documentation?
+    refute_predicate fixture_blob("project/doc/foo.html"), :documentation?
+    refute_predicate fixture_blob("project/docs/foo.html"), :documentation?
+
+    assert_predicate fixture_blob("Documentation/foo.md"), :documentation?
+    assert_predicate fixture_blob("documentation/foo.md"), :documentation?
+    assert_predicate fixture_blob("project/Documentation/foo.md"), :documentation?
+    assert_predicate fixture_blob("project/documentation/foo.md"), :documentation?
+
+    assert_predicate fixture_blob("javadoc/foo.html"), :documentation?
+    assert_predicate fixture_blob("project/javadoc/foo.html"), :documentation?
+
+    assert_predicate fixture_blob("man/foo.html"), :documentation?
+    refute_predicate fixture_blob("project/man/foo.html"), :documentation?
+
+    assert_predicate fixture_blob("README"), :documentation?
+    assert_predicate fixture_blob("README.md"), :documentation?
+    assert_predicate fixture_blob("README.txt"), :documentation?
+    assert_predicate fixture_blob("foo/README"), :documentation?
+
+    assert_predicate fixture_blob("CONTRIBUTING"), :documentation?
+    assert_predicate fixture_blob("CONTRIBUTING.md"), :documentation?
+    assert_predicate fixture_blob("CONTRIBUTING.txt"), :documentation?
+    assert_predicate fixture_blob("foo/CONTRIBUTING"), :documentation?
+
+    assert_predicate fixture_blob("LICENSE"), :documentation?
+    assert_predicate fixture_blob("LICENCE.md"), :documentation?
+    assert_predicate fixture_blob("LICENSE.txt"), :documentation?
+    assert_predicate fixture_blob("foo/LICENSE"), :documentation?
+
+    assert_predicate fixture_blob("COPYING"), :documentation?
+    assert_predicate fixture_blob("COPYING.md"), :documentation?
+    assert_predicate fixture_blob("COPYING.txt"), :documentation?
+    assert_predicate fixture_blob("foo/COPYING"), :documentation?
+
+    assert_predicate fixture_blob("INSTALL"), :documentation?
+    assert_predicate fixture_blob("INSTALL.md"), :documentation?
+    assert_predicate fixture_blob("INSTALL.txt"), :documentation?
+    assert_predicate fixture_blob("foo/INSTALL"), :documentation?
+
+    refute_predicate fixture_blob("foo.md"), :documentation?
  end

  def test_language
@@ -485,4 +548,29 @@ class TestBlob < Minitest::Test
    refute blob.new(" ").empty?
    refute blob.new("nope").empty?
  end
+
+  def test_include_in_language_stats
+    vendored = sample_blob("bower_components/custom/custom.js")
+    assert_predicate vendored, :vendored?
+    refute_predicate vendored, :include_in_language_stats?
+
+    documentation = fixture_blob("README")
+    assert_predicate documentation, :documentation?
+    refute_predicate documentation, :include_in_language_stats?
+
+    generated = sample_blob("CSS/bootstrap.min.css")
+    assert_predicate generated, :generated?
+    refute_predicate generated, :include_in_language_stats?
+
+    data = sample_blob("Ant Build System/filenames/ant.xml")
+    assert_equal :data, data.language.type
+    refute_predicate data, :include_in_language_stats?
+
+    prose = sample_blob("Markdown/tender.md")
+    assert_equal :prose, prose.language.type
+    refute_predicate prose, :include_in_language_stats?
+
+    included = sample_blob("HTML/pages.html")
+    assert_predicate included, :include_in_language_stats?
+  end
 end
--- a/test/test_grammars.rb
+++ b/test/test_grammars.rb
@@ -3,12 +3,18 @@ require_relative "./helper"
 class TestGrammars < Minitest::Test
  ROOT = File.expand_path("../..", __FILE__)

-  # These grammars have no license but have been grandfathered in. New grammars
-  # must have a license that allows redistribution.
-  UNLICENSED_GRAMMARS_WHITELIST = %w[
-    vendor/grammars/Sublime-Lasso
-    vendor/grammars/Sublime-REBOL
-    vendor/grammars/x86-assembly-textmate-bundle
+  LICENSE_WHITELIST = [
+    # This grammar's MIT license is inside a subdirectory.
+    "vendor/grammars/SublimePapyrus",
+
+    # This grammar has a nonstandard but acceptable license.
+    "vendor/grammars/gap-tmbundle",
+
+    # These grammars have no license but have been grandfathered in. New grammars
+    # must have a license that allows redistribution.
+    "vendor/grammars/Sublime-Lasso",
+    "vendor/grammars/Sublime-REBOL",
+    "vendor/grammars/x86-assembly-textmate-bundle"
  ].freeze

  def setup
@@ -77,9 +83,9 @@ class TestGrammars < Minitest::Test

    unlicensed = categories[:unlicensed] || []
    unrecognized = categories[:unrecognized] || []
-    disallowed_unlicensed = unlicensed - UNLICENSED_GRAMMARS_WHITELIST
-    disallowed_unrecognized = unrecognized - UNLICENSED_GRAMMARS_WHITELIST
-    extra_whitelist_entries = UNLICENSED_GRAMMARS_WHITELIST - (unlicensed | unrecognized)
+    disallowed_unlicensed = unlicensed - LICENSE_WHITELIST
+    disallowed_unrecognized = unrecognized - LICENSE_WHITELIST
+    extra_whitelist_entries = LICENSE_WHITELIST - (unlicensed | unrecognized)

    message = ""
    if disallowed_unlicensed.any?
@@ -93,7 +99,7 @@ class TestGrammars < Minitest::Test
    end
    if extra_whitelist_entries.any?
      message << "\n\n" unless message.empty?
-      message << "The following grammar submodules are listed in UNLICENSED_GRAMMARS_WHITELIST but either have a license (yay!)\n"
+      message << "The following grammar submodules are listed in LICENSE_WHITELIST but either have a license (yay!)\n"
      message << "or have been removed from the repository. Please remove them from the whitelist.\n"
      message << extra_whitelist_entries.sort.join("\n")
    end
@@ -131,6 +137,8 @@ class TestGrammars < Minitest::Test
      "unlicense"
    elsif content.include?("http://www.wtfpl.net/txt/copying/")
      "WTFPL"
+    elsif content.include?("zlib") && content.include?("license") && content.include?("2. Altered source versions must be plainly marked as such")
+      "zlib"
    end
  end
 end
--- a/test/test_heuristics.rb
+++ b/test/test_heuristics.rb
@@ -47,8 +47,10 @@ class TestHeuristcs < Minitest::Test
  # Candidate languages = ["Perl", "Prolog"]
  def test_pl_prolog_perl_by_heuristics
    assert_heuristics({
-      "Prolog" => "Prolog/turing.pl",
-      "Perl" => "Perl/perl-test.t",
+      "Prolog" => all_fixtures("Prolog/*.pl"),
+      "Perl" => all_fixtures("Perl/*.pl"),
+      "Perl" => ["Perl/perl-test.t"],
+      "Perl6" => all_fixtures("Perl6/*.pl")
    })
  end

@@ -60,19 +62,22 @@ class TestHeuristcs < Minitest::Test
    })
  end

-  # Candidate languages = ["IDL", "Prolog"]
-  def test_pro_prolog_idl_by_heuristics
+  # Candidate languages = ["IDL", "Prolog", "QMake", "INI"]
+  def test_pro_by_heuristics
    assert_heuristics({
-      "Prolog" => "Prolog/logic-problem.pro",
-      "IDL" => "IDL/mg_acosh.pro"
+      "Prolog" => all_fixtures("Prolog", "*.pro"),
+      "IDL" => all_fixtures("IDL", "*.pro"),
+      "INI" => all_fixtures("INI", "*.pro"),
+      "QMake" => all_fixtures("QMake", "*.pro")
    })
  end

-  # Candidate languages = ["AGS Script", "AsciiDoc"]
-  def test_asc_asciidoc_by_heuristics
+  # Candidate languages = ["AGS Script", "AsciiDoc", "Public Key"]
+  def test_asc_by_heuristics
    assert_heuristics({
      "AsciiDoc" => "AsciiDoc/list.asc",
-      "AGS Script" => nil
+      "AGS Script" => "AGS Script/GlobalScript.asc",
+      "Public Key" => "Public Key/sunCert.asc"
    })
  end

--- a/test/test_instrumentation.rb
+++ b/test/test_instrumentation.rb
@@ -0,0 +1,50 @@
+require_relative "./helper"
+
+class TestInstrumentation < Minitest::Test
+  include Linguist
+
+  class LocalInstrumenter
+    Event = Struct.new(:name, :args)
+
+    attr_reader :events
+
+    def initialize
+      @events = []
+    end
+
+    def instrument(name, *args)
+      @events << Event.new(name, args)
+      yield if block_given?
+    end
+  end
+
+  def setup
+    Linguist.instrumenter = LocalInstrumenter.new
+  end
+
+  def teardown
+    Linguist.instrumenter = nil
+  end
+
+  def test_detection_instrumentation_with_binary_blob
+    binary_blob = fixture_blob("Binary/octocat.ai")
+    Language.detect(binary_blob)
+
+    # Shouldn't instrument this (as it's binary)
+    assert_equal 0, Linguist.instrumenter.events.size
+  end
+
+  def test_modeline_instrumentation
+    blob = fixture_blob("Data/Modelines/ruby")
+    Language.detect(blob)
+
+    detect_event = Linguist.instrumenter.events.last
+    detect_event_payload = detect_event[:args].first
+
+    assert_equal 3, Linguist.instrumenter.events.size
+    assert_equal "linguist.detected", detect_event.name
+    assert_equal Language['Ruby'], detect_event_payload[:language]
+    assert_equal blob, detect_event_payload[:blob]
+    assert_equal Linguist::Strategy::Modeline, detect_event_payload[:strategy]
+  end
+end
--- a/test/test_language.rb
+++ b/test/test_language.rb
@@ -155,10 +155,6 @@ class TestLanguage < Minitest::Test
    assert_equal :prose, Language['Org'].type
  end

-  def test_other
-    assert_nil Language['Brainfuck'].type
-  end
-
  def test_searchable
    assert Language['Ruby'].searchable?
    assert !Language['Gettext Catalog'].searchable?
@@ -192,7 +188,7 @@ class TestLanguage < Minitest::Test
    assert_equal [], Language.find_by_extension('foo.rb')
    assert_equal [Language['Ruby']], Language.find_by_extension('rb')
    assert_equal [Language['Ruby']], Language.find_by_extension('.rb')
-    assert_equal [Language['M'], Language['Mathematica'], Language['Matlab'], Language['Mercury'], Language['Objective-C']], Language.find_by_extension('.m')
+    assert_equal [Language['M'], Language['MUF'], Language['Mathematica'], Language['Matlab'], Language['Mercury'], Language['Objective-C']], Language.find_by_extension('.m')
  end

  def test_find_all_by_extension
@@ -358,6 +354,15 @@ class TestLanguage < Minitest::Test
    assert missing.empty?, message
  end

+  def test_all_languages_have_type
+    missing = Language.all.select { |language| language.type.nil? }
+    message = "The following languages do not have a type listed in grammars.yml. Please add types for all new languages.\n"
+
+    width = missing.map { |language| language.name.length }.max
+    message << missing.map { |language| sprintf("%-#{width}s", language.name) }.sort.join("\n")
+    assert missing.empty?, message
+  end
+
  def test_all_languages_have_a_valid_ace_mode
    ace_fixture_path = File.join('test', 'fixtures', 'ace_modes.json')
    skip("No ace_modes.json file") unless File.exist?(ace_fixture_path)
--- a/test/test_pedantic.rb
+++ b/test/test_pedantic.rb
@@ -12,7 +12,7 @@ class TestPedantic < Minitest::Test
  def test_extensions_are_sorted
    LANGUAGES.each do |name, language|
      extensions = language['extensions']
-      assert_sorted extensions[1..-1] if extensions && extensions.size > 1
+      assert_sorted extensions[1..-1].map(&:downcase) if extensions && extensions.size > 1
    end
  end

--- a/test/test_repository.rb
+++ b/test/test_repository.rb
@@ -99,4 +99,16 @@ class TestRepository < Minitest::Test
    # overridden .gitattributes
    assert !override_unvendored.vendored?
  end
+
+  def test_linguist_override_documentation?
+    attr_commit = "d4c8fb8a28e91f97a7e53428a365c0abbac36d3d"
+    repo = linguist_repo(attr_commit).read_index
+
+    readme = Linguist::LazyBlob.new(rugged_repository, attr_commit, "README.md")
+    arduino = Linguist::LazyBlob.new(rugged_repository, attr_commit, "samples/Arduino/hello.ino")
+
+    # overridden by .gitattributes
+    refute_predicate readme, :documentation?
+    assert_predicate arduino, :documentation?
+  end
 end
--- a/test/test_samples.rb
+++ b/test/test_samples.rb
@@ -43,7 +43,7 @@ class TestSamples < Minitest::Test
      if extnames = Samples.cache['extnames'][name]
        extnames.each do |extname|
          next if extname == '.script!'
-          assert options['extensions'].index { |x| x.end_with? extname }, "#{name} has a sample with extension (#{extname}) that isn't explicitly defined in languages.yml"
+          assert options['extensions'].index { |x| x.downcase.end_with? extname.downcase }, "#{name} has a sample with extension (#{extname.downcase}) that isn't explicitly defined in languages.yml"
        end
      end

@@ -67,7 +67,7 @@ class TestSamples < Minitest::Test
        if language_matches.length > 1
          language_matches.each do |match|
            samples = "samples/#{match.name}/*#{extension}"
-            assert Dir.glob(samples).any?, "Missing samples in #{samples.inspect}. See https://github.com/github/linguist/blob/master/CONTRIBUTING.md"
+            assert Dir.glob(samples, File::FNM_CASEFOLD).any?, "Missing samples in #{samples.inspect}. See https://github.com/github/linguist/blob/master/CONTRIBUTING.md"
          end
        end
      end
--- a/vendor/grammars/BrightScript.tmbundle
+++ b/vendor/grammars/BrightScript.tmbundle
--- a/vendor/grammars/CLIPS-sublime
+++ b/vendor/grammars/CLIPS-sublime
--- a/vendor/grammars/G-Code
+++ b/vendor/grammars/G-Code
--- a/vendor/grammars/GDScript-sublime
+++ b/vendor/grammars/GDScript-sublime
--- a/vendor/grammars/InnoSetup
+++ b/vendor/grammars/InnoSetup
--- a/vendor/grammars/Lean.tmbundle
+++ b/vendor/grammars/Lean.tmbundle
--- a/vendor/grammars/Modelica
+++ b/vendor/grammars/Modelica
--- a/vendor/grammars/NimLime
+++ b/vendor/grammars/NimLime
--- a/vendor/grammars/PHP-Twig.tmbundle
+++ b/vendor/grammars/PHP-Twig.tmbundle
--- a/vendor/grammars/PogoScript.tmbundle
+++ b/vendor/grammars/PogoScript.tmbundle
--- a/vendor/grammars/Stylus
+++ b/vendor/grammars/Stylus
--- a/vendor/grammars/Sublime-Logos
+++ b/vendor/grammars/Sublime-Logos
--- a/vendor/grammars/Sublime-SQF-Language
+++ b/vendor/grammars/Sublime-SQF-Language
--- a/Show More
+++ b/Show More