Merge branch 'master' into ascii

.gitmodules

@@ -249,7 +249,7 @@
 	url = https://github.com/shellderp/sublime-robot-plugin
 [submodule "vendor/grammars/actionscript3-tmbundle"]
 	path = vendor/grammars/actionscript3-tmbundle
-	url = https://github.com/simongregory/actionscript3-tmbundle
+	url = https://github.com/honzabrecka/actionscript3-tmbundle
 [submodule "vendor/grammars/Sublime-QML"]
 	path = vendor/grammars/Sublime-QML
 	url = https://github.com/skozlovf/Sublime-QML
@@ -552,9 +552,6 @@
 [submodule "vendor/grammars/liquid.tmbundle"]
 	path = vendor/grammars/liquid.tmbundle
 	url = https://github.com/bastilian/validcode-textmate-bundles
-[submodule "vendor/grammars/AutoHotkey"]
-	path = vendor/grammars/AutoHotkey
-	url = https://github.com/ahkscript/AutoHotkey
 [submodule "vendor/grammars/ats.sublime"]
 	path = vendor/grammars/ats.sublime
 	url = https://github.com/steinwaywhw/ats-mode-sublimetext
@@ -579,3 +576,42 @@
 [submodule "vendor/grammars/JSyntax"]
 	path = vendor/grammars/JSyntax
 	url = https://github.com/bcj/JSyntax
+[submodule "vendor/grammars/TXL"]
+	path = vendor/grammars/TXL
+	url = https://github.com/MikeHoffert/Sublime-Text-TXL-syntax
+[submodule "vendor/grammars/G-Code"]
+	path = vendor/grammars/G-Code
+	url = https://github.com/robotmaster/sublime-text-syntax-highlighting
+[submodule "vendor/grammars/grace-tmbundle"]
+	path = vendor/grammars/grace-tmbundle
+	url = https://github.com/zmthy/grace-tmbundle
+[submodule "vendor/grammars/sublime-text-ox"]
+	path = vendor/grammars/sublime-text-ox
+	url = https://github.com/andreashetland/sublime-text-ox
+[submodule "vendor/grammars/AutoHotkey"]
+	path = vendor/grammars/AutoHotkey
+	url = https://github.com/ahkscript/SublimeAutoHotkey
+[submodule "vendor/grammars/ec.tmbundle"]
+	path = vendor/grammars/ec.tmbundle
+	url = https://github.com/ecere/ec.tmbundle
+[submodule "vendor/grammars/InnoSetup"]
+	path = vendor/grammars/InnoSetup
+	url = https://github.com/idleberg/InnoSetup-Sublime-Text
+[submodule "vendor/grammars/gap-tmbundle"]
+	path = vendor/grammars/gap-tmbundle
+	url = https://github.com/dhowden/gap-tmbundle
+[submodule "vendor/grammars/SublimePapyrus"]
+	path = vendor/grammars/SublimePapyrus
+	url = https://github.com/Kapiainen/SublimePapyrus
+[submodule "vendor/grammars/sublime-spintools"]
+	path = vendor/grammars/sublime-spintools
+	url = https://github.com/bitbased/sublime-spintools
+[submodule "vendor/grammars/PogoScript.tmbundle"]
+	path = vendor/grammars/PogoScript.tmbundle
+	url = https://github.com/featurist/PogoScript.tmbundle
+[submodule "vendor/grammars/sublime-opal"]
+	path = vendor/grammars/sublime-opal
+	url = https://github.com/artifactz/sublime-opal
+[submodule "vendor/grammars/mediawiki.tmbundle"]
+	path = vendor/grammars/mediawiki.tmbundle
+	url = https://github.com/textmate/mediawiki.tmbundle

CONTRIBUTING.md

@@ -1,41 +1,81 @@
-## Contributing
+# Contributing
 
-The majority of contributions won't need to touch any Ruby code at all. The [master language list][languages] is just a YAML configuration file.
+Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great. The majority of contributions won't need to touch any Ruby code at all.
 
-Almost all bug fixes or new language additions should come with some additional code samples. Just drop them under [`samples/`][samples] in the correct subdirectory and our test suite will automatically test them. In most cases you shouldn't need to add any new assertions.
+## Adding a language
 
-### My code is detected as the wrong language
+We try only to add languages once they have some usage on GitHub. In most cases we prefer that languages be in use in hundreds of repositories before supporting them in Linguist.
 
-This can usually be solved either by adding a new filename or file name extension to the language's entry in [`languages.yml`][languages] or adding more [samples][samples] for your language to the repository to make Linguist's classifier smarter.
-
-### Syntax highlighting looks wrong
-
-Assuming your code is being detected as the right language (see above), in most cases this is due to a bug in the language grammar rather than a bug in Linguist. [`grammars.yml`][grammars] lists all the grammars we use for syntax highlighting on github.com. Find the one corresponding to your code's programming language and submit a bug report upstream. If you can, try to reproduce the highlighting problem in the text editor that the grammar is designed for (TextMate, Sublime Text, or Atom) and include that information in your bug report.
-
-You can also try to fix the bug yourself and submit a Pull Request. [This piece from TextMate's documentation](http://manual.macromates.com/en/language_grammars) offers a good introduction on how to work with TextMate-compatible grammars. You can test grammars using [Lightshow](https://lightshow.githubapp.com).
-
-Once the bug has been fixed upstream, please let us know and we'll pick it up for GitHub.
-
-### I want to add support for the `X` programming language
-
-Great! You'll need to:
+To add support for a new language:
 
 0. Add an entry for your language to [`languages.yml`][languages].
 0. Add a grammar for your language. Please only add grammars that have a license that permits redistribution.
-    0. Add your grammar as a submodule: `git submodule add https://github.com/JaneSmith/MyGrammar vendor/grammars/MyGrammar`.
-    0. Add your grammar to [`grammars.yml`][grammars] by running `script/convert-grammars --add vendor/grammars/MyGrammar`.
-0. Add samples for your language to the [samples directory][samples].
+  0. Add your grammar as a submodule: `git submodule add https://github.com/JaneSmith/MyGrammar vendor/grammars/MyGrammar`.
+  0. Add your grammar to [`grammars.yml`][grammars] by running `script/convert-grammars --add vendor/grammars/MyGrammar`.
+0. Add samples for your language to the [samples directory][samples] in the correct subdirectory.
+0. Open a pull request, linking to a [GitHub search result](https://github.com/search?utf8=%E2%9C%93&q=extension%3Aboot+NOT+nothack&type=Code&ref=searchresults) showing in-the-wild usage.
 
 In addition, if your new language defines an extension that's already listed in [`languages.yml`][languages] (such as `.foo`) then sometimes a few more steps will need to be taken:
 
-0. Make sure that example `.foo` files are present in the [samples directory][samples] for each language that uses `.foo`. 
+0. Make sure that example `.foo` files are present in the [samples directory][samples] for each language that uses `.foo`.
 0. Test the performance of the Bayesian classifier with a relatively large number (1000s) of sample `.foo` files. (ping @arfon or @bkeepers to help with this) to ensure we're not misclassifying files.
 0. If the Bayesian classifier does a bad job with the sample `.foo` files then a [heuristic](https://github.com/github/linguist/blob/master/lib/linguist/heuristics.rb) may need to be written to help.
 
 Remember, the goal here is to try and avoid false positives!
 
-We try only to add languages once they have some usage on GitHub, so please note in-the-wild usage examples in your pull request. In most cases we prefer that languages already be in use in hundreds of repositories before supporting them in Linguist.
+## Fixing a misclassified language
+
+Most languages are detected by their file extension defined in [languages.yml][languages].  For disambiguating between files with common extensions, linguist applies some [heuristics](/lib/linguist/heuristics.rb) and a [statistical classifier](lib/linguist/classifier.rb). This process can help differentiate between, for example, `.h` files which could be either C, C++, or Obj-C.
+
+Misclassifications can often be solved by either adding a new filename or extension for the language or adding more [samples][samples] to make the classifier smarter.
+
+## Fixing syntax highlighting
+
+Syntax highlighting in GitHub is performed using TextMate-compatible grammars. These are the same grammars that TextMate, Sublime Text and Atom use. Every language in [languages.yml][languages] is mapped to its corresponding TM `scope`. This scope will be used when picking up a grammar for highlighting.
+
+Assuming your code is being detected as the right language, in most cases this is due to a bug in the language grammar rather than a bug in Linguist. [`grammars.yml`][grammars] lists all the grammars we use for syntax highlighting on github.com. Find the one corresponding to your code's programming language and submit a bug report upstream. If you can, try to reproduce the highlighting problem in the text editor that the grammar is designed for (TextMate, Sublime Text, or Atom) and include that information in your bug report.
+
+You can also try to fix the bug yourself and submit a Pull Request. [TextMate's documentation](http://manual.macromates.com/en/language_grammars) offers a good introduction on how to work with TextMate-compatible grammars. You can test grammars using [Lightshow](https://github-lightshow.herokuapp.com).
+
+Once the bug has been fixed upstream, please let us know and we'll pick it up for GitHub.
+
+## Testing
+
+For development you are going to want to checkout out the source. To get it, clone the repo and run [Bundler](http://gembundler.com/) to install its dependencies.
+
+    git clone https://github.com/github/linguist.git
+    cd linguist/
+    script/bootstrap
+
+To run the tests:
+
+    bundle exec rake test
+
+Sometimes getting the tests running can be too much work, especially if you don't have much Ruby experience. It's okay: be lazy and let our build bot [Travis](http://travis-ci.org/#!/github/linguist) run the tests for you. Just open a pull request and the bot will start cranking away.
+
+Here's our current build status: [![Build Status](https://secure.travis-ci.org/github/linguist.png?branch=master)](http://travis-ci.org/github/linguist)
+
+## Releasing
+
+If you are the current maintainer of this gem:
+
+0. Create a branch for the release: `git checkout -b cut-release-vxx.xx.xx`
+0. Make sure your local dependencies are up to date: `script/bootstrap`
+0. If grammar submodules have not been updated recently, update them: `git submodule update --remote && git commit -a`
+0. Ensure that samples are updated: `bundle exec rake samples`
+0. Ensure that tests are green: `bundle exec rake test`
+0. Bump gem version in `lib/linguist/version.rb`, [like this](https://github.com/github/linguist/commit/8d2ea90a5ba3b2fe6e1508b7155aa4632eea2985).
+0. Make a PR to github/linguist, [like this](https://github.com/github/linguist/pull/1238).
+0. Build a local gem: `bundle exec rake build_gem`
+0. Test the gem:
+  0. Bump the Gemfile and Gemfile.lock versions for an app which relies on this gem
+  0. Install the new gem locally
+  0. Test behavior locally, branch deploy, whatever needs to happen
+0. Merge github/linguist PR
+0. Tag and push: `git tag vx.xx.xx; git push --tags`
+0. Push to rubygems.org -- `gem push github-linguist-3.0.0.gem`
 
 [grammars]: /grammars.yml
 [languages]: /lib/linguist/languages.yml
 [samples]: /samples
+[new-issue]: https://github.com/github/linguist/issues/new

README.md

@@ -1,45 +1,80 @@
 # Linguist
 
-We use this library at GitHub to detect blob languages, ignore binary files, suppress generated files in diffs, and generate language breakdown graphs.
+[issues]: https://github.com/github/linguist/issues
+[new-issue]: https://github.com/github/linguist/issues/new
 
-Tips for filing issues and creating pull requests can be found in [`CONTRIBUTING.md`](/CONTRIBUTING.md).
+This library is used on GitHub.com to detect blob languages, ignore binary or vendored files, suppress generated files in diffs, and generate language breakdown graphs.
 
-## Features
+See [Troubleshooting](#troubleshooting) and [`CONTRIBUTING.md`](/CONTRIBUTING.md) before filing an issue or creating a pull request.
 
-### Language detection
+## Troubleshooting
 
-Linguist defines a list of all languages known to GitHub in a [yaml file](https://github.com/github/linguist/blob/master/lib/linguist/languages.yml).
+### My repository is detected as the wrong language
 
-Most languages are detected by their file extension. For disambiguating between files with common extensions, we first apply some common-sense heuristics to pick out obvious languages. After that, we use a
-[statistical
-classifier](https://github.com/github/linguist/blob/master/lib/linguist/classifier.rb).
-This process can help us tell the difference between, for example, `.h` files which could be either C, C++, or Obj-C.
+![language stats bar](https://cloud.githubusercontent.com/assets/173/5562290/48e24654-8ddf-11e4-8fe7-735b0ce3a0d3.png)
 
-```ruby
+The Language stats bar is built by aggregating the languages of each file in that repository. If it is reporting a language that you don't expect:
 
-Linguist::FileBlob.new("lib/linguist.rb").language.name #=> "Ruby"
+0. Click on the name of the language in the stats bar to see a list of the files that are identified as that language.
+0. If you see files that you didn't write, consider moving the files into one of the [paths for vendored  code](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml), or use the [manual overrides](#overrides) feature to ignore them.
+0. If the files are being misclassified, search for [open issues][issues] to see if anyone else has already reported the issue. Any information you an add, especially links to public repositories, is helpful.
+0. If there are no reported issues of this misclassification, [open an issue][new-issue] and include a link to the repository or a sample of the code that is being misclassified.
 
-Linguist::FileBlob.new("bin/linguist").language.name #=> "Ruby"
+## Overrides
+
+Linguist supports a number of different custom overrides strategies for language definitions and vendored paths.
+
+### Using gitattributes
+
+Add a `.gitattributes` file to your project and use standard git-style path matchers for the files you want to override to set `linguist-documentation`, `linguist-language`, and `linguist-vendored`.
+
+```
+$ cat .gitattributes
+*.rb linguist-language=Java
 ```
 
-See [lib/linguist/language.rb](https://github.com/github/linguist/blob/master/lib/linguist/language.rb) and [lib/linguist/languages.yml](https://github.com/github/linguist/blob/master/lib/linguist/languages.yml).
+Checking code you didn't write, such as JavaScript libraries, into your git repo is a common practice, but this often inflates your project's language stats and may even cause your project to be labeled as another language. By default, Linguist treats all of the paths defined in [lib/linguist/vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml) as vendored and therefore doesn't include them in the language statistics for a repository. Vendored files are also hidden by default in diffs on github.com.
 
-### Syntax Highlighting
+Use the `linguist-vendored` attribute to vendor or un-vendor paths.
 
-Syntax highlighting in GitHub is performed using TextMate-compatible grammars. These are the same grammars that TextMate, Sublime Text and Atom use.
+```
+$ cat .gitattributes
+special-vendored-path/* linguist-vendored
+jquery.js linguist-vendored=false
+```
 
-Every language in `languages.yml` is mapped to its corresponding TM `scope`. This scope will be used when picking up a grammar for highlighting. **When adding a new language to Linguist, please add its corresponding scope too (assuming there's an existing TextMate bundle, Sublime Text package, or Atom package) so syntax highlighting works for it**.
+Similar to vendored files, Linguist excludes documentation files from your project's language stats. (Unlike vendored files, documentation files are displayed in diffs on github.com.) [lib/linguist/documentation.yml](lib/linguist/documentation.yml) lists common documentation paths and excludes them from the language statistics for your repository.
 
-### Stats
+Use the `linguist-documentation` attribute to mark or unmark paths as documentation.
 
-The Language stats bar that you see on every repository is built by aggregating the languages of each file in that repository. The top language in the graph determines the project's primary language.
+```
+$ cat .gitattributes
+project-docs/* linguist-documentation
+docs/formatter.rb linguist-documentation=false
+```
 
-The repository stats API, accessed through `#languages`, can be used on a directory:
+### Using Emacs and Vim modelines
 
-***API UPDATE***
+Alternatively, you can use Vim and Emacs style modelines to set the language for a single file. Modelines can be placed anywhere within a file and are respected when determining how to syntax-highlight a file on GitHub.com
 
-Since [Version 3.0.0](https://github.com/github/linguist/releases/tag/v3.0.0) Linguist expects a git repository (in the form of a [Rugged::Repository](https://github.com/libgit2/rugged#repositories)) to be passed when initializing `Linguist::Repository`.
+```
+Vim
+vim: set filetype=prolog:
+vim: set ft=cpp:
 
+Emacs
+-*- mode: php;-*-
+```
+
+## Usage
+
+Install the gem:
+
+```
+$ gem install github-linguist
+```
+
+Then use it in your application:
 
 ```ruby
 require 'rugged'
@@ -51,147 +86,27 @@ project.language       #=> "Ruby"
 project.languages      #=> { "Ruby" => 119387 }
 ```
 
-These stats are also printed out by the `linguist` binary. You can use the
+These stats are also printed out by the `linguist` executable. You can use the
 `--breakdown` flag, and the binary will also output the breakdown of files by language.
 
 You can try running `linguist` on the root directory in this repository itself:
 
-    $ bundle exec linguist --breakdown
+```
+$ bundle exec linguist --breakdown
 
-    100.00% Ruby
+100.00% Ruby
 
-    Ruby:
-    Gemfile
-    Rakefile
-    bin/linguist
-    github-linguist.gemspec
-    lib/linguist.rb
-    lib/linguist/blob_helper.rb
-    lib/linguist/classifier.rb
-    lib/linguist/file_blob.rb
-    lib/linguist/generated.rb
-    lib/linguist/heuristics.rb
-    lib/linguist/language.rb
-    lib/linguist/lazy_blob.rb
-    lib/linguist/md5.rb
-    lib/linguist/repository.rb
-    lib/linguist/samples.rb
-    lib/linguist/tokenizer.rb
-    lib/linguist/version.rb
-    test/test_blob.rb
-    test/test_classifier.rb
-    test/test_heuristics.rb
-    test/test_language.rb
-    test/test_md5.rb
-    test/test_pedantic.rb
-    test/test_repository.rb
-    test/test_samples.rb
-    test/test_tokenizer.rb
-
-#### Ignore vendored files
-
-Checking other code into your git repo is a common practice. But this often inflates your project's language stats and may even cause your project to be labeled as another language. We are able to identify some of these files and directories and exclude them.
-
-```ruby
-Linguist::FileBlob.new("vendor/plugins/foo.rb").vendored? # => true
+Ruby:
+Gemfile
+Rakefile
+bin/linguist
+github-linguist.gemspec
+lib/linguist.rb
+…
 ```
 
-See [Linguist::BlobHelper#vendored?](https://github.com/github/linguist/blob/master/lib/linguist/blob_helper.rb) and [lib/linguist/vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml).
+## Contributing
 
-#### Generated file detection
+Please check out our [contributing guidelines](CONTRIBUTING.md).
 
-Not all plain text files are true source files. Generated files like minified js and compiled CoffeeScript can be detected and excluded from language stats. As an extra bonus, these files are suppressed in diffs.
-
-```ruby
-Linguist::FileBlob.new("underscore.min.js").generated? # => true
-```
-
-See [Linguist::Generated#generated?](https://github.com/github/linguist/blob/master/lib/linguist/generated.rb).
-
-## Overrides
-
-Linguist supports custom overrides for language definitions and vendored paths. Add a `.gitattributes` file to your project using the keys `linguist-language` and `linguist-vendored` with the standard git-style path matchers for the files you want to override.
-
-Please note that the overrides currently only affect the language statistics for a repository and not the syntax-highlighting of files.
-
-```
-$ cat .gitattributes
-*.rb linguist-language=Java
-
-$ linguist --breakdown
-100.00% Java
-
-Java:
-ruby_file.rb
-```
-
-By default, Linguist treats all of the paths defined in [lib/linguist/vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml) as vendored and therefore doesn't include them in the language statistics for a repository. Use the `linguist-vendored` attribute to vendor or un-vendor paths.
-
-```
-$ cat .gitattributes
-special-vendored-path/* linguist-vendored
-jquery.js linguist-vendored=false
-```
-
-## Installation
-
-Github.com is usually running the latest version of the `github-linguist` gem that is released on [RubyGems.org](http://rubygems.org/gems/github-linguist).
-
-But for development you are going to want to checkout out the source. To get it, clone the repo and run [Bundler](http://gembundler.com/) to install its dependencies.
-
-    git clone https://github.com/github/linguist.git
-    cd linguist/
-    script/bootstrap
-
-To run the tests:
-
-    bundle exec rake test
-
-### A note on language extensions
-
-Linguist has a number of methods available to it for identifying the language of a particular file. The initial lookup is based upon the extension of the file, possible file extensions are defined in an array called `extensions`. Take a look at this example for example for `Perl`:
-
-```
-Perl:
-  type: programming
-  ace_mode: perl
-  color: "#0298c3"
-  extensions:
-  - .pl
-  - .PL
-  - .perl
-  - .ph
-  - .plx
-  - .pm
-  - .pod
-  - .psgi
-  interpreters:
-  - perl
-```
-Any of the extensions defined are valid but the first in this array should be the most popular.
-
-### Testing
-
-Sometimes getting the tests running can be too much work, especially if you don't have much Ruby experience. It's okay: be lazy and let our build bot [Travis](http://travis-ci.org/#!/github/linguist) run the tests for you. Just open a pull request and the bot will start cranking away.
-
-Here's our current build status, which is hopefully green: [![Build Status](https://secure.travis-ci.org/github/linguist.png?branch=master)](http://travis-ci.org/github/linguist)
-
-### Releasing
-
-If you are the current maintainer of this gem:
-
- 0. Create a branch for the release: `git checkout -b cut-release-vxx.xx.xx`
- 0. Make sure your local dependencies are up to date: `script/bootstrap`
- 0. If grammar submodules have not been updated recently, update them: `git submodule update --remote && git commit -a`
- 0. Ensure that samples are updated: `bundle exec rake samples`
- 0. Ensure that tests are green: `bundle exec rake test`
- 0. Bump gem version in `lib/linguist/version.rb`.  For example, [like this](https://github.com/github/linguist/commit/8d2ea90a5ba3b2fe6e1508b7155aa4632eea2985).
- 0. Make a PR to github/linguist.  For example, [#1238](https://github.com/github/linguist/pull/1238).
- 0. Build a local gem: `bundle exec rake build_gem`
- 0. Testing:
-   0. Bump the Gemfile and Gemfile.lock versions for an app which relies on this gem
-   0. Install the new gem locally
-   0. Test behavior locally, branch deploy, whatever needs to happen
- 0. Merge github/linguist PR
- 0. Tag and push: `git tag vx.xx.xx; git push --tags`
- 0. Push to rubygems.org -- `gem push github-linguist-3.0.0.gem`
+##

grammars.yml

@@ -24,7 +24,7 @@ vendor/grammars/Agda.tmbundle:
 - source.agda
 vendor/grammars/Alloy.tmbundle:
 - source.alloy
-vendor/grammars/AutoHotkey:
+vendor/grammars/AutoHotkey/:
 - source.ahk
 vendor/grammars/CLIPS-sublime:
 - source.clips
@@ -39,12 +39,20 @@ vendor/grammars/Docker.tmbundle:
 - source.dockerfile
 vendor/grammars/Elm.tmLanguage:
 - source.elm
+vendor/grammars/G-Code/:
+- source.LS
+- source.MCPOST
+- source.MOD
+- source.apt
+- source.gcode
 vendor/grammars/GDScript-sublime/:
 - source.gdscript
 vendor/grammars/Handlebars:
 - text.html.handlebars
 vendor/grammars/IDL-Syntax:
 - source.webidl
+vendor/grammars/InnoSetup/:
+- source.inno
 vendor/grammars/Isabelle.tmbundle:
 - source.isabelle.root
 - source.isabelle.theory
@@ -64,6 +72,8 @@ vendor/grammars/NimLime:
 - source.nimcfg
 vendor/grammars/PHP-Twig.tmbundle:
 - text.html.twig
+vendor/grammars/PogoScript.tmbundle/:
+- source.pogoscript
 vendor/grammars/RDoc.tmbundle:
 - text.rdoc
 vendor/grammars/Racket:
@@ -104,8 +114,14 @@ vendor/grammars/Sublime-VimL:
 - source.viml
 vendor/grammars/SublimeBrainfuck:
 - source.bf
+vendor/grammars/SublimePapyrus/:
+- source.compiled-papyrus
+- source.papyrus
+- source.papyrus-assembly
 vendor/grammars/SublimeXtend:
 - source.xtend
+vendor/grammars/TXL/:
+- source.txl
 vendor/grammars/Textmate-Gosu-Bundle:
 - source.gosu.2
 vendor/grammars/VBDotNetSyntax:
@@ -192,6 +208,8 @@ vendor/grammars/dylan.tmbundle:
 - source.makegen
 vendor/grammars/ebundles/Bundles/MSDOS batch file.tmbundle:
 - source.dosbatch
+vendor/grammars/ec.tmbundle/:
+- source.c.ec
 vendor/grammars/eiffel.tmbundle:
 - source.eiffel
 vendor/grammars/elixir-tmbundle:
@@ -213,12 +231,16 @@ vendor/grammars/fortran.tmbundle:
 - source.fortran.modern
 vendor/grammars/fsharpbinding:
 - source.fsharp
+vendor/grammars/gap-tmbundle/:
+- source.gap
 vendor/grammars/gettext.tmbundle:
 - source.po
 vendor/grammars/gnuplot-tmbundle:
 - source.gnuplot
 vendor/grammars/go-tmbundle:
 - source.go
+vendor/grammars/grace-tmbundle/:
+- source.grace
 vendor/grammars/gradle.tmbundle:
 - source.groovy.gradle
 vendor/grammars/graphviz.tmbundle:
@@ -324,6 +346,8 @@ vendor/grammars/matlab.tmbundle:
 - source.octave
 vendor/grammars/maven.tmbundle:
 - text.xml.pom
+vendor/grammars/mediawiki.tmbundle/:
+- text.html.mediawiki
 vendor/grammars/mercury-tmlanguage:
 - source.mercury
 vendor/grammars/monkey.tmbundle:
@@ -439,14 +463,22 @@ vendor/grammars/sublime-nginx:
 - source.nginx
 vendor/grammars/sublime-nix:
 - source.nix
+vendor/grammars/sublime-opal/:
+- source.opal
+- source.opalsysdefs
 vendor/grammars/sublime-robot-plugin:
 - text.robot
 vendor/grammars/sublime-rust:
 - source.rust
 vendor/grammars/sublime-sourcepawn:
 - source.sp
+vendor/grammars/sublime-spintools/:
+- source.regexp.spin
+- source.spin
 vendor/grammars/sublime-tea:
 - source.tea
+vendor/grammars/sublime-text-ox/:
+- source.ox
 vendor/grammars/sublime_cobol:
 - source.acucobol
 - source.cobol

lib/linguist/blob_helper.rb

@@ -236,6 +236,21 @@ module Linguist
       name =~ VendoredRegexp ? true : false
     end
 
+    documentation_paths = YAML.load_file(File.expand_path("../documentation.yml", __FILE__))
+    DocumentationRegexp = Regexp.new(documentation_paths.join('|'))
+
+    # Public: Is the blob in a documentation directory?
+    #
+    # Documentation files are ignored by language statistics.
+    #
+    # See "documentation.yml" for a list of documentation conventions that match
+    # this pattern.
+    #
+    # Return true or false
+    def documentation?
+      name =~ DocumentationRegexp ? true : false
+    end
+
     # Public: Get each line of data
     #
     # Requires Blob#data
@@ -317,5 +332,15 @@ module Linguist
     def tm_scope
       language && language.tm_scope
     end
+
+    DETECTABLE_TYPES = [:programming, :markup].freeze
+
+    # Internal: Should this blob be included in repository language statistics?
+    def include_in_language_stats?
+      !vendored? &&
+      !documentation? &&
+      !generated? &&
+      language && DETECTABLE_TYPES.include?(language.type)
+    end
   end
 end

lib/linguist/documentation.yml

@@ -0,0 +1,19 @@
+# Documentation files and directories are excluded from language
+# statistics.
+#
+# Lines in this file are Regexps that are matched against the file
+# pathname.
+#
+# Please add additional test coverage to
+# `test/test_blob.rb#test_documentation` if you make any changes.
+
+## Documentation Conventions ##
+
+- ^docs?/
+- ^Documentation/
+
+- (^|/)CONTRIBUTING(\.|$)
+- (^|/)COPYING(\.|$)
+- (^|/)INSTALL(\.|$)
+- (^|/)LICEN[CS]E(\.|$)
+- (^|/)README(\.|$)

lib/linguist/heuristics.rb

@@ -92,7 +92,7 @@ module Linguist
     disambiguate "Perl", "Perl6", "Prolog" do |data|
       if data.include?("use v6")
         Language["Perl6"]
-      elsif data.include?("use strict")
+      elsif data.match(/use strict|use\s+v?5\./)
         Language["Perl"]
       elsif data.include?(":-")
         Language["Prolog"]
@@ -163,7 +163,7 @@ module Linguist
     disambiguate "FORTRAN", "Forth" do |data|
       if /^: /.match(data)
         Language["Forth"]
-      elsif /^([c*][^a-z]|      (subroutine|program)\s|!)/i.match(data)
+      elsif /^([c*][^a-z]|      (subroutine|program)\s|\s*!)/i.match(data)
         Language["FORTRAN"]
       end
     end

lib/linguist/language.rb

@@ -32,13 +32,6 @@ module Linguist
     # Valid Languages types
     TYPES = [:data, :markup, :programming, :prose]
 
-    # Names of non-programming languages that we will still detect
-    #
-    # Returns an array
-    def self.detectable_markup
-      ["CSS", "Less", "Sass", "SCSS", "Stylus", "TeX"]
-    end
-
     # Detect languages by a specific type
     #
     # type - A symbol that exists within TYPES
@@ -96,8 +89,8 @@ module Linguist
 
     STRATEGIES = [
       Linguist::Strategy::Modeline,
-      Linguist::Strategy::Filename,
       Linguist::Shebang,
+      Linguist::Strategy::Filename,
       Linguist::Heuristics,
       Linguist::Classifier
     ]

lib/linguist/languages.yml

@@ -206,6 +206,7 @@ Assembly:
   - .asm
   - .ASM
   - .a51
+  - .nasm
   tm_scope: source.asm.x86
   ace_mode: assembly_x86
 
@@ -438,6 +439,8 @@ COBOL:
   ace_mode: cobol
 
 CSS:
+  type: markup
+  tm_scope: source.css
   ace_mode: css
   color: "#563d7c"
   extensions:
@@ -918,6 +921,7 @@ Forth:
   color: "#341708"
   extensions:
   - .fth
+  - .4TH
   - .4th
   - .F
   - .f
@@ -942,7 +946,7 @@ G-code:
   - .g
   - .gco
   - .gcode
-  tm_scope: none
+  tm_scope: source.gcode
   ace_mode: gcode
 
 GAMS:
@@ -960,7 +964,7 @@ GAP:
   - .gd
   - .gi
   - .tst
-  tm_scope: none
+  tm_scope: source.gap
   ace_mode: text
 
 GAS:
@@ -1092,7 +1096,7 @@ Grace:
   type: programming
   extensions:
   - .grace
-  tm_scope: none
+  tm_scope: source.grace
   ace_mode: text
 
 Gradle:
@@ -1171,6 +1175,7 @@ HTML:
   type: markup
   tm_scope: text.html.basic
   ace_mode: html
+  color: "#e44b23"
   aliases:
   - xhtml
   extensions:
@@ -1345,7 +1350,7 @@ Inform 7:
 Inno Setup:
   extensions:
   - .iss
-  tm_scope: none
+  tm_scope: source.inno
   ace_mode: text
 
 Io:
@@ -1806,7 +1811,7 @@ MediaWiki:
   wrap: true
   extensions:
   - .mediawiki
-  tm_scope: none
+  tm_scope: text.html.mediawiki
   ace_mode: text
 
 Mercury:
@@ -2051,7 +2056,7 @@ Opal:
   color: "#f7ede0"
   extensions:
   - .opal
-  tm_scope: none
+  tm_scope: source.opal
   ace_mode: text
 
 OpenCL:
@@ -2096,7 +2101,7 @@ Ox:
   - .ox
   - .oxh
   - .oxo
-  tm_scope: none
+  tm_scope: source.ox
   ace_mode: text
 
 Oxygene:
@@ -2157,7 +2162,7 @@ Papyrus:
   color: "#6600cc"
   extensions:
   - .psc
-  tm_scope: none
+  tm_scope: source.papyrus
   ace_mode: text
 
 Parrot:
@@ -2275,7 +2280,7 @@ PogoScript:
   color: "#d80074"
   extensions:
   - .pogo
-  tm_scope: none
+  tm_scope: source.pogoscript
   ace_mode: text
 
 PostScript:
@@ -2322,7 +2327,7 @@ Propeller Spin:
   color: "#2b446d"
   extensions:
   - .spin
-  tm_scope: none
+  tm_scope: source.spin
   ace_mode: text
 
 Protocol Buffer:
@@ -2682,6 +2687,13 @@ STON:
   tm_scope: source.smalltalk
   ace_mode: text
 
+SVG:
+  type: data
+  extensions:
+  - .svg
+  tm_scope: text.xml
+  ace_mode: xml
+
 Sage:
   type: programming
   group: Python
@@ -2921,7 +2933,7 @@ TXL:
   type: programming
   extensions:
   - .txl
-  tm_scope: none
+  tm_scope: source.txl
   ace_mode: text
 
 Tcl:
@@ -3197,7 +3209,6 @@ XML:
   - .srdf
   - .stTheme
   - .sublime-snippet
-  - .svg
   - .targets
   - .tmCommand
   - .tmLanguage
@@ -3331,7 +3342,7 @@ eC:
   extensions:
   - .ec
   - .eh
-  tm_scope: none
+  tm_scope: source.c.ec
   ace_mode: text
 
 edn:

lib/linguist/lazy_blob.rb

@@ -4,7 +4,7 @@ require 'rugged'
 
 module Linguist
   class LazyBlob
-    GIT_ATTR = ['linguist-language', 'linguist-vendored']
+    GIT_ATTR = ['linguist-documentation', 'linguist-language', 'linguist-vendored']
     GIT_ATTR_OPTS = { :priority => [:index], :skip_system => true }
     GIT_ATTR_FLAGS = Rugged::Repository::Attributes.parse_opts(GIT_ATTR_OPTS)
 
@@ -37,6 +37,14 @@ module Linguist
       end
     end
 
+    def documentation?
+      if attr = git_attributes['linguist-documentation']
+        boolean_attribute(attr)
+      else
+        super
+      end
+    end
+
     def language
       return @language if defined?(@language)
 

lib/linguist/repository.rb

@@ -156,13 +156,8 @@ module Linguist
 
           blob = Linguist::LazyBlob.new(repository, delta.new_file[:oid], new, mode.to_s(8))
 
-          # Skip vendored or generated blobs
-          next if blob.vendored? || blob.generated? || blob.language.nil?
-
-          # Only include programming languages and acceptable markup languages
-          if blob.language.type == :programming || Language.detectable_markup.include?(blob.language.name)
-            file_map[new] = [blob.language.group.name, blob.size]
-          end
+          next unless blob.include_in_language_stats?
+          file_map[new] = [blob.language.group.name, blob.size]
         end
       end
 

lib/linguist/strategy/modeline.rb

@@ -1,8 +1,8 @@
 module Linguist
   module Strategy
     class Modeline
-      EmacsModeline = /-\*-\s*(?:mode:)?\s*(\w+);?\s*-\*-/
-      VimModeline = /\/\*\s*vim:\s*set\s*(?:ft|filetype)=(\w+):\s*\*\//
+      EmacsModeline = /-\*-\s*mode:\s*(\w+);?\s*-\*-/i
+      VimModeline = /\/\*\s*vim:\s*set\s*(?:ft|filetype)=(\w+):\s*\*\//i
 
       # Public: Detects language based on Vim and Emacs modelines
       #

lib/linguist/vendor.yml

@@ -40,7 +40,7 @@
 # Minified JavaScript and CSS
 - (\.|-)min\.(js|css)$
 
-#Stylesheets imported from packages
+# Stylesheets imported from packages
 - ([^\s]*)import\.(css|less|scss|styl)$
 
 # Bootstrap css and js

lib/linguist/version.rb

@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "4.3.0"
+  VERSION = "4.4.0"
 end

package.json

@@ -1,6 +1,6 @@
 {
   "repository": "https://github.com/github/linguist",
   "dependencies": {
-    "season": "~>3.0"
+    "season": "~>5.0"
   }
 }

samples/Forth/tools.4TH

@@ -0,0 +1,133 @@
+\ -*- forth -*- Copyright 2004, 2013 Lars Brinkhoff
+
+( Tools words. )
+
+: .s ( -- )
+    [char] < emit  depth (.)  ." > "
+    'SP @ >r r@ depth 1- cells +
+    begin
+	dup r@ <>
+    while
+	dup @ .
+	/cell -
+    repeat r> 2drop ;
+
+: ?   @ . ;
+
+: c?   c@ . ;
+
+: dump   bounds do i ? /cell +loop cr ;
+
+: cdump   bounds do i c? loop cr ;
+
+: again   postpone branch , ; immediate
+
+: see-find ( caddr -- end xt )
+    >r here lastxt @
+    begin
+	dup 0= abort" Undefined word"
+	dup r@ word= if r> drop exit then
+	nip dup >nextxt
+    again ;
+
+: cabs ( char -- |char| )   dup 127 > if 256 swap - then ;
+
+: xt. ( xt -- )
+    ( >name ) count cabs type ;
+
+: xt? ( xt -- flag )
+    >r lastxt @ begin
+	?dup
+    while
+	dup r@ = if r> 2drop -1 exit then
+	>nextxt
+    repeat r> drop 0 ;
+
+: disassemble ( x -- )
+    dup xt? if
+        ( >name ) count
+        dup 127 > if ." postpone " then
+        cabs type
+    else
+        .
+    then ;
+
+: .addr  dup . ;
+
+: see-line ( addr -- )
+    cr ."     ( " .addr ." ) "  @ disassemble ;
+
+: see-word ( end xt -- )
+    >r ." : " r@ xt.
+    r@ >body do i see-line /cell +loop
+    ."  ;" r> c@ 127 > if ."  immediate" then ;
+
+: see   bl word see-find see-word cr ;
+
+: #body   bl word see-find >body - ;
+
+: type-word ( end xt -- flag )
+    xt. space drop 0 ;
+
+: traverse-dictionary ( in.. xt -- out.. )
+    \ xt execution: ( in.. end xt2 -- in.. 0 | in.. end xt2 -- out.. true )
+    >r  here lastxt @  begin
+	?dup
+    while
+	r> 2dup >r >r execute
+	if r> r> 2drop exit then
+	r> dup >nextxt
+    repeat r> 2drop ;
+
+: words ( -- )
+    ['] type-word traverse-dictionary cr ;
+
+\ ----------------------------------------------------------------------
+
+( Tools extension words. )
+
+\ ;code
+
+\ assembler
+
+\ in kernel: bye
+
+\ code
+
+\ cs-pick
+
+\ cs-roll
+
+\ editor
+
+: forget   ' dup >nextxt lastxt !  'here !  reveal ;
+
+\ Kernel: state
+
+\ [else]
+
+\ [if]
+
+\ [then]
+
+\ ----------------------------------------------------------------------
+
+( Forth2012 tools extension words. )
+
+\ TODO: n>r
+
+\ TODO: nr>
+
+\ TODO: synonym
+
+: [undefined]   bl-word find nip 0= ; immediate
+
+: [defined]   postpone [undefined] invert ; immediate
+
+\ ----------------------------------------------------------------------
+
+: @+ ( addr -- addr+/cell x )   dup cell+ swap @ ;
+
+: !+ ( x addr -- addr+/cell )   tuck ! cell+ ;
+
+: -rot   swap >r swap r> ;

samples/Inno Setup/expat.iss

@@ -0,0 +1,69 @@
+; Basic setup script for the Inno Setup installer builder.  For more
+; information on the free installer builder, see www.jrsoftware.org.
+;
+; This script was contributed by Tim Peters.
+; It was designed for Inno Setup 2.0.19 but works with later versions as well.
+
+[Setup]
+AppName=Expat
+AppId=expat
+AppVersion=2.1.0
+AppVerName=Expat 2.1.0
+AppCopyright=Copyright 1998-2012 Thai Open Source Software Center, Clark Cooper, and the Expat maintainers
+AppPublisher=The Expat Developers
+AppPublisherURL=http://www.libexpat.org/
+AppSupportURL=http://www.libexpat.org/
+AppUpdatesURL=http://www.libexpat.org/
+UninstallDisplayName=Expat XML Parser 2.1.0
+VersionInfoVersion=2.1.0
+
+DefaultDirName={pf}\Expat 2.1.0
+UninstallFilesDir={app}\Uninstall
+
+Compression=lzma
+SolidCompression=yes
+SourceDir=..
+OutputDir=win32
+DisableStartupPrompt=yes
+AllowNoIcons=yes
+DisableProgramGroupPage=yes
+DisableReadyPage=yes
+
+[Files]
+Flags: ignoreversion; Source: win32\bin\Release\xmlwf.exe;  DestDir: "{app}\Bin"
+Flags: ignoreversion; Source: win32\MANIFEST.txt;           DestDir: "{app}"
+Flags: ignoreversion; Source: Changes;                      DestDir: "{app}"; DestName: Changes.txt
+Flags: ignoreversion; Source: COPYING;                      DestDir: "{app}"; DestName: COPYING.txt
+Flags: ignoreversion; Source: README;                       DestDir: "{app}"; DestName: README.txt
+Flags: ignoreversion; Source: doc\*.html;                   DestDir: "{app}\Doc"
+Flags: ignoreversion; Source: doc\*.css;                    DestDir: "{app}\Doc"
+Flags: ignoreversion; Source: doc\*.png;                    DestDir: "{app}\Doc"
+Flags: ignoreversion; Source: win32\bin\Release\*.dll;      DestDir: "{app}\Bin"
+Flags: ignoreversion; Source: win32\bin\Release\*.lib;      DestDir: "{app}\Bin"
+Flags: ignoreversion; Source: expat.dsw;                    DestDir: "{app}\Source"
+Flags: ignoreversion; Source: win32\README.txt;             DestDir: "{app}\Source"
+Flags: ignoreversion; Source: bcb5\*.bp*;                   DestDir: "{app}\Source\bcb5"
+Flags: ignoreversion; Source: bcb5\*.mak;                   DestDir: "{app}\Source\bcb5"
+Flags: ignoreversion; Source: bcb5\*.def;                   DestDir: "{app}\Source\bcb5"
+Flags: ignoreversion; Source: bcb5\*.txt;                   DestDir: "{app}\Source\bcb5"
+Flags: ignoreversion; Source: bcb5\*.bat;                   DestDir: "{app}\Source\bcb5"
+Flags: ignoreversion; Source: lib\*.c;                      DestDir: "{app}\Source\lib"
+Flags: ignoreversion; Source: lib\*.h;                      DestDir: "{app}\Source\lib"
+Flags: ignoreversion; Source: lib\*.def;                    DestDir: "{app}\Source\lib"
+Flags: ignoreversion; Source: lib\*.dsp;                    DestDir: "{app}\Source\lib"
+Flags: ignoreversion; Source: examples\*.c;                 DestDir: "{app}\Source\examples"
+Flags: ignoreversion; Source: examples\*.dsp;               DestDir: "{app}\Source\examples"
+Flags: ignoreversion; Source: tests\*.c;                    DestDir: "{app}\Source\tests"
+Flags: ignoreversion; Source: tests\*.cpp;                  DestDir: "{app}\Source\tests"
+Flags: ignoreversion; Source: tests\*.h;                    DestDir: "{app}\Source\tests"
+Flags: ignoreversion; Source: tests\README.txt;             DestDir: "{app}\Source\tests"
+Flags: ignoreversion; Source: tests\benchmark\*.c;          DestDir: "{app}\Source\tests\benchmark"
+Flags: ignoreversion; Source: tests\benchmark\*.ds*;        DestDir: "{app}\Source\tests\benchmark"
+Flags: ignoreversion; Source: tests\benchmark\README.txt;   DestDir: "{app}\Source\tests\benchmark"
+Flags: ignoreversion; Source: xmlwf\*.c*;                   DestDir: "{app}\Source\xmlwf"
+Flags: ignoreversion; Source: xmlwf\*.h;                    DestDir: "{app}\Source\xmlwf"
+Flags: ignoreversion; Source: xmlwf\*.dsp;                  DestDir: "{app}\Source\xmlwf"
+
+[Messages]
+WelcomeLabel1=Welcome to the Expat XML Parser Setup Wizard
+WelcomeLabel2=This will install [name/ver] on your computer.%n%nExpat is an XML parser with a C-language API, and is primarily made available to allow developers to build applications which use XML using a portable API and fast implementation.%n%nIt is strongly recommended that you close all other applications you have running before continuing. This will help prevent any conflicts during the installation process.

samples/Perl/use5.pl

@@ -0,0 +1,3 @@
+use Mojolicious::Lite;
+use 5.20.0;
+use experimental 'signatures';

samples/eC/Designer.ec

@@ -0,0 +1,337 @@
+import "ide"
+
+class Designer : DesignerBase
+{
+   ~Designer()
+   {
+      if(GetActiveDesigner() == this)
+      {
+         SetActiveDesigner(null);
+      }
+      if(classDesigner)
+         delete classDesigner;
+   }
+
+   // *** DesignerBase Implementation ***
+
+   void ModifyCode()
+   {
+      codeEditor.ModifyCode();
+   }
+
+   void UpdateProperties()
+   {
+      codeEditor.DesignerModifiedObject();
+   }
+
+   void CodeAddObject(Instance instance, ObjectInfo * object)
+   {
+      codeEditor.AddObject(instance, object);
+   }
+
+   void SheetAddObject(ObjectInfo object)
+   {
+      codeEditor.sheet.AddObject(object, object.name, typeData, true); //className, true);
+   }
+
+   void AddToolBoxClass(Class _class)
+   {
+      ((IDEWorkSpace)master).toolBox.AddControl(_class);
+   }
+
+   void AddDefaultMethod(Instance instance, Instance classInstance)
+   {
+      Class _class = instance._class;
+      Method defaultMethod = null;
+
+      for( ; _class; _class = _class.base)
+      {
+         Method method;
+         int minID = MAXINT;
+         for(method = (Method)_class.methods.first; method; method = (Method)((BTNode)method).next)
+         {
+            if(method.type == virtualMethod)
+            {
+               if(!method.dataType)
+                  method.dataType = ProcessTypeString(method.dataTypeString, false);
+               if(method.vid < minID && (instance == classInstance || (method.dataType.thisClass && eClass_IsDerived(classInstance._class, method.dataType.thisClass.registered))))
+               {
+                  defaultMethod = method;
+                  minID = method.vid;
+               }
+            }
+         }
+         if(defaultMethod)
+            break;
+      }
+      codeEditor.AddMethod(defaultMethod);
+   }
+
+   bool ObjectContainsCode(ObjectInfo object)
+   {
+      // Confirmation if control contains code
+      if(object.instCode)
+      {
+         MembersInit members;
+         if(object.instCode.members)
+         {
+            for(members = object.instCode.members->first; members; members = members.next)
+            {
+               if(members.type == methodMembersInit)
+               {
+                  //if(!Code_IsFunctionEmpty(members.function))
+                  {
+                     return true;
+                  }
+               }
+            }
+         }
+      }
+      return false;
+   }
+
+   void DeleteObject(ObjectInfo object)
+   {
+      if(codeEditor)
+         codeEditor.DeleteObject(object);
+   }
+
+   void RenameObject(ObjectInfo object, const char * name)
+   {
+      if(object && (name || !object.classDefinition))
+         codeEditor.RenameObject(object, name);
+   }
+
+   bool FindObject(Instance * object, const char * string)
+   {
+      ObjectInfo classObject;
+      for(classObject = codeEditor.classes.first; classObject; classObject = classObject.next)
+      {
+         ObjectInfo check;
+         if(classObject.name && !strcmp(string, classObject.name))
+         {
+            *object = classObject.instance;
+            break;
+         }
+         for(check = classObject.instances.first; check; check = check.next)
+         {
+            if(check.name && !strcmp(string, check.name))
+            {
+               *object = check.instance;
+               break;
+            }
+         }
+         if(check)
+            return true;
+      }
+      return false;
+   }
+
+   void SelectObjectFromDesigner(ObjectInfo object)
+   {
+      codeEditor.SelectObjectFromDesigner(object);
+   }
+
+   borderStyle = sizable;
+   isActiveClient = true;
+   hasVertScroll = true;
+   hasHorzScroll = true;
+   hasClose = true;
+   hasMaximize = true;
+   hasMinimize = true;
+   text = $"Designer";
+   menu = Menu { };
+   anchor = Anchor { left = 300, right = 150, top = 0, bottom = 0 };
+
+   ToolBox toolBox;
+   CodeEditor codeEditor;
+
+   Menu fileMenu { menu, $"File", f };
+   MenuItem fileSaveItem
+   {
+      fileMenu, $"Save", s, ctrlS;
+      bool NotifySelect(MenuItem selection, Modifiers mods)
+      {
+         return codeEditor.MenuFileSave(selection, mods);
+      }
+   };
+   MenuItem fileSaveAsItem
+   {
+      fileMenu, $"Save As...", a;
+      bool NotifySelect(MenuItem selection, Modifiers mods)
+      {
+         return codeEditor.MenuFileSaveAs(selection, mods);
+      }
+   };
+   bool debugClosing;
+
+   bool OnClose(bool parentClosing)
+   {
+      if(!parentClosing)
+      {
+         if(codeEditor && codeEditor.inUseDebug && !debugClosing)
+         {
+            debugClosing = true;
+            closing = false;
+            if(CloseConfirmation(false))
+            {
+               visible = false;
+               if(modifiedDocument)
+                  OnFileModified({ modified = true }, null);
+            }
+            debugClosing = false;
+            return false;
+         }
+         if(codeEditor && !codeEditor.closing && !debugClosing)
+         {
+            if(!codeEditor.visible)
+            {
+               if(!codeEditor.Destroy(0))
+                  return false;
+               else
+                  codeEditor = null;
+            }
+            else
+            {
+               visible = false;
+               return false;
+            }
+         }
+      }
+      return true;
+   }
+
+   bool OnActivate(bool active, Window previous, bool * goOnWithActivation, bool direct)
+   {
+      if(active)
+      {
+         codeEditor.EnsureUpToDate();
+         codeEditor.fixCaret = true;
+         /*
+         if(classDesigner)
+            classDesigner.Activate();
+         */
+      }
+      return true;
+   }
+
+   bool OnKeyHit(Key key, unichar ch)
+   {
+      return codeEditor.sheet.OnKeyHit(key, ch);
+   }
+
+   watch(modifiedDocument)
+   {
+      fileSaveItem.disabled = !modifiedDocument && codeEditor.fileName;
+   };
+
+   // *** METHODS ACCESSED FROM PROPERTY SHEET/TOOLBOX/CODE EDITOR ***
+   void Reset()
+   {
+      if(classDesigner)
+      {
+         classDesigner.Reset();
+         classDesigner.SelectObject(null, null);
+         classDesigner.Destroy(0);
+         delete classDesigner;
+      }
+   }
+
+   void FillToolBox()
+   {
+      if(this && classDesigner)
+         classDesigner.ListToolBoxClasses(this);
+   }
+
+   void SelectObject(ObjectInfo object, Instance instance)
+   {
+      ClassDesignerBase classDesigner = this.classDesigner;
+#ifdef _DEBUG
+      if(instance && instance._class.module.application != codeEditor.privateModule)
+         printf("warning: SelectObject: instance._class.module.application != codeEditor.privateModule\n");
+#endif
+      if(!classDesigner || !instance || classDesigner._class != (Class)eInstance_GetDesigner(instance))
+      {
+         if(classDesigner)
+         {
+            classDesigner.SelectObject(null, null);
+            classDesigner.Destroy(0);
+            classDesigner = null;
+            delete this.classDesigner;
+         }
+         if(instance)
+         {
+            this.classDesigner = classDesigner = eInstance_New(eInstance_GetDesigner(instance));
+            incref classDesigner;
+            //if(!classDesigner.parent)
+            {
+               classDesigner.parent = this;
+               classDesigner.anchor = Anchor { left = 0, right = 0, top = 0, bottom = 0 };
+            }
+            classDesigner.Create();
+         }
+      }
+      // Call class editor SelectObject
+      if(classDesigner)
+         classDesigner.SelectObject(object, instance);
+   }
+
+   void AddObject()
+   {
+      // Call class editor AddObject
+      if(classDesigner)
+         classDesigner.AddObject();
+      if(visible)
+         Activate();
+      else
+         codeEditor.Activate();
+   }
+
+   void CreateObject(Instance instance, ObjectInfo object, bool isClass, Instance iclass)
+   {
+      subclass(ClassDesignerBase) designerClass = eInstance_GetDesigner(instance);
+
+      // Call class editor CreateObject
+      if(designerClass)
+         designerClass.CreateObject(this, instance, object, isClass, iclass);
+   }
+
+   void ::PostCreateObject(Instance instance, ObjectInfo object, bool isClass, Instance iclass)
+   {
+      subclass(ClassDesignerBase) designerClass = eInstance_GetDesigner(instance);
+
+      // Call class editor PostCreateObject
+      if(designerClass)
+         designerClass.PostCreateObject(instance, object, isClass, iclass);
+   }
+
+   void ::DroppedObject(Instance instance, ObjectInfo object, bool isClass, Instance iclass)
+   {
+      subclass(ClassDesignerBase) designerClass = eInstance_GetDesigner(instance);
+
+      // Call class editor PostCreateObject
+      if(designerClass)
+         designerClass.DroppedObject(instance, object, isClass, iclass);
+   }
+
+   void PrepareTestObject(Instance instance)
+   {
+      subclass(ClassDesignerBase) designerClass = eInstance_GetDesigner(instance);
+      if(designerClass)
+         designerClass.PrepareTestObject(this, instance);
+   }
+
+   void ::DestroyObject(Instance instance)
+   {
+      subclass(ClassDesignerBase) designerClass = eInstance_GetDesigner(instance);
+      if(designerClass)
+         designerClass.DestroyObject(instance);
+   }
+
+   void ::FixProperty(Property prop, Instance instance)
+   {
+      subclass(ClassDesignerBase) designerClass = eInstance_GetDesigner(instance);
+      if(designerClass)
+         designerClass.FixProperty(prop, instance);
+   }
+}

test/fixtures/Data/Modelines/example_smalltalk.md

@@ -1 +1 @@
-; -*-Smalltalk-*-
+; -*-mode:Smalltalk-*-

test/fixtures/Data/Modelines/iamphp.inc

@@ -1 +1 @@
-; -*- mode: php;-*-
+; -*- MoDe: PhP;-*-

test/fixtures/Data/Modelines/not_perl.pl

@@ -1,3 +1,3 @@
-/* vim: set filetype=prolog: */
+/* vim: set filEtype=pRoloG: */
 
 # I am Prolog

test/fixtures/Shell/crossbuild_liblua5.1

@@ -0,0 +1,57 @@
+#! /bin/sh
+#
+# Builds and installs liblua5.1 for the cross toolchain.
+# Executed by build-uqm-dependencies.chroot
+
+# Include our common functions
+. /usr/lib/crossbuild/crossbuild.subr
+
+# envvar LIBLUA51_URL
+#
+# Specifies the URL of the liblua5.1 source tarball you want to use.
+export LIBLUA51_URL="http://www.lua.org/ftp/lua-5.1.5.tar.gz"
+
+# envvar INSTALL_TOP
+#
+# This determines where lua's makefiles install everything (we don't want to use
+# /usr/local!).
+export INSTALL_TOP="/usr/${HOST_TRIPLET}"
+
+# envvar TO_BIN
+#
+# Names of the binary files to install (that's right, lua's makefiles don't
+# determine this automatically, and since we end up with files named according
+# to Windows conventions the install chokes without these)
+export TO_BIN="lua.exe luac.exe"
+
+# envvar TO_LIB
+#
+# Names of the libraries to install, see TO_BIN
+export TO_LIB="liblua.a lua51.dll"
+
+
+# liblua5.1 uses custom makefiles and does not natively support cross-building.
+# However, with our cross toolchain in its PATH it successfully builds the mingw
+# target.
+export PATH=/usr/${HOST_TRIPLET}/bin:${PATH}
+
+echo "*************************************************************************"
+echo "--- BEGIN: crossbuild_liblua5.1 ---"
+
+get_tarball "liblua5.1" "${LIBLUA51_URL}" gz
+
+cd ${SRC_ROOT_DIR}/liblua5.1/*
+
+if [ -f Makefile ]; then
+  make clean
+  make --environment-overrides mingw install
+
+
+else
+  echo "crossbuild_liblua5.1 failed: Could not find Makefile"
+  echo "(is the liblua5.1 source tarball sane?)"
+  exit 1
+fi
+
+echo "--- END: crossbuild_liblua5.1 ---"
+echo "*************************************************************************"

test/fixtures/Shell/graylog2-server.init.d

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+### BEGIN INIT INFO
+# Provides:        graylog2-server
+# Required-Start:  $network
+# Required-Stop:   $network
+# Default-Start:   2 3 4 5
+# Default-Stop:    1
+# Short-Description: Start Graylog2 server
+### END INIT INFO
+
+# Written by Lital Natan <litaln@gmail.com>
+
+PREFIX=/usr
+SHAREDIR=$PREFIX/share/graylog2-server
+SERVER_JAR=$SHAREDIR/graylog2-server.jar
+SYSLOG4J_JAR=$SHAREDIR/syslog4j-0.9.46-bin.jar
+SVCNAME="graylog2-server"
+
+CONFIG="/etc/graylog2.conf"
+LOGFILE="/var/log/graylog2.log"
+PIDFILE="/var/run/graylog2.pid"
+
+start() {
+    if [ ! -e $CONFIG ]; then
+        echo "Config file $CONFIG does not exist"
+        return 1
+    fi
+
+    echo "Starting ${SVCNAME}"
+        nohup `which java` -cp $SERVER_JAR:$SYSLOG4J_JAR org.graylog2.Main \
+      -p ${PIDFILE} -f ${CONFIG} > $LOGFILE 2>&1 &
+
+    # Sleep before testing the service
+    sleep 2
+
+    graylog2_test || return 1
+}
+
+stop() {
+    pid=`< $PIDFILE`
+    kill $pid
+    rm -f ${PIDFILE} # just in case
+}
+
+graylog2_test() {
+    # Graylog2 only deletes its PID file if it hits a config error
+    if [ ! -e ${PIDFILE} ]; then
+        echo "Configuration error, check ${CONFIG}"
+        return 1
+    fi
+
+    local pid=`cat ${PIDFILE}`
+
+    # Graylog2 isn't running, so that means there was a problem
+    if [ ! -e /proc/$pid ]; then
+        echo "Something went wrong, check ${LOGFILE}"
+        rm -f ${PIDFILE}
+        return 1
+    else
+        return 0
+    fi
+}
+
+
+status() {
+    graylog2_test > /dev/null 2>&1
+    if [ "$?" == "0" ]; then
+       echo "Graylog2 server is up"
+       return 0
+    else
+       echo "Graylog2 server is down"
+       return 1
+    fi
+}
+
+
+restart() {
+    stop
+    start
+}
+
+case "$1" in
+    start)
+  start
+  ;;
+    stop)
+  stop
+  ;;
+    status)
+        status
+        ;;
+    restart)
+  restart
+  ;;
+    *)
+  echo "Usage $0 {start|stop|restart|status}"
+  RETVAL=1
+esac

test/test_blob.rb

@@ -441,6 +441,43 @@ class TestBlob < Minitest::Test
     assert sample_blob("subproject/activator.bat").vendored?
   end
 
+  def test_documentation
+    assert_predicate fixture_blob("doc/foo.html"), :documentation?
+    assert_predicate fixture_blob("docs/foo.html"), :documentation?
+    refute_predicate fixture_blob("project/doc/foo.html"), :documentation?
+    refute_predicate fixture_blob("project/docs/foo.html"), :documentation?
+
+    assert_predicate fixture_blob("Documentation/foo.md"), :documentation?
+    refute_predicate fixture_blob("project/Documentation/foo.md"), :documentation?
+
+    assert_predicate fixture_blob("README"), :documentation?
+    assert_predicate fixture_blob("README.md"), :documentation?
+    assert_predicate fixture_blob("README.txt"), :documentation?
+    assert_predicate fixture_blob("foo/README"), :documentation?
+
+    assert_predicate fixture_blob("CONTRIBUTING"), :documentation?
+    assert_predicate fixture_blob("CONTRIBUTING.md"), :documentation?
+    assert_predicate fixture_blob("CONTRIBUTING.txt"), :documentation?
+    assert_predicate fixture_blob("foo/CONTRIBUTING"), :documentation?
+
+    assert_predicate fixture_blob("LICENSE"), :documentation?
+    assert_predicate fixture_blob("LICENCE.md"), :documentation?
+    assert_predicate fixture_blob("LICENSE.txt"), :documentation?
+    assert_predicate fixture_blob("foo/LICENSE"), :documentation?
+
+    assert_predicate fixture_blob("COPYING"), :documentation?
+    assert_predicate fixture_blob("COPYING.md"), :documentation?
+    assert_predicate fixture_blob("COPYING.txt"), :documentation?
+    assert_predicate fixture_blob("foo/COPYING"), :documentation?
+
+    assert_predicate fixture_blob("INSTALL"), :documentation?
+    assert_predicate fixture_blob("INSTALL.md"), :documentation?
+    assert_predicate fixture_blob("INSTALL.txt"), :documentation?
+    assert_predicate fixture_blob("foo/INSTALL"), :documentation?
+
+    refute_predicate fixture_blob("foo.md"), :documentation?
+  end
+
   def test_language
     Samples.each do |sample|
       blob = sample_blob(sample[:path])
@@ -485,4 +522,29 @@ class TestBlob < Minitest::Test
     refute blob.new(" ").empty?
     refute blob.new("nope").empty?
   end
+
+  def test_include_in_language_stats
+    vendored = sample_blob("bower_components/custom/custom.js")
+    assert_predicate vendored, :vendored?
+    refute_predicate vendored, :include_in_language_stats?
+
+    documentation = fixture_blob("README")
+    assert_predicate documentation, :documentation?
+    refute_predicate documentation, :include_in_language_stats?
+
+    generated = sample_blob("CSS/bootstrap.min.css")
+    assert_predicate generated, :generated?
+    refute_predicate generated, :include_in_language_stats?
+
+    data = sample_blob("Ant Build System/filenames/ant.xml")
+    assert_equal :data, data.language.type
+    refute_predicate data, :include_in_language_stats?
+
+    prose = sample_blob("Markdown/tender.md")
+    assert_equal :prose, prose.language.type
+    refute_predicate prose, :include_in_language_stats?
+
+    included = sample_blob("HTML/pages.html")
+    assert_predicate included, :include_in_language_stats?
+  end
 end

test/test_grammars.rb

@@ -3,12 +3,18 @@ require_relative "./helper"
 class TestGrammars < Minitest::Test
   ROOT = File.expand_path("../..", __FILE__)
 
-  # These grammars have no license but have been grandfathered in. New grammars
-  # must have a license that allows redistribution.
-  UNLICENSED_GRAMMARS_WHITELIST = %w[
-    vendor/grammars/Sublime-Lasso
-    vendor/grammars/Sublime-REBOL
-    vendor/grammars/x86-assembly-textmate-bundle
+  LICENSE_WHITELIST = [
+    # This grammar's MIT license is inside a subdirectory.
+    "vendor/grammars/SublimePapyrus",
+
+    # This grammar has a nonstandard but acceptable license.
+    "vendor/grammars/gap-tmbundle",
+
+    # These grammars have no license but have been grandfathered in. New grammars
+    # must have a license that allows redistribution.
+    "vendor/grammars/Sublime-Lasso",
+    "vendor/grammars/Sublime-REBOL",
+    "vendor/grammars/x86-assembly-textmate-bundle",
   ].freeze
 
   def setup
@@ -77,9 +83,9 @@ class TestGrammars < Minitest::Test
 
     unlicensed = categories[:unlicensed] || []
     unrecognized = categories[:unrecognized] || []
-    disallowed_unlicensed = unlicensed - UNLICENSED_GRAMMARS_WHITELIST
-    disallowed_unrecognized = unrecognized - UNLICENSED_GRAMMARS_WHITELIST
-    extra_whitelist_entries = UNLICENSED_GRAMMARS_WHITELIST - (unlicensed | unrecognized)
+    disallowed_unlicensed = unlicensed - LICENSE_WHITELIST
+    disallowed_unrecognized = unrecognized - LICENSE_WHITELIST
+    extra_whitelist_entries = LICENSE_WHITELIST - (unlicensed | unrecognized)
 
     message = ""
     if disallowed_unlicensed.any?
@@ -93,7 +99,7 @@ class TestGrammars < Minitest::Test
     end
     if extra_whitelist_entries.any?
       message << "\n\n" unless message.empty?
-      message << "The following grammar submodules are listed in UNLICENSED_GRAMMARS_WHITELIST but either have a license (yay!)\n"
+      message << "The following grammar submodules are listed in LICENSE_WHITELIST but either have a license (yay!)\n"
       message << "or have been removed from the repository. Please remove them from the whitelist.\n"
       message << extra_whitelist_entries.sort.join("\n")
     end
@@ -131,6 +137,8 @@ class TestGrammars < Minitest::Test
       "unlicense"
     elsif content.include?("http://www.wtfpl.net/txt/copying/")
       "WTFPL"
+    elsif content.include?("zlib") && content.include?("license") && content.include?("2. Altered source versions must be plainly marked as such")
+      "zlib"
     end
   end
 end

test/test_heuristics.rb

@@ -48,7 +48,7 @@ class TestHeuristcs < Minitest::Test
   def test_pl_prolog_perl_by_heuristics
     assert_heuristics({
       "Prolog" => "Prolog/turing.pl",
-      "Perl" => "Perl/perl-test.t",
+      "Perl" => ["Perl/perl-test.t", "Perl/use5.pl"]
     })
   end
 

test/test_repository.rb

@@ -99,4 +99,16 @@ class TestRepository < Minitest::Test
     # overridden .gitattributes
     assert !override_unvendored.vendored?
   end
+
+  def test_linguist_override_documentation?
+    attr_commit = "d4c8fb8a28e91f97a7e53428a365c0abbac36d3d"
+    repo = linguist_repo(attr_commit).read_index
+
+    readme = Linguist::LazyBlob.new(rugged_repository, attr_commit, "README.md")
+    arduino = Linguist::LazyBlob.new(rugged_repository, attr_commit, "samples/Arduino/hello.ino")
+
+    # overridden by .gitattributes
+    refute_predicate readme, :documentation?
+    assert_predicate arduino, :documentation?
+  end
 end