diff --git a/.gitmodules b/.gitmodules
index 5f07d011..98dbcfb2 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -253,9 +253,6 @@
[submodule "vendor/grammars/SublimeXtend"]
path = vendor/grammars/SublimeXtend
url = https://github.com/staltz/SublimeXtend
-[submodule "vendor/grammars/Stata.tmbundle"]
- path = vendor/grammars/Stata.tmbundle
- url = https://github.com/statatmbundle/Stata.tmbundle
[submodule "vendor/grammars/Vala-TMBundle"]
path = vendor/grammars/Vala-TMBundle
url = https://github.com/technosophos/Vala-TMBundle
@@ -677,3 +674,9 @@
[submodule "vendor/grammars/X10"]
path = vendor/grammars/X10
url = git@github.com:x10-lang/x10-highlighting.git
+[submodule "vendor/grammars/language-babel"]
+ path = vendor/grammars/language-babel
+ url = https://github.com/gandm/language-babel
+[submodule "vendor/grammars/Stata.tmbundle"]
+ path = vendor/grammars/Stata.tmbundle
+ url = https://github.com/pschumm/Stata.tmbundle
diff --git a/README.md b/README.md
index 4ca08af1..2a4d3c31 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@ See [Troubleshooting](#troubleshooting) and [`CONTRIBUTING.md`](/CONTRIBUTING.md
-The Language stats bar is built by aggregating the languages of each file in that repository. If it is reporting a language that you don't expect:
+The Language stats bar displays languages percentages for the files in the repository. The percentages are calculated based on the bytes of code for each language as reported by the [List Languages](https://developer.github.com/v3/repos/#list-languages) API. If the bar is reporting a language that you don't expect:
0. Click on the name of the language in the stats bar to see a list of the files that are identified as that language.
0. If you see files that you didn't write, consider moving the files into one of the [paths for vendored code](/lib/linguist/vendor.yml), or use the [manual overrides](#overrides) feature to ignore them.
diff --git a/bin/git-linguist b/bin/git-linguist
new file mode 100755
index 00000000..9541cf82
--- /dev/null
+++ b/bin/git-linguist
@@ -0,0 +1,137 @@
+#!/usr/bin/env ruby
+
+require 'linguist'
+require 'rugged'
+require 'optparse'
+require 'json'
+require 'tmpdir'
+require 'zlib'
+
+class GitLinguist
+ def initialize(path, commit_oid, incremental = true)
+ @repo_path = path
+ @commit_oid = commit_oid
+ @incremental = incremental
+ end
+
+ def linguist
+ if @commit_oid.nil?
+ raise "git-linguist must be called with a specific commit OID to perform language computation"
+ end
+ repo = Linguist::Repository.new(rugged, @commit_oid)
+
+ if @incremental && stats = load_language_stats
+ old_commit_oid, old_stats = stats
+
+ # A cache with NULL oid means that we want to froze
+ # these language stats in place and stop computing
+ # them (for performance reasons)
+ return old_stats if old_commit_oid == NULL_OID
+ repo.load_existing_stats(old_commit_oid, old_stats)
+ end
+
+ result = yield repo
+
+ save_language_stats(@commit_oid, repo.cache)
+ result
+ end
+
+ def load_language_stats
+ version, oid, stats = load_cache
+ if version == LANGUAGE_STATS_CACHE_VERSION && oid && stats
+ [oid, stats]
+ end
+ end
+
+ def save_language_stats(oid, stats)
+ cache = [LANGUAGE_STATS_CACHE_VERSION, oid, stats]
+ write_cache(cache)
+ end
+
+ def clear_language_stats
+ File.unlink(cache_file)
+ end
+
+ def disable_language_stats
+ save_language_stats(NULL_OID, {})
+ end
+
+ protected
+ NULL_OID = ("0" * 40).freeze
+
+ LANGUAGE_STATS_CACHE = 'language-stats.cache'
+ LANGUAGE_STATS_CACHE_VERSION = "v3:#{Linguist::VERSION}"
+
+ def rugged
+ @rugged ||= Rugged::Repository.bare(@repo_path)
+ end
+
+ def cache_file
+ File.join(@repo_path, LANGUAGE_STATS_CACHE)
+ end
+
+ def write_cache(object)
+ return unless File.directory? @repo_path
+
+ begin
+ tmp_path = Dir::Tmpname.make_tmpname(cache_file, nil)
+ File.open(tmp_path, "wb") do |f|
+ marshal = Marshal.dump(object)
+ f.write(Zlib::Deflate.deflate(marshal))
+ end
+
+ File.rename(tmp_path, cache_file)
+ rescue => e
+ (File.unlink(tmp_path) rescue nil)
+ raise e
+ end
+ end
+
+ def load_cache
+ marshal = File.open(cache_file, "rb") { |f| Zlib::Inflate.inflate(f.read) }
+ Marshal.load(marshal)
+ rescue SystemCallError, ::Zlib::DataError, ::Zlib::BufError, TypeError
+ nil
+ end
+end
+
+
+def git_linguist(args)
+ incremental = true
+ commit = nil
+
+ parser = OptionParser.new do |opts|
+ opts.banner = "Usage: git-linguist [OPTIONS] stats|breakdown|dump-cache|clear|disable"
+
+ opts.on("-f", "--force", "Force a full rescan") { incremental = false }
+ opts.on("--commit=COMMIT", "Commit to index") { |v| commit = v}
+ end
+
+ parser.parse!(args)
+
+ git_dir = `git rev-parse --git-dir`.strip
+ raise "git-linguist must be ran in a Git repository" unless $?.success?
+ wrapper = GitLinguist.new(git_dir, commit, incremental)
+
+ case args.pop
+ when "stats"
+ wrapper.linguist do |linguist|
+ puts JSON.dump(linguist.languages)
+ end
+ when "breakdown"
+ wrapper.linguist do |linguist|
+ puts JSON.dump(linguist.breakdown_by_file)
+ end
+ when "dump-cache"
+ puts JSON.dump(wrapper.load_language_stats)
+ when "clear"
+ wrapper.clear_language_stats
+ when "disable"
+ wrapper.disable_language_stats
+ else
+ $stderr.print(parser.help)
+ exit 1
+ end
+end
+
+git_linguist(ARGV)
diff --git a/github-linguist.gemspec b/github-linguist.gemspec
index 87bbc8bf..bdba3e6b 100644
--- a/github-linguist.gemspec
+++ b/github-linguist.gemspec
@@ -10,8 +10,8 @@ Gem::Specification.new do |s|
s.homepage = "https://github.com/github/linguist"
s.license = "MIT"
- s.files = Dir['lib/**/*'] - ['lib/linguist/grammars.rb']
- s.executables << 'linguist'
+ s.files = Dir['lib/**/*'] - ['lib/linguist/grammars.rb'] + ['LICENSE']
+ s.executables = ['linguist', 'git-linguist']
s.add_dependency 'charlock_holmes', '~> 0.7.3'
s.add_dependency 'escape_utils', '~> 1.1.0'
diff --git a/grammars.yml b/grammars.yml
index 42be3967..4bd66f12 100644
--- a/grammars.yml
+++ b/grammars.yml
@@ -92,7 +92,7 @@ vendor/grammars/Scalate.tmbundle:
- text.html.ssp
vendor/grammars/Slash.tmbundle:
- text.html.slash
-vendor/grammars/Stata.tmbundle:
+vendor/grammars/Stata.tmbundle/:
- source.mata
- source.stata
vendor/grammars/Stylus/:
@@ -178,6 +178,8 @@ vendor/grammars/assembly.tmbundle:
- source.x86asm
vendor/grammars/atom-fsharp/:
- source.fsharp
+- source.fsharp.fsi
+- source.fsharp.fsx
vendor/grammars/atom-language-purescript/:
- source.purescript
vendor/grammars/atom-salt:
@@ -314,6 +316,9 @@ vendor/grammars/json.tmbundle:
- source.json
vendor/grammars/kotlin-sublime-package:
- source.Kotlin
+vendor/grammars/language-babel/:
+- source.js.jsx
+- source.regexp.babel
vendor/grammars/language-clojure:
- source.clojure
vendor/grammars/language-coffee-script:
diff --git a/lib/linguist/generated.rb b/lib/linguist/generated.rb
index f1fb2d19..d107e737 100644
--- a/lib/linguist/generated.rb
+++ b/lib/linguist/generated.rb
@@ -241,22 +241,26 @@ module Linguist
return lines[0].include?("Code generated by")
end
+ PROTOBUF_EXTENSIONS = ['.py', '.java', '.h', '.cc', '.cpp']
+
# Internal: Is the blob a C++, Java or Python source file generated by the
# Protocol Buffer compiler?
#
# Returns true of false.
def generated_protocol_buffer?
- return false unless ['.py', '.java', '.h', '.cc', '.cpp'].include?(extname)
+ return false unless PROTOBUF_EXTENSIONS.include?(extname)
return false unless lines.count > 1
return lines[0].include?("Generated by the protocol buffer compiler. DO NOT EDIT!")
end
+ APACHE_THRIFT_EXTENSIONS = ['.rb', '.py', '.go', '.js', '.m', '.java', '.h', '.cc', '.cpp']
+
# Internal: Is the blob generated by Apache Thrift compiler?
#
# Returns true or false
def generated_apache_thrift?
- return false unless ['.rb', '.py', '.go', '.js', '.m', '.java', '.h', '.cc', '.cpp'].include?(extname)
+ return false unless APACHE_THRIFT_EXTENSIONS.include?(extname)
return false unless lines.count > 1
return lines[0].include?("Autogenerated by Thrift Compiler") || lines[1].include?("Autogenerated by Thrift Compiler")
diff --git a/lib/linguist/heuristics.rb b/lib/linguist/heuristics.rb
index dd1f8ee6..11f58b28 100644
--- a/lib/linguist/heuristics.rb
+++ b/lib/linguist/heuristics.rb
@@ -56,7 +56,8 @@ module Linguist
# Internal: Check if this heuristic matches the candidate languages.
def matches?(filename)
- @extensions.any? { |ext| filename.downcase.end_with?(ext) }
+ filename = filename.downcase
+ @extensions.any? { |ext| filename.end_with?(ext) }
end
# Internal: Perform the heuristic
@@ -65,7 +66,7 @@ module Linguist
end
# Common heuristics
- ObjectiveCRegex = /^[ \t]*@(interface|class|protocol|property|end|synchronized|selector|implementation)\b/
+ ObjectiveCRegex = /^\s*(@(interface|class|protocol|property|end|synchronised|selector|implementation)\b|#import\s+.+\.h[">])/
disambiguate ".asc" do |data|
if /^(----[- ]BEGIN|ssh-(rsa|dss)) /.match(data)
diff --git a/lib/linguist/languages.yml b/lib/linguist/languages.yml
index b2d75fa1..d7a3a18b 100644
--- a/lib/linguist/languages.yml
+++ b/lib/linguist/languages.yml
@@ -8,7 +8,8 @@
# Use "text" if a mode does not exist.
# wrap - Boolean wrap to enable line wrapping (default: false)
# extensions - An Array of associated extensions (the first one is
-# considered the primary extension)
+# considered the primary extension, the others should be
+# listed alphabetically)
# interpreters - An Array of associated interpreters
# searchable - Boolean flag to enable searching (defaults to true)
# search_term - Deprecated: Some languages maybe indexed under a
@@ -1367,6 +1368,7 @@ Haml:
Handlebars:
type: markup
color: "#01a9d6"
+ group: HTML
aliases:
- hbs
- htmlbars
@@ -1575,6 +1577,14 @@ JSONiq:
- .jq
tm_scope: source.jq
+JSX:
+ type: programming
+ group: JavaScript
+ extensions:
+ - .jsx
+ tm_scope: source.js.jsx
+ ace_mode: javascript
+
Jade:
group: HTML
type: markup
@@ -1628,7 +1638,6 @@ JavaScript:
- .jsfl
- .jsm
- .jss
- - .jsx
- .njs
- .pac
- .sjs
@@ -2448,6 +2457,7 @@ PLSQL:
color: "#dad8d8"
extensions:
- .pls
+ - .pck
- .pkb
- .pks
- .plb
@@ -3433,6 +3443,7 @@ TypeScript:
- ts
extensions:
- .ts
+ - .tsx
tm_scope: source.ts
ace_mode: typescript
@@ -3649,6 +3660,7 @@ XML:
- .tmSnippet
- .tmTheme
- .ts
+ - .tsx
- .ui
- .urdf
- .vbproj
diff --git a/lib/linguist/lazy_blob.rb b/lib/linguist/lazy_blob.rb
index 55c10309..28fb78f3 100644
--- a/lib/linguist/lazy_blob.rb
+++ b/lib/linguist/lazy_blob.rb
@@ -79,6 +79,10 @@ module Linguist
@size
end
+ def cleanup!
+ @data.clear if @data
+ end
+
protected
# Returns true if the attribute is present and not the string "false".
diff --git a/lib/linguist/repository.rb b/lib/linguist/repository.rb
index 181ddf0e..51040734 100644
--- a/lib/linguist/repository.rb
+++ b/lib/linguist/repository.rb
@@ -126,12 +126,13 @@ module Linguist
end
protected
+ MAX_TREE_SIZE = 100_000
def compute_stats(old_commit_oid, cache = nil)
+ return {} if current_tree.count_recursive(MAX_TREE_SIZE) >= MAX_TREE_SIZE
+
old_tree = old_commit_oid && Rugged::Commit.lookup(repository, old_commit_oid).tree
-
read_index
-
diff = Rugged::Tree.diff(repository, old_tree, current_tree)
# Clear file map and fetch full diff if any .gitattributes files are changed
@@ -157,8 +158,11 @@ module Linguist
blob = Linguist::LazyBlob.new(repository, delta.new_file[:oid], new, mode.to_s(8))
- next unless blob.include_in_language_stats?
- file_map[new] = [blob.language.group.name, blob.size]
+ if blob.include_in_language_stats?
+ file_map[new] = [blob.language.group.name, blob.size]
+ end
+
+ blob.cleanup!
end
end
diff --git a/lib/linguist/version.rb b/lib/linguist/version.rb
index dbdb7d7f..c5dfb5f9 100644
--- a/lib/linguist/version.rb
+++ b/lib/linguist/version.rb
@@ -1,3 +1,3 @@
module Linguist
- VERSION = "4.5.15"
+ VERSION = "4.6.4"
end
diff --git a/samples/JSX/sample.jsx b/samples/JSX/sample.jsx
new file mode 100644
index 00000000..2020618d
--- /dev/null
+++ b/samples/JSX/sample.jsx
@@ -0,0 +1,23 @@
+'use strict';
+
+const React = require('react')
+
+module.exports = React.createClass({
+ render: function() {
+ let {feeds, log} = this.props;
+
+ log.info(feeds);
+ return
+
News Feed's
+
+ {feeds.map(function(feed) {
+ return -
+ {feed.data && feed.data.length > 0 ?
+ {feed.name} ({feed.data.length})
+ : 'feed.name' }
+
+ })}
+
+
+
+
+ | # |
+ Name |
+ Description |
+ Quantity |
+ Price |
+
+');
+
+-- Fill out the parts table
+for row in (select * from parts) loop
+ htp.prn('
+
+ | '||row.pid||' |
+ '||row.name||' |
+ '||row.description||' |
+ '||row.quantity||' |
+ '||row.price||' |
+
+ ');
+end loop;
+
+htp.prn('
+
+