Merge pull request #2654 from github/vmg/git-linguist-fixes

Small fixes for git-linguist

.gitmodules

@@ -677,3 +677,6 @@
 [submodule "vendor/grammars/X10"]
 	path = vendor/grammars/X10
 	url = git@github.com:x10-lang/x10-highlighting.git
+[submodule "vendor/grammars/language-babel"]
+	path = vendor/grammars/language-babel
+	url = https://github.com/gandm/language-babel

README.md

@@ -13,7 +13,7 @@ See [Troubleshooting](#troubleshooting) and [`CONTRIBUTING.md`](/CONTRIBUTING.md
 
 ![language stats bar](https://cloud.githubusercontent.com/assets/173/5562290/48e24654-8ddf-11e4-8fe7-735b0ce3a0d3.png)
 
-The Language stats bar is built by aggregating the languages of each file in that repository. If it is reporting a language that you don't expect:
+The Language stats bar displays languages percentages for the files in the repository. The percentages are calculated based on the bytes of code for each language as reported by the [List Languages](https://developer.github.com/v3/repos/#list-languages) API. If the bar is reporting a language that you don't expect:
 
 0. Click on the name of the language in the stats bar to see a list of the files that are identified as that language.
 0. If you see files that you didn't write, consider moving the files into one of the [paths for vendored  code](/lib/linguist/vendor.yml), or use the [manual overrides](#overrides) feature to ignore them.

bin/git-linguist

@@ -0,0 +1,137 @@
+#!/usr/bin/env ruby
+
+require 'linguist'
+require 'rugged'
+require 'optparse'
+require 'json'
+require 'tmpdir'
+require 'zlib'
+
+class GitLinguist
+  def initialize(path, commit_oid, incremental = true)
+    @repo_path = path
+    @commit_oid = commit_oid
+    @incremental = incremental
+  end
+
+  def linguist
+    if @commit_oid.nil?
+      raise "git-linguist must be called with a specific commit OID to perform language computation"
+    end
+    repo = Linguist::Repository.new(rugged, @commit_oid)
+
+    if @incremental && stats = load_language_stats
+      old_commit_oid, old_stats = stats
+
+      # A cache with NULL oid means that we want to froze
+      # these language stats in place and stop computing
+      # them (for performance reasons)
+      return old_stats if old_commit_oid == NULL_OID
+      repo.load_existing_stats(old_commit_oid, old_stats)
+    end
+
+    result = yield repo
+
+    save_language_stats(@commit_oid, repo.cache)
+    result
+  end
+
+  def load_language_stats
+    version, oid, stats = load_cache
+    if version == LANGUAGE_STATS_CACHE_VERSION && oid && stats
+      [oid, stats]
+    end
+  end
+
+  def save_language_stats(oid, stats)
+    cache = [LANGUAGE_STATS_CACHE_VERSION, oid, stats]
+    write_cache(cache)
+  end
+
+  def clear_language_stats
+    File.unlink(cache_file)
+  end
+
+  def disable_language_stats
+    save_language_stats(NULL_OID, {})
+  end
+
+  protected
+  NULL_OID = ("0" * 40).freeze
+
+  LANGUAGE_STATS_CACHE = 'language-stats.cache'
+  LANGUAGE_STATS_CACHE_VERSION = "v3:#{Linguist::VERSION}"
+
+  def rugged
+    @rugged ||= Rugged::Repository.bare(@repo_path)
+  end
+
+  def cache_file
+    File.join(@repo_path, LANGUAGE_STATS_CACHE)
+  end
+
+  def write_cache(object)
+    return unless File.directory? @repo_path
+
+    begin
+      tmp_path = Dir::Tmpname.make_tmpname(cache_file, nil)
+      File.open(tmp_path, "wb") do |f|
+        marshal = Marshal.dump(object)
+        f.write(Zlib::Deflate.deflate(marshal))
+      end
+
+      File.rename(tmp_path, cache_file)
+    rescue => e
+      (File.unlink(tmp_path) rescue nil)
+      raise e
+    end
+  end
+
+  def load_cache
+    marshal = File.open(cache_file, "rb") { |f| Zlib::Inflate.inflate(f.read) }
+    Marshal.load(marshal)
+  rescue SystemCallError, ::Zlib::DataError, ::Zlib::BufError, TypeError
+    nil
+  end
+end
+
+
+def git_linguist(args)
+  incremental = true
+  commit = nil
+
+  parser = OptionParser.new do |opts|
+    opts.banner = "Usage: git-linguist [OPTIONS] stats|breakdown|dump-cache|clear|disable"
+
+    opts.on("-f", "--force", "Force a full rescan") { incremental = false }
+    opts.on("--commit=COMMIT", "Commit to index") { |v| commit = v}
+  end
+
+  parser.parse!(args)
+
+  git_dir = `git rev-parse --git-dir`.strip
+  raise "git-linguist must be ran in a Git repository" unless $?.success?
+  wrapper = GitLinguist.new(git_dir, commit, incremental)
+
+  case args.pop
+  when "stats"
+    wrapper.linguist do |linguist|
+      puts JSON.dump(linguist.languages)
+    end
+  when "breakdown"
+    wrapper.linguist do |linguist|
+      puts JSON.dump(linguist.breakdown_by_file)
+    end
+  when "dump-cache"
+    puts JSON.dump(wrapper.load_language_stats)
+  when "clear"
+    wrapper.clear_language_stats
+  when "disable"
+    wrapper.disable_language_stats
+  else
+    $stderr.print(parser.help)
+    exit 1
+  end
+end
+
+git_linguist(ARGV)

github-linguist.gemspec

@@ -10,8 +10,8 @@ Gem::Specification.new do |s|
   s.homepage = "https://github.com/github/linguist"
   s.license  = "MIT"
 
-  s.files = Dir['lib/**/*'] - ['lib/linguist/grammars.rb']
-  s.executables << 'linguist'
+  s.files = Dir['lib/**/*'] - ['lib/linguist/grammars.rb'] + ['LICENSE']
+  s.executables = ['linguist', 'git-linguist']
 
   s.add_dependency 'charlock_holmes', '~> 0.7.3'
   s.add_dependency 'escape_utils',    '~> 1.1.0'

grammars.yml

@@ -314,6 +314,9 @@ vendor/grammars/json.tmbundle:
 - source.json
 vendor/grammars/kotlin-sublime-package:
 - source.Kotlin
+vendor/grammars/language-babel/:
+- source.js.jsx
+- source.regexp.babel
 vendor/grammars/language-clojure:
 - source.clojure
 vendor/grammars/language-coffee-script:

lib/linguist/generated.rb

@@ -241,22 +241,26 @@ module Linguist
       return lines[0].include?("Code generated by")
     end
 
+    PROTOBUF_EXTENSIONS = ['.py', '.java', '.h', '.cc', '.cpp']
+
     # Internal: Is the blob a C++, Java or Python source file generated by the
     # Protocol Buffer compiler?
     #
     # Returns true of false.
     def generated_protocol_buffer?
-      return false unless ['.py', '.java', '.h', '.cc', '.cpp'].include?(extname)
+      return false unless PROTOBUF_EXTENSIONS.include?(extname)
       return false unless lines.count > 1
 
       return lines[0].include?("Generated by the protocol buffer compiler.  DO NOT EDIT!")
     end
 
+    APACHE_THRIFT_EXTENSIONS = ['.rb', '.py', '.go', '.js', '.m', '.java', '.h', '.cc', '.cpp']
+
     # Internal: Is the blob generated by Apache Thrift compiler?
     #
     # Returns true or false
     def generated_apache_thrift?
-      return false unless ['.rb', '.py', '.go', '.js', '.m', '.java', '.h', '.cc', '.cpp'].include?(extname)
+      return false unless APACHE_THRIFT_EXTENSIONS.include?(extname)
       return false unless lines.count > 1
 
       return lines[0].include?("Autogenerated by Thrift Compiler") || lines[1].include?("Autogenerated by Thrift Compiler")

lib/linguist/heuristics.rb

@@ -56,7 +56,8 @@ module Linguist
 
     # Internal: Check if this heuristic matches the candidate languages.
     def matches?(filename)
-      @extensions.any? { |ext| filename.downcase.end_with?(ext) }
+      filename = filename.downcase
+      @extensions.any? { |ext| filename.end_with?(ext) }
     end
 
     # Internal: Perform the heuristic
@@ -65,7 +66,7 @@ module Linguist
     end
 
     # Common heuristics
-    ObjectiveCRegex = /^[ \t]*@(interface|class|protocol|property|end|synchronized|selector|implementation)\b/
+    ObjectiveCRegex = /^\s*(@(interface|class|protocol|property|end|synchronised|selector|implementation)\b|#import\s+.+\.h[">])/
 
     disambiguate ".asc" do |data|
       if /^(----[- ]BEGIN|ssh-(rsa|dss)) /.match(data)

lib/linguist/languages.yml

@@ -8,7 +8,8 @@
 #                     Use "text" if a mode does not exist.
 # wrap              - Boolean wrap to enable line wrapping (default: false)
 # extensions        - An Array of associated extensions (the first one is
-#                     considered the primary extension)
+#                     considered the primary extension, the others should be
+#                     listed alphabetically)
 # interpreters      - An Array of associated interpreters
 # searchable        - Boolean flag to enable searching (defaults to true)
 # search_term       - Deprecated: Some languages maybe indexed under a
@@ -1367,6 +1368,7 @@ Haml:
 Handlebars:
   type: markup
   color: "#01a9d6"
+  group: HTML
   aliases:
   - hbs
   - htmlbars
@@ -1545,7 +1547,9 @@ JSON:
   searchable: false
   extensions:
   - .json
+  - .geojson
   - .lock
+  - .topojson
   filenames:
   - .jshintrc
   - composer.lock
@@ -1573,6 +1577,14 @@ JSONiq:
   - .jq
   tm_scope: source.jq
 
+JSX:
+  type: programming
+  group: JavaScript
+  extensions:
+  - .jsx
+  tm_scope: source.js.jsx
+  ace_mode: javascript
+
 Jade:
   group: HTML
   type: markup
@@ -1626,7 +1638,6 @@ JavaScript:
   - .jsfl
   - .jsm
   - .jss
-  - .jsx
   - .njs
   - .pac
   - .sjs
@@ -2127,7 +2138,7 @@ Myghty:
 
 NCL:
   type: programming
-  color: #28431f
+  color: "#28431f"
   extensions:
   - .ncl
   tm_scope: source.ncl

lib/linguist/lazy_blob.rb

@@ -4,7 +4,11 @@ require 'rugged'
 
 module Linguist
   class LazyBlob
-    GIT_ATTR = ['linguist-documentation', 'linguist-language', 'linguist-vendored']
+    GIT_ATTR = ['linguist-documentation',
+                'linguist-language',
+                'linguist-vendored',
+                'linguist-generated']
+
     GIT_ATTR_OPTS = { :priority => [:index], :skip_system => true }
     GIT_ATTR_FLAGS = Rugged::Repository::Attributes.parse_opts(GIT_ATTR_OPTS)
 
@@ -31,14 +35,6 @@ module Linguist
         name, GIT_ATTR, GIT_ATTR_FLAGS)
     end
 
-    def vendored?
-      if attr = git_attributes['linguist-vendored']
-        return boolean_attribute(attr)
-      else
-        return super
-      end
-    end
-
     def documentation?
       if attr = git_attributes['linguist-documentation']
         boolean_attribute(attr)
@@ -47,6 +43,22 @@ module Linguist
       end
     end
 
+    def generated?
+      if attr = git_attributes['linguist-generated']
+        boolean_attribute(attr)
+      else
+        super
+      end
+    end
+
+    def vendored?
+      if attr = git_attributes['linguist-vendored']
+        return boolean_attribute(attr)
+      else
+        super
+      end
+    end
+
     def language
       return @language if defined?(@language)
 
@@ -67,6 +79,10 @@ module Linguist
       @size
     end
 
+    def cleanup!
+      @data.clear if @data
+    end
+
     protected
 
     # Returns true if the attribute is present and not the string "false".

lib/linguist/repository.rb

@@ -126,12 +126,13 @@ module Linguist
     end
 
     protected
+    MAX_TREE_SIZE = 100_000
 
     def compute_stats(old_commit_oid, cache = nil)
+      return {} if current_tree.count_recursive(MAX_TREE_SIZE) >= MAX_TREE_SIZE
+
       old_tree = old_commit_oid && Rugged::Commit.lookup(repository, old_commit_oid).tree
-
       read_index
-
       diff = Rugged::Tree.diff(repository, old_tree, current_tree)
 
       # Clear file map and fetch full diff if any .gitattributes files are changed
@@ -157,9 +158,12 @@ module Linguist
 
           blob = Linguist::LazyBlob.new(repository, delta.new_file[:oid], new, mode.to_s(8))
 
-          next unless blob.include_in_language_stats?
+          if blob.include_in_language_stats?
             file_map[new] = [blob.language.group.name, blob.size]
           end
+
+          blob.cleanup!
+        end
       end
 
       file_map

lib/linguist/vendor.yml

@@ -266,6 +266,7 @@
 
 # Test fixtures
 - ^[Tt]ests?/fixtures/
+- ^[Ss]pecs?/fixtures/
 
 # PhoneGap/Cordova
 - (^|/)cordova([^.]*)\.js$

lib/linguist/version.rb

@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "4.5.15"
+  VERSION = "4.6.3"
 end

samples/JSON/geo.geojson

@@ -0,0 +1,82 @@
+{
+	"type": "FeatureCollection",
+	"features": [
+		{
+			"type": "Feature",
+			"properties": {
+				"name": "Australia Post - North Ryde BC",
+				"geo": [-33.787792, 151.13288],
+				"streetAddress": "11 Waterloo Road",
+				"addressLocality": "Macquarie Park",
+				"addressRegion": "New South Wales",
+				"addressCountry": "Australia",
+				"postalCode": "2113"
+			},
+			"geometry": {
+				"type": "Point",
+				"coordinates": [151.13288, -33.787792, 0]
+			}
+		},
+
+
+		{
+			"type": "Feature",
+			"properties": {
+				"name": "George Weston Foods Limited",
+				"geo": [-37.8263884, 144.9105381],
+				"streetAddress": "Level 3, 187 Todd Road",
+				"addressLocality": "Port Melbourne",
+				"addressRegion": "Victoria",
+				"addressCountry": "Australia",
+				"postalCode": "3207"
+			},
+			"geometry": {
+				"type": "Polygon",
+				"coordinates": [
+					[
+						[144.9097088901841, -37.82622654171794, 0],
+						[144.9099724266943, -37.82679388891783, 0],
+						[144.9110127325916, -37.82651526396403, 0],
+						[144.9112227645738, -37.82655667152123, 0],
+						[144.9113739439796, -37.82618552508767, 0],
+						[144.9112740633105, -37.82615750100924, 0],
+						[144.9111355846674, -37.82584493693527, 0],
+						[144.9097088901841, -37.82622654171794, 0]
+					]
+				]
+			}
+		},
+
+
+		{
+			"type": "Feature",
+			"properties": {
+				"name": "George Weston Foods Limited",
+				"geo": [-37.05202791502396, 144.2085614999388],
+				"streetAddress": "67 Richards Road",
+				"addressLocality": "Castlemaine",
+				"addressRegion": "Victoria",
+				"addressCountry": "Australia",
+				"postalCode": "3450"
+			},
+			"geometry": {
+				"type": "Polygon",
+				"coordinates": [
+					[
+						[144.2052428913937, -37.04906391287216, 0],
+						[144.205540392692,  -37.05049727485623, 0],
+						[144.2059800881858, -37.05066835966983, 0],
+						[144.206490656024,  -37.05279538900776, 0],
+						[144.2064525845008, -37.05366195881602, 0],
+						[144.2084322301922, -37.0538920493147,  0],
+						[144.2084811895712, -37.05266519735124, 0],
+						[144.2079784002005, -37.05041270555773, 0],
+						[144.2074017905817, -37.04817406993293, 0],
+						[144.2061363939852, -37.04834972871226, 0],
+						[144.2052428913937, -37.04906391287216, 0]
+					]
+				]
+			}
+		}
+	]
+}

samples/JSX/sample.jsx

@@ -0,0 +1,23 @@
+'use strict';
+
+const React = require('react')
+
+module.exports = React.createClass({
+  render: function() {
+    let {feeds, log} = this.props;
+
+    log.info(feeds);
+    return <div className="feed-list">
+      <h3>News Feed's</h3>
+      <ul>
+        {feeds.map(function(feed) {
+          return <li key={feed.name} className={feed.fetched ? 'loaded' : 'loading'}>
+            {feed.data && feed.data.length > 0 ?
+              <span>{feed.name} <span className='light'>({feed.data.length})</span></span>
+              : 'feed.name' }
+          </li>
+        })}
+      </ul>
+    </div>;
+  }
+});

samples/Objective-C/Siesta.h

@@ -0,0 +1,16 @@
+//
+//  Siesta.h
+//  Siesta
+//
+//  Created by Paul on 2015/6/14.
+//  Copyright © 2015 Bust Out Solutions. MIT license.
+//
+
+#import <UIKit/UIKit.h>
+
+//! Project version number for Siesta.
+FOUNDATION_EXPORT double SiestaVersionNumber;
+
+//! Project version string for Siesta.
+FOUNDATION_EXPORT const unsigned char SiestaVersionString[];
+

test/test_pedantic.rb

@@ -9,7 +9,7 @@ class TestPedantic < Minitest::Test
     assert_sorted LANGUAGES.keys
   end
 
-  def test_extensions_are_sorted
+  def test_nonprimary_extensions_are_sorted
     LANGUAGES.each do |name, language|
       extensions = language['extensions']
       assert_sorted extensions[1..-1].map(&:downcase) if extensions && extensions.size > 1

test/test_repository.rb

@@ -111,4 +111,14 @@ class TestRepository < Minitest::Test
     refute_predicate readme, :documentation?
     assert_predicate arduino, :documentation?
   end
+
+  def test_linguist_override_generated?
+    attr_commit = "351c1cc8fd57340839bdb400d7812332af80e9bd"
+    repo = linguist_repo(attr_commit).read_index
+
+    rakefile = Linguist::LazyBlob.new(rugged_repository, attr_commit, "Rakefile")
+
+    # overridden .gitattributes
+    assert rakefile.generated?
+  end
 end