KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into newlisp

Browse Source
This commit is contained in:
Paul Chaignon
2014-09-25 10:47:16 -04:00
parent e4975fc476 5152bd7124
commit 7b44baa417
242 changed files with 151969 additions and 57041 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
lib/linguist/blob_helper.rb
View File

@@ -1,6 +1,4 @@
require 'linguist/generated'
require 'linguist/language'
require 'charlock_holmes'
require 'escape_utils'
require 'mime/types'
@@ -112,6 +110,12 @@ module Linguist
end
end
def ruby_encoding
if hash = detect_encoding
hash[:ruby_encoding]
end
end
# Try to guess the encoding
#
# Returns: a Hash, with :encoding, :confidence, :type
@@ -241,7 +245,31 @@ module Linguist
def lines
@lines ||=
if viewable? && data
data.split(/\r\n|\r|\n/, -1)
# `data` is usually encoded as ASCII-8BIT even when the content has
# been detected as a different encoding. However, we are not allowed
# to change the encoding of `data` because we've made the implicit
# guarantee that each entry in `lines` is encoded the same way as
# `data`.
#
# Instead, we re-encode each possible newline sequence as the
# detected encoding, then force them back to the encoding of `data`
# (usually a binary encoding like ASCII-8BIT). This means that the
# byte sequence will match how newlines are likely encoded in the
# file, but we don't have to change the encoding of `data` as far as
# Ruby is concerned. This allows us to correctly parse out each line
# without changing the encoding of `data`, and
# also--importantly--without having to duplicate many (potentially
# large) strings.
begin
encoded_newlines = ["\r\n", "\r", "\n"].
map { |nl| nl.encode(ruby_encoding, "ASCII-8BIT").force_encoding(data.encoding) }
data.split(Regexp.union(encoded_newlines), -1)
rescue Encoding::ConverterNotFoundError
# The data is not splittable in the detected encoding. Assume it's
# one big line.
[data]
end
else
[]
end
@@ -283,15 +311,7 @@ module Linguist
#
# Returns a Language or nil if none is detected
def language
return @language if defined? @language
if defined?(@data) && @data.is_a?(String)
data = @data
else
data = lambda { (binary_mime_type? || binary?) ? "" : self.data }
end
@language = Language.detect(name.to_s, data, mode)
@language ||= Language.detect(self)
end
# Internal: Get the lexer of the blob.

15
lib/linguist/file_blob.rb
View File

@@ -52,5 +52,20 @@ module Linguist
def size
File.size(@path)
end
# Public: Get file extension.
#
# Returns a String.
def extension
# File.extname returns nil if the filename is an extension.
extension = File.extname(name)
basename = File.basename(name)
# Checks if the filename is an extension.
if extension.empty? && basename[0] == "."
basename
else
extension
end
end
end
end

32
lib/linguist/generated.rb
View File

@@ -54,7 +54,7 @@ module Linguist
name == 'Gemfile.lock' ||
minified_files? ||
compiled_coffeescript? ||
xcode_project_file? ||
xcode_file? ||
generated_parser? ||
generated_net_docfile? ||
generated_net_designer_file? ||
@@ -63,17 +63,19 @@ module Linguist
generated_jni_header? ||
composer_lock? ||
node_modules? ||
vcr_cassette?
godeps? ||
vcr_cassette? ||
generated_by_zephir?
end
# Internal: Is the blob an XCode project file?
# Internal: Is the blob an Xcode file?
#
# Generated if the file extension is an XCode project
# Generated if the file extension is an Xcode
# file extension.
#
# Returns true of false.
def xcode_project_file?
['.xib', '.nib', '.storyboard', '.pbxproj', '.xcworkspacedata', '.xcuserstate'].include?(extname)
def xcode_file?
['.nib', '.xcworkspacedata', '.xcuserstate'].include?(extname)
end
# Internal: Is the blob minified files?
@@ -230,11 +232,26 @@ module Linguist
!!name.match(/node_modules\//)
end
# Internal: Is the blob part of Godeps/,
# which are not meant for humans in pull requests.
#
# Returns true or false.
def godeps?
!!name.match(/Godeps\//)
end
# Internal: Is the blob a generated php composer lock file?
#
# Returns true or false.
def composer_lock?
!!name.match(/composer.lock/)
!!name.match(/composer\.lock/)
end
# Internal: Is the blob a generated by Zephir
#
# Returns true or false.
def generated_by_zephir?
!!name.match(/.\.zep\.(?:c|h|php)$/)
end
# Is the blob a VCR Cassette file?
@@ -248,3 +265,4 @@ module Linguist
end
end
end

20
lib/linguist/heuristics.rb
View File

@@ -1,7 +1,7 @@
module Linguist
# A collection of simple heuristics that can be used to better analyze languages.
class Heuristics
ACTIVE = false
ACTIVE = true
# Public: Given an array of String language names,
# apply heuristics against the given data and return an array
@@ -13,31 +13,23 @@ module Linguist
# Returns an array of Languages or []
def self.find_by_heuristics(data, languages)
if active?
if languages.all? { |l| ["Objective-C", "C++"].include?(l) }
disambiguate_c(data, languages)
end
if languages.all? { |l| ["Perl", "Prolog"].include?(l) }
disambiguate_pl(data, languages)
result = disambiguate_pl(data, languages)
end
if languages.all? { |l| ["ECL", "Prolog"].include?(l) }
disambiguate_ecl(data, languages)
end
if languages.all? { |l| ["TypeScript", "XML"].include?(l) }
disambiguate_ts(data, languages)
result = disambiguate_ecl(data, languages)
end
if languages.all? { |l| ["Common Lisp", "NewLisp"].include?(l) }
disambiguate_lsp(data, languages)
end
if languages.all? { |l| ["Common Lisp", "OpenCL"].include?(l) }
disambiguate_cl(data, languages)
end
if languages.all? { |l| ["Rebol", "R"].include?(l) }
disambiguate_r(data, languages)
result = disambiguate_cl(data, languages)
end
return result
end
end
# .h extensions are ambigious between C, C++, and Objective-C.
# .h extensions are ambiguous between C, C++, and Objective-C.
# We want to shortcut look for Objective-C _and_ now C++ too!
#
# Returns an array of Languages or []

95
lib/linguist/language.rb
View File

@@ -9,6 +9,8 @@ end
require 'linguist/classifier'
require 'linguist/heuristics'
require 'linguist/samples'
require 'linguist/file_blob'
require 'linguist/blob_helper'
module Linguist
# Language names that are recognizable by GitHub. Defined languages
@@ -24,7 +26,6 @@ module Linguist
@extension_index = Hash.new { |h,k| h[k] = [] }
@interpreter_index = Hash.new { |h,k| h[k] = [] }
@filename_index = Hash.new { |h,k| h[k] = [] }
@primary_extension_index = {}
# Valid Languages types
TYPES = [:data, :markup, :programming, :prose]
@@ -80,12 +81,6 @@ module Linguist
@extension_index[extension] << language
end
if @primary_extension_index.key?(language.primary_extension)
raise ArgumentError, "Duplicate primary extension: #{language.primary_extension}"
end
@primary_extension_index[language.primary_extension] = language
language.interpreters.each do |interpreter|
@interpreter_index[interpreter] << language
end
@@ -99,18 +94,25 @@ module Linguist
# Public: Detects the Language of the blob.
#
# name - String filename
# data - String blob data. A block also maybe passed in for lazy
# loading. This behavior is deprecated and you should always
# pass in a String.
# mode - Optional String mode (defaults to nil)
# blob - an object that includes the Linguist `BlobHelper` interface;
# see Linguist::LazyBlob and Linguist::FileBlob for examples
#
# Returns Language or nil.
def self.detect(name, data, mode = nil)
def self.detect(blob)
name = blob.name.to_s
# Check if the blob is possibly binary and bail early; this is a cheap
# test that uses the extension name to guess a binary binary mime type.
#
# We'll perform a more comprehensive test later which actually involves
# looking for binary characters in the blob
return nil if blob.likely_binary? || blob.binary?
# A bit of an elegant hack. If the file is executable but extensionless,
# append a "magic" extension so it can be classified with other
# languages that have shebang scripts.
if File.extname(name).empty? && mode && (mode.to_i(8) & 05) == 05
extension = FileBlob.new(name).extension
if extension.empty? && blob.mode && (blob.mode.to_i(8) & 05) == 05
name += ".script!"
end
@@ -121,10 +123,10 @@ module Linguist
# extension at all, in the case of extensionless scripts), we need to continue
# our detection work
if possible_languages.length > 1
data = data.call() if data.respond_to?(:call)
data = blob.data
possible_language_names = possible_languages.map(&:name)
# Don't bother with emptiness
# Don't bother with binary contents or an empty file
if data.nil? || data == ""
nil
# Check if there's a shebang line and use that as authoritative
@@ -133,8 +135,8 @@ module Linguist
# No shebang. Still more work to do. Try to find it with our heuristics.
elsif (determined = Heuristics.find_by_heuristics(data, possible_language_names)) && !determined.empty?
determined.first
# Lastly, fall back to the probablistic classifier.
elsif classified = Classifier.classify(Samples::DATA, data, possible_language_names ).first
# Lastly, fall back to the probabilistic classifier.
elsif classified = Classifier.classify(Samples.cache, data, possible_language_names).first
# Return the actual Language object based of the string language name (i.e., first element of `#classify`)
Language[classified[0]]
end
@@ -190,9 +192,9 @@ module Linguist
#
# Returns all matching Languages or [] if none were found.
def self.find_by_filename(filename)
basename, extname = File.basename(filename), File.extname(filename)
langs = [@primary_extension_index[extname]] +
@filename_index[basename] +
basename = File.basename(filename)
extname = FileBlob.new(filename).extension
langs = @filename_index[basename] +
@extension_index[extname]
langs.compact.uniq
end
@@ -299,15 +301,6 @@ module Linguist
@interpreters = attributes[:interpreters] || []
@filenames = attributes[:filenames] || []
unless @primary_extension = attributes[:primary_extension]
raise ArgumentError, "#{@name} is missing primary extension"
end
# Prepend primary extension unless its already included
if primary_extension && !extensions.include?(primary_extension)
@extensions = [primary_extension] + extensions
end
# Set popular, and searchable flags
@popular = attributes.key?(:popular) ? attributes[:popular] : false
@searchable = attributes.key?(:searchable) ? attributes[:searchable] : true
@@ -395,20 +388,6 @@ module Linguist
# Returns the extensions Array
attr_reader :extensions
# Deprecated: Get primary extension
#
# Defaults to the first extension but can be overridden
# in the languages.yml.
#
# The primary extension can not be nil. Tests should verify this.
#
# This attribute is only used by app/helpers/gists_helper.rb for
# creating the language dropdown. It really should be using `name`
# instead. Would like to drop primary extension.
#
# Returns the extension String.
attr_reader :primary_extension
# Public: Get interpreters
#
# Examples
@@ -426,12 +405,28 @@ module Linguist
#
# Returns the extensions Array
attr_reader :filenames
# Public: Return all possible extensions for language
def all_extensions
(extensions + [primary_extension]).uniq
end
# Deprecated: Get primary extension
#
# Defaults to the first extension but can be overridden
# in the languages.yml.
#
# The primary extension can not be nil. Tests should verify this.
#
# This method is only used by app/helpers/gists_helper.rb for creating
# the language dropdown. It really should be using `name` instead.
# Would like to drop primary extension.
#
# Returns the extension String.
def primary_extension
extensions.first
end
# Public: Get URL escaped name.
#
# Examples
@@ -515,9 +510,9 @@ module Linguist
end
end
extensions = Samples::DATA['extnames']
interpreters = Samples::DATA['interpreters']
filenames = Samples::DATA['filenames']
extensions = Samples.cache['extnames']
interpreters = Samples.cache['interpreters']
filenames = Samples.cache['filenames']
popular = YAML.load_file(File.expand_path("../popular.yml", __FILE__))
languages_yml = File.expand_path("../languages.yml", __FILE__)
@@ -537,6 +532,7 @@ module Linguist
if extnames = extensions[name]
extnames.each do |extname|
if !options['extensions'].include?(extname)
warn "#{name} has a sample with extension (#{extname}) that isn't explicitly defined in languages.yml" unless extname == '.script!'
options['extensions'] << extname
end
end
@@ -573,9 +569,8 @@ module Linguist
:group_name => options['group'],
:searchable => options.key?('searchable') ? options['searchable'] : true,
:search_term => options['search_term'],
:extensions => options['extensions'].sort,
:extensions => [options['extensions'].first] + options['extensions'][1..-1].sort,
:interpreters => options['interpreters'].sort,
:primary_extension => options['primary_extension'],
:filenames => options['filenames'],
:popular => popular.include?(name)
)

1122
lib/linguist/languages.yml
View File

File diff suppressed because it is too large Load Diff

37
lib/linguist/lazy_blob.rb Normal file
View File

@@ -0,0 +1,37 @@
require 'linguist/blob_helper'
require 'rugged'
module Linguist
class LazyBlob
include BlobHelper
MAX_SIZE = 128 * 1024
attr_reader :repository
attr_reader :oid
attr_reader :name
attr_reader :mode
def initialize(repo, oid, name, mode = nil)
@repository = repo
@oid = oid
@name = name
@mode = mode
end
def data
load_blob!
@data
end
def size
load_blob!
@size
end
protected
def load_blob!
@data, @size = Rugged::Blob.to_buffer(repository, oid, MAX_SIZE) if @data.nil?
end
end
end

159
lib/linguist/repository.rb
View File

@@ -1,4 +1,5 @@
require 'linguist/file_blob'
require 'linguist/lazy_blob'
require 'rugged'
module Linguist
# A Repository is an abstraction of a Grit::Repo or a basic file
@@ -7,100 +8,146 @@ module Linguist
# Its primary purpose is for gathering language statistics across
# the entire project.
class Repository
# Public: Initialize a new Repository from a File directory
#
# base_path - A path String
#
# Returns a Repository
def self.from_directory(base_path)
new Dir["#{base_path}/**/*"].
select { |f| File.file?(f) }.
map { |path| FileBlob.new(path, base_path) }
attr_reader :repository
# Public: Create a new Repository based on the stats of
# an existing one
def self.incremental(repo, commit_oid, old_commit_oid, old_stats)
repo = self.new(repo, commit_oid)
repo.load_existing_stats(old_commit_oid, old_stats)
repo
end
# Public: Initialize a new Repository
# Public: Initialize a new Repository to be analyzed for language
# data
#
# enum - Enumerator that responds to `each` and
# yields Blob objects
# repo - a Rugged::Repository object
# commit_oid - the sha1 of the commit that will be analyzed;
# this is usually the master branch
#
# Returns a Repository
def initialize(enum)
@enum = enum
@computed_stats = false
@language = @size = nil
@sizes = Hash.new { 0 }
@file_breakdown = Hash.new { |h,k| h[k] = Array.new }
def initialize(repo, commit_oid)
@repository = repo
@commit_oid = commit_oid
raise TypeError, 'commit_oid must be a commit SHA1' unless commit_oid.is_a?(String)
end
# Public: Load the results of a previous analysis on this repository
# to speed up the new scan.
#
# The new analysis will be performed incrementally as to only take
# into account the file changes since the last time the repository
# was scanned
#
# old_commit_oid - the sha1 of the commit that was previously analyzed
# old_stats - the result of the previous analysis, obtained by calling
# Repository#cache on the old repository
#
# Returns nothing
def load_existing_stats(old_commit_oid, old_stats)
@old_commit_oid = old_commit_oid
@old_stats = old_stats
nil
end
# Public: Returns a breakdown of language stats.
#
# Examples
#
# # => { Language['Ruby'] => 46319,
# Language['JavaScript'] => 258 }
# # => { 'Ruby' => 46319,
# 'JavaScript' => 258 }
#
# Returns a Hash of Language keys and Integer size values.
# Returns a Hash of language names and Integer size values.
def languages
compute_stats
@sizes
@sizes ||= begin
sizes = Hash.new { 0 }
cache.each do |_, (language, size)|
sizes[language] += size
end
sizes
end
end
# Public: Get primary Language of repository.
#
# Returns a Language
# Returns a language name
def language
compute_stats
@language
@language ||= begin
primary = languages.max_by { |(_, size)| size }
primary && primary[0]
end
end
# Public: Get the total size of the repository.
#
# Returns a byte size Integer
def size
compute_stats
@size
@size ||= languages.inject(0) { |s,(_,v)| s + v }
end
# Public: Return the language breakdown of this repository by file
#
# Returns a map of language names => [filenames...]
def breakdown_by_file
compute_stats
@file_breakdown
@file_breakdown ||= begin
breakdown = Hash.new { |h,k| h[k] = Array.new }
cache.each do |filename, (language, _)|
breakdown[language] << filename
end
breakdown
end
end
# Internal: Compute language breakdown for each blob in the Repository.
# Public: Return the cached results of the analysis
#
# Returns nothing
def compute_stats
return if @computed_stats
# This is a per-file breakdown that can be passed to other instances
# of Linguist::Repository to perform incremental scans
#
# Returns a map of filename => [language, size]
def cache
@cache ||= begin
if @old_commit_oid == @commit_oid
@old_stats
else
compute_stats(@old_commit_oid, @commit_oid, @old_stats)
end
end
end
@enum.each do |blob|
# Skip files that are likely binary
next if blob.likely_binary?
protected
def compute_stats(old_commit_oid, commit_oid, cache = nil)
file_map = cache ? cache.dup : {}
old_tree = old_commit_oid && Rugged::Commit.lookup(repository, old_commit_oid).tree
new_tree = Rugged::Commit.lookup(repository, commit_oid).tree
# Skip vendored or generated blobs
next if blob.vendored? || blob.generated? || blob.language.nil?
diff = Rugged::Tree.diff(repository, old_tree, new_tree)
# Only include programming languages and acceptable markup languages
if blob.language.type == :programming || Language.detectable_markup.include?(blob.language.name)
diff.each_delta do |delta|
old = delta.old_file[:path]
new = delta.new_file[:path]
# Build up the per-file breakdown stats
@file_breakdown[blob.language.group.name] << blob.name
file_map.delete(old)
next if delta.binary
@sizes[blob.language.group] += blob.size
if [:added, :modified].include? delta.status
# Skip submodules
mode = delta.new_file[:mode]
next if (mode & 040000) != 0
blob = Linguist::LazyBlob.new(repository, delta.new_file[:oid], new, mode.to_s(8))
# Skip vendored or generated blobs
next if blob.vendored? || blob.generated? || blob.language.nil?
# Only include programming languages and acceptable markup languages
if blob.language.type == :programming || Language.detectable_markup.include?(blob.language.name)
file_map[new] = [blob.language.group.name, blob.size]
end
end
end
# Compute total size
@size = @sizes.inject(0) { |s,(_,v)| s + v }
# Get primary language
if primary = @sizes.max_by { |(_, size)| size }
@language = primary[0]
end
@computed_stats = true
nil
file_map
end
end
end

124570
lib/linguist/samples.json
View File

File diff suppressed because it is too large Load Diff

10
lib/linguist/samples.rb
View File

@@ -17,9 +17,11 @@ module Linguist
PATH = File.expand_path('../samples.json', __FILE__)
# Hash of serialized samples object
if File.exist?(PATH)
serializer = defined?(JSON) ? JSON : YAML
DATA = serializer.load(File.read(PATH))
def self.cache
@cache ||= begin
serializer = defined?(JSON) ? JSON : YAML
serializer.load(File.read(PATH))
end
end
# Public: Iterate over each sample.
@@ -28,7 +30,7 @@ module Linguist
#
# Returns nothing.
def self.each(&block)
Dir.entries(ROOT).each do |category|
Dir.entries(ROOT).sort!.each do |category|
next if category == '.' || category == '..'
# Skip text and binary for now

57
lib/linguist/vendor.yml
View File

@@ -33,16 +33,36 @@
# Erlang bundles
- ^rebar$
# Go dependencies
- Godeps/_workspace/
# Bootstrap minified css and js
- (^|/)bootstrap([^.]*)(\.min)?\.(js|css)$
# Font Awesome
- font-awesome.min.css
- font-awesome.css
# Foundation css
- foundation.min.css
- foundation.css
# Normalize.css
- normalize.css
# Bourbon SCSS
- (^|/)[Bb]ourbon/.*\.css$
- (^|/)[Bb]ourbon/.*\.scss$
# Animate.css
- animate.css
- animate.min.css
# Vendored dependencies
- thirdparty/
- third[-_]?party/
- 3rd[-_]?party/
- vendors?/
- extern(al)?/
# Debian packaging
- ^debian/
@@ -98,9 +118,20 @@
# AngularJS
- (^|/)angular([^.]*)(\.min)?\.js$
# D3.js
- (^|\/)d3(\.v\d+)?([^.]*)(\.min)?\.js$
# React
- (^|/)react(-[^.]*)?(\.min)?\.js$
# Modernizr
- (^|/)modernizr\-\d\.\d+(\.\d+)?(\.min)?\.js$
- (^|/)modernizr\.custom\.\d+\.js$
# Knockout
- (^|/)knockout-(\d+\.){3}(debug\.)?js$
- knockout-min.js
## Python ##
# django
@@ -117,6 +148,9 @@
## Obj-C ##
# Cocoapods
- ^Pods/
# Sparkle
- (^|/)Sparkle/
@@ -141,7 +175,7 @@
- (^|/)[Mm]icrosoft([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?\.js$
# NuGet
- ^[Pp]ackages/
- ^[Pp]ackages\/.+\.\d+\/
# ExtJS
- (^|/)extjs/.*?\.js$
@@ -161,6 +195,9 @@
- (^|/)extjs/src/
- (^|/)extjs/welcome/
# Html5shiv
- (^|/)html5shiv(\.min)?\.js$
# Samples folders
- ^[Ss]amples/
@@ -181,6 +218,9 @@
- (^|/)cordova([^.]*)(\.min)?\.js$
- (^|/)cordova\-\d\.\d(\.\d)?(\.min)?\.js$
# Foundation js
- foundation(\..*)?\.js$
# Vagrant
- ^Vagrantfile$
@@ -189,3 +229,16 @@
# Mercury --use-subdirs
- Mercury/
# R packages
- ^vignettes/
- ^inst/extdata/
# Octicons
- octicons.css
- octicons.min.css
- sprockets-octicons.scss
# Typesafe Activator
- (^|/)activator$
- (^|/)activator\.bat$

2
lib/linguist/version.rb
View File

@@ -1,3 +1,3 @@
module Linguist
VERSION = "2.10.15"
VERSION = "3.1.5"
end