KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into 1267-local

Browse Source 
Conflicts:
	lib/linguist/samples.json
This commit is contained in:
Arfon Smith
2014-09-24 16:43:42 -05:00
parent 5580f39df2 a38f77683b
commit 1d2a6c38c7
131 changed files with 73214 additions and 67747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
lib/linguist/blob_helper.rb
View File

@@ -1,6 +1,4 @@
require 'linguist/generated'
require 'linguist/language'
require 'charlock_holmes'
require 'escape_utils'
require 'mime/types'
@@ -313,15 +311,7 @@ module Linguist
#
# Returns a Language or nil if none is detected
def language
return @language if defined? @language
if defined?(@data) && @data.is_a?(String)
data = @data
else
data = lambda { (binary_mime_type? || binary?) ? "" : self.data }
end
@language = Language.detect(name.to_s, data, mode)
@language ||= Language.detect(self)
end
# Internal: Get the lexer of the blob.

15
lib/linguist/file_blob.rb
View File

@@ -52,5 +52,20 @@ module Linguist
def size
File.size(@path)
end
# Public: Get file extension.
#
# Returns a String.
def extension
# File.extname returns nil if the filename is an extension.
extension = File.extname(name)
basename = File.basename(name)
# Checks if the filename is an extension.
if extension.empty? && basename[0] == "."
basename
else
extension
end
end
end
end

32
lib/linguist/generated.rb
View File

@@ -54,7 +54,7 @@ module Linguist
name == 'Gemfile.lock' ||
minified_files? ||
compiled_coffeescript? ||
xcode_project_file? ||
xcode_file? ||
generated_parser? ||
generated_net_docfile? ||
generated_net_designer_file? ||
@@ -63,17 +63,19 @@ module Linguist
generated_jni_header? ||
composer_lock? ||
node_modules? ||
vcr_cassette?
godeps? ||
vcr_cassette? ||
generated_by_zephir?
end
# Internal: Is the blob an XCode project file?
# Internal: Is the blob an Xcode file?
#
# Generated if the file extension is an XCode project
# Generated if the file extension is an Xcode
# file extension.
#
# Returns true of false.
def xcode_project_file?
['.xib', '.nib', '.storyboard', '.pbxproj', '.xcworkspacedata', '.xcuserstate'].include?(extname)
def xcode_file?
['.nib', '.xcworkspacedata', '.xcuserstate'].include?(extname)
end
# Internal: Is the blob minified files?
@@ -230,11 +232,26 @@ module Linguist
!!name.match(/node_modules\//)
end
# Internal: Is the blob part of Godeps/,
# which are not meant for humans in pull requests.
#
# Returns true or false.
def godeps?
!!name.match(/Godeps\//)
end
# Internal: Is the blob a generated php composer lock file?
#
# Returns true or false.
def composer_lock?
!!name.match(/composer.lock/)
!!name.match(/composer\.lock/)
end
# Internal: Is the blob a generated by Zephir
#
# Returns true or false.
def generated_by_zephir?
!!name.match(/.\.zep\.(?:c|h|php)$/)
end
# Is the blob a VCR Cassette file?
@@ -248,3 +265,4 @@ module Linguist
end
end
end

20
lib/linguist/heuristics.rb
View File

@@ -1,7 +1,7 @@
module Linguist
# A collection of simple heuristics that can be used to better analyze languages.
class Heuristics
ACTIVE = false
ACTIVE = true
# Public: Given an array of String language names,
# apply heuristics against the given data and return an array
@@ -13,28 +13,20 @@ module Linguist
# Returns an array of Languages or []
def self.find_by_heuristics(data, languages)
if active?
if languages.all? { |l| ["Objective-C", "C++"].include?(l) }
disambiguate_c(data, languages)
end
if languages.all? { |l| ["Perl", "Prolog"].include?(l) }
disambiguate_pl(data, languages)
result = disambiguate_pl(data, languages)
end
if languages.all? { |l| ["ECL", "Prolog"].include?(l) }
disambiguate_ecl(data, languages)
end
if languages.all? { |l| ["TypeScript", "XML"].include?(l) }
disambiguate_ts(data, languages)
result = disambiguate_ecl(data, languages)
end
if languages.all? { |l| ["Common Lisp", "OpenCL"].include?(l) }
disambiguate_cl(data, languages)
end
if languages.all? { |l| ["Rebol", "R"].include?(l) }
disambiguate_r(data, languages)
result = disambiguate_cl(data, languages)
end
return result
end
end
# .h extensions are ambigious between C, C++, and Objective-C.
# .h extensions are ambiguous between C, C++, and Objective-C.
# We want to shortcut look for Objective-C _and_ now C++ too!
#
# Returns an array of Languages or []

43
lib/linguist/language.rb
View File

@@ -9,6 +9,8 @@ end
require 'linguist/classifier'
require 'linguist/heuristics'
require 'linguist/samples'
require 'linguist/file_blob'
require 'linguist/blob_helper'
module Linguist
# Language names that are recognizable by GitHub. Defined languages
@@ -92,18 +94,25 @@ module Linguist
# Public: Detects the Language of the blob.
#
# name - String filename
# data - String blob data. A block also maybe passed in for lazy
# loading. This behavior is deprecated and you should always
# pass in a String.
# mode - Optional String mode (defaults to nil)
# blob - an object that includes the Linguist `BlobHelper` interface;
# see Linguist::LazyBlob and Linguist::FileBlob for examples
#
# Returns Language or nil.
def self.detect(name, data, mode = nil)
def self.detect(blob)
name = blob.name.to_s
# Check if the blob is possibly binary and bail early; this is a cheap
# test that uses the extension name to guess a binary binary mime type.
#
# We'll perform a more comprehensive test later which actually involves
# looking for binary characters in the blob
return nil if blob.likely_binary? || blob.binary?
# A bit of an elegant hack. If the file is executable but extensionless,
# append a "magic" extension so it can be classified with other
# languages that have shebang scripts.
if File.extname(name).empty? && mode && (mode.to_i(8) & 05) == 05
extension = FileBlob.new(name).extension
if extension.empty? && blob.mode && (blob.mode.to_i(8) & 05) == 05
name += ".script!"
end
@@ -114,10 +123,10 @@ module Linguist
# extension at all, in the case of extensionless scripts), we need to continue
# our detection work
if possible_languages.length > 1
data = data.call() if data.respond_to?(:call)
data = blob.data
possible_language_names = possible_languages.map(&:name)
# Don't bother with emptiness
# Don't bother with binary contents or an empty file
if data.nil? || data == ""
nil
# Check if there's a shebang line and use that as authoritative
@@ -126,8 +135,8 @@ module Linguist
# No shebang. Still more work to do. Try to find it with our heuristics.
elsif (determined = Heuristics.find_by_heuristics(data, possible_language_names)) && !determined.empty?
determined.first
# Lastly, fall back to the probablistic classifier.
elsif classified = Classifier.classify(Samples::DATA, data, possible_language_names ).first
# Lastly, fall back to the probabilistic classifier.
elsif classified = Classifier.classify(Samples.cache, data, possible_language_names).first
# Return the actual Language object based of the string language name (i.e., first element of `#classify`)
Language[classified[0]]
end
@@ -183,7 +192,8 @@ module Linguist
#
# Returns all matching Languages or [] if none were found.
def self.find_by_filename(filename)
basename, extname = File.basename(filename), File.extname(filename)
basename = File.basename(filename)
extname = FileBlob.new(filename).extension
langs = @filename_index[basename] +
@extension_index[extname]
langs.compact.uniq
@@ -395,7 +405,7 @@ module Linguist
#
# Returns the extensions Array
attr_reader :filenames
# Public: Return all possible extensions for language
def all_extensions
(extensions + [primary_extension]).uniq
@@ -500,9 +510,9 @@ module Linguist
end
end
extensions = Samples::DATA['extnames']
interpreters = Samples::DATA['interpreters']
filenames = Samples::DATA['filenames']
extensions = Samples.cache['extnames']
interpreters = Samples.cache['interpreters']
filenames = Samples.cache['filenames']
popular = YAML.load_file(File.expand_path("../popular.yml", __FILE__))
languages_yml = File.expand_path("../languages.yml", __FILE__)
@@ -522,6 +532,7 @@ module Linguist
if extnames = extensions[name]
extnames.each do |extname|
if !options['extensions'].include?(extname)
warn "#{name} has a sample with extension (#{extname}) that isn't explicitly defined in languages.yml" unless extname == '.script!'
options['extensions'] << extname
end
end

301
lib/linguist/languages.yml
View File

@@ -28,6 +28,16 @@ ABAP:
extensions:
- .abap
AGS Script:
type: programming
lexer: C++
color: "#B9D9FF"
aliases:
- ags
extensions:
- .asc
- .ash
ANTLR:
type: programming
color: "#9DC3FF"
@@ -35,6 +45,12 @@ ANTLR:
extensions:
- .g4
APL:
type: programming
color: "#8a0707"
extensions:
- .apl
ASP:
type: programming
color: "#6a40fd"
@@ -89,7 +105,7 @@ Agda:
Alloy:
type: programming # 'modeling' would be more appropiate
lexer: Text only
lexer: Alloy
color: "#cc5c24"
extensions:
- .als
@@ -103,7 +119,7 @@ ApacheConf:
Apex:
type: programming
lexer: Text only
lexer: Java
extensions:
- .cls
@@ -221,6 +237,8 @@ BlitzBasic:
- .decls
BlitzMax:
type: programming
color: "#cd6400"
extensions:
- .bmx
@@ -258,13 +276,14 @@ C:
extensions:
- .c
- .cats
- .h
- .w
C#:
type: programming
ace_mode: csharp
search_term: csharp
color: "#5a25a2"
color: "#178600"
aliases:
- csharp
extensions:
@@ -286,6 +305,7 @@ C++:
- .cc
- .cxx
- .H
- .h
- .h++
- .hh
- .hpp
@@ -319,7 +339,7 @@ CLIPS:
CMake:
extensions:
- .cmake
- .cmake.in
- .in
filenames:
- CMakeLists.txt
@@ -344,6 +364,14 @@ Ceylon:
extensions:
- .ceylon
Chapel:
type: programming
color: "#8dc63f"
aliases:
- chpl
extensions:
- .chpl
ChucK:
lexer: Java
extensions:
@@ -352,9 +380,8 @@ ChucK:
Cirru:
type: programming
color: "#aaaaff"
# ace_mode: cirru
# lexer: Cirru
lexer: Text only
ace_mode: cirru
lexer: Cirru
extensions:
- .cirru
@@ -378,7 +405,7 @@ Clojure:
- .cljscm
- .cljx
- .hic
- .cljs.hl
- .hl
filenames:
- riemann.config
@@ -401,14 +428,27 @@ CoffeeScript:
ColdFusion:
type: programming
group: ColdFusion
lexer: Coldfusion HTML
ace_mode: coldfusion
color: "#ed2cd6"
search_term: cfm
aliases:
- cfm
- cfml
extensions:
- .cfm
ColdFusion CFC:
type: programming
group: ColdFusion
lexer: Coldfusion CFC
ace_mode: coldfusion
color: "#ed2cd6"
search_term: cfc
aliases:
- cfc
extensions:
- .cfc
Common Lisp:
@@ -442,6 +482,7 @@ Coq:
type: programming
extensions:
- .coq
- .v
Cpp-ObjDump:
type: data
@@ -477,6 +518,12 @@ Cuda:
- .cu
- .cuh
Cycript:
type: programming
lexer: JavaScript
extensions:
- .cy
Cython:
type: programming
group: Python
@@ -528,18 +575,10 @@ Dart:
extensions:
- .dart
DCPU-16 ASM:
type: programming
lexer: dasm16
extensions:
- .dasm16
- .dasm
aliases:
- dasm16
Diff:
extensions:
- .diff
- .patch
Dogescript:
type: programming
@@ -588,7 +627,7 @@ Eagle:
Eiffel:
type: programming
lexer: Text only
lexer: Eiffel
color: "#946d57"
extensions:
- .e
@@ -608,7 +647,7 @@ Elm:
Emacs Lisp:
type: programming
lexer: Scheme
lexer: Common Lisp
color: "#c065db"
aliases:
- elisp
@@ -619,11 +658,20 @@ Emacs Lisp:
- .el
- .emacs
EmberScript:
type: programming
color: "#f64e3e"
lexer: CoffeeScript
extensions:
- .em
- .emberscript
Erlang:
type: programming
color: "#0faf8d"
extensions:
- .erl
- .escript
- .hrl
F#:
@@ -699,6 +747,7 @@ Forth:
extensions:
- .fth
- .4th
- .forth
Frege:
type: programming
@@ -707,6 +756,14 @@ Frege:
extensions:
- .fr
G-code:
type: data
lexer: Text only
extensions:
- .g
- .gco
- .gcode
Game Maker Language:
type: programming
color: "#8ad353"
@@ -736,6 +793,12 @@ GAS:
- .s
- .S
GDScript:
type: programming
lexer: Text only
extensions:
- .gd
GLSL:
group: C
type: programming
@@ -743,12 +806,14 @@ GLSL:
- .glsl
- .fp
- .frag
- .frg
- .fshader
- .geom
- .glslv
- .gshader
- .shader
- .vert
- .vrx
- .vshader
Genshi:
@@ -805,6 +870,15 @@ Gosu:
color: "#82937f"
extensions:
- .gs
- .gst
- .gsx
- .vark
Grace:
type: programming
lexer: Text only
extensions:
- .grace
Grammatical Framework:
type: programming
@@ -834,6 +908,7 @@ Groovy:
color: "#e69f56"
extensions:
- .groovy
- .gradle
- .grt
- .gtpl
- .gvy
@@ -856,7 +931,6 @@ HTML:
extensions:
- .html
- .htm
- .html.hl
- .st
- .xhtml
@@ -876,9 +950,7 @@ HTML+ERB:
- erb
extensions:
- .erb
- .erb.deface
- .html.erb
- .html.erb.deface
- .deface
HTML+PHP:
type: markup
@@ -896,17 +968,14 @@ Haml:
type: markup
extensions:
- .haml
- .haml.deface
- .html.haml.deface
- .deface
Handlebars:
type: markup
lexer: Text only
lexer: Handlebars
extensions:
- .handlebars
- .hbs
- .html.handlebars
- .html.hbs
Harbour:
type: programming
@@ -932,7 +1001,7 @@ Haxe:
Hy:
type: programming
lexer: Clojure
lexer: Hy
ace_mode: clojure
color: "#7891b1"
extensions:
@@ -940,10 +1009,17 @@ Hy:
IDL:
type: programming
lexer: Text only
lexer: IDL
color: "#e3592c"
extensions:
- .pro
- .dlm
IGOR Pro:
type: programming
lexer: Igor
extensions:
- .ipf
INI:
type: data
@@ -959,14 +1035,14 @@ Inno Setup:
Idris:
type: programming
lexer: Text only
lexer: Idris
extensions:
- .idr
- .lidr
Inform 7:
type: programming
lexer: Text only
lexer: Inform 7
wrap: true
extensions:
- .ni
@@ -998,6 +1074,13 @@ Ioke:
extensions:
- .ik
Isabelle:
type: programming
lexer: Text only
color: "#fdcd00"
extensions:
- .thy
J:
type: programming
lexer: Text only
@@ -1011,6 +1094,7 @@ JSON:
searchable: false
extensions:
- .json
- .lock
- .sublime-keymap
- .sublime-mousemap
- .sublime-project
@@ -1079,6 +1163,7 @@ JavaScript:
- .es6
- .frag
- .jake
- .jsb
- .jsfl
- .jsm
- .jss
@@ -1087,6 +1172,8 @@ JavaScript:
- .pac
- .sjs
- .ssjs
- .xsjs
- .xsjslib
filenames:
- Jakefile
interpreters:
@@ -1131,12 +1218,31 @@ LLVM:
extensions:
- .ll
LSL:
type: programming
lexer: LSL
ace_mode: lsl
extensions:
- .lsl
interpreters:
- lsl
color: '#3d9970'
LabVIEW:
type: programming
lexer: Text only
extensions:
- .lvproj
Lasso:
type: programming
lexer: Lasso
color: "#2584c3"
extensions:
- .lasso
- .las
- .lasso9
- .ldml
Latte:
type: markup
@@ -1215,6 +1321,14 @@ Logtalk:
- .lgt
- .logtalk
LookML:
type: programming
lexer: YAML
ace_mode: yaml
color: "#652B81"
extensions:
- .lookml
Lua:
type: programming
ace_mode: lua
@@ -1222,6 +1336,7 @@ Lua:
extensions:
- .lua
- .nse
- .pd_lua
- .rbxs
interpreters:
- lua
@@ -1285,7 +1400,9 @@ Mathematica:
type: programming
extensions:
- .mathematica
lexer: Text only
- .m
- .nb
lexer: Mathematica
Matlab:
type: programming
@@ -1333,7 +1450,7 @@ MiniD: # Legacy
Mirah:
type: programming
lexer: Ruby
search_term: ruby
search_term: mirah
color: "#c7a938"
extensions:
- .druby
@@ -1365,6 +1482,7 @@ Myghty:
NSIS:
extensions:
- .nsi
- .nsh
Nemerle:
type: programming
@@ -1392,6 +1510,19 @@ Nimrod:
- .nim
- .nimrod
Nit:
type: programming
lexer: Text only
color: "#0d8921"
extensions:
- .nit
Nix:
type: programming
lexer: Nix
extensions:
- .nix
Nu:
type: programming
lexer: Scheme
@@ -1416,6 +1547,7 @@ OCaml:
color: "#3be133"
extensions:
- .ml
- .eliom
- .eliomi
- .ml4
- .mli
@@ -1436,6 +1568,7 @@ Objective-C:
- objc
extensions:
- .m
- .h
Objective-C++:
type: programming
@@ -1467,6 +1600,13 @@ Opa:
extensions:
- .opa
Opal:
type: programming
color: "#f7ede0"
lexer: Text only
extensions:
- .opal
OpenCL:
type: programming
group: C
@@ -1483,6 +1623,13 @@ OpenEdge ABL:
- abl
extensions:
- .p
- .cls
OpenSCAD:
type: programming
lexer: Text only
extensions:
- .scad
Org:
type: prose
@@ -1521,16 +1668,19 @@ PHP:
- .php
- .aw
- .ctp
- .module
- .php3
- .php4
- .php5
- .phpt
filenames:
- Phakefile
interpreters:
- php
Pan:
type: programming
lexer: Text only
lexer: Pan
color: '#cc0000'
extensions:
- .pan
@@ -1567,6 +1717,7 @@ Pascal:
extensions:
- .pas
- .dfm
- .dpr
- .lpr
- .pp
@@ -1577,12 +1728,15 @@ Perl:
extensions:
- .pl
- .PL
- .cgi
- .fcgi
- .perl
- .ph
- .plx
- .pm
- .pod
- .psgi
- .t
interpreters:
- perl
@@ -1599,10 +1753,17 @@ Perl6:
- .pl6
- .pm6
PigLatin:
type: programming
color: "#fcd7de"
lexer: Text only
extensions:
- .pig
Pike:
type: programming
color: "#066ab2"
lexer: C
lexer: Pike
extensions:
- .pike
- .pmod
@@ -1647,11 +1808,12 @@ Processing:
Prolog:
type: programming
lexer: Logtalk
color: "#74283c"
extensions:
- .prolog
- .ecl
- .pl
- .ecl
- .prolog
Propeller Spin:
type: programming
@@ -1696,9 +1858,11 @@ Python:
color: "#3581ba"
extensions:
- .py
- .cgi
- .gyp
- .lmi
- .pyde
- .pyp
- .pyt
- .pyw
- .wsgi
@@ -1724,6 +1888,12 @@ QML:
extensions:
- .qml
QMake:
lexer: Text only
extensions:
- .pro
- .pri
R:
type: programming
color: "#198ce7"
@@ -1744,7 +1914,7 @@ R:
RDoc:
type: prose
lexer: Text only
lexer: Rd
ace_mode: rdoc
wrap: true
extensions:
@@ -1784,6 +1954,7 @@ Racket:
- .rkt
- .rktd
- .rktl
- .scrbl
Ragel in Ruby Host:
type: programming
@@ -1853,7 +2024,10 @@ Ruby:
- .god
- .irbrc
- .mspec
- .pluginspec
- .podspec
- .rabl
- .rake
- .rbuild
- .rbw
- .rbx
@@ -1863,13 +2037,17 @@ Ruby:
interpreters:
- ruby
filenames:
- .pryrc
- Appraisals
- Berksfile
- Buildfile
- Gemfile
- Gemfile.lock
- Guardfile
- Jarfile
- Mavenfile
- Podfile
- Puppetfile
- Thorfile
- Vagrantfile
- buildfile
@@ -1894,6 +2072,14 @@ SCSS:
extensions:
- .scss
SQF:
type: programming
color: "#FFCB1F"
lexer: C++
extensions:
- .sqf
- .hqf
SQL:
type: data
ace_mode: sql
@@ -1923,6 +2109,7 @@ Sass:
group: CSS
extensions:
- .sass
- .scss
Scala:
type: programming
@@ -1930,6 +2117,7 @@ Scala:
color: "#7dd3b0"
extensions:
- .scala
- .sbt
- .sc
Scaml:
@@ -1945,6 +2133,7 @@ Scheme:
- .scm
- .sld
- .sls
- .sps
- .ss
interpreters:
- guile
@@ -1956,6 +2145,8 @@ Scilab:
type: programming
extensions:
- .sci
- .sce
- .tst
Self:
type: programming
@@ -1975,8 +2166,11 @@ Shell:
- zsh
extensions:
- .sh
- .bash
- .bats
- .cgi
- .tmux
- .zsh
interpreters:
- bash
- sh
@@ -2003,6 +2197,14 @@ Slash:
extensions:
- .sl
Slim:
group: HTML
type: markup
lexer: Slim
color: "#ff8877"
extensions:
- .slim
Smalltalk:
type: programming
color: "#596706"
@@ -2035,6 +2237,7 @@ Standard ML:
extensions:
- .ML
- .fun
- .sig
- .sml
Stata:
@@ -2065,8 +2268,8 @@ SuperCollider:
Swift:
type: programming
lexer: Swift
color: "#ffac45"
lexer: Text only
extensions:
- .swift
@@ -2115,10 +2318,13 @@ TeX:
extensions:
- .tex
- .aux
- .bbx
- .bib
- .cbx
- .cls
- .dtx
- .ins
- .lbx
- .ltx
- .mkii
- .mkiv
@@ -2235,6 +2441,7 @@ Visual Basic:
extensions:
- .vb
- .bas
- .cls
- .frm
- .frx
- .vba
@@ -2263,6 +2470,7 @@ XML:
- wsdl
extensions:
- .xml
- .ant
- .axml
- .ccxml
- .clixml
@@ -2276,11 +2484,13 @@ XML:
- .fsproj
- .glade
- .grxml
- .ivy
- .jelly
- .kml
- .launch
- .mxml
- .nproj
- .nuspec
- .osm
- .plist
- .pluginspec
@@ -2352,6 +2562,17 @@ XSLT:
- .xslt
- .xsl
Xojo:
type: programming
lexer: VB.net
extensions:
- .xojo_code
- .xojo_menu
- .xojo_report
- .xojo_script
- .xojo_toolbar
- .xojo_window
Xtend:
type: programming
extensions:

37
lib/linguist/lazy_blob.rb Normal file
View File

@@ -0,0 +1,37 @@
require 'linguist/blob_helper'
require 'rugged'
module Linguist
class LazyBlob
include BlobHelper
MAX_SIZE = 128 * 1024
attr_reader :repository
attr_reader :oid
attr_reader :name
attr_reader :mode
def initialize(repo, oid, name, mode = nil)
@repository = repo
@oid = oid
@name = name
@mode = mode
end
def data
load_blob!
@data
end
def size
load_blob!
@size
end
protected
def load_blob!
@data, @size = Rugged::Blob.to_buffer(repository, oid, MAX_SIZE) if @data.nil?
end
end
end

159
lib/linguist/repository.rb
View File

@@ -1,4 +1,5 @@
require 'linguist/file_blob'
require 'linguist/lazy_blob'
require 'rugged'
module Linguist
# A Repository is an abstraction of a Grit::Repo or a basic file
@@ -7,100 +8,146 @@ module Linguist
# Its primary purpose is for gathering language statistics across
# the entire project.
class Repository
# Public: Initialize a new Repository from a File directory
#
# base_path - A path String
#
# Returns a Repository
def self.from_directory(base_path)
new Dir["#{base_path}/**/*"].
select { |f| File.file?(f) }.
map { |path| FileBlob.new(path, base_path) }
attr_reader :repository
# Public: Create a new Repository based on the stats of
# an existing one
def self.incremental(repo, commit_oid, old_commit_oid, old_stats)
repo = self.new(repo, commit_oid)
repo.load_existing_stats(old_commit_oid, old_stats)
repo
end
# Public: Initialize a new Repository
# Public: Initialize a new Repository to be analyzed for language
# data
#
# enum - Enumerator that responds to `each` and
# yields Blob objects
# repo - a Rugged::Repository object
# commit_oid - the sha1 of the commit that will be analyzed;
# this is usually the master branch
#
# Returns a Repository
def initialize(enum)
@enum = enum
@computed_stats = false
@language = @size = nil
@sizes = Hash.new { 0 }
@file_breakdown = Hash.new { |h,k| h[k] = Array.new }
def initialize(repo, commit_oid)
@repository = repo
@commit_oid = commit_oid
raise TypeError, 'commit_oid must be a commit SHA1' unless commit_oid.is_a?(String)
end
# Public: Load the results of a previous analysis on this repository
# to speed up the new scan.
#
# The new analysis will be performed incrementally as to only take
# into account the file changes since the last time the repository
# was scanned
#
# old_commit_oid - the sha1 of the commit that was previously analyzed
# old_stats - the result of the previous analysis, obtained by calling
# Repository#cache on the old repository
#
# Returns nothing
def load_existing_stats(old_commit_oid, old_stats)
@old_commit_oid = old_commit_oid
@old_stats = old_stats
nil
end
# Public: Returns a breakdown of language stats.
#
# Examples
#
# # => { Language['Ruby'] => 46319,
# Language['JavaScript'] => 258 }
# # => { 'Ruby' => 46319,
# 'JavaScript' => 258 }
#
# Returns a Hash of Language keys and Integer size values.
# Returns a Hash of language names and Integer size values.
def languages
compute_stats
@sizes
@sizes ||= begin
sizes = Hash.new { 0 }
cache.each do |_, (language, size)|
sizes[language] += size
end
sizes
end
end
# Public: Get primary Language of repository.
#
# Returns a Language
# Returns a language name
def language
compute_stats
@language
@language ||= begin
primary = languages.max_by { |(_, size)| size }
primary && primary[0]
end
end
# Public: Get the total size of the repository.
#
# Returns a byte size Integer
def size
compute_stats
@size
@size ||= languages.inject(0) { |s,(_,v)| s + v }
end
# Public: Return the language breakdown of this repository by file
#
# Returns a map of language names => [filenames...]
def breakdown_by_file
compute_stats
@file_breakdown
@file_breakdown ||= begin
breakdown = Hash.new { |h,k| h[k] = Array.new }
cache.each do |filename, (language, _)|
breakdown[language] << filename
end
breakdown
end
end
# Internal: Compute language breakdown for each blob in the Repository.
# Public: Return the cached results of the analysis
#
# Returns nothing
def compute_stats
return if @computed_stats
# This is a per-file breakdown that can be passed to other instances
# of Linguist::Repository to perform incremental scans
#
# Returns a map of filename => [language, size]
def cache
@cache ||= begin
if @old_commit_oid == @commit_oid
@old_stats
else
compute_stats(@old_commit_oid, @commit_oid, @old_stats)
end
end
end
@enum.each do |blob|
# Skip files that are likely binary
next if blob.likely_binary?
protected
def compute_stats(old_commit_oid, commit_oid, cache = nil)
file_map = cache ? cache.dup : {}
old_tree = old_commit_oid && Rugged::Commit.lookup(repository, old_commit_oid).tree
new_tree = Rugged::Commit.lookup(repository, commit_oid).tree
# Skip vendored or generated blobs
next if blob.vendored? || blob.generated? || blob.language.nil?
diff = Rugged::Tree.diff(repository, old_tree, new_tree)
# Only include programming languages and acceptable markup languages
if blob.language.type == :programming || Language.detectable_markup.include?(blob.language.name)
diff.each_delta do |delta|
old = delta.old_file[:path]
new = delta.new_file[:path]
# Build up the per-file breakdown stats
@file_breakdown[blob.language.group.name] << blob.name
file_map.delete(old)
next if delta.binary
@sizes[blob.language.group] += blob.size
if [:added, :modified].include? delta.status
# Skip submodules
mode = delta.new_file[:mode]
next if (mode & 040000) != 0
blob = Linguist::LazyBlob.new(repository, delta.new_file[:oid], new, mode.to_s(8))
# Skip vendored or generated blobs
next if blob.vendored? || blob.generated? || blob.language.nil?
# Only include programming languages and acceptable markup languages
if blob.language.type == :programming || Language.detectable_markup.include?(blob.language.name)
file_map[new] = [blob.language.group.name, blob.size]
end
end
end
# Compute total size
@size = @sizes.inject(0) { |s,(_,v)| s + v }
# Get primary language
if primary = @sizes.max_by { |(_, size)| size }
@language = primary[0]
end
@computed_stats = true
nil
file_map
end
end
end

67559
lib/linguist/samples.json
View File

File diff suppressed because it is too large Load Diff

10
lib/linguist/samples.rb
View File

@@ -17,9 +17,11 @@ module Linguist
PATH = File.expand_path('../samples.json', __FILE__)
# Hash of serialized samples object
if File.exist?(PATH)
serializer = defined?(JSON) ? JSON : YAML
DATA = serializer.load(File.read(PATH))
def self.cache
@cache ||= begin
serializer = defined?(JSON) ? JSON : YAML
serializer.load(File.read(PATH))
end
end
# Public: Iterate over each sample.
@@ -28,7 +30,7 @@ module Linguist
#
# Returns nothing.
def self.each(&block)
Dir.entries(ROOT).each do |category|
Dir.entries(ROOT).sort!.each do |category|
next if category == '.' || category == '..'
# Skip text and binary for now

36
lib/linguist/vendor.yml
View File

@@ -33,16 +33,36 @@
# Erlang bundles
- ^rebar$
# Go dependencies
- Godeps/_workspace/
# Bootstrap minified css and js
- (^|/)bootstrap([^.]*)(\.min)?\.(js|css)$
# Font Awesome
- font-awesome.min.css
- font-awesome.css
# Foundation css
- foundation.min.css
- foundation.css
# Normalize.css
- normalize.css
# Bourbon SCSS
- (^|/)[Bb]ourbon/.*\.css$
- (^|/)[Bb]ourbon/.*\.scss$
# Animate.css
- animate.css
- animate.min.css
# Vendored dependencies
- thirdparty/
- third[-_]?party/
- 3rd[-_]?party/
- vendors?/
- extern(al)?/
# Debian packaging
- ^debian/
@@ -108,6 +128,10 @@
- (^|/)modernizr\-\d\.\d+(\.\d+)?(\.min)?\.js$
- (^|/)modernizr\.custom\.\d+\.js$
# Knockout
- (^|/)knockout-(\d+\.){3}(debug\.)?js$
- knockout-min.js
## Python ##
# django
@@ -124,6 +148,9 @@
## Obj-C ##
# Cocoapods
- ^Pods/
# Sparkle
- (^|/)Sparkle/
@@ -191,6 +218,9 @@
- (^|/)cordova([^.]*)(\.min)?\.js$
- (^|/)cordova\-\d\.\d(\.\d)?(\.min)?\.js$
# Foundation js
- foundation(\..*)?\.js$
# Vagrant
- ^Vagrantfile$
@@ -204,3 +234,7 @@
- ^vignettes/
- ^inst/extdata/
# Octicons
- octicons.css
- octicons.min.css
- sprockets-octicons.scss

2
lib/linguist/version.rb
View File

@@ -1,3 +1,3 @@
module Linguist
VERSION = "2.11.4"
VERSION = "3.1.5"
end