KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-12-29 13:21:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'refactor-heuristics' into 1036-local

Browse Source 
* refactor-heuristics: (43 commits)
  update docs
  Clean up heuristic logic
  Allow disambiguate to return an Array
  Rename .create to .disambiguate
  docs
  Remove inactive heuristics
  Refactor heuristics
  Not going back
  docs
  Move call method into existing Classifier class
  Try strategies until one language is returned
  Remove unneded empty blob check
  Add F# and GLSL samples.  Add Forth and GLSL extension .fs. Add heuristic to disambiguate between F#, Forth, and GLSL.
  byebug requires ruby 2.0
  Remove test for removed extension
  Fix typo in test
  add rake interpreter
  add python3 interpreter
  Remove old wrong_shebang.rb sample
  Add byebug
  ...

Conflicts:
	lib/linguist/heuristics.rb
	test/test_heuristics.rb
This commit is contained in:
Brandon Keepers
2014-11-28 17:58:00 -06:00
parent 8de2cd15ed 770a1d4553
commit 7b41346db8
32 changed files with 2199 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
lib/linguist/classifier.rb
View File

@@ -3,6 +3,25 @@ require 'linguist/tokenizer'
module Linguist
# Language bayesian classifier.
class Classifier
# Public: Use the classifier to detect language of the blob.
#
# blob - An object that quacks like a blob.
# possible_languages - Array of Language objects
#
# Examples
#
# Classifier.call(FileBlob.new("path/to/file"), [
# Language["Ruby"], Language["Python"]
# ])
#
# Returns an Array of Language objects, most probable first.
def self.call(blob, possible_languages)
language_names = possible_languages.map(&:name)
classify(Samples.cache, blob.data, language_names).map do |name, _|
Language[name] # Return the actual Language objects
end
end
# Public: Train classifier that data is a certain language.
#
# db - Hash classifier database object

22
lib/linguist/file_blob.rb
View File

@@ -57,14 +57,20 @@ module Linguist
#
# Returns a String.
def extension
# File.extname returns nil if the filename is an extension.
extension = File.extname(name)
basename = File.basename(name)
# Checks if the filename is an extension.
if extension.empty? && basename[0] == "."
basename
else
extension
extensions.last || ""
end
# Public: Return an array of the file extensions
#
# >> Linguist::FileBlob.new("app/views/things/index.html.erb").extensions
# => [".html.erb", ".erb"]
#
# Returns an Array
def extensions
basename, *segments = File.basename(name).split(".")
segments.map.with_index do |segment, index|
"." + segments[index..-1].join(".")
end
end
end

201
lib/linguist/heuristics.rb
View File

@@ -1,162 +1,143 @@
module Linguist
# A collection of simple heuristics that can be used to better analyze languages.
class Heuristics
ACTIVE = true
# Public: Given an array of String language names,
# apply heuristics against the given data and return an array
# of matching languages, or nil.
# Public: Use heuristics to detect language of the blob.
#
# data - Array of tokens or String data to analyze.
# languages - Array of language name Strings to restrict to.
# blob - An object that quacks like a blob.
# possible_languages - Array of Language objects
#
# Returns an array of Languages or []
def self.find_by_heuristics(data, languages)
if active?
result = []
# Examples
#
# Heuristics.call(FileBlob.new("path/to/file"), [
# Language["Ruby"], Language["Python"]
# ])
#
# Returns an Array of languages, or empty if none matched or were inconclusive.
def self.call(blob, languages)
data = blob.data
if languages.all? { |l| ["Objective-C", "C++", "C"].include?(l) }
result = disambiguate_c(data)
end
if languages.all? { |l| ["Perl", "Prolog"].include?(l) }
result = disambiguate_pl(data)
end
if languages.all? { |l| ["ECL", "Prolog"].include?(l) }
result = disambiguate_ecl(data)
end
if languages.all? { |l| ["IDL", "Prolog"].include?(l) }
result = disambiguate_pro(data)
end
if languages.all? { |l| ["Common Lisp", "OpenCL"].include?(l) }
result = disambiguate_cl(data)
end
if languages.all? { |l| ["Hack", "PHP"].include?(l) }
result = disambiguate_hack(data)
end
if languages.all? { |l| ["Scala", "SuperCollider"].include?(l) }
result = disambiguate_sc(data)
end
if languages.all? { |l| ["AsciiDoc", "AGS Script"].include?(l) }
result = disambiguate_asc(data)
end
if languages.all? { |l| ["FORTRAN", "Forth"].include?(l) }
result = disambiguate_f(data)
end
return result
@heuristics.each do |heuristic|
return Array(heuristic.call(data)) if heuristic.matches?(languages)
end
[] # No heuristics matched
end
# .h extensions are ambiguous between C, C++, and Objective-C.
# We want to shortcut look for Objective-C _and_ now C++ too!
# Internal: Define a new heuristic.
#
# Returns an array of Languages or []
def self.disambiguate_c(data)
matches = []
# languages - String names of languages to disambiguate.
# heuristic - Block which takes data as an argument and returns a Language or nil.
#
# Examples
#
# disambiguate "Perl", "Prolog" do |data|
# if data.include?("use strict")
# Language["Perl"]
# elsif data.include?(":-")
# Language["Prolog"]
# end
# end
#
def self.disambiguate(*languages, &heuristic)
@heuristics << new(languages, &heuristic)
end
# Internal: Array of defined heuristics
@heuristics = []
# Internal
def initialize(languages, &heuristic)
@languages = languages
@heuristic = heuristic
end
# Internal: Check if this heuristic matches the candidate languages.
def matches?(candidates)
candidates.all? { |l| @languages.include?(l.name) }
end
# Internal: Perform the heuristic
def call(data)
@heuristic.call(data)
end
disambiguate "Objective-C", "C++", "C" do |data|
if (/@(interface|class|protocol|property|end|synchronised|selector|implementation)\b/.match(data))
matches << Language["Objective-C"]
Language["Objective-C"]
elsif (/^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>/.match(data) ||
/^\s*template\s*</.match(data) || /^[^@]class\s+\w+/.match(data) || /^[^@](private|public|protected):$/.match(data) || /std::.+$/.match(data))
matches << Language["C++"]
/^\s*template\s*</.match(data) || /^[^@]class\s+\w+/.match(data) || /^[^@](private|public|protected):$/.match(data) || /std::.+$/.match(data))
Language["C++"]
end
matches
end
def self.disambiguate_pl(data)
matches = []
disambiguate "Perl", "Prolog" do |data|
if data.include?("use strict")
matches << Language["Perl"]
Language["Perl"]
elsif data.include?(":-")
matches << Language["Prolog"]
Language["Prolog"]
end
matches
end
def self.disambiguate_ecl(data)
matches = []
disambiguate "ECL", "Prolog" do |data|
if data.include?(":-")
matches << Language["Prolog"]
Language["Prolog"]
elsif data.include?(":=")
matches << Language["ECL"]
Language["ECL"]
end
matches
end
def self.disambiguate_pro(data)
matches = []
if (data.include?(":-"))
matches << Language["Prolog"]
disambiguate "IDL", "Prolog" do |data|
if data.include?(":-")
Language["Prolog"]
else
matches << Language["IDL"]
Language["IDL"]
end
matches
end
def self.disambiguate_ts(data)
matches = []
if (data.include?("</translation>"))
matches << Language["XML"]
else
matches << Language["TypeScript"]
end
matches
end
def self.disambiguate_cl(data)
matches = []
disambiguate "Common Lisp", "OpenCL" do |data|
if data.include?("(defun ")
matches << Language["Common Lisp"]
Language["Common Lisp"]
elsif /\/\* |\/\/ |^\}/.match(data)
matches << Language["OpenCL"]
Language["OpenCL"]
end
matches
end
def self.disambiguate_r(data)
matches = []
matches << Language["Rebol"] if /\bRebol\b/i.match(data)
matches << Language["R"] if data.include?("<-")
matches
end
def self.disambiguate_hack(data)
matches = []
disambiguate "Hack", "PHP" do |data|
if data.include?("<?hh")
matches << Language["Hack"]
Language["Hack"]
elsif /<?[^h]/.match(data)
matches << Language["PHP"]
Language["PHP"]
end
matches
end
def self.disambiguate_sc(data)
matches = []
if (/\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data))
matches << Language["SuperCollider"]
disambiguate "Scala", "SuperCollider" do |data|
if /\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data)
Language["SuperCollider"]
elsif /^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data)
Language["Scala"]
end
if (/^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data))
matches << Language["Scala"]
end
matches
end
def self.disambiguate_asc(data)
matches = []
matches << Language["AsciiDoc"] if /^=+(\s|\n)/.match(data)
matches
disambiguate "AsciiDoc", "AGS Script" do |data|
Language["AsciiDoc"] if /^=+(\s|\n)/.match(data)
end
def self.disambiguate_f(data)
matches = []
disambiguate "FORTRAN", "Forth" do |data|
if /^: /.match(data)
matches << Language["Forth"]
Language["Forth"]
elsif /^([c*][^a-z]| subroutine\s)/i.match(data)
matches << Language["FORTRAN"]
Language["FORTRAN"]
end
matches
end
def self.active?
!!ACTIVE
disambiguate "F#", "Forth", "GLSL" do |data|
if /^(: |new-device)/.match(data)
Language["Forth"]
elsif /^(#light|import|let|module|namespace|open|type)/.match(data)
Language["F#"]
elsif /^(#include|#pragma|precision|uniform|varying|void)/.match(data)
Language["GLSL"]
end
end
end
end

69
lib/linguist/language.rb
View File

@@ -10,6 +10,8 @@ require 'linguist/heuristics'
require 'linguist/samples'
require 'linguist/file_blob'
require 'linguist/blob_helper'
require 'linguist/strategy/filename'
require 'linguist/strategy/shebang'
module Linguist
# Language names that are recognizable by GitHub. Defined languages
@@ -91,6 +93,13 @@ module Linguist
language
end
STRATEGIES = [
Linguist::Strategy::Filename,
Linguist::Strategy::Shebang,
Linguist::Heuristics,
Linguist::Classifier
]
# Public: Detects the Language of the blob.
#
# blob - an object that includes the Linguist `BlobHelper` interface;
@@ -98,49 +107,22 @@ module Linguist
#
# Returns Language or nil.
def self.detect(blob)
name = blob.name.to_s
# Bail early if the blob is binary or empty.
return nil if blob.likely_binary? || blob.binary? || blob.empty?
# A bit of an elegant hack. If the file is executable but extensionless,
# append a "magic" extension so it can be classified with other
# languages that have shebang scripts.
extension = FileBlob.new(name).extension
if extension.empty? && blob.mode && (blob.mode.to_i(8) & 05) == 05
name += ".script!"
end
# First try to find languages that match based on filename.
possible_languages = find_by_filename(name)
# If there is more than one possible language with that extension (or no
# extension at all, in the case of extensionless scripts), we need to continue
# our detection work
if possible_languages.length > 1
data = blob.data
possible_language_names = possible_languages.map(&:name)
heuristic_languages = Heuristics.find_by_heuristics(data, possible_language_names)
if heuristic_languages.size > 1
possible_language_names = heuristic_languages.map(&:name)
# Call each strategy until one candidate is returned
STRATEGIES.reduce([]) do |languages, strategy|
candidates = strategy.call(blob, languages)
if candidates.size == 1
return candidates.first
elsif candidates.size > 1
# More than one candidate was found, pass them to the next strategy
candidates
else
# Strategy couldn't find any candidates, so pass on the original list
languages
end
# Check if there's a shebang line and use that as authoritative
if (result = find_by_shebang(data)) && !result.empty?
result.first
# No shebang. Still more work to do. Try to find it with our heuristics.
elsif heuristic_languages.size == 1
heuristic_languages.first
# Lastly, fall back to the probabilistic classifier.
elsif classified = Classifier.classify(Samples.cache, data, possible_language_names).first
# Return the actual Language object based of the string language name (i.e., first element of `#classify`)
Language[classified[0]]
end
else
# Simplest and most common case, we can just return the one match based on extension
possible_languages.first
end
end.first
end
# Public: Get all Languages
@@ -190,8 +172,13 @@ module Linguist
# Returns all matching Languages or [] if none were found.
def self.find_by_filename(filename)
basename = File.basename(filename)
extname = FileBlob.new(filename).extension
(@filename_index[basename] + find_by_extension(extname)).compact.uniq
# find the first extension with language definitions
extname = FileBlob.new(filename).extensions.detect do |e|
!@extension_index[e].empty?
end
(@filename_index[basename] + @extension_index[extname]).compact.uniq
end
# Public: Look up Languages by file extension.

28
lib/linguist/languages.yml
View File

@@ -558,6 +558,8 @@ Crystal:
- .cr
ace_mode: ruby
tm_scope: source.ruby
interpreters:
- crystal
Cucumber:
extensions:
@@ -735,6 +737,8 @@ Erlang:
- .es
- .escript
- .hrl
interpreters:
- escript
F#:
type: programming
@@ -814,6 +818,7 @@ Forth:
- .for
- .forth
- .frt
- .fs
Frege:
type: programming
@@ -867,6 +872,7 @@ GLSL:
- .fp
- .frag
- .frg
- .fs
- .fshader
- .geo
- .geom
@@ -930,6 +936,8 @@ Gnuplot:
- .gnuplot
- .plot
- .plt
interpreters:
- gnuplot
Go:
type: programming
@@ -1195,6 +1203,8 @@ Ioke:
color: "#078193"
extensions:
- .ik
interpreters:
- ioke
Isabelle:
type: programming
@@ -1702,6 +1712,8 @@ Nu:
filenames:
- Nukefile
tm_scope: source.scheme
interpreters:
- nush
NumPy:
group: Python
@@ -1888,6 +1900,8 @@ Parrot Assembly:
- pasm
extensions:
- .pasm
interpreters:
- parrot
tm_scope: none
Parrot Internal Representation:
@@ -1898,6 +1912,8 @@ Parrot Internal Representation:
- pir
extensions:
- .pir
interpreters:
- parrot
Pascal:
type: programming
@@ -1940,6 +1956,8 @@ Perl6:
- .p6m
- .pl6
- .pm6
interpreters:
- perl6
tm_scope: none
PigLatin:
@@ -2004,6 +2022,8 @@ Prolog:
- .ecl
- .pro
- .prolog
interpreters:
- swipl
Propeller Spin:
type: programming
@@ -2067,6 +2087,8 @@ Python:
- wscript
interpreters:
- python
- python2
- python3
Python traceback:
type: data
@@ -2087,6 +2109,8 @@ QMake:
extensions:
- .pro
- .pri
interpreters:
- qmake
R:
type: programming
@@ -2241,6 +2265,8 @@ Ruby:
- .watchr
interpreters:
- ruby
- macruby
- rake
filenames:
- .pryrc
- Appraisals
@@ -2327,6 +2353,8 @@ Scala:
- .scala
- .sbt
- .sc
interpreters:
- scala
Scaml:
group: HTML

21
lib/linguist/samples.rb
View File

@@ -52,14 +52,16 @@ module Linguist
})
end
else
path = File.join(dirname, filename)
if File.extname(filename) == ""
raise "#{File.join(dirname, filename)} is missing an extension, maybe it belongs in filenames/ subdir"
raise "#{path} is missing an extension, maybe it belongs in filenames/ subdir"
end
yield({
:path => File.join(dirname, filename),
:path => path,
:language => category,
:interpreter => File.exist?(filename) ? Linguist.interpreter_from_shebang(File.read(filename)) : nil,
:interpreter => Linguist.interpreter_from_shebang(File.read(path)),
:extname => File.extname(filename)
})
end
@@ -131,18 +133,19 @@ module Linguist
script = script == 'env' ? tokens[1] : script
# "python2.6" -> "python"
if script =~ /((?:\d+\.?)+)/
script.sub! $1, ''
end
# If script has an invalid shebang, we might get here
return unless script
# "python2.6" -> "python2"
script.sub! $1, '' if script =~ /(\.\d+)$/
# Check for multiline shebang hacks that call `exec`
if script == 'sh' &&
lines[0...5].any? { |l| l.match(/exec (\w+).+\$0.+\$@/) }
script = $1
end
script
File.basename(script)
else
nil
end

20
lib/linguist/strategy/filename.rb Normal file
View File

@@ -0,0 +1,20 @@
module Linguist
module Strategy
# Detects language based on filename and/or extension
class Filename
def self.call(blob, _)
name = blob.name.to_s
# A bit of an elegant hack. If the file is executable but extensionless,
# append a "magic" extension so it can be classified with other
# languages that have shebang scripts.
extensions = FileBlob.new(name).extensions
if extensions.empty? && blob.mode && (blob.mode.to_i(8) & 05) == 05
name += ".script!"
end
Language.find_by_filename(name)
end
end
end
end

10
lib/linguist/strategy/shebang.rb Normal file
View File

@@ -0,0 +1,10 @@
module Linguist
module Strategy
# Check if there's a shebang line and use that as authoritative
class Shebang
def self.call(blob, _)
Language.find_by_shebang(blob.data)
end
end
end
end