KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1788 from github/refactor-heuristics

Browse Source 
Refactor heuristics (updated)
This commit is contained in:
Brandon Keepers
2014-11-28 17:59:43 -06:00
parent a903123cb8 770a1d4553
commit 3acbf06beb
2 changed files with 128 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
lib/linguist/heuristics.rb
View File

@@ -1,8 +1,6 @@
module Linguist module Linguist
# A collection of simple heuristics that can be used to better analyze languages. # A collection of simple heuristics that can be used to better analyze languages.
class Heuristics class Heuristics
ACTIVE = true
# Public: Use heuristics to detect language of the blob. # Public: Use heuristics to detect language of the blob.
# #
# blob - An object that quacks like a blob. # blob - An object that quacks like a blob.
@@ -14,177 +12,123 @@ module Linguist
# Language["Ruby"], Language["Python"] # Language["Ruby"], Language["Python"]
# ]) # ])
# #
# Returns an Array with one Language if a heuristic matched, or empty if # Returns an Array of languages, or empty if none matched or were inconclusive.
# none matched or were inconclusive.
def self.call(blob, languages) def self.call(blob, languages)
find_by_heuristics(blob.data, languages.map(&:name)) data = blob.data
end
# Public: Given an array of String language names, @heuristics.each do |heuristic|
# apply heuristics against the given data and return an array return Array(heuristic.call(data)) if heuristic.matches?(languages)
# of matching languages, or nil.
#
# data - Array of tokens or String data to analyze.
# languages - Array of language name Strings to restrict to.
#
# Returns an array of Languages or []
def self.find_by_heuristics(data, languages)
if active?
result = []
if languages.all? { |l| ["Perl", "Prolog"].include?(l) }
result = disambiguate_pl(data)
end
if languages.all? { |l| ["ECL", "Prolog"].include?(l) }
result = disambiguate_ecl(data)
end
if languages.all? { |l| ["IDL", "Prolog"].include?(l) }
result = disambiguate_pro(data)
end
if languages.all? { |l| ["Common Lisp", "OpenCL"].include?(l) }
result = disambiguate_cl(data)
end
if languages.all? { |l| ["Hack", "PHP"].include?(l) }
result = disambiguate_hack(data)
end
if languages.all? { |l| ["Scala", "SuperCollider"].include?(l) }
result = disambiguate_sc(data)
end
if languages.all? { |l| ["AsciiDoc", "AGS Script"].include?(l) }
result = disambiguate_asc(data)
end
if languages.all? { |l| ["FORTRAN", "Forth"].include?(l) }
result = disambiguate_f(data)
end
if languages.all? { |l| ["F#", "Forth", "GLSL"].include?(l) }
result = disambiguate_fs(data)
end
return result
end end
[] # No heuristics matched
end end
# .h extensions are ambiguous between C, C++, and Objective-C. # Internal: Define a new heuristic.
# We want to shortcut look for Objective-C _and_ now C++ too!
# #
# Returns an array of Languages or [] # languages - String names of languages to disambiguate.
def self.disambiguate_c(data) # heuristic - Block which takes data as an argument and returns a Language or nil.
matches = [] #
if data.include?("@interface") # Examples
matches << Language["Objective-C"] #
elsif data.include?("#include <cstdint>") # disambiguate "Perl", "Prolog" do |data|
matches << Language["C++"] # if data.include?("use strict")
end # Language["Perl"]
matches # elsif data.include?(":-")
# Language["Prolog"]
# end
# end
#
def self.disambiguate(*languages, &heuristic)
@heuristics << new(languages, &heuristic)
end end
def self.disambiguate_pl(data) # Internal: Array of defined heuristics
matches = [] @heuristics = []
# Internal
def initialize(languages, &heuristic)
@languages = languages
@heuristic = heuristic
end
# Internal: Check if this heuristic matches the candidate languages.
def matches?(candidates)
candidates.all? { |l| @languages.include?(l.name) }
end
# Internal: Perform the heuristic
def call(data)
@heuristic.call(data)
end
disambiguate "Perl", "Prolog" do |data|
if data.include?("use strict") if data.include?("use strict")
matches << Language["Perl"] Language["Perl"]
elsif data.include?(":-") elsif data.include?(":-")
matches << Language["Prolog"] Language["Prolog"]
end end
matches
end end
def self.disambiguate_ecl(data) disambiguate "ECL", "Prolog" do |data|
matches = []
if data.include?(":-") if data.include?(":-")
matches << Language["Prolog"] Language["Prolog"]
elsif data.include?(":=") elsif data.include?(":=")
matches << Language["ECL"] Language["ECL"]
end end
matches
end end
def self.disambiguate_pro(data) disambiguate "IDL", "Prolog" do |data|
matches = [] if data.include?(":-")
if (data.include?(":-")) Language["Prolog"]
matches << Language["Prolog"]
else else
matches << Language["IDL"] Language["IDL"]
end end
matches
end end
def self.disambiguate_ts(data) disambiguate "Common Lisp", "OpenCL" do |data|
matches = []
if (data.include?("</translation>"))
matches << Language["XML"]
else
matches << Language["TypeScript"]
end
matches
end
def self.disambiguate_cl(data)
matches = []
if data.include?("(defun ") if data.include?("(defun ")
matches << Language["Common Lisp"] Language["Common Lisp"]
elsif /\/\* |\/\/ |^\}/.match(data) elsif /\/\* |\/\/ |^\}/.match(data)
matches << Language["OpenCL"] Language["OpenCL"]
end end
matches
end end
def self.disambiguate_r(data) disambiguate "Hack", "PHP" do |data|
matches = []
matches << Language["Rebol"] if /\bRebol\b/i.match(data)
matches << Language["R"] if data.include?("<-")
matches
end
def self.disambiguate_hack(data)
matches = []
if data.include?("<?hh") if data.include?("<?hh")
matches << Language["Hack"] Language["Hack"]
elsif /<?[^h]/.match(data) elsif /<?[^h]/.match(data)
matches << Language["PHP"] Language["PHP"]
end end
matches
end end
def self.disambiguate_sc(data) disambiguate "Scala", "SuperCollider" do |data|
matches = [] if /\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data)
if (/\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data)) Language["SuperCollider"]
matches << Language["SuperCollider"] elsif /^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data)
Language["Scala"]
end end
if (/^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data))
matches << Language["Scala"]
end
matches
end end
def self.disambiguate_asc(data) disambiguate "AsciiDoc", "AGS Script" do |data|
matches = [] Language["AsciiDoc"] if /^=+(\s|\n)/.match(data)
matches << Language["AsciiDoc"] if /^=+(\s|\n)/.match(data)
matches
end end
def self.disambiguate_f(data) disambiguate "FORTRAN", "Forth" do |data|
matches = []
if /^: /.match(data) if /^: /.match(data)
matches << Language["Forth"] Language["Forth"]
elsif /^([c*][^a-z]| subroutine\s)/i.match(data) elsif /^([c*][^a-z]| subroutine\s)/i.match(data)
matches << Language["FORTRAN"] Language["FORTRAN"]
end end
matches
end end
def self.disambiguate_fs(data) disambiguate "F#", "Forth", "GLSL" do |data|
matches = []
if /^(: |new-device)/.match(data) if /^(: |new-device)/.match(data)
matches << Language["Forth"] Language["Forth"]
elsif /^(#light|import|let|module|namespace|open|type)/.match(data) elsif /^(#light|import|let|module|namespace|open|type)/.match(data)
matches << Language["F#"] Language["F#"]
elsif /^(#include|#pragma|precision|uniform|varying|void)/.match(data) elsif /^(#include|#pragma|precision|uniform|varying|void)/.match(data)
matches << Language["GLSL"] Language["GLSL"]
end end
matches
end
def self.active?
!!ACTIVE
end end
end end
end end

139
test/test_heuristics.rb
View File

@@ -11,25 +11,15 @@ class TestHeuristcs < Test::Unit::TestCase
File.read(File.join(samples_path, name)) File.read(File.join(samples_path, name))
end end
def file_blob(name)
path = File.exist?(name) ? name : File.join(samples_path, name)
FileBlob.new(path)
end
def all_fixtures(language_name, file="*") def all_fixtures(language_name, file="*")
Dir.glob("#{samples_path}/#{language_name}/#{file}") Dir.glob("#{samples_path}/#{language_name}/#{file}")
end end
# Candidate languages = ["C++", "Objective-C"]
def test_obj_c_by_heuristics
# Only calling out '.h' filenames as these are the ones causing issues
all_fixtures("Objective-C", "*.h").each do |fixture|
results = Heuristics.disambiguate_c(fixture("Objective-C/#{File.basename(fixture)}"))
assert_equal Language["Objective-C"], results.first
end
end
# Candidate languages = ["C++", "Objective-C"]
def test_cpp_by_heuristics
results = Heuristics.disambiguate_c(fixture("C++/render_adapter.cpp"))
assert_equal Language["C++"], results.first
end
def test_detect_still_works_if_nothing_matches def test_detect_still_works_if_nothing_matches
blob = Linguist::FileBlob.new(File.join(samples_path, "Objective-C/hello.m")) blob = Linguist::FileBlob.new(File.join(samples_path, "Objective-C/hello.m"))
match = Language.detect(blob) match = Language.detect(blob)
@@ -37,103 +27,88 @@ class TestHeuristcs < Test::Unit::TestCase
end end
# Candidate languages = ["Perl", "Prolog"] # Candidate languages = ["Perl", "Prolog"]
def test_pl_prolog_by_heuristics def test_pl_prolog_perl_by_heuristics
results = Heuristics.disambiguate_pl(fixture("Prolog/turing.pl")) assert_heuristics({
assert_equal Language["Prolog"], results.first "Prolog" => "Prolog/turing.pl",
end "Perl" => "Perl/perl-test.t",
})
# Candidate languages = ["Perl", "Prolog"]
def test_pl_perl_by_heuristics
results = Heuristics.disambiguate_pl(fixture("Perl/perl-test.t"))
assert_equal Language["Perl"], results.first
end end
# Candidate languages = ["ECL", "Prolog"] # Candidate languages = ["ECL", "Prolog"]
def test_ecl_prolog_by_heuristics def test_ecl_prolog_by_heuristics
results = Heuristics.disambiguate_ecl(fixture("Prolog/or-constraint.ecl")) results = Heuristics.call(file_blob("Prolog/or-constraint.ecl"), [Language["ECL"], Language["Prolog"]])
assert_equal Language["Prolog"], results.first assert_equal [Language["Prolog"]], results
end end
# Candidate languages = ["ECL", "Prolog"] # Candidate languages = ["ECL", "Prolog"]
def test_ecl_ecl_by_heuristics def test_ecl_prolog_by_heuristics
results = Heuristics.disambiguate_ecl(fixture("ECL/sample.ecl")) assert_heuristics({
assert_equal Language["ECL"], results.first "ECL" => "ECL/sample.ecl",
"Prolog" => "Prolog/or-constraint.ecl"
})
end end
# Candidate languages = ["IDL", "Prolog"] # Candidate languages = ["IDL", "Prolog"]
def test_pro_prolog_by_heuristics def test_pro_prolog_idl_by_heuristics
results = Heuristics.disambiguate_pro(fixture("Prolog/logic-problem.pro")) assert_heuristics({
assert_equal Language["Prolog"], results.first "Prolog" => "Prolog/logic-problem.pro",
end "IDL" => "IDL/mg_acosh.pro"
})
# Candidate languages = ["IDL", "Prolog"]
def test_pro_idl_by_heuristics
results = Heuristics.disambiguate_pro(fixture("IDL/mg_acosh.pro"))
assert_equal Language["IDL"], results.first
end end
# Candidate languages = ["AGS Script", "AsciiDoc"] # Candidate languages = ["AGS Script", "AsciiDoc"]
def test_asc_asciidoc_by_heuristics def test_asc_asciidoc_by_heuristics
results = Heuristics.disambiguate_asc(fixture("AsciiDoc/list.asc")) assert_heuristics({
assert_equal Language["AsciiDoc"], results.first "AsciiDoc" => "AsciiDoc/list.asc",
end "AGS Script" => nil
})
# Candidate languages = ["TypeScript", "XML"]
def test_ts_typescript_by_heuristics
results = Heuristics.disambiguate_ts(fixture("TypeScript/classes.ts"))
assert_equal Language["TypeScript"], results.first
end
# Candidate languages = ["TypeScript", "XML"]
def test_ts_xml_by_heuristics
results = Heuristics.disambiguate_ts(fixture("XML/pt_BR.xml"))
assert_equal Language["XML"], results.first
end end
def test_cl_by_heuristics def test_cl_by_heuristics
languages = ["Common Lisp", "OpenCL"] assert_heuristics({
languages.each do |language| "Common Lisp" => all_fixtures("Common Lisp"),
all_fixtures(language).each do |fixture| "OpenCL" => all_fixtures("OpenCL")
results = Heuristics.disambiguate_cl(fixture("#{language}/#{File.basename(fixture)}")) })
assert_equal Language[language], results.first
end
end
end end
def test_f_by_heuristics def test_f_by_heuristics
languages = ["FORTRAN", "Forth"] assert_heuristics({
languages.each do |language| "FORTRAN" => all_fixtures("FORTRAN"),
all_fixtures(language).each do |fixture| "Forth" => all_fixtures("Forth")
results = Heuristics.disambiguate_f(fixture("#{language}/#{File.basename(fixture)}")) })
assert_equal Language[language], results.first
end
end
end end
# Candidate languages = ["Hack", "PHP"] # Candidate languages = ["Hack", "PHP"]
def test_hack_by_heuristics def test_hack_by_heuristics
results = Heuristics.disambiguate_hack(fixture("Hack/funs.php")) assert_heuristics({
assert_equal Language["Hack"], results.first "Hack" => "Hack/funs.php",
"PHP" => "PHP/Model.php"
})
end end
# Candidate languages = ["Scala", "SuperCollider"] # Candidate languages = ["Scala", "SuperCollider"]
def test_sc_supercollider_by_heuristics def test_sc_supercollider_scala_by_heuristics
results = Heuristics.disambiguate_sc(fixture("SuperCollider/WarpPreset.sc")) assert_heuristics({
assert_equal Language["SuperCollider"], results.first "SuperCollider" => "SuperCollider/WarpPreset.sc",
end "Scala" => "Scala/node11.sc"
})
# Candidate languages = ["Scala", "SuperCollider"]
def test_sc_scala_by_heuristics
results = Heuristics.disambiguate_sc(fixture("Scala/node11.sc"))
assert_equal Language["Scala"], results.first
end end
def test_fs_by_heuristics def test_fs_by_heuristics
languages = ["F#", "Forth", "GLSL"] assert_heuristics({
languages.each do |language| "F#" => all_fixtures("F#"),
all_fixtures(language).each do |fixture| "Forth" => all_fixtures("Forth"),
results = Heuristics.disambiguate_fs(fixture("#{language}/#{File.basename(fixture)}")) "GLSL" => all_fixtures("GLSL")
assert_equal Language[language], results.first })
end
def assert_heuristics(hash)
candidates = hash.keys.map { |l| Language[l] }
hash.each do |language, blobs|
Array(blobs).each do |blob|
result = Heuristics.call(file_blob(blob), candidates)
assert_equal [Language[language]], result
end end
end end
end end