mirror of
https://github.com/KevinMidboe/linguist.git
synced 2025-10-29 17:50:22 +00:00
Merge pull request #1788 from github/refactor-heuristics
Refactor heuristics (updated)
This commit is contained in:
Brandon Keepers
@@ -1,8 +1,6 @@
|
||||
module Linguist
|
||||
# A collection of simple heuristics that can be used to better analyze languages.
|
||||
class Heuristics
|
||||
ACTIVE = true
|
||||
|
||||
# Public: Use heuristics to detect language of the blob.
|
||||
#
|
||||
# blob - An object that quacks like a blob.
|
||||
@@ -14,177 +12,123 @@ module Linguist
|
||||
# Language["Ruby"], Language["Python"]
|
||||
# ])
|
||||
#
|
||||
# Returns an Array with one Language if a heuristic matched, or empty if
|
||||
# none matched or were inconclusive.
|
||||
# Returns an Array of languages, or empty if none matched or were inconclusive.
|
||||
def self.call(blob, languages)
|
||||
find_by_heuristics(blob.data, languages.map(&:name))
|
||||
data = blob.data
|
||||
|
||||
@heuristics.each do |heuristic|
|
||||
return Array(heuristic.call(data)) if heuristic.matches?(languages)
|
||||
end
|
||||
|
||||
# Public: Given an array of String language names,
|
||||
# apply heuristics against the given data and return an array
|
||||
# of matching languages, or nil.
|
||||
[] # No heuristics matched
|
||||
end
|
||||
|
||||
# Internal: Define a new heuristic.
|
||||
#
|
||||
# data - Array of tokens or String data to analyze.
|
||||
# languages - Array of language name Strings to restrict to.
|
||||
# languages - String names of languages to disambiguate.
|
||||
# heuristic - Block which takes data as an argument and returns a Language or nil.
|
||||
#
|
||||
# Returns an array of Languages or []
|
||||
def self.find_by_heuristics(data, languages)
|
||||
if active?
|
||||
result = []
|
||||
|
||||
if languages.all? { |l| ["Perl", "Prolog"].include?(l) }
|
||||
result = disambiguate_pl(data)
|
||||
end
|
||||
if languages.all? { |l| ["ECL", "Prolog"].include?(l) }
|
||||
result = disambiguate_ecl(data)
|
||||
end
|
||||
if languages.all? { |l| ["IDL", "Prolog"].include?(l) }
|
||||
result = disambiguate_pro(data)
|
||||
end
|
||||
if languages.all? { |l| ["Common Lisp", "OpenCL"].include?(l) }
|
||||
result = disambiguate_cl(data)
|
||||
end
|
||||
if languages.all? { |l| ["Hack", "PHP"].include?(l) }
|
||||
result = disambiguate_hack(data)
|
||||
end
|
||||
if languages.all? { |l| ["Scala", "SuperCollider"].include?(l) }
|
||||
result = disambiguate_sc(data)
|
||||
end
|
||||
if languages.all? { |l| ["AsciiDoc", "AGS Script"].include?(l) }
|
||||
result = disambiguate_asc(data)
|
||||
end
|
||||
if languages.all? { |l| ["FORTRAN", "Forth"].include?(l) }
|
||||
result = disambiguate_f(data)
|
||||
end
|
||||
if languages.all? { |l| ["F#", "Forth", "GLSL"].include?(l) }
|
||||
result = disambiguate_fs(data)
|
||||
end
|
||||
return result
|
||||
end
|
||||
end
|
||||
|
||||
# .h extensions are ambiguous between C, C++, and Objective-C.
|
||||
# We want to shortcut look for Objective-C _and_ now C++ too!
|
||||
# Examples
|
||||
#
|
||||
# Returns an array of Languages or []
|
||||
def self.disambiguate_c(data)
|
||||
matches = []
|
||||
if data.include?("@interface")
|
||||
matches << Language["Objective-C"]
|
||||
elsif data.include?("#include <cstdint>")
|
||||
matches << Language["C++"]
|
||||
end
|
||||
matches
|
||||
# disambiguate "Perl", "Prolog" do |data|
|
||||
# if data.include?("use strict")
|
||||
# Language["Perl"]
|
||||
# elsif data.include?(":-")
|
||||
# Language["Prolog"]
|
||||
# end
|
||||
# end
|
||||
#
|
||||
def self.disambiguate(*languages, &heuristic)
|
||||
@heuristics << new(languages, &heuristic)
|
||||
end
|
||||
|
||||
def self.disambiguate_pl(data)
|
||||
matches = []
|
||||
# Internal: Array of defined heuristics
|
||||
@heuristics = []
|
||||
|
||||
# Internal
|
||||
def initialize(languages, &heuristic)
|
||||
@languages = languages
|
||||
@heuristic = heuristic
|
||||
end
|
||||
|
||||
# Internal: Check if this heuristic matches the candidate languages.
|
||||
def matches?(candidates)
|
||||
candidates.all? { |l| @languages.include?(l.name) }
|
||||
end
|
||||
|
||||
# Internal: Perform the heuristic
|
||||
def call(data)
|
||||
@heuristic.call(data)
|
||||
end
|
||||
|
||||
disambiguate "Perl", "Prolog" do |data|
|
||||
if data.include?("use strict")
|
||||
matches << Language["Perl"]
|
||||
Language["Perl"]
|
||||
elsif data.include?(":-")
|
||||
matches << Language["Prolog"]
|
||||
Language["Prolog"]
|
||||
end
|
||||
matches
|
||||
end
|
||||
|
||||
def self.disambiguate_ecl(data)
|
||||
matches = []
|
||||
disambiguate "ECL", "Prolog" do |data|
|
||||
if data.include?(":-")
|
||||
matches << Language["Prolog"]
|
||||
Language["Prolog"]
|
||||
elsif data.include?(":=")
|
||||
matches << Language["ECL"]
|
||||
Language["ECL"]
|
||||
end
|
||||
matches
|
||||
end
|
||||
|
||||
def self.disambiguate_pro(data)
|
||||
matches = []
|
||||
if (data.include?(":-"))
|
||||
matches << Language["Prolog"]
|
||||
disambiguate "IDL", "Prolog" do |data|
|
||||
if data.include?(":-")
|
||||
Language["Prolog"]
|
||||
else
|
||||
matches << Language["IDL"]
|
||||
Language["IDL"]
|
||||
end
|
||||
matches
|
||||
end
|
||||
|
||||
def self.disambiguate_ts(data)
|
||||
matches = []
|
||||
if (data.include?("</translation>"))
|
||||
matches << Language["XML"]
|
||||
else
|
||||
matches << Language["TypeScript"]
|
||||
end
|
||||
matches
|
||||
end
|
||||
|
||||
def self.disambiguate_cl(data)
|
||||
matches = []
|
||||
disambiguate "Common Lisp", "OpenCL" do |data|
|
||||
if data.include?("(defun ")
|
||||
matches << Language["Common Lisp"]
|
||||
Language["Common Lisp"]
|
||||
elsif /\/\* |\/\/ |^\}/.match(data)
|
||||
matches << Language["OpenCL"]
|
||||
Language["OpenCL"]
|
||||
end
|
||||
matches
|
||||
end
|
||||
|
||||
def self.disambiguate_r(data)
|
||||
matches = []
|
||||
matches << Language["Rebol"] if /\bRebol\b/i.match(data)
|
||||
matches << Language["R"] if data.include?("<-")
|
||||
matches
|
||||
end
|
||||
|
||||
def self.disambiguate_hack(data)
|
||||
matches = []
|
||||
disambiguate "Hack", "PHP" do |data|
|
||||
if data.include?("<?hh")
|
||||
matches << Language["Hack"]
|
||||
Language["Hack"]
|
||||
elsif /<?[^h]/.match(data)
|
||||
matches << Language["PHP"]
|
||||
Language["PHP"]
|
||||
end
|
||||
matches
|
||||
end
|
||||
|
||||
def self.disambiguate_sc(data)
|
||||
matches = []
|
||||
if (/\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data))
|
||||
matches << Language["SuperCollider"]
|
||||
disambiguate "Scala", "SuperCollider" do |data|
|
||||
if /\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data)
|
||||
Language["SuperCollider"]
|
||||
elsif /^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data)
|
||||
Language["Scala"]
|
||||
end
|
||||
if (/^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data))
|
||||
matches << Language["Scala"]
|
||||
end
|
||||
matches
|
||||
end
|
||||
|
||||
def self.disambiguate_asc(data)
|
||||
matches = []
|
||||
matches << Language["AsciiDoc"] if /^=+(\s|\n)/.match(data)
|
||||
matches
|
||||
disambiguate "AsciiDoc", "AGS Script" do |data|
|
||||
Language["AsciiDoc"] if /^=+(\s|\n)/.match(data)
|
||||
end
|
||||
|
||||
def self.disambiguate_f(data)
|
||||
matches = []
|
||||
disambiguate "FORTRAN", "Forth" do |data|
|
||||
if /^: /.match(data)
|
||||
matches << Language["Forth"]
|
||||
Language["Forth"]
|
||||
elsif /^([c*][^a-z]| subroutine\s)/i.match(data)
|
||||
matches << Language["FORTRAN"]
|
||||
Language["FORTRAN"]
|
||||
end
|
||||
matches
|
||||
end
|
||||
|
||||
def self.disambiguate_fs(data)
|
||||
matches = []
|
||||
disambiguate "F#", "Forth", "GLSL" do |data|
|
||||
if /^(: |new-device)/.match(data)
|
||||
matches << Language["Forth"]
|
||||
Language["Forth"]
|
||||
elsif /^(#light|import|let|module|namespace|open|type)/.match(data)
|
||||
matches << Language["F#"]
|
||||
Language["F#"]
|
||||
elsif /^(#include|#pragma|precision|uniform|varying|void)/.match(data)
|
||||
matches << Language["GLSL"]
|
||||
end
|
||||
matches
|
||||
end
|
||||
|
||||
def self.active?
|
||||
!!ACTIVE
|
||||
Language["GLSL"]
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -11,25 +11,15 @@ class TestHeuristcs < Test::Unit::TestCase
|
||||
File.read(File.join(samples_path, name))
|
||||
end
|
||||
|
||||
def file_blob(name)
|
||||
path = File.exist?(name) ? name : File.join(samples_path, name)
|
||||
FileBlob.new(path)
|
||||
end
|
||||
|
||||
def all_fixtures(language_name, file="*")
|
||||
Dir.glob("#{samples_path}/#{language_name}/#{file}")
|
||||
end
|
||||
|
||||
# Candidate languages = ["C++", "Objective-C"]
|
||||
def test_obj_c_by_heuristics
|
||||
# Only calling out '.h' filenames as these are the ones causing issues
|
||||
all_fixtures("Objective-C", "*.h").each do |fixture|
|
||||
results = Heuristics.disambiguate_c(fixture("Objective-C/#{File.basename(fixture)}"))
|
||||
assert_equal Language["Objective-C"], results.first
|
||||
end
|
||||
end
|
||||
|
||||
# Candidate languages = ["C++", "Objective-C"]
|
||||
def test_cpp_by_heuristics
|
||||
results = Heuristics.disambiguate_c(fixture("C++/render_adapter.cpp"))
|
||||
assert_equal Language["C++"], results.first
|
||||
end
|
||||
|
||||
def test_detect_still_works_if_nothing_matches
|
||||
blob = Linguist::FileBlob.new(File.join(samples_path, "Objective-C/hello.m"))
|
||||
match = Language.detect(blob)
|
||||
@@ -37,103 +27,88 @@ class TestHeuristcs < Test::Unit::TestCase
|
||||
end
|
||||
|
||||
# Candidate languages = ["Perl", "Prolog"]
|
||||
def test_pl_prolog_by_heuristics
|
||||
results = Heuristics.disambiguate_pl(fixture("Prolog/turing.pl"))
|
||||
assert_equal Language["Prolog"], results.first
|
||||
end
|
||||
|
||||
# Candidate languages = ["Perl", "Prolog"]
|
||||
def test_pl_perl_by_heuristics
|
||||
results = Heuristics.disambiguate_pl(fixture("Perl/perl-test.t"))
|
||||
assert_equal Language["Perl"], results.first
|
||||
def test_pl_prolog_perl_by_heuristics
|
||||
assert_heuristics({
|
||||
"Prolog" => "Prolog/turing.pl",
|
||||
"Perl" => "Perl/perl-test.t",
|
||||
})
|
||||
end
|
||||
|
||||
# Candidate languages = ["ECL", "Prolog"]
|
||||
def test_ecl_prolog_by_heuristics
|
||||
results = Heuristics.disambiguate_ecl(fixture("Prolog/or-constraint.ecl"))
|
||||
assert_equal Language["Prolog"], results.first
|
||||
results = Heuristics.call(file_blob("Prolog/or-constraint.ecl"), [Language["ECL"], Language["Prolog"]])
|
||||
assert_equal [Language["Prolog"]], results
|
||||
end
|
||||
|
||||
# Candidate languages = ["ECL", "Prolog"]
|
||||
def test_ecl_ecl_by_heuristics
|
||||
results = Heuristics.disambiguate_ecl(fixture("ECL/sample.ecl"))
|
||||
assert_equal Language["ECL"], results.first
|
||||
def test_ecl_prolog_by_heuristics
|
||||
assert_heuristics({
|
||||
"ECL" => "ECL/sample.ecl",
|
||||
"Prolog" => "Prolog/or-constraint.ecl"
|
||||
})
|
||||
end
|
||||
|
||||
# Candidate languages = ["IDL", "Prolog"]
|
||||
def test_pro_prolog_by_heuristics
|
||||
results = Heuristics.disambiguate_pro(fixture("Prolog/logic-problem.pro"))
|
||||
assert_equal Language["Prolog"], results.first
|
||||
end
|
||||
|
||||
# Candidate languages = ["IDL", "Prolog"]
|
||||
def test_pro_idl_by_heuristics
|
||||
results = Heuristics.disambiguate_pro(fixture("IDL/mg_acosh.pro"))
|
||||
assert_equal Language["IDL"], results.first
|
||||
def test_pro_prolog_idl_by_heuristics
|
||||
assert_heuristics({
|
||||
"Prolog" => "Prolog/logic-problem.pro",
|
||||
"IDL" => "IDL/mg_acosh.pro"
|
||||
})
|
||||
end
|
||||
|
||||
# Candidate languages = ["AGS Script", "AsciiDoc"]
|
||||
def test_asc_asciidoc_by_heuristics
|
||||
results = Heuristics.disambiguate_asc(fixture("AsciiDoc/list.asc"))
|
||||
assert_equal Language["AsciiDoc"], results.first
|
||||
end
|
||||
|
||||
# Candidate languages = ["TypeScript", "XML"]
|
||||
def test_ts_typescript_by_heuristics
|
||||
results = Heuristics.disambiguate_ts(fixture("TypeScript/classes.ts"))
|
||||
assert_equal Language["TypeScript"], results.first
|
||||
end
|
||||
|
||||
# Candidate languages = ["TypeScript", "XML"]
|
||||
def test_ts_xml_by_heuristics
|
||||
results = Heuristics.disambiguate_ts(fixture("XML/pt_BR.xml"))
|
||||
assert_equal Language["XML"], results.first
|
||||
assert_heuristics({
|
||||
"AsciiDoc" => "AsciiDoc/list.asc",
|
||||
"AGS Script" => nil
|
||||
})
|
||||
end
|
||||
|
||||
def test_cl_by_heuristics
|
||||
languages = ["Common Lisp", "OpenCL"]
|
||||
languages.each do |language|
|
||||
all_fixtures(language).each do |fixture|
|
||||
results = Heuristics.disambiguate_cl(fixture("#{language}/#{File.basename(fixture)}"))
|
||||
assert_equal Language[language], results.first
|
||||
end
|
||||
end
|
||||
assert_heuristics({
|
||||
"Common Lisp" => all_fixtures("Common Lisp"),
|
||||
"OpenCL" => all_fixtures("OpenCL")
|
||||
})
|
||||
end
|
||||
|
||||
def test_f_by_heuristics
|
||||
languages = ["FORTRAN", "Forth"]
|
||||
languages.each do |language|
|
||||
all_fixtures(language).each do |fixture|
|
||||
results = Heuristics.disambiguate_f(fixture("#{language}/#{File.basename(fixture)}"))
|
||||
assert_equal Language[language], results.first
|
||||
end
|
||||
end
|
||||
assert_heuristics({
|
||||
"FORTRAN" => all_fixtures("FORTRAN"),
|
||||
"Forth" => all_fixtures("Forth")
|
||||
})
|
||||
end
|
||||
|
||||
# Candidate languages = ["Hack", "PHP"]
|
||||
def test_hack_by_heuristics
|
||||
results = Heuristics.disambiguate_hack(fixture("Hack/funs.php"))
|
||||
assert_equal Language["Hack"], results.first
|
||||
assert_heuristics({
|
||||
"Hack" => "Hack/funs.php",
|
||||
"PHP" => "PHP/Model.php"
|
||||
})
|
||||
end
|
||||
|
||||
# Candidate languages = ["Scala", "SuperCollider"]
|
||||
def test_sc_supercollider_by_heuristics
|
||||
results = Heuristics.disambiguate_sc(fixture("SuperCollider/WarpPreset.sc"))
|
||||
assert_equal Language["SuperCollider"], results.first
|
||||
end
|
||||
|
||||
# Candidate languages = ["Scala", "SuperCollider"]
|
||||
def test_sc_scala_by_heuristics
|
||||
results = Heuristics.disambiguate_sc(fixture("Scala/node11.sc"))
|
||||
assert_equal Language["Scala"], results.first
|
||||
def test_sc_supercollider_scala_by_heuristics
|
||||
assert_heuristics({
|
||||
"SuperCollider" => "SuperCollider/WarpPreset.sc",
|
||||
"Scala" => "Scala/node11.sc"
|
||||
})
|
||||
end
|
||||
|
||||
def test_fs_by_heuristics
|
||||
languages = ["F#", "Forth", "GLSL"]
|
||||
languages.each do |language|
|
||||
all_fixtures(language).each do |fixture|
|
||||
results = Heuristics.disambiguate_fs(fixture("#{language}/#{File.basename(fixture)}"))
|
||||
assert_equal Language[language], results.first
|
||||
assert_heuristics({
|
||||
"F#" => all_fixtures("F#"),
|
||||
"Forth" => all_fixtures("Forth"),
|
||||
"GLSL" => all_fixtures("GLSL")
|
||||
})
|
||||
end
|
||||
|
||||
def assert_heuristics(hash)
|
||||
candidates = hash.keys.map { |l| Language[l] }
|
||||
|
||||
hash.each do |language, blobs|
|
||||
Array(blobs).each do |blob|
|
||||
result = Heuristics.call(file_blob(blob), candidates)
|
||||
assert_equal [Language[language]], result
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user