diff --git a/lib/linguist/heuristics.rb b/lib/linguist/heuristics.rb index 0930305c..0b09bd1b 100644 --- a/lib/linguist/heuristics.rb +++ b/lib/linguist/heuristics.rb @@ -1,8 +1,6 @@ module Linguist # A collection of simple heuristics that can be used to better analyze languages. class Heuristics - ACTIVE = true - # Public: Use heuristics to detect language of the blob. # # blob - An object that quacks like a blob. @@ -14,177 +12,123 @@ module Linguist # Language["Ruby"], Language["Python"] # ]) # - # Returns an Array with one Language if a heuristic matched, or empty if - # none matched or were inconclusive. + # Returns an Array of languages, or empty if none matched or were inconclusive. def self.call(blob, languages) - find_by_heuristics(blob.data, languages.map(&:name)) - end + data = blob.data - # Public: Given an array of String language names, - # apply heuristics against the given data and return an array - # of matching languages, or nil. - # - # data - Array of tokens or String data to analyze. - # languages - Array of language name Strings to restrict to. - # - # Returns an array of Languages or [] - def self.find_by_heuristics(data, languages) - if active? - result = [] - - if languages.all? { |l| ["Perl", "Prolog"].include?(l) } - result = disambiguate_pl(data) - end - if languages.all? { |l| ["ECL", "Prolog"].include?(l) } - result = disambiguate_ecl(data) - end - if languages.all? { |l| ["IDL", "Prolog"].include?(l) } - result = disambiguate_pro(data) - end - if languages.all? { |l| ["Common Lisp", "OpenCL"].include?(l) } - result = disambiguate_cl(data) - end - if languages.all? { |l| ["Hack", "PHP"].include?(l) } - result = disambiguate_hack(data) - end - if languages.all? { |l| ["Scala", "SuperCollider"].include?(l) } - result = disambiguate_sc(data) - end - if languages.all? { |l| ["AsciiDoc", "AGS Script"].include?(l) } - result = disambiguate_asc(data) - end - if languages.all? { |l| ["FORTRAN", "Forth"].include?(l) } - result = disambiguate_f(data) - end - if languages.all? { |l| ["F#", "Forth", "GLSL"].include?(l) } - result = disambiguate_fs(data) - end - return result + @heuristics.each do |heuristic| + return Array(heuristic.call(data)) if heuristic.matches?(languages) end + + [] # No heuristics matched end - # .h extensions are ambiguous between C, C++, and Objective-C. - # We want to shortcut look for Objective-C _and_ now C++ too! + # Internal: Define a new heuristic. # - # Returns an array of Languages or [] - def self.disambiguate_c(data) - matches = [] - if data.include?("@interface") - matches << Language["Objective-C"] - elsif data.include?("#include ") - matches << Language["C++"] - end - matches + # languages - String names of languages to disambiguate. + # heuristic - Block which takes data as an argument and returns a Language or nil. + # + # Examples + # + # disambiguate "Perl", "Prolog" do |data| + # if data.include?("use strict") + # Language["Perl"] + # elsif data.include?(":-") + # Language["Prolog"] + # end + # end + # + def self.disambiguate(*languages, &heuristic) + @heuristics << new(languages, &heuristic) end - def self.disambiguate_pl(data) - matches = [] + # Internal: Array of defined heuristics + @heuristics = [] + + # Internal + def initialize(languages, &heuristic) + @languages = languages + @heuristic = heuristic + end + + # Internal: Check if this heuristic matches the candidate languages. + def matches?(candidates) + candidates.all? { |l| @languages.include?(l.name) } + end + + # Internal: Perform the heuristic + def call(data) + @heuristic.call(data) + end + + disambiguate "Perl", "Prolog" do |data| if data.include?("use strict") - matches << Language["Perl"] + Language["Perl"] elsif data.include?(":-") - matches << Language["Prolog"] + Language["Prolog"] end - matches end - def self.disambiguate_ecl(data) - matches = [] + disambiguate "ECL", "Prolog" do |data| if data.include?(":-") - matches << Language["Prolog"] + Language["Prolog"] elsif data.include?(":=") - matches << Language["ECL"] + Language["ECL"] end - matches end - def self.disambiguate_pro(data) - matches = [] - if (data.include?(":-")) - matches << Language["Prolog"] + disambiguate "IDL", "Prolog" do |data| + if data.include?(":-") + Language["Prolog"] else - matches << Language["IDL"] + Language["IDL"] end - matches end - def self.disambiguate_ts(data) - matches = [] - if (data.include?("")) - matches << Language["XML"] - else - matches << Language["TypeScript"] - end - matches - end - - def self.disambiguate_cl(data) - matches = [] + disambiguate "Common Lisp", "OpenCL" do |data| if data.include?("(defun ") - matches << Language["Common Lisp"] + Language["Common Lisp"] elsif /\/\* |\/\/ |^\}/.match(data) - matches << Language["OpenCL"] + Language["OpenCL"] end - matches end - def self.disambiguate_r(data) - matches = [] - matches << Language["Rebol"] if /\bRebol\b/i.match(data) - matches << Language["R"] if data.include?("<-") - matches - end - - def self.disambiguate_hack(data) - matches = [] + disambiguate "Hack", "PHP" do |data| if data.include?(" "Prolog/turing.pl", + "Perl" => "Perl/perl-test.t", + }) end # Candidate languages = ["ECL", "Prolog"] def test_ecl_prolog_by_heuristics - results = Heuristics.disambiguate_ecl(fixture("Prolog/or-constraint.ecl")) - assert_equal Language["Prolog"], results.first + results = Heuristics.call(file_blob("Prolog/or-constraint.ecl"), [Language["ECL"], Language["Prolog"]]) + assert_equal [Language["Prolog"]], results end # Candidate languages = ["ECL", "Prolog"] - def test_ecl_ecl_by_heuristics - results = Heuristics.disambiguate_ecl(fixture("ECL/sample.ecl")) - assert_equal Language["ECL"], results.first + def test_ecl_prolog_by_heuristics + assert_heuristics({ + "ECL" => "ECL/sample.ecl", + "Prolog" => "Prolog/or-constraint.ecl" + }) end # Candidate languages = ["IDL", "Prolog"] - def test_pro_prolog_by_heuristics - results = Heuristics.disambiguate_pro(fixture("Prolog/logic-problem.pro")) - assert_equal Language["Prolog"], results.first - end - - # Candidate languages = ["IDL", "Prolog"] - def test_pro_idl_by_heuristics - results = Heuristics.disambiguate_pro(fixture("IDL/mg_acosh.pro")) - assert_equal Language["IDL"], results.first + def test_pro_prolog_idl_by_heuristics + assert_heuristics({ + "Prolog" => "Prolog/logic-problem.pro", + "IDL" => "IDL/mg_acosh.pro" + }) end # Candidate languages = ["AGS Script", "AsciiDoc"] def test_asc_asciidoc_by_heuristics - results = Heuristics.disambiguate_asc(fixture("AsciiDoc/list.asc")) - assert_equal Language["AsciiDoc"], results.first - end - - # Candidate languages = ["TypeScript", "XML"] - def test_ts_typescript_by_heuristics - results = Heuristics.disambiguate_ts(fixture("TypeScript/classes.ts")) - assert_equal Language["TypeScript"], results.first - end - - # Candidate languages = ["TypeScript", "XML"] - def test_ts_xml_by_heuristics - results = Heuristics.disambiguate_ts(fixture("XML/pt_BR.xml")) - assert_equal Language["XML"], results.first + assert_heuristics({ + "AsciiDoc" => "AsciiDoc/list.asc", + "AGS Script" => nil + }) end def test_cl_by_heuristics - languages = ["Common Lisp", "OpenCL"] - languages.each do |language| - all_fixtures(language).each do |fixture| - results = Heuristics.disambiguate_cl(fixture("#{language}/#{File.basename(fixture)}")) - assert_equal Language[language], results.first - end - end + assert_heuristics({ + "Common Lisp" => all_fixtures("Common Lisp"), + "OpenCL" => all_fixtures("OpenCL") + }) end def test_f_by_heuristics - languages = ["FORTRAN", "Forth"] - languages.each do |language| - all_fixtures(language).each do |fixture| - results = Heuristics.disambiguate_f(fixture("#{language}/#{File.basename(fixture)}")) - assert_equal Language[language], results.first - end - end + assert_heuristics({ + "FORTRAN" => all_fixtures("FORTRAN"), + "Forth" => all_fixtures("Forth") + }) end # Candidate languages = ["Hack", "PHP"] def test_hack_by_heuristics - results = Heuristics.disambiguate_hack(fixture("Hack/funs.php")) - assert_equal Language["Hack"], results.first + assert_heuristics({ + "Hack" => "Hack/funs.php", + "PHP" => "PHP/Model.php" + }) end # Candidate languages = ["Scala", "SuperCollider"] - def test_sc_supercollider_by_heuristics - results = Heuristics.disambiguate_sc(fixture("SuperCollider/WarpPreset.sc")) - assert_equal Language["SuperCollider"], results.first - end - - # Candidate languages = ["Scala", "SuperCollider"] - def test_sc_scala_by_heuristics - results = Heuristics.disambiguate_sc(fixture("Scala/node11.sc")) - assert_equal Language["Scala"], results.first + def test_sc_supercollider_scala_by_heuristics + assert_heuristics({ + "SuperCollider" => "SuperCollider/WarpPreset.sc", + "Scala" => "Scala/node11.sc" + }) end def test_fs_by_heuristics - languages = ["F#", "Forth", "GLSL"] - languages.each do |language| - all_fixtures(language).each do |fixture| - results = Heuristics.disambiguate_fs(fixture("#{language}/#{File.basename(fixture)}")) - assert_equal Language[language], results.first + assert_heuristics({ + "F#" => all_fixtures("F#"), + "Forth" => all_fixtures("Forth"), + "GLSL" => all_fixtures("GLSL") + }) + end + + def assert_heuristics(hash) + candidates = hash.keys.map { |l| Language[l] } + + hash.each do |language, blobs| + Array(blobs).each do |blob| + result = Heuristics.call(file_blob(blob), candidates) + assert_equal [Language[language]], result end end end