Refactor heuristics

lib/linguist/heuristics.rb

@@ -1,7 +1,6 @@
 module Linguist
   # A collection of simple heuristics that can be used to better analyze languages.
   class Heuristics
-
     # Public: Use heuristics to detect language of the blob.
     #
     # blob               - An object that quacks like a blob.
@@ -16,48 +15,105 @@ module Linguist
     # Returns an Array with one Language if a heuristic matched, or empty if
     # none matched or were inconclusive.
     def self.call(blob, languages)
-      find_by_heuristics(blob.data, languages.map(&:name))
+      data = blob.data
+
+      @heuristics.each do |heuristic|
+        if heuristic.matches?(languages)
+          language = heuristic.call(data)
+          return [language] if language
+        end
+      end
+
+      [] # No heuristics matched
     end
 
-    # Public: Given an array of String language names,
+    @heuristics = []
-    # apply heuristics against the given data and return an array
-    # of matching languages, or nil.
-    #
-    # data      - Array of tokens or String data to analyze.
-    # languages - Array of language name Strings to restrict to.
-    #
-    # Returns an array of Languages or []
-    def self.find_by_heuristics(data, languages)
-      result = []
 
-      if languages.all? { |l| ["Perl", "Prolog"].include?(l) }
+    def self.create(*languages, &heuristic)
-        result = disambiguate_pl(data)
+      @heuristics << new(languages, &heuristic)
+    end
+
+    def initialize(languages, &heuristic)
+      @languages = languages
+      @heuristic = heuristic
+    end
+
+    def matches?(candidates)
+      candidates.all? { |l| @languages.include?(l.name) }
+    end
+
+    def call(data)
+      @heuristic.call(data)
+    end
+
+    create "Perl", "Prolog" do |data|
+      if data.include?("use strict")
+        Language["Perl"]
+      elsif data.include?(":-")
+        Language["Prolog"]
       end
-      if languages.all? { |l| ["ECL", "Prolog"].include?(l) }
+    end
-        result = disambiguate_ecl(data)
+
+    create "ECL", "Prolog" do |data|
+      if data.include?(":-")
+        Language["Prolog"]
+      elsif data.include?(":=")
+        Language["ECL"]
       end
-      if languages.all? { |l| ["IDL", "Prolog"].include?(l) }
+    end
-        result = disambiguate_pro(data)
+
+    create "IDL", "Prolog" do |data|
+      if data.include?(":-")
+        Language["Prolog"]
+      else
+        Language["IDL"]
       end
-      if languages.all? { |l| ["Common Lisp", "OpenCL"].include?(l) }
+    end
-        result = disambiguate_cl(data)
+
+    create "Common Lisp", "OpenCL" do |data|
+      if data.include?("(defun ")
+        Language["Common Lisp"]
+      elsif /\/\* |\/\/ |^\}/.match(data)
+        Language["OpenCL"]
       end
-      if languages.all? { |l| ["Hack", "PHP"].include?(l) }
+    end
-        result = disambiguate_hack(data)
+
+    create "Hack", "PHP" do |data|
+      if data.include?("<?hh")
+        Language["Hack"]
+      elsif /<?[^h]/.match(data)
+        Language["PHP"]
       end
-      if languages.all? { |l| ["Scala", "SuperCollider"].include?(l) }
+    end
-        result = disambiguate_sc(data)
+
+    create "Scala", "SuperCollider" do |data|
+      if /\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data)
+        Language["SuperCollider"]
+      elsif /^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data)
+        Language["Scala"]
       end
-      if languages.all? { |l| ["AsciiDoc", "AGS Script"].include?(l) }
+    end
-        result = disambiguate_asc(data)
+
+    create "AsciiDoc", "AGS Script" do |data|
+      Language["AsciiDoc"] if /^=+(\s|\n)/.match(data)
+    end
+
+    create "FORTRAN", "Forth" do |data|
+      if /^: /.match(data)
+        Language["Forth"]
+      elsif /^([c*][^a-z]|      subroutine\s)/i.match(data)
+        Language["FORTRAN"]
       end
-      if languages.all? { |l| ["FORTRAN", "Forth"].include?(l) }
+    end
-        result = disambiguate_f(data)
+
+    create "F#", "Forth", "GLSL" do |data|
+      if /^(: |new-device)/.match(data)
+        Language["Forth"]
+      elsif /^(#light|import|let|module|namespace|open|type)/.match(data)
+        Language["F#"]
+      elsif /^(#include|#pragma|precision|uniform|varying|void)/.match(data)
+        Language["GLSL"]
       end
-      if languages.all? { |l| ["F#", "Forth", "GLSL"].include?(l) }
-        result = disambiguate_fs(data)
-      end
-      return result
     end
 
     # .h extensions are ambiguous between C, C++, and Objective-C.
@@ -74,36 +130,6 @@ module Linguist
       matches
     end
 
-    def self.disambiguate_pl(data)
-      matches = []
-      if data.include?("use strict")
-        matches << Language["Perl"]
-      elsif data.include?(":-")
-        matches << Language["Prolog"]
-      end
-      matches
-    end
-
-    def self.disambiguate_ecl(data)
-      matches = []
-      if data.include?(":-")
-        matches << Language["Prolog"]
-      elsif data.include?(":=")
-        matches << Language["ECL"]
-      end
-      matches
-    end
-
-    def self.disambiguate_pro(data)
-      matches = []
-      if (data.include?(":-"))
-        matches << Language["Prolog"]
-      else
-        matches << Language["IDL"]
-      end
-      matches
-    end
-
     def self.disambiguate_ts(data)
       matches = []
       if (data.include?("</translation>"))
@@ -114,16 +140,6 @@ module Linguist
       matches
     end
 
-    def self.disambiguate_cl(data)
-      matches = []
-      if data.include?("(defun ")
-        matches << Language["Common Lisp"]
-      elsif /\/\* |\/\/ |^\}/.match(data)
-        matches << Language["OpenCL"]
-      end
-      matches
-    end
-
     def self.disambiguate_r(data)
       matches = []
       matches << Language["Rebol"] if /\bRebol\b/i.match(data)
@@ -131,57 +147,5 @@ module Linguist
       matches
     end
 
-    def self.disambiguate_hack(data)
-      matches = []
-      if data.include?("<?hh")
-        matches << Language["Hack"]
-      elsif /<?[^h]/.match(data)
-        matches << Language["PHP"]
-      end
-      matches
-    end
-
-    def self.disambiguate_sc(data)
-      matches = []
-      if (/\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data))
-        matches << Language["SuperCollider"]
-      end
-      if (/^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data))
-        matches << Language["Scala"]
-      end
-      matches
-    end
-
-    def self.disambiguate_asc(data)
-      matches = []
-      matches << Language["AsciiDoc"] if /^=+(\s|\n)/.match(data)
-      matches
-    end
-
-    def self.disambiguate_f(data)
-      matches = []
-      if /^: /.match(data)
-        matches << Language["Forth"]
-      elsif /^([c*][^a-z]|      subroutine\s)/i.match(data)
-        matches << Language["FORTRAN"]
-      end
-      matches
-    end
-
-    def self.disambiguate_fs(data)
-      matches = []
-      if /^(: |new-device)/.match(data)
-        matches << Language["Forth"]
-      elsif /^(#light|import|let|module|namespace|open|type)/.match(data)
-        matches << Language["F#"]
-      elsif /^(#include|#pragma|precision|uniform|varying|void)/.match(data)
-        matches << Language["GLSL"]
-      end
-      matches
-    end
-
-    def self.active?
-      !!ACTIVE
-    end
   end
 end

test/test_heuristics.rb

@@ -11,6 +11,11 @@ class TestHeuristcs < Test::Unit::TestCase
     File.read(File.join(samples_path, name))
   end
 
+  def file_blob(name)
+    path = File.exist?(name) ? name : File.join(samples_path, name)
+    FileBlob.new(path)
+  end
+
   def all_fixtures(language_name, file="*")
     Dir.glob("#{samples_path}/#{language_name}/#{file}")
   end
@@ -37,45 +42,41 @@ class TestHeuristcs < Test::Unit::TestCase
   end
 
   # Candidate languages = ["Perl", "Prolog"]
-  def test_pl_prolog_by_heuristics
+  def test_pl_prolog_perl_by_heuristics
-    results = Heuristics.disambiguate_pl(fixture("Prolog/turing.pl"))
+    assert_heuristics({
-    assert_equal Language["Prolog"], results.first
+      "Prolog" => "Prolog/turing.pl",
-  end
+      "Perl" => "Perl/perl-test.t",
-
+    })
-  # Candidate languages = ["Perl", "Prolog"]
-  def test_pl_perl_by_heuristics
-    results = Heuristics.disambiguate_pl(fixture("Perl/perl-test.t"))
-    assert_equal Language["Perl"], results.first
   end
 
   # Candidate languages = ["ECL", "Prolog"]
   def test_ecl_prolog_by_heuristics
-    results = Heuristics.disambiguate_ecl(fixture("Prolog/or-constraint.ecl"))
+    results = Heuristics.call(file_blob("Prolog/or-constraint.ecl"), [Language["ECL"], Language["Prolog"]])
-    assert_equal Language["Prolog"], results.first
+    assert_equal [Language["Prolog"]], results
   end
 
   # Candidate languages = ["ECL", "Prolog"]
-  def test_ecl_ecl_by_heuristics
+  def test_ecl_prolog_by_heuristics
-    results = Heuristics.disambiguate_ecl(fixture("ECL/sample.ecl"))
+    assert_heuristics({
-    assert_equal Language["ECL"], results.first
+      "ECL" => "ECL/sample.ecl",
+      "Prolog" => "Prolog/or-constraint.ecl"
+    })
   end
 
   # Candidate languages = ["IDL", "Prolog"]
-  def test_pro_prolog_by_heuristics
+  def test_pro_prolog_idl_by_heuristics
-    results = Heuristics.disambiguate_pro(fixture("Prolog/logic-problem.pro"))
+    assert_heuristics({
-    assert_equal Language["Prolog"], results.first
+      "Prolog" => "Prolog/logic-problem.pro",
-  end
+      "IDL" => "IDL/mg_acosh.pro"
-
+    })
-  # Candidate languages = ["IDL", "Prolog"]
-  def test_pro_idl_by_heuristics
-    results = Heuristics.disambiguate_pro(fixture("IDL/mg_acosh.pro"))
-    assert_equal Language["IDL"], results.first
   end
 
   # Candidate languages = ["AGS Script", "AsciiDoc"]
   def test_asc_asciidoc_by_heuristics
-    results = Heuristics.disambiguate_asc(fixture("AsciiDoc/list.asc"))
+    assert_heuristics({
-    assert_equal Language["AsciiDoc"], results.first
+      "AsciiDoc" => "AsciiDoc/list.asc",
+      "AGS Script" => nil
+    })
   end
 
   # Candidate languages = ["TypeScript", "XML"]
@@ -91,49 +92,50 @@ class TestHeuristcs < Test::Unit::TestCase
   end
 
   def test_cl_by_heuristics
-    languages = ["Common Lisp", "OpenCL"]
+    assert_heuristics({
-    languages.each do |language|
+      "Common Lisp" => all_fixtures("Common Lisp"),
-      all_fixtures(language).each do |fixture|
+      "OpenCL" => all_fixtures("OpenCL")
-        results = Heuristics.disambiguate_cl(fixture("#{language}/#{File.basename(fixture)}"))
+    })
-        assert_equal Language[language], results.first
-      end
-    end
   end
 
   def test_f_by_heuristics
-    languages = ["FORTRAN", "Forth"]
+    assert_heuristics({
-    languages.each do |language|
+      "FORTRAN" => all_fixtures("FORTRAN"),
-      all_fixtures(language).each do |fixture|
+      "Forth" => all_fixtures("Forth")
-        results = Heuristics.disambiguate_f(fixture("#{language}/#{File.basename(fixture)}"))
+    })
-        assert_equal Language[language], results.first
-      end
-    end
   end
 
   # Candidate languages = ["Hack", "PHP"]
   def test_hack_by_heuristics
-    results = Heuristics.disambiguate_hack(fixture("Hack/funs.php"))
+    assert_heuristics({
-    assert_equal Language["Hack"], results.first
+      "Hack" => "Hack/funs.php",
+      "PHP" => "PHP/Model.php"
+    })
   end
 
   # Candidate languages = ["Scala", "SuperCollider"]
-  def test_sc_supercollider_by_heuristics
+  def test_sc_supercollider_scala_by_heuristics
-    results = Heuristics.disambiguate_sc(fixture("SuperCollider/WarpPreset.sc"))
+    assert_heuristics({
-    assert_equal Language["SuperCollider"], results.first
+      "SuperCollider" => "SuperCollider/WarpPreset.sc",
-  end
+      "Scala" => "Scala/node11.sc"
-
+    })
-  # Candidate languages = ["Scala", "SuperCollider"]
-  def test_sc_scala_by_heuristics
-    results = Heuristics.disambiguate_sc(fixture("Scala/node11.sc"))
-    assert_equal Language["Scala"], results.first
   end
 
   def test_fs_by_heuristics
-    languages = ["F#", "Forth", "GLSL"]
+    assert_heuristics({
-    languages.each do |language|
+      "F#" => all_fixtures("F#"),
-      all_fixtures(language).each do |fixture|
+      "Forth" => all_fixtures("Forth"),
-        results = Heuristics.disambiguate_fs(fixture("#{language}/#{File.basename(fixture)}"))
+      "GLSL" => all_fixtures("GLSL")
-        assert_equal Language[language], results.first
+    })
+  end
+
+  def assert_heuristics(hash)
+    candidates = hash.keys.map { |l| Language[l] }
+
+    hash.each do |language, blobs|
+      Array(blobs).each do |blob|
+        result = Heuristics.call(file_blob(blob), candidates)
+        assert_equal [Language[language]], result
       end
     end
   end