KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Alphabetise heuristics.

Browse Source
This commit is contained in:
Lars Brinkhoff
2015-08-30 12:32:33 +02:00
parent e24efad5ff
commit dd8eaf2893
2 changed files with 290 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

366
lib/linguist/heuristics.rb
View File

@@ -67,6 +67,16 @@ module Linguist
# Common heuristics
ObjectiveCRegex = /^[ \t]*@(interface|class|protocol|property|end|synchronized|selector|implementation)\b/
disambiguate ".asc" do |data|
if /^(----[- ]BEGIN|ssh-(rsa|dss)) /.match(data)
Language["Public Key"]
elsif /^[=-]+(\s|\n)|{{[A-Za-z]/.match(data)
Language["AsciiDoc"]
elsif /^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])/.match(data)
Language["AGS Script"]
end
end
disambiguate ".bb" do |data|
if /^\s*; /.match(data) || data.include?("End Function")
Language["BlitzBasic"]
@@ -75,67 +85,9 @@ module Linguist
end
end
disambiguate ".cs" do |data|
if /![\w\s]+methodsFor: /.match(data)
Language["Smalltalk"]
elsif /^\s*namespace\s*[\w\.]+\s*{/.match(data) || /^\s*\/\//.match(data)
Language["C#"]
end
end
disambiguate ".h" do |data|
if ObjectiveCRegex.match(data)
Language["Objective-C"]
elsif (/^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>/.match(data) ||
/^\s*template\s*</.match(data) || /^[ \t]*try/.match(data) || /^[ \t]*catch\s*\(/.match(data) || /^[ \t]*(class|(using[ \t]+)?namespace)\s+\w+/.match(data) || /^[ \t]*(private|public|protected):$/.match(data) || /std::\w+/.match(data))
Language["C++"]
end
end
disambiguate ".pl" do |data|
if /^(use v6|(my )?class|module)/.match(data)
Language["Perl6"]
elsif /use strict|use\s+v?5\./.match(data)
Language["Perl"]
elsif /^[^#]+:-/.match(data)
Language["Prolog"]
end
end
disambiguate ".pm", ".t" do |data|
if /^(use v6|(my )?class|module)/.match(data)
Language["Perl6"]
elsif /use strict|use\s+v?5\./.match(data)
Language["Perl"]
end
end
disambiguate ".ecl" do |data|
if /^[^#]+:-/.match(data)
Language["ECLiPSe"]
elsif data.include?(":=")
Language["ECL"]
end
end
disambiguate ".pro" do |data|
if /^[^#]+:-/.match(data)
Language["Prolog"]
elsif data.include?("last_client=")
Language["INI"]
elsif data.include?("HEADERS") && data.include?("SOURCES")
Language["QMake"]
elsif /^\s*function[ \w,]+$/.match(data)
Language["IDL"]
end
end
disambiguate ".tst" do |data|
if (data.include?("gap> "))
Language["GAP"]
# Heads up - we don't usually write heuristics like this (with no regex match)
else
Language["Scilab"]
disambiguate ".ch" do |data|
if /^\s*#\s*(if|ifdef|ifndef|define|command|xcommand|translate|xtranslate|include|pragma|undef)\b/i.match(data)
Language["xBase"]
end
end
@@ -149,29 +101,29 @@ module Linguist
end
end
disambiguate ".php" do |data|
if data.include?("<?hh")
Language["Hack"]
elsif /<?[^h]/.match(data)
Language["PHP"]
disambiguate ".cs" do |data|
if /![\w\s]+methodsFor: /.match(data)
Language["Smalltalk"]
elsif /^\s*namespace\s*[\w\.]+\s*{/.match(data) || /^\s*\/\//.match(data)
Language["C#"]
end
end
disambiguate ".sc" do |data|
if /\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data)
Language["SuperCollider"]
elsif /^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data)
Language["Scala"]
disambiguate ".d" do |data|
if /^module /.match(data)
Language["D"]
elsif /^((dtrace:::)?BEGIN|provider |#pragma (D (option|attributes)|ident)\s)/.match(data)
Language["DTrace"]
elsif /(\/.*:( .* \\)$| : \\$|^ : |: \\$)/.match(data)
Language["Makefile"]
end
end
disambiguate ".asc" do |data|
if /^(----[- ]BEGIN|ssh-(rsa|dss)) /.match(data)
Language["Public Key"]
elsif /^[=-]+(\s|\n)|{{[A-Za-z]/.match(data)
Language["AsciiDoc"]
elsif /^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])/.match(data)
Language["AGS Script"]
disambiguate ".ecl" do |data|
if /^[^#]+:-/.match(data)
Language["ECLiPSe"]
elsif data.include?(":=")
Language["ECL"]
end
end
@@ -183,6 +135,16 @@ module Linguist
end
end
disambiguate ".fr" do |data|
if /^(: |also |new-device|previous )/.match(data)
Language["Forth"]
elsif /^\s*(import|module|package|data|type) /.match(data)
Language["Frege"]
else
Language["Text"]
end
end
disambiguate ".fs" do |data|
if /^(: |new-device)/.match(data)
Language["Forth"]
@@ -195,6 +157,47 @@ module Linguist
end
end
disambiguate ".gs" do |data|
Language["Gosu"] if /^uses java\./.match(data)
end
disambiguate ".h" do |data|
if ObjectiveCRegex.match(data)
Language["Objective-C"]
elsif (/^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>/.match(data) ||
/^\s*template\s*</.match(data) || /^[ \t]*try/.match(data) || /^[ \t]*catch\s*\(/.match(data) || /^[ \t]*(class|(using[ \t]+)?namespace)\s+\w+/.match(data) || /^[ \t]*(private|public|protected):$/.match(data) || /std::\w+/.match(data))
Language["C++"]
end
end
disambiguate ".l" do |data|
if /\(def(un|macro)\s/.match(data)
Language["Common Lisp"]
elsif /^(%[%{}]xs|<.*>)/.match(data)
Language["Lex"]
elsif /^\.[a-z][a-z](\s|$)/i.match(data)
Language["Groff"]
elsif /^\((de|class|rel|code|data|must)\s/.match(data)
Language["PicoLisp"]
end
end
disambiguate ".ls" do |data|
if /^\s*package\s*[\w\.\/\*\s]*\s*{/.match(data)
Language["LoomScript"]
else
Language["LiveScript"]
end
end
disambiguate ".lsp", ".lisp" do |data|
if /^\s*\((defun|in-package|defpackage) /i.match(data)
Language["Common Lisp"]
elsif /^\s*\(define /.match(data)
Language["NewLisp"]
end
end
disambiguate ".m" do |data|
if ObjectiveCRegex.match(data)
Language["Objective-C"]
@@ -213,41 +216,113 @@ module Linguist
end
end
disambiguate ".gs" do |data|
Language["Gosu"] if /^uses java\./.match(data)
disambiguate ".ml" do |data|
if /(^\s*module)|let rec |match\s+(\S+\s)+with/.match(data)
Language["OCaml"]
elsif /=> |case\s+(\S+\s)+of/.match(data)
Language["Standard ML"]
end
end
disambiguate ".ls" do |data|
if /^\s*package\s*[\w\.\/\*\s]*\s*{/.match(data)
Language["LoomScript"]
disambiguate ".mod" do |data|
if data.include?('<!ENTITY ')
Language["XML"]
elsif /MODULE\s\w+\s*;/i.match(data) || /^\s*END \w+;$/i.match(data)
Language["Modula-2"]
else
Language["LiveScript"]
[Language["Linux Kernel Module"], Language["AMPL"]]
end
end
disambiguate ".lsp", ".lisp" do |data|
if /^\s*\((defun|in-package|defpackage) /i.match(data)
Language["Common Lisp"]
elsif /^\s*\(define /.match(data)
disambiguate ".ms" do |data|
if /^[.'][a-z][a-z](\s|$)/i.match(data)
Language["Groff"]
elsif /((^|\s)move?[. ])|\.(include|globa?l)\s/.match(data)
Language["GAS"]
end
end
disambiguate ".n" do |data|
if /^[.']/.match(data)
Language["Groff"]
elsif /^(module|namespace|using)\s/.match(data)
Language["Nemerle"]
end
end
disambiguate ".ncl" do |data|
if data.include?("THE_TITLE")
Language["Text"]
end
end
disambiguate ".nl" do |data|
if /^(b|g)[0-9]+ /.match(data)
Language["NL"]
else
Language["NewLisp"]
end
end
disambiguate ".ts" do |data|
if data.include?("<TS ")
Language["XML"]
else
Language["TypeScript"]
disambiguate ".php" do |data|
if data.include?("<?hh")
Language["Hack"]
elsif /<?[^h]/.match(data)
Language["PHP"]
end
end
disambiguate ".fr" do |data|
if /^(: |also |new-device|previous )/.match(data)
Language["Forth"]
elsif /^\s*(import|module|package|data|type) /.match(data)
Language["Frege"]
else
Language["Text"]
disambiguate ".pl" do |data|
if /^(use v6|(my )?class|module)/.match(data)
Language["Perl6"]
elsif /use strict|use\s+v?5\./.match(data)
Language["Perl"]
elsif /^[^#]+:-/.match(data)
Language["Prolog"]
end
end
disambiguate ".pm", ".t" do |data|
if /^(use v6|(my )?class|module)/.match(data)
Language["Perl6"]
elsif /use strict|use\s+v?5\./.match(data)
Language["Perl"]
end
end
disambiguate ".pro" do |data|
if /^[^#]+:-/.match(data)
Language["Prolog"]
elsif data.include?("last_client=")
Language["INI"]
elsif data.include?("HEADERS") && data.include?("SOURCES")
Language["QMake"]
elsif /^\s*function[ \w,]+$/.match(data)
Language["IDL"]
end
end
disambiguate ".r" do |data|
if /\bRebol\b/i.match(data)
Language["Rebol"]
elsif data.include?("<-")
Language["R"]
end
end
disambiguate ".rs" do |data|
if /^(use |fn |mod |pub |macro_rules|impl|#!?\[)/.match(data)
Language["Rust"]
elsif /#include|#pragma\s+(rs|version)|__attribute__/.match(data)
Language["RenderScript"]
end
end
disambiguate ".sc" do |data|
if /\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data)
Language["SuperCollider"]
elsif /^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data)
Language["Scala"]
end
end
@@ -267,95 +342,20 @@ module Linguist
end
end
disambiguate ".d" do |data|
if /^module /.match(data)
Language["D"]
elsif /^((dtrace:::)?BEGIN|provider |#pragma (D (option|attributes)|ident)\s)/.match(data)
Language["DTrace"]
elsif /(\/.*:( .* \\)$| : \\$|^ : |: \\$)/.match(data)
Language["Makefile"]
end
end
disambiguate ".ml" do |data|
if /(^\s*module)|let rec |match\s+(\S+\s)+with/.match(data)
Language["OCaml"]
elsif /=> |case\s+(\S+\s)+of/.match(data)
Language["Standard ML"]
end
end
disambiguate ".mod" do |data|
if data.include?('<!ENTITY ')
disambiguate ".ts" do |data|
if data.include?("<TS ")
Language["XML"]
elsif /MODULE\s\w+\s*;/i.match(data) || /^\s*END \w+;$/i.match(data)
Language["Modula-2"]
else
[Language["Linux Kernel Module"], Language["AMPL"]]
Language["TypeScript"]
end
end
disambiguate ".ncl" do |data|
if data.include?("THE_TITLE")
Language["Text"]
end
end
disambiguate ".nl" do |data|
if /^(b|g)[0-9]+ /.match(data)
Language["NL"]
disambiguate ".tst" do |data|
if (data.include?("gap> "))
Language["GAP"]
# Heads up - we don't usually write heuristics like this (with no regex match)
else
Language["NewLisp"]
end
end
disambiguate ".rs" do |data|
if /^(use |fn |mod |pub |macro_rules|impl|#!?\[)/.match(data)
Language["Rust"]
elsif /#include|#pragma\s+(rs|version)|__attribute__/.match(data)
Language["RenderScript"]
end
end
disambiguate ".l" do |data|
if /\(def(un|macro)\s/.match(data)
Language["Common Lisp"]
elsif /^(%[%{}]xs|<.*>)/.match(data)
Language["Lex"]
elsif /^\.[a-z][a-z](\s|$)/i.match(data)
Language["Groff"]
elsif /^\((de|class|rel|code|data|must)\s/.match(data)
Language["PicoLisp"]
end
end
disambiguate ".n" do |data|
if /^[.']/.match(data)
Language["Groff"]
elsif /^(module|namespace|using)\s/.match(data)
Language["Nemerle"]
end
end
disambiguate ".ms" do |data|
if /^[.'][a-z][a-z](\s|$)/i.match(data)
Language["Groff"]
elsif /((^|\s)move?[. ])|\.(include|globa?l)\s/.match(data)
Language["GAS"]
end
end
disambiguate ".ch" do |data|
if /^\s*#\s*(if|ifdef|ifndef|define|command|xcommand|translate|xtranslate|include|pragma|undef)\b/i.match(data)
Language["xBase"]
end
end
disambiguate ".r" do |data|
if /\bRebol\b/i.match(data)
Language["Rebol"]
elsif data.include?("<-")
Language["R"]
Language["Scilab"]
end
end
end

208
test/test_heuristics.rb
View File

@@ -33,6 +33,101 @@ class TestHeuristcs < Minitest::Test
end
end
def test_detect_still_works_if_nothing_matches
blob = Linguist::FileBlob.new(File.join(samples_path, "Objective-C/hello.m"))
match = Language.detect(blob)
assert_equal Language["Objective-C"], match
end
# Candidate languages = ["AGS Script", "AsciiDoc", "Public Key"]
def test_asc_by_heuristics
assert_heuristics({
"AsciiDoc" => all_fixtures("AsciiDoc", "*.asc"),
"AGS Script" => all_fixtures("AGS Script", "*.asc"),
"Public Key" => all_fixtures("Public Key", "*.asc")
})
end
def test_bb_by_heuristics
assert_heuristics({
"BitBake" => all_fixtures("BitBake", "*.bb"),
"BlitzBasic" => all_fixtures("BlitzBasic", "*.bb")
})
end
def test_ch_by_heuristics
assert_heuristics({
"xBase" => all_fixtures("xBase", ".ch")
})
end
def test_cl_by_heuristics
assert_heuristics({
"Common Lisp" => all_fixtures("Common Lisp", "*.cl"),
"OpenCL" => all_fixtures("OpenCL", "*.cl")
})
end
def test_cs_by_heuristics
assert_heuristics({
"C#" => all_fixtures("C#", "*.cs"),
"Smalltalk" => all_fixtures("Smalltalk", "*.cs")
})
end
# Candidate languages = ["ECL", "ECLiPSe"]
def test_ecl_by_heuristics
assert_heuristics({
"ECL" => all_fixtures("ECL", "*.ecl"),
"ECLiPSe" => all_fixtures("ECLiPSe", "*.ecl")
})
end
def test_f_by_heuristics
assert_heuristics({
"FORTRAN" => all_fixtures("FORTRAN", "*.f") + all_fixtures("FORTRAN", "*.for"),
"Forth" => all_fixtures("Forth", "*.f") + all_fixtures("Forth", "*.for")
})
end
def test_fr_by_heuristics
assert_heuristics({
"Frege" => all_fixtures("Frege", "*.fr"),
"Forth" => all_fixtures("Forth", "*.fr"),
"Text" => all_fixtures("Text", "*.fr")
})
end
def test_fs_by_heuristics
assert_heuristics({
"F#" => all_fixtures("F#", "*.fs"),
"Forth" => all_fixtures("Forth", "*.fs"),
"GLSL" => all_fixtures("GLSL", "*.fs")
})
end
# Candidate languages = ["Hack", "PHP"]
def test_hack_by_heuristics
assert_heuristics({
"Hack" => all_fixtures("Hack", "*.php"),
"PHP" => all_fixtures("PHP", "*.php")
})
end
def test_ls_by_heuristics
assert_heuristics({
"LiveScript" => all_fixtures("LiveScript", "*.ls"),
"LoomScript" => all_fixtures("LoomScript", "*.ls")
})
end
def test_lsp_by_heuristics
assert_heuristics({
"Common Lisp" => all_fixtures("Common Lisp", "*.lsp") + all_fixtures("Common Lisp", "*.lisp"),
"NewLisp" => all_fixtures("NewLisp", "*.lsp") + all_fixtures("NewLisp", "*.lisp")
})
end
# Candidate languages = ["C++", "Objective-C"]
def test_obj_c_by_heuristics
# Only calling out '.h' filenames as these are the ones causing issues
@@ -43,12 +138,6 @@ class TestHeuristcs < Minitest::Test
})
end
def test_detect_still_works_if_nothing_matches
blob = Linguist::FileBlob.new(File.join(samples_path, "Objective-C/hello.m"))
match = Language.detect(blob)
assert_equal Language["Objective-C"], match
end
# Candidate languages = ["Perl", "Perl6", "Prolog"]
def test_pl_prolog_perl_by_heuristics
assert_heuristics({
@@ -66,24 +155,6 @@ class TestHeuristcs < Minitest::Test
})
end
# Candidate languages = ["Perl", "Perl6"]
def test_t_perl_by_heuristics
assert_heuristics({
"Perl" => all_fixtures("Perl", "*.t"),
"Perl6" => ["Perl6/01-dash-uppercase-i.t", "Perl6/01-parse.t", "Perl6/advent2009-day16.t",
"Perl6/basic-open.t", "Perl6/calendar.t", "Perl6/for.t", "Perl6/hash.t",
"Perl6/listquote-whitespace.t"]
})
end
# Candidate languages = ["ECL", "ECLiPSe"]
def test_ecl_by_heuristics
assert_heuristics({
"ECL" => all_fixtures("ECL", "*.ecl"),
"ECLiPSe" => all_fixtures("ECLiPSe", "*.ecl")
})
end
# Candidate languages = ["IDL", "Prolog", "QMake", "INI"]
def test_pro_by_heuristics
assert_heuristics({
@@ -94,34 +165,10 @@ class TestHeuristcs < Minitest::Test
})
end
# Candidate languages = ["AGS Script", "AsciiDoc", "Public Key"]
def test_asc_by_heuristics
def test_r_by_heuristics
assert_heuristics({
"AsciiDoc" => all_fixtures("AsciiDoc", "*.asc"),
"AGS Script" => all_fixtures("AGS Script", "*.asc"),
"Public Key" => all_fixtures("Public Key", "*.asc")
})
end
def test_cl_by_heuristics
assert_heuristics({
"Common Lisp" => all_fixtures("Common Lisp", "*.cl"),
"OpenCL" => all_fixtures("OpenCL", "*.cl")
})
end
def test_f_by_heuristics
assert_heuristics({
"FORTRAN" => all_fixtures("FORTRAN", "*.f") + all_fixtures("FORTRAN", "*.for"),
"Forth" => all_fixtures("Forth", "*.f") + all_fixtures("Forth", "*.for")
})
end
# Candidate languages = ["Hack", "PHP"]
def test_hack_by_heuristics
assert_heuristics({
"Hack" => all_fixtures("Hack", "*.php"),
"PHP" => all_fixtures("PHP", "*.php")
"R" => all_fixtures("R", "*.r") + all_fixtures("R", "*.R"),
"Rebol" => all_fixtures("Rebol", "*.r")
})
end
@@ -133,47 +180,13 @@ class TestHeuristcs < Minitest::Test
})
end
def test_fs_by_heuristics
# Candidate languages = ["Perl", "Perl6"]
def test_t_perl_by_heuristics
assert_heuristics({
"F#" => all_fixtures("F#", "*.fs"),
"Forth" => all_fixtures("Forth", "*.fs"),
"GLSL" => all_fixtures("GLSL", "*.fs")
})
end
def test_fr_by_heuristics
assert_heuristics({
"Frege" => all_fixtures("Frege", "*.fr"),
"Forth" => all_fixtures("Forth", "*.fr"),
"Text" => all_fixtures("Text", "*.fr")
})
end
def test_bb_by_heuristics
assert_heuristics({
"BitBake" => all_fixtures("BitBake", "*.bb"),
"BlitzBasic" => all_fixtures("BlitzBasic", "*.bb")
})
end
def test_lsp_by_heuristics
assert_heuristics({
"Common Lisp" => all_fixtures("Common Lisp", "*.lsp") + all_fixtures("Common Lisp", "*.lisp"),
"NewLisp" => all_fixtures("NewLisp", "*.lsp") + all_fixtures("NewLisp", "*.lisp")
})
end
def test_cs_by_heuristics
assert_heuristics({
"C#" => all_fixtures("C#", "*.cs"),
"Smalltalk" => all_fixtures("Smalltalk", "*.cs")
})
end
def test_ls_by_heuristics
assert_heuristics({
"LiveScript" => all_fixtures("LiveScript", "*.ls"),
"LoomScript" => all_fixtures("LoomScript", "*.ls")
"Perl" => all_fixtures("Perl", "*.t"),
"Perl6" => ["Perl6/01-dash-uppercase-i.t", "Perl6/01-parse.t", "Perl6/advent2009-day16.t",
"Perl6/basic-open.t", "Perl6/calendar.t", "Perl6/for.t", "Perl6/hash.t",
"Perl6/listquote-whitespace.t"]
})
end
@@ -183,17 +196,4 @@ class TestHeuristcs < Minitest::Test
"XML" => all_fixtures("XML", "*.ts")
})
end
def test_ch_by_heuristics
assert_heuristics({
"xBase" => all_fixtures("xBase", ".ch")
})
end
def test_r_by_heuristics
assert_heuristics({
"R" => all_fixtures("R", "*.r") + all_fixtures("R", "*.R"),
"Rebol" => all_fixtures("Rebol", "*.r")
})
end
end