KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Nicer debug factoring

Browse Source
This commit is contained in:
Ted Nyman
2013-12-14 15:24:26 -08:00
parent e0c1a84821
commit 6a8de63d2d
2 changed files with 16 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
lib/linguist/classifier.rb
View File

@@ -78,18 +78,13 @@ module Linguist
def classify(tokens, languages) def classify(tokens, languages)
return [] if tokens.nil? return [] if tokens.nil?
tokens = Tokenizer.tokenize(tokens) if tokens.is_a?(String) tokens = Tokenizer.tokenize(tokens) if tokens.is_a?(String)
scores = {} scores = {}
if verbosity >= 2
dump_all_tokens(tokens, languages) debug_dump_all_tokens(tokens, languages) if verbosity >= 2
end
languages.each do |language| languages.each do |language|
scores[language] = tokens_probability(tokens, language) + debug_dump_probabilities(tokens, language) if verbosity >= 1
language_probability(language) scores[language] = tokens_probability(tokens, language) + language_probability(language)
if verbosity >= 1
printf "%10s = %10.3f + %7.3f = %10.3f\n",
language, tokens_probability(tokens, language), language_probability(language), scores[language]
end
end end
scores.sort { |a, b| b[1] <=> a[1] }.map { |score| [score[0], score[1]] } scores.sort { |a, b| b[1] <=> a[1] }.map { |score| [score[0], score[1]] }
@@ -135,6 +130,11 @@ module Linguist
@verbosity ||= (ENV['LINGUIST_DEBUG'] || 0).to_i @verbosity ||= (ENV['LINGUIST_DEBUG'] || 0).to_i
end end
def debug_dump_probabilities
printf("%10s = %10.3f + %7.3f = %10.3f\n",
language, tokens_probability(tokens, language), language_probability(language), scores[language])
end
# Internal: show a table of probabilities for each <token,language> pair. # Internal: show a table of probabilities for each <token,language> pair.
# #
# The number in each table entry is the number of "points" that each # The number in each table entry is the number of "points" that each
@@ -145,7 +145,7 @@ module Linguist
# how much more likely (log of probability ratio) that token is to # how much more likely (log of probability ratio) that token is to
# appear in one language vs. the least-likely language. Dashes # appear in one language vs. the least-likely language. Dashes
# indicate the least-likely language (and zero points) for each token. # indicate the least-likely language (and zero points) for each token.
def dump_all_tokens(tokens, languages) def debug_dump_all_tokens(tokens, languages)
maxlen = tokens.map { |tok| tok.size }.max maxlen = tokens.map { |tok| tok.size }.max
printf "%#{maxlen}s", "" printf "%#{maxlen}s", ""

1
lib/linguist/language.rb
View File

@@ -117,6 +117,7 @@ module Linguist
if possible_languages.length > 1 if possible_languages.length > 1
data = data.call() if data.respond_to?(:call) data = data.call() if data.respond_to?(:call)
if data.nil? || data == "" if data.nil? || data == ""
nil nil
elsif (result = find_by_shebang(data)) && !result.empty? elsif (result = find_by_shebang(data)) && !result.empty?