KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #836 from github/debug

Browse Source 
Nicer debug factoring
This commit is contained in:
Ted Nyman
2013-12-14 15:25:33 -08:00
parent e0c1a84821 6a8de63d2d
commit 7c636c4f65
2 changed files with 16 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
lib/linguist/classifier.rb
View File

@@ -78,18 +78,13 @@ module Linguist
def classify(tokens, languages) def classify(tokens, languages)
return [] if tokens.nil? return [] if tokens.nil?
tokens = Tokenizer.tokenize(tokens) if tokens.is_a?(String) tokens = Tokenizer.tokenize(tokens) if tokens.is_a?(String)
scores = {} scores = {}
if verbosity >= 2
dump_all_tokens(tokens, languages) debug_dump_all_tokens(tokens, languages) if verbosity >= 2
end
languages.each do |language| languages.each do |language|
scores[language] = tokens_probability(tokens, language) + debug_dump_probabilities(tokens, language) if verbosity >= 1
language_probability(language) scores[language] = tokens_probability(tokens, language) + language_probability(language)
if verbosity >= 1
printf "%10s = %10.3f + %7.3f = %10.3f\n",
language, tokens_probability(tokens, language), language_probability(language), scores[language]
end
end end
scores.sort { |a, b| b[1] <=> a[1] }.map { |score| [score[0], score[1]] } scores.sort { |a, b| b[1] <=> a[1] }.map { |score| [score[0], score[1]] }
@@ -135,6 +130,11 @@ module Linguist
@verbosity ||= (ENV['LINGUIST_DEBUG'] || 0).to_i @verbosity ||= (ENV['LINGUIST_DEBUG'] || 0).to_i
end end
def debug_dump_probabilities
printf("%10s = %10.3f + %7.3f = %10.3f\n",
language, tokens_probability(tokens, language), language_probability(language), scores[language])
end
# Internal: show a table of probabilities for each <token,language> pair. # Internal: show a table of probabilities for each <token,language> pair.
# #
# The number in each table entry is the number of "points" that each # The number in each table entry is the number of "points" that each
@@ -145,22 +145,22 @@ module Linguist
# how much more likely (log of probability ratio) that token is to # how much more likely (log of probability ratio) that token is to
# appear in one language vs. the least-likely language. Dashes # appear in one language vs. the least-likely language. Dashes
# indicate the least-likely language (and zero points) for each token. # indicate the least-likely language (and zero points) for each token.
def dump_all_tokens(tokens, languages) def debug_dump_all_tokens(tokens, languages)
maxlen = tokens.map { |tok| tok.size }.max maxlen = tokens.map { |tok| tok.size }.max
printf "%#{maxlen}s", "" printf "%#{maxlen}s", ""
puts " #" + languages.map { |lang| sprintf("%10s", lang) }.join puts " #" + languages.map { |lang| sprintf("%10s", lang) }.join
token_map = Hash.new(0) token_map = Hash.new(0)
tokens.each { |tok| token_map[tok] += 1 } tokens.each { |tok| token_map[tok] += 1 }
token_map.sort.each { |tok, count| token_map.sort.each { |tok, count|
arr = languages.map { |lang| [lang, token_probability(tok, lang)] } arr = languages.map { |lang| [lang, token_probability(tok, lang)] }
min = arr.map { |a,b| b }.min min = arr.map { |a,b| b }.min
minlog = Math.log(min) minlog = Math.log(min)
if !arr.inject(true) { |result, n| result && n[1] == arr[0][1] } if !arr.inject(true) { |result, n| result && n[1] == arr[0][1] }
printf "%#{maxlen}s%5d", tok, count printf "%#{maxlen}s%5d", tok, count
puts arr.map { |ent| puts arr.map { |ent|
ent[1] == min ? " -" : sprintf("%10.3f", count * (Math.log(ent[1]) - minlog)) ent[1] == min ? " -" : sprintf("%10.3f", count * (Math.log(ent[1]) - minlog))
}.join }.join

1
lib/linguist/language.rb
View File

@@ -117,6 +117,7 @@ module Linguist
if possible_languages.length > 1 if possible_languages.length > 1
data = data.call() if data.respond_to?(:call) data = data.call() if data.respond_to?(:call)
if data.nil? || data == "" if data.nil? || data == ""
nil nil
elsif (result = find_by_shebang(data)) && !result.empty? elsif (result = find_by_shebang(data)) && !result.empty?