KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

some air

Browse Source
This commit is contained in:
Joshua Peek
2013-06-10 10:37:55 -05:00
parent 9822b153eb
commit 490afdddd1
1 changed files with 10 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
lib/linguist/classifier.rb
View File

@@ -132,7 +132,7 @@ module Linguist
private private
def verbosity def verbosity
@verbosity ||= (ENV['LINGUIST_DEBUG']||0).to_i @verbosity ||= (ENV['LINGUIST_DEBUG'] || 0).to_i
end end
# Internal: show a table of probabilities for each <token,language> pair. # Internal: show a table of probabilities for each <token,language> pair.
@@ -146,19 +146,19 @@ module Linguist
# appear in one language vs. the least-likely language. Dashes # appear in one language vs. the least-likely language. Dashes
# indicate the least-likely language (and zero points) for each token. # indicate the least-likely language (and zero points) for each token.
def dump_all_tokens(tokens, languages) def dump_all_tokens(tokens, languages)
maxlen = tokens.map{|tok| tok.size}.max maxlen = tokens.map { |tok| tok.size }.max
printf "%#{maxlen}s", "" printf "%#{maxlen}s", ""
puts " #" + languages.map{|lang| sprintf("%10s", lang)}.join puts " #" + languages.map { |lang| sprintf("%10s", lang) }.join
tokmap = Hash.new(0) tokmap = Hash.new(0)
tokens.each{|tok| tokmap[tok] += 1} tokens.each { |tok| tokmap[tok] += 1 }
tokmap.sort.each{|tok, count| tokmap.sort.each { |tok, count|
arr = languages.map{|lang| [lang, token_probability(tok, lang)] } arr = languages.map { |lang| [lang, token_probability(tok, lang)] }
min = arr.map{|a,b| b}.min min = arr.map { |a,b| b }.min
minlog = Math.log(min) minlog = Math.log(min)
if !arr.inject(true) {|result, n| result && n[1] == arr[0][1]} # if not all the same if !arr.inject(true) { |result, n| result && n[1] == arr[0][1] }
printf "%#{maxlen}s%5d", tok, count printf "%#{maxlen}s%5d", tok, count
puts arr.map{|ent| puts arr.map { |ent|
ent[1] == min ? " -" : sprintf("%10.3f", count*(Math.log(ent[1])-minlog)) ent[1] == min ? " -" : sprintf("%10.3f", count * (Math.log(ent[1]) - minlog))
}.join }.join
end end
} }