KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add tokenize helper to Tokenize class

Browse Source
This commit is contained in:
Joshua Peek
2012-07-20 15:14:58 -05:00
parent 79be8b8c67
commit 2637d8dc55
4 changed files with 8 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lib/linguist/classifier.rb
View File

@@ -55,7 +55,7 @@ module Linguist
# Returns nothing.
def train(language, data)
language = language.name
tokens = Tokenizer.new(data).tokens
tokens = Tokenizer.tokenize(data)
tokens.each do |token|
@tokens[language][token] += 1
@@ -98,7 +98,7 @@ module Linguist
# Language and a Float score.
def classify(tokens, languages = @languages.keys)
return [] if tokens.nil?
tokens = Tokenizer.new(tokens).tokens if tokens.is_a?(String)
tokens = Tokenizer.tokenize(tokens) if tokens.is_a?(String)
scores = {}
languages.each do |language|