Add tokenize helper to Tokenize class

This commit is contained in:
Joshua Peek
2012-07-20 15:14:58 -05:00
parent 79be8b8c67
commit 2637d8dc55
4 changed files with 8 additions and 18 deletions

View File

@@ -32,7 +32,7 @@ class TestClassifier < Test::Unit::TestCase
results = classifier.classify(fixture("objective-c/hello.m"))
assert_equal Language["Objective-C"], results.first[0]
tokens = Tokenizer.new(fixture("objective-c/hello.m")).tokens
tokens = Tokenizer.tokenize(fixture("objective-c/hello.m"))
results = classifier.classify(tokens)
assert_equal Language["Objective-C"], results.first[0]
end