Allow tokens to be passed directly to classify

test/test_classifier.rb

@@ -1,6 +1,7 @@
 require 'linguist/classifier'
 require 'linguist/language'
 require 'linguist/sample'
+require 'linguist/tokenizer'
 
 require 'test/unit'
 
@@ -23,6 +24,10 @@ class TestClassifier < Test::Unit::TestCase
 
     results = classifier.classify(fixture("objective-c/hello.m"))
     assert_equal Language["Objective-C"], results.first[0]
+
+    tokens  = Tokenizer.new(fixture("objective-c/hello.m")).tokens
+    results = classifier.classify(tokens)
+    assert_equal Language["Objective-C"], results.first[0]
   end
 
   def test_instance_classify_empty