Allow tokens to be passed directly to classify

lib/linguist/classifier.rb

@@ -74,7 +74,7 @@ module Linguist
 
     # Public: Guess language of data.
     #
-    # data - String data to analyze.
+    # data - Array of tokens or String data to analyze.
     #
     # Examples
     #
@@ -83,8 +83,8 @@ module Linguist
     #
     # Returns sorted Array of result pairs. Each pair contains the
     # Language and a Float score.
-    def classify(data)
+    def classify(tokens)
-      tokens = Tokenizer.new(data).tokens
+      tokens = Tokenizer.new(tokens).tokens if tokens.is_a?(String)
 
       scores = {}
       @languages.keys.each do |language|

test/test_classifier.rb

@@ -1,6 +1,7 @@
 require 'linguist/classifier'
 require 'linguist/language'
 require 'linguist/sample'
+require 'linguist/tokenizer'
 
 require 'test/unit'
 
@@ -23,6 +24,10 @@ class TestClassifier < Test::Unit::TestCase
 
     results = classifier.classify(fixture("objective-c/hello.m"))
     assert_equal Language["Objective-C"], results.first[0]
+
+    tokens  = Tokenizer.new(fixture("objective-c/hello.m")).tokens
+    results = classifier.classify(tokens)
+    assert_equal Language["Objective-C"], results.first[0]
   end
 
   def test_instance_classify_empty