From 53300ca5812a3f19b4ea1c670e286d68b900c7cf Mon Sep 17 00:00:00 2001
From: Joshua Peek <josh@joshpeek.com>
Date: Tue, 24 Jul 2012 11:28:27 -0500
Subject: [PATCH] Add brackets to tokens

---
 lib/linguist/tokenizer.rb | 2 +-
 test/test_tokenizer.rb    | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb
index e5912149..bf05dcd8 100644
--- a/lib/linguist/tokenizer.rb
+++ b/lib/linguist/tokenizer.rb
@@ -76,7 +76,7 @@ module Linguist
           extract_sgml_tokens(token).each { |t| tokens << t }
 
         # Common programming punctuation
-        elsif token = s.scan(/;|\{|\}|\(|\)/)
+        elsif token = s.scan(/;|\{|\}|\(|\)|\[|\]/)
           tokens << token
 
         # Regular token
diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb
index 55271ba6..81654a0c 100644
--- a/test/test_tokenizer.rb
+++ b/test/test_tokenizer.rb
@@ -78,13 +78,17 @@ class TestTokenizer < Test::Unit::TestCase
   def test_objective_c_tokens
     assert_equal %w(#import <Foundation/Foundation.h> @interface Foo NSObject { } @end), tokenize(:"Objective-C/Foo.h")
     assert_equal %w(#import @implementation Foo @end), tokenize(:"Objective-C/Foo.m")
-    assert_equal %w(#import <Cocoa/Cocoa.h> int main \( int argc char *argv \) { NSLog \( @ \) ; return ; }), tokenize(:"Objective-C/hello.m")
+    assert_equal %w(#import <Cocoa/Cocoa.h> int main \( int argc char *argv [ ] \) { NSLog \( @ \) ; return ; }), tokenize(:"Objective-C/hello.m")
   end
 
   def test_javascript_tokens
     assert_equal %w( \( function \( \) { console.log \( \) ; } \) .call \( this \) ;), tokenize(:"JavaScript/hello.js")
   end
 
+  def test_json_tokens
+    assert_equal %w( { [ ] { } } ), tokenize(:"JSON/product.json")
+  end
+
   def test_ruby_tokens
     assert_equal %w(module Foo end), tokenize(:"Ruby/foo.rb")
     assert_equal %w(# /usr/bin/env ruby puts), tokenize(:"Ruby/script.rb")