diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb index e5912149..bf05dcd8 100644 --- a/lib/linguist/tokenizer.rb +++ b/lib/linguist/tokenizer.rb @@ -76,7 +76,7 @@ module Linguist extract_sgml_tokens(token).each { |t| tokens << t } # Common programming punctuation - elsif token = s.scan(/;|\{|\}|\(|\)/) + elsif token = s.scan(/;|\{|\}|\(|\)|\[|\]/) tokens << token # Regular token diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb index 55271ba6..81654a0c 100644 --- a/test/test_tokenizer.rb +++ b/test/test_tokenizer.rb @@ -78,13 +78,17 @@ class TestTokenizer < Test::Unit::TestCase def test_objective_c_tokens assert_equal %w(#import @interface Foo NSObject { } @end), tokenize(:"Objective-C/Foo.h") assert_equal %w(#import @implementation Foo @end), tokenize(:"Objective-C/Foo.m") - assert_equal %w(#import int main \( int argc char *argv \) { NSLog \( @ \) ; return ; }), tokenize(:"Objective-C/hello.m") + assert_equal %w(#import int main \( int argc char *argv [ ] \) { NSLog \( @ \) ; return ; }), tokenize(:"Objective-C/hello.m") end def test_javascript_tokens assert_equal %w( \( function \( \) { console.log \( \) ; } \) .call \( this \) ;), tokenize(:"JavaScript/hello.js") end + def test_json_tokens + assert_equal %w( { [ ] { } } ), tokenize(:"JSON/product.json") + end + def test_ruby_tokens assert_equal %w(module Foo end), tokenize(:"Ruby/foo.rb") assert_equal %w(# /usr/bin/env ruby puts), tokenize(:"Ruby/script.rb")