KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add brackets to tokens

Browse Source
This commit is contained in:
Joshua Peek
2012-07-24 11:28:27 -05:00
parent 52833b58d5
commit 53300ca581
2 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/linguist/tokenizer.rb
View File

@@ -76,7 +76,7 @@ module Linguist
extract_sgml_tokens(token).each { |t| tokens << t }
# Common programming punctuation
elsif token = s.scan(/;|\{|\}|\(|\)/)
elsif token = s.scan(/;|\{|\}|\(|\)|\[|\]/)
tokens << token
# Regular token

6
test/test_tokenizer.rb
View File

@@ -78,13 +78,17 @@ class TestTokenizer < Test::Unit::TestCase
def test_objective_c_tokens
assert_equal %w(#import <Foundation/Foundation.h> @interface Foo NSObject { } @end), tokenize(:"Objective-C/Foo.h")
assert_equal %w(#import @implementation Foo @end), tokenize(:"Objective-C/Foo.m")
assert_equal %w(#import <Cocoa/Cocoa.h> int main \( int argc char *argv \) { NSLog \( @ \) ; return ; }), tokenize(:"Objective-C/hello.m")
assert_equal %w(#import <Cocoa/Cocoa.h> int main \( int argc char *argv [ ] \) { NSLog \( @ \) ; return ; }), tokenize(:"Objective-C/hello.m")
end
def test_javascript_tokens
assert_equal %w( \( function \( \) { console.log \( \) ; } \) .call \( this \) ;), tokenize(:"JavaScript/hello.js")
end
def test_json_tokens
assert_equal %w( { [ ] { } } ), tokenize(:"JSON/product.json")
end
def test_ruby_tokens
assert_equal %w(module Foo end), tokenize(:"Ruby/foo.rb")
assert_equal %w(# /usr/bin/env ruby puts), tokenize(:"Ruby/script.rb")