KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix tokenzing empty strings

Browse Source
This commit is contained in:
Joshua Peek
2012-07-24 11:49:29 -05:00
parent 7aac87681b
commit e5d302459f
2 changed files with 14 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
lib/linguist/tokenizer.rb
View File

@@ -64,9 +64,17 @@ module Linguist
# Skip single or double quoted strings
elsif s.scan(/"/)
if s.peek(1) == "\""
s.getch
else
s.skip_until(/[^\\]"/)
end
elsif s.scan(/'/)
if s.peek(1) == "'"
s.getch
else
s.skip_until(/[^\\]'/)
end
# Skip number literals
elsif s.scan(/(0x)?\d(\d|\.)*/)

4
test/test_tokenizer.rb
View File

@@ -20,6 +20,10 @@ class TestTokenizer < Test::Unit::TestCase
assert_equal %w(print), tokenize("print 'Josh'")
assert_equal %w(print), tokenize('print "Hello \"Josh\""')
assert_equal %w(print), tokenize("print 'Hello \\'Josh\\''")
assert_equal %w(print), tokenize("print \"Hello\", \"Josh\"")
assert_equal %w(print), tokenize("print 'Hello', 'Josh'")
assert_equal %w(print), tokenize("print \"Hello\", \"\", \"Josh\"")
assert_equal %w(print), tokenize("print 'Hello', '', 'Josh'")
end
def test_skip_number_literals