diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb index bf05dcd8..ed06a9fe 100644 --- a/lib/linguist/tokenizer.rb +++ b/lib/linguist/tokenizer.rb @@ -64,9 +64,17 @@ module Linguist # Skip single or double quoted strings elsif s.scan(/"/) - s.skip_until(/[^\\]"/) + if s.peek(1) == "\"" + s.getch + else + s.skip_until(/[^\\]"/) + end elsif s.scan(/'/) - s.skip_until(/[^\\]'/) + if s.peek(1) == "'" + s.getch + else + s.skip_until(/[^\\]'/) + end # Skip number literals elsif s.scan(/(0x)?\d(\d|\.)*/) diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb index 81654a0c..d57726c3 100644 --- a/test/test_tokenizer.rb +++ b/test/test_tokenizer.rb @@ -20,6 +20,10 @@ class TestTokenizer < Test::Unit::TestCase assert_equal %w(print), tokenize("print 'Josh'") assert_equal %w(print), tokenize('print "Hello \"Josh\""') assert_equal %w(print), tokenize("print 'Hello \\'Josh\\''") + assert_equal %w(print), tokenize("print \"Hello\", \"Josh\"") + assert_equal %w(print), tokenize("print 'Hello', 'Josh'") + assert_equal %w(print), tokenize("print \"Hello\", \"\", \"Josh\"") + assert_equal %w(print), tokenize("print 'Hello', '', 'Josh'") end def test_skip_number_literals