Fix tokenzing empty strings

2025-10-29 17:50:22 +00:00 · 2012-07-24 11:49:29 -05:00
parent 7aac87681b
commit e5d302459f
2 changed files with 14 additions and 2 deletions
--- a/lib/linguist/tokenizer.rb
+++ b/lib/linguist/tokenizer.rb
@@ -64,9 +64,17 @@ module Linguist

        # Skip single or double quoted strings
        elsif s.scan(/"/)
-          s.skip_until(/[^\\]"/)
+          if s.peek(1) == "\""
+            s.getch
+          else
+            s.skip_until(/[^\\]"/)
+          end
        elsif s.scan(/'/)
-          s.skip_until(/[^\\]'/)
+          if s.peek(1) == "'"
+            s.getch
+          else
+            s.skip_until(/[^\\]'/)
+          end

        # Skip number literals
        elsif s.scan(/(0x)?\d(\d|\.)*/)