From fea8bb21a0a340a559f2ef904d862cc8e83a3ba0 Mon Sep 17 00:00:00 2001
From: Adam Roben <adam@roben.org>
Date: Thu, 5 Nov 2015 10:18:44 -0500
Subject: [PATCH] Use negative lookbehind when tokenizing string literals

This can double the speed of tokenizing large RTF files that use \'hh
escape sequences.
---
 lib/linguist/tokenizer.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb
index 64a82e67..7b618049 100644
--- a/lib/linguist/tokenizer.rb
+++ b/lib/linguist/tokenizer.rb
@@ -86,13 +86,13 @@ module Linguist
           if s.peek(1) == "\""
             s.getch
           else
-            s.skip_until(/[^\\]"/)
+            s.skip_until(/(?<!\\)"/)
           end
         elsif s.scan(/'/)
           if s.peek(1) == "'"
             s.getch
           else
-            s.skip_until(/[^\\]'/)
+            s.skip_until(/(?<!\\)'/)
           end
 
         # Skip number literals