KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve operator tokenizing

Browse Source
This commit is contained in:
Joshua Peek
2012-06-20 17:16:53 -05:00
parent c353d3a050
commit 5cdd5e206c
2 changed files with 24 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
lib/linguist/tokenizer.rb
View File

@@ -74,18 +74,22 @@ module Linguist
s.skip_until(/[^\\]'/)
# Skip number literals
elsif s.scan(/\d+/)
elsif s.scan(/(0x)?\d+/)
# SGML style brackets
elsif token = s.scan(/<[^>]+>/)
elsif token = s.scan(/<[^\s<>][^<>]*>/)
extract_sgml_tokens(token).each { |t| tokens << t }
# Common programming punctuation
elsif token = s.scan(/;|\{|\}|\(|\)|<<?|\+/)
elsif token = s.scan(/;|\{|\}|\(|\)/)
tokens << token
# Regular token
elsif token = s.scan(/[\w\.@#\/]+/)
elsif token = s.scan(/[\w\.@#\/\*]+/)
tokens << token
# Common operators
elsif token = s.scan(/<<?|\+|\-|\*|\/|&&?|\|\|?/)
tokens << token
else