KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-12-08 20:38:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2524 from github/more-encompassing-number-skips

Browse Source 
More encompassing number skips
This commit is contained in:
Arfon Smith
2015-07-29 14:07:04 +01:00
parent e869f6c173 90a293727d
commit 24ca98b1a3
2 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/linguist/tokenizer.rb
View File

@@ -96,7 +96,7 @@ module Linguist
end
# Skip number literals
elsif s.scan(/(0x)?\d(\d|\.)*/)
elsif s.scan(/(0x\h(\h|\.)*|\d(\d|\.)*)([uU][lL]{0,2}|([eE][-+]\d*)?[fFlL]*)/)
# SGML style brackets
elsif token = s.scan(/<[^\s<>][^<>]*>/)

4
test/test_tokenizer.rb
View File

@@ -25,6 +25,10 @@ class TestTokenizer < Minitest::Test
assert_equal %w(add \( \)), tokenize('add(123, 456)')
assert_equal %w(|), tokenize('0x01 | 0x10')
assert_equal %w(*), tokenize('500.42 * 1.0')
assert_equal %w(), tokenize('1.23e-04')
assert_equal %w(), tokenize('1.0f')
assert_equal %w(), tokenize('1234ULL')
assert_equal %w(G1 X55 Y5 F2000), tokenize('G1 X55 Y5 F2000')
end
def test_skip_comments