KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Parse float tokens

Browse Source
This commit is contained in:
Joshua Peek
2012-07-24 11:23:06 -05:00
parent e2a91bba3e
commit f5705eaf38
2 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/linguist/tokenizer.rb
View File

@@ -69,7 +69,7 @@ module Linguist
s.skip_until(/[^\\]'/) s.skip_until(/[^\\]'/)
# Skip number literals # Skip number literals
elsif s.scan(/(0x)?\d+/) elsif s.scan(/(0x)?\d(\d|\.)*/)
# SGML style brackets # SGML style brackets
elsif token = s.scan(/<[^\s<>][^<>]*>/) elsif token = s.scan(/<[^\s<>][^<>]*>/)

1
test/test_tokenizer.rb
View File

@@ -26,6 +26,7 @@ class TestTokenizer < Test::Unit::TestCase
assert_equal %w(+), tokenize('1 + 1') assert_equal %w(+), tokenize('1 + 1')
assert_equal %w(add \( \)), tokenize('add(123, 456)') assert_equal %w(add \( \)), tokenize('add(123, 456)')
assert_equal %w(|), tokenize('0x01 | 0x10') assert_equal %w(|), tokenize('0x01 | 0x10')
assert_equal %w(*), tokenize('500.42 * 1.0')
end end
def test_skip_comments def test_skip_comments