diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb index 520ed5c9..e5912149 100644 --- a/lib/linguist/tokenizer.rb +++ b/lib/linguist/tokenizer.rb @@ -69,7 +69,7 @@ module Linguist s.skip_until(/[^\\]'/) # Skip number literals - elsif s.scan(/(0x)?\d+/) + elsif s.scan(/(0x)?\d(\d|\.)*/) # SGML style brackets elsif token = s.scan(/<[^\s<>][^<>]*>/) diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb index 644a2c32..55271ba6 100644 --- a/test/test_tokenizer.rb +++ b/test/test_tokenizer.rb @@ -26,6 +26,7 @@ class TestTokenizer < Test::Unit::TestCase assert_equal %w(+), tokenize('1 + 1') assert_equal %w(add \( \)), tokenize('add(123, 456)') assert_equal %w(|), tokenize('0x01 | 0x10') + assert_equal %w(*), tokenize('500.42 * 1.0') end def test_skip_comments