From f5705eaf3872fa5872dd7500483f8c01e53d9768 Mon Sep 17 00:00:00 2001
From: Joshua Peek <josh@joshpeek.com>
Date: Tue, 24 Jul 2012 11:23:06 -0500
Subject: [PATCH] Parse float tokens

---
 lib/linguist/tokenizer.rb | 2 +-
 test/test_tokenizer.rb    | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb
index 520ed5c9..e5912149 100644
--- a/lib/linguist/tokenizer.rb
+++ b/lib/linguist/tokenizer.rb
@@ -69,7 +69,7 @@ module Linguist
           s.skip_until(/[^\\]'/)
 
         # Skip number literals
-        elsif s.scan(/(0x)?\d+/)
+        elsif s.scan(/(0x)?\d(\d|\.)*/)
 
         # SGML style brackets
         elsif token = s.scan(/<[^\s<>][^<>]*>/)
diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb
index 644a2c32..55271ba6 100644
--- a/test/test_tokenizer.rb
+++ b/test/test_tokenizer.rb
@@ -26,6 +26,7 @@ class TestTokenizer < Test::Unit::TestCase
     assert_equal %w(+), tokenize('1 + 1')
     assert_equal %w(add \( \)), tokenize('add(123, 456)')
     assert_equal %w(|), tokenize('0x01 | 0x10')
+    assert_equal %w(*), tokenize('500.42 * 1.0')
   end
 
   def test_skip_comments