Skip number literals

test/test_tokenizer.rb

@@ -14,7 +14,7 @@ class TestTokenizer < Test::Unit::TestCase
     Tokenizer.new(data).tokens
   end
 
-  def test_skip_strings
+  def test_skip_string_literals
     assert_equal %w(print), tokenize('print ""')
     assert_equal %w(print), tokenize('print "Josh"')
     assert_equal %w(print), tokenize("print 'Josh'")
@@ -22,6 +22,11 @@ class TestTokenizer < Test::Unit::TestCase
     assert_equal %w(print), tokenize("print 'Hello \\'Josh\\''")
   end
 
+  def test_skip_number_literals
+    assert_equal %w(+), tokenize('1 + 1')
+    assert_equal %w(add \( \)), tokenize('add(123, 456)')
+  end
+
   def test_skip_comments
     assert_equal %w(foo #), tokenize("foo # Comment")
     assert_equal %w(foo # bar), tokenize("foo # Comment\nbar")
@@ -43,7 +48,8 @@ class TestTokenizer < Test::Unit::TestCase
   end
 
   def test_c_tokens
-    assert_equal %w(#include <stdio.h> int main \( \) { printf \( \) ; return 0 ; }), tokenize(:"c/hello.c")
+    assert_equal %w(#ifndef HELLO_H #define HELLO_H void hello \( \) ; #endif), tokenize(:"c/hello.h")
+    assert_equal %w(#include <stdio.h> int main \( \) { printf \( \) ; return ; }), tokenize(:"c/hello.c")
   end
 
   def test_cpp_tokens
@@ -54,7 +60,7 @@ class TestTokenizer < Test::Unit::TestCase
   def test_objective_c_tokens
     assert_equal %w(#import <Foundation/Foundation.h> @interface Foo NSObject { } @end), tokenize(:"objective-c/Foo.h")
     assert_equal %w(#import @implementation Foo @end), tokenize(:"objective-c/Foo.m")
-    assert_equal %w(#import <Cocoa/Cocoa.h> int main \( int argc char argv \) { NSLog \( @ \) ; return 0 ; }), tokenize(:"objective-c/hello.m")
+    assert_equal %w(#import <Cocoa/Cocoa.h> int main \( int argc char argv \) { NSLog \( @ \) ; return ; }), tokenize(:"objective-c/hello.m")
   end
 
   def test_javascript_tokens