KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor tokenizer test helper

Browse Source
This commit is contained in:
Joshua Peek
2012-06-19 13:12:17 -05:00
parent 555573071e
commit d5fa8cbcb7
1 changed files with 32 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
test/test_tokenizer.rb
View File

@@ -9,62 +9,62 @@ class TestTokenizer < Test::Unit::TestCase
File.expand_path("../fixtures", __FILE__)
end
def tokenize(name)
data = File.read(File.join(fixtures_path, name))
def tokenize(data)
data = File.read(File.join(fixtures_path, data.to_s)) if data.is_a?(Symbol)
Tokenizer.new(data).tokens
end
def test_skip_strings
assert_equal %w(print), Tokenizer.new('print ""').tokens
assert_equal %w(print), Tokenizer.new('print "Josh"').tokens
assert_equal %w(print), Tokenizer.new("print 'Josh'").tokens
assert_equal %w(print), Tokenizer.new('print "Hello \"Josh\""').tokens
assert_equal %w(print), Tokenizer.new("print 'Hello \\'Josh\\''").tokens
assert_equal %w(print), tokenize('print ""')
assert_equal %w(print), tokenize('print "Josh"')
assert_equal %w(print), tokenize("print 'Josh'")
assert_equal %w(print), tokenize('print "Hello \"Josh\""')
assert_equal %w(print), tokenize("print 'Hello \\'Josh\\''")
end
def test_skip_comments
assert_equal %w(foo #), Tokenizer.new("foo # Comment").tokens
assert_equal %w(foo # bar), Tokenizer.new("foo # Comment\nbar").tokens
assert_equal %w(foo //), Tokenizer.new("foo // Comment").tokens
assert_equal %w(foo /* */), Tokenizer.new("foo /* Comment */").tokens
assert_equal %w(foo /* */), Tokenizer.new("foo /* \nComment\n */").tokens
assert_equal %w(foo <!-- -->), Tokenizer.new("foo <!-- Comment -->").tokens
assert_equal %w(foo {- -}), Tokenizer.new("foo {- Comment -}").tokens
assert_equal %w(foo #), tokenize("foo # Comment")
assert_equal %w(foo # bar), tokenize("foo # Comment\nbar")
assert_equal %w(foo //), tokenize("foo // Comment")
assert_equal %w(foo /* */), tokenize("foo /* Comment */")
assert_equal %w(foo /* */), tokenize("foo /* \nComment\n */")
assert_equal %w(foo <!-- -->), tokenize("foo <!-- Comment -->")
assert_equal %w(foo {- -}), tokenize("foo {- Comment -}")
end
def test_sgml_tags
assert_equal %w(<html> </html>), Tokenizer.new("<html></html>").tokens
assert_equal %w(<div> id </div>), Tokenizer.new("<div id></div>").tokens
assert_equal %w(<div> id= </div>), Tokenizer.new("<div id=foo></div>").tokens
assert_equal %w(<div> id class </div>), Tokenizer.new("<div id class></div>").tokens
assert_equal %w(<div> id= </div>), Tokenizer.new("<div id=\"foo bar\"></div>").tokens
assert_equal %w(<div> id= </div>), Tokenizer.new("<div id='foo bar'></div>").tokens
assert_equal %w(<?xml> version=), Tokenizer.new("<?xml version=\"1.0\"?>").tokens
assert_equal %w(<html> </html>), tokenize("<html></html>")
assert_equal %w(<div> id </div>), tokenize("<div id></div>")
assert_equal %w(<div> id= </div>), tokenize("<div id=foo></div>")
assert_equal %w(<div> id class </div>), tokenize("<div id class></div>")
assert_equal %w(<div> id= </div>), tokenize("<div id=\"foo bar\"></div>")
assert_equal %w(<div> id= </div>), tokenize("<div id='foo bar'></div>")
assert_equal %w(<?xml> version=), tokenize("<?xml version=\"1.0\"?>")
end
def test_c_tokens
assert_equal %w(#include <stdio.h> int main \( \) { printf \( \) ; return 0 ; }), tokenize("c/hello.c")
assert_equal %w(#ifndef HELLO_H #define HELLO_H void hello \( \) ; #endif), tokenize("c/hello.h")
assert_equal %w(#include <stdio.h> int main \( \) { printf \( \) ; return 0 ; }), tokenize(:"c/hello.c")
assert_equal %w(#ifndef HELLO_H #define HELLO_H void hello \( \) ; #endif), tokenize(:"c/hello.h")
end
def test_cpp_tokens
assert_equal %w(class Bar { protected char name ; public void hello \( \) ; }), tokenize("cpp/bar.h")
assert_equal %w(#include <iostream> using namespace std ; int main \( \) { cout << << endl ; }), tokenize("cpp/hello.cpp")
assert_equal %w(class Bar { protected char name ; public void hello \( \) ; }), tokenize(:"cpp/bar.h")
assert_equal %w(#include <iostream> using namespace std ; int main \( \) { cout << << endl ; }), tokenize(:"cpp/hello.cpp")
end
def test_objective_c_tokens
assert_equal %w(#import <Foundation/Foundation.h> @interface Foo NSObject { } @end), tokenize("objective-c/Foo.h")
assert_equal %w(#import @implementation Foo @end), tokenize("objective-c/Foo.m")
assert_equal %w(#import <Cocoa/Cocoa.h> int main \( int argc char argv \) { NSLog \( @ \) ; return 0 ; }), tokenize("objective-c/hello.m")
assert_equal %w(#import <Foundation/Foundation.h> @interface Foo NSObject { } @end), tokenize(:"objective-c/Foo.h")
assert_equal %w(#import @implementation Foo @end), tokenize(:"objective-c/Foo.m")
assert_equal %w(#import <Cocoa/Cocoa.h> int main \( int argc char argv \) { NSLog \( @ \) ; return 0 ; }), tokenize(:"objective-c/hello.m")
end
def test_javascript_tokens
assert_equal %w( \( function \( \) { console.log \( \) ; } \) .call \( this \) ;), tokenize("javascript/hello.js")
assert_equal %w( \( function \( \) { console.log \( \) ; } \) .call \( this \) ;), tokenize(:"javascript/hello.js")
end
def test_ruby_tokens
assert_equal %w(module Foo end), tokenize("ruby/foo.rb")
assert_equal %w(# /usr/bin/env ruby puts), tokenize("ruby/script.rb")
assert_equal %w(task default do puts end), tokenize("ruby/Rakefile")
assert_equal %w(module Foo end), tokenize(:"ruby/foo.rb")
assert_equal %w(# /usr/bin/env ruby puts), tokenize(:"ruby/script.rb")
assert_equal %w(task default do puts end), tokenize(:"ruby/Rakefile")
end
end