From c5a654e692240557a05a2de76119aad006da87df Mon Sep 17 00:00:00 2001 From: Paul Chaignon Date: Tue, 25 Nov 2014 20:01:24 -0500 Subject: [PATCH] Tests for Python multiline comments during tokenization --- test/test_tokenizer.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb index 0521f4da..a460afc3 100644 --- a/test/test_tokenizer.rb +++ b/test/test_tokenizer.rb @@ -43,6 +43,8 @@ class TestTokenizer < Test::Unit::TestCase assert_equal %w(foo), tokenize("foo {- Comment -}") assert_equal %w(foo), tokenize("foo (* Comment *)") assert_equal %w(%), tokenize("2 % 10\n% Comment") + assert_equal %w(foo bar), tokenize("foo\n\"\"\"\nComment\n\"\"\"\nbar") + assert_equal %w(foo bar), tokenize("foo\n'''\nComment\n'''\nbar") end def test_sgml_tags