diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb index 4b2ea607..05882649 100644 --- a/lib/linguist/tokenizer.rb +++ b/lib/linguist/tokenizer.rb @@ -33,7 +33,8 @@ module Linguist [''], # XML ['{-', '-}'], # Haskell ['(*', '*)'], # Coq - ['"""', '"""'] # Python + ['"""', '"""'], # Python + ["'''", "'''"] # Python ] START_SINGLE_LINE_COMMENT = Regexp.compile(SINGLE_LINE_COMMENTS.map { |c| diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb index 24a74105..339d5485 100644 --- a/test/test_tokenizer.rb +++ b/test/test_tokenizer.rb @@ -41,6 +41,8 @@ class TestTokenizer < Minitest::Test assert_equal %w(foo), tokenize("foo {- Comment -}") assert_equal %w(foo), tokenize("foo (* Comment *)") assert_equal %w(%), tokenize("2 % 10\n% Comment") + assert_equal %w(foo bar), tokenize("foo\n\"\"\"\nComment\n\"\"\"\nbar") + assert_equal %w(foo bar), tokenize("foo\n'''\nComment\n'''\nbar") end def test_sgml_tags