KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Extract single and multi line comment parser

Browse Source
This commit is contained in:
Joshua Peek
2012-07-20 15:06:21 -05:00
parent d063089430
commit 175d4244c2
2 changed files with 33 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
lib/linguist/tokenizer.rb
View File

@@ -24,6 +24,27 @@ module Linguist
extract_tokens(data) extract_tokens(data)
end end
SINGLE_LINE_COMMENTS = [
'//', # C
'#', # Ruby
'%', # Tex
]
MULTI_LINE_COMMENTS = [
['/*', '*/'], # C
['<!--', '-->'], # XML
['{-', '-}'], # Haskell
['(*', '*)'] # Coq
]
START_SINGLE_LINE_COMMENT = Regexp.compile(SINGLE_LINE_COMMENTS.map { |c|
"^\s*#{Regexp.escape(c)} "
}.join("|"))
START_MULTI_LINE_COMMENT = Regexp.compile(MULTI_LINE_COMMENTS.map { |c|
Regexp.escape(c[0])
}.join("|"))
# Internal: Extract generic tokens from data. # Internal: Extract generic tokens from data.
# #
# data - String to scan. # data - String to scan.
@@ -39,44 +60,17 @@ module Linguist
tokens = [] tokens = []
until s.eos? until s.eos?
# Ruby single line comment # Single line comment
if token = s.scan(/# /) if token = s.scan(START_SINGLE_LINE_COMMENT)
tokens << "#" tokens << token.strip
s.skip_until(/\n|\Z/) s.skip_until(/\n|\Z/)
# C style single line comment # Multiline comments
elsif token = s.scan(/\/\/ /) elsif token = s.scan(START_MULTI_LINE_COMMENT)
tokens << "//" tokens << token
s.skip_until(/\n|\Z/) close_token = MULTI_LINE_COMMENTS.assoc(token)[1]
s.skip_until(Regexp.compile(Regexp.escape(close_token)))
# Leading Tex or Matlab comments tokens << close_token
elsif token = s.scan(/\n%/)
tokens << "%"
s.skip_until(/\n|\Z/)
# C multiline comments
elsif token = s.scan(/\/\*/)
tokens << "/*"
s.skip_until(/\*\//)
tokens << "*/"
# Haskell multiline comments
elsif token = s.scan(/\{-/)
tokens << "{-"
s.skip_until(/-\}/)
tokens << "-}"
# XML multiline comments
elsif token = s.scan(/<!--/)
tokens << "<!--"
s.skip_until(/-->/)
tokens << "-->"
# Coq multiline comments
elsif token = s.scan(/\(\*/)
tokens << "(*"
s.skip_until(/\*\)/)
tokens << "*)"
# Skip single or double quoted strings # Skip single or double quoted strings
elsif s.scan(/"/) elsif s.scan(/"/)

6
test/test_tokenizer.rb
View File

@@ -29,9 +29,9 @@ class TestTokenizer < Test::Unit::TestCase
end end
def test_skip_comments def test_skip_comments
assert_equal %w(foo #), tokenize("foo # Comment") assert_equal %w(foo #), tokenize("foo\n# Comment")
assert_equal %w(foo # bar), tokenize("foo # Comment\nbar") assert_equal %w(foo # bar), tokenize("foo\n# Comment\nbar")
assert_equal %w(foo //), tokenize("foo // Comment") assert_equal %w(foo //), tokenize("foo\n// Comment")
assert_equal %w(foo /* */), tokenize("foo /* Comment */") assert_equal %w(foo /* */), tokenize("foo /* Comment */")
assert_equal %w(foo /* */), tokenize("foo /* \nComment\n */") assert_equal %w(foo /* */), tokenize("foo /* \nComment\n */")
assert_equal %w(foo <!-- -->), tokenize("foo <!-- Comment -->") assert_equal %w(foo <!-- -->), tokenize("foo <!-- Comment -->")