KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 17:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Only read up to 100KB

Browse Source
This commit is contained in:
Joshua Peek
2012-08-27 11:30:38 -05:00
parent 684a57dbc0
commit b183fcca05
1 changed files with 5 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
lib/linguist/tokenizer.rb
View File

@@ -16,6 +16,9 @@ module Linguist
new.extract_tokens(data) new.extract_tokens(data)
end end
# Read up to 100KB
BYTE_LIMIT = 100_000
# Start state on token, ignore anything till the next newline # Start state on token, ignore anything till the next newline
SINGLE_LINE_COMMENTS = [ SINGLE_LINE_COMMENTS = [
'//', # C '//', # C
@@ -55,6 +58,8 @@ module Linguist
tokens = [] tokens = []
until s.eos? until s.eos?
break if s.pos >= BYTE_LIMIT
if token = s.scan(/^#!.+$/) if token = s.scan(/^#!.+$/)
if name = extract_shebang(token) if name = extract_shebang(token)
tokens << "SHEBANG#!#{name}" tokens << "SHEBANG#!#{name}"