KevinMidboe/linguist
1
0
Fork 0
mirror of https://github.com/KevinMidboe/linguist.git synced 2025-10-29 09:40:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace Lexer with Pygments::Lexer

Browse Source
This commit is contained in:
Joshua Peek
2011-08-23 12:41:06 -05:00
parent bb11317546
commit 0864b9847f
12 changed files with 33 additions and 1995 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Gemfile
View File

@@ -1,2 +1,4 @@
source :rubygems
gemspec
gem 'pygments.rb', :git => 'git://github.com/tmm1/pygments.rb.git'

14
Rakefile
View File

@@ -1,4 +1,3 @@
require 'rake/clean'
require 'rake/testtask'
task :default => :test
@@ -6,16 +5,3 @@ task :default => :test
Rake::TestTask.new do |t|
t.warning = true
end
CLOBBER.include 'lib/linguist/lexers.yml'
file 'lib/linguist/lexers.yml' do |f|
# GitHub vendored pygments path
# vendor/python/pygments
path = File.expand_path('../../../python/pygments', __FILE__)
ENV['PYTHONPATH'] = path if File.directory?(path)
sh "python ./bin/pygments-lexers > #{f.name}"
end
task :lexers => 'lib/linguist/lexers.yml'

27
bin/pygments-lexers
View File

@@ -1,27 +0,0 @@
#!/usr/bin/env python
from pygments import __version__
from pygments.lexers import get_all_lexers
print "# AUTOMATICALLY GENERATED BY bin/pygments-lexers"
print "# DO NOT MODIFIY. YOUR PULL WILL BE REJECTED."
print "# Pygments version %s" % __version__
print "---"
for name, aliases, filenames, mimetypes in get_all_lexers():
print "%s:" % name
if any(aliases):
print " aliases:"
for alias in aliases:
print " - '%s'" % alias
if any(filenames):
print " filenames:"
for filename in filenames:
print " - '%s'" % filename
if any(mimetypes):
print " mimetypes:"
for mimetype in mimetypes:
print " - '%s'" % mimetype

19
lib/linguist/blob_helper.rb
View File

@@ -3,6 +3,7 @@ require 'linguist/mime'
require 'linguist/pathname'
require 'escape_utils'
require 'pygments'
require 'yaml'
module Linguist
@@ -343,7 +344,7 @@ module Linguist
#
# Returns a Lexer.
def lexer
language ? language.lexer : Lexer['Text only']
language ? language.lexer : Pygments::Lexer.find_by_name('Text only')
end
# Internal: Disambiguates between multiple language extensions.
@@ -512,19 +513,27 @@ module Linguist
# Public: Highlight syntax of blob
#
# options - A Hash of options (defaults to {})
#
# Returns html String
def colorize
def colorize(options = {})
return if !text? || large?
lexer.colorize(data)
lexer.highlight(data, options)
end
# Public: Highlight syntax of blob without the outer highlight div
# wrapper.
#
# options - A Hash of options (defaults to {})
#
# Returns html String
def colorize_without_wrapper
def colorize_without_wrapper(options = {})
return if !text? || large?
lexer.colorize_without_wrapper(data)
if text = lexer.highlight(data, options)
text[%r{<div class="highlight"><pre>(.*?)</pre>\s*</div>}m, 1]
else
''
end
end
Language.overridden_extensions.each do |extension|

22
lib/linguist/language.rb
View File

@@ -1,6 +1,5 @@
require 'linguist/lexer'
require 'yaml'
require 'pygments'
module Linguist
# Language names that are recognizable by GitHub. Defined languages
@@ -211,7 +210,7 @@ module Linguist
@aliases = [default_alias_name] + (attributes[:aliases] || [])
# Lookup Lexer object
@lexer = Lexer.find_by_name(attributes[:lexer] || name) ||
@lexer = Pygments::Lexer.find_by_name(attributes[:lexer] || name) ||
raise(ArgumentError, "#{@name} is missing lexer")
# Set legacy search term
@@ -370,21 +369,12 @@ module Linguist
# Public: Highlight syntax of text
#
# text - String of code to be highlighted
# text - String of code to be highlighted
# options - A Hash of options (defaults to {})
#
# Returns html String
def colorize(text)
lexer.colorize(text)
end
# Public: Highlight syntax of text without the outer highlight div
# wrapper.
#
# text - String of code to be highlighted
#
# Returns html String
def colorize_without_wrapper(text)
lexer.colorize_without_wrapper(text)
def colorize(text, options = {})
lexer.highlight(text, options = {})
end
# Public: Return name as String representation

167
lib/linguist/lexer.rb
View File

@@ -1,167 +0,0 @@
require 'pygments'
require 'yaml'
module Linguist
# Mirror of Pygments Lexer structure.
#
# name - Proper lexer name (JavaScript, Ruby, Python)
# aliases - Aliases for lookup (js, javascript)
# filenames - Filename globs (*.js)
# mimetypes - Mime types (application/javascript)
class Lexer < Struct.new(:name, :aliases, :filenames, :mimetypes)
@lexers = []
@index = {}
@name_index = {}
@alias_index = {}
@mimetypes_index = {}
# Internal: Create a new Lexer object
#
# name - Name of Lexer
# attrs - A hash of attributes
#
# Returns a Lexer object
def self.create(name, attrs)
name = name
aliases = attrs['aliases'] || []
filenames = attrs['filenames'] || []
mimetypes = attrs['mimetypes'] || []
@lexers << lexer = new(name, aliases, filenames, mimetypes)
# All Lexer names should be unique. Warn if there is a duplicate.
if @name_index.key?(lexer.name)
warn "Duplicate lexer name: #{lexer.name}"
end
@index[lexer.name] = @name_index[lexer.name] = lexer
lexer.aliases.each do |name|
# All Lexer aliases should be unique. Warn if there is a duplicate.
if @alias_index.key?(name)
warn "Duplicate alias: #{name}"
end
@index[name] = @alias_index[name] = lexer
end
lexer.mimetypes.each do |type|
# All Lexer mimetypes should be unique. Warn if there is a duplicate.
if @mimetypes_index.key?(name)
warn "Duplicate mimetype: #{name}"
end
@mimetypes_index[type] = lexer
end
end
# Public: Get all Lexers
#
# Returns an Array of Lexers
def self.all
@lexers
end
# Public: Look up Lexer by name or alias.
#
# name - A String name or alias
#
# Lexer['Ruby']
# => #<Lexer name="Ruby">
#
# Returns the Lexer or nil if none was found.
def self.[](name)
@index[name]
end
# Public: Look up Lexer by its proper name.
#
# name - The String name of the Lexer
#
# Examples
#
# Lexer.find_by_name('Ruby')
# # => #<Lexer name="Ruby">
#
# Returns the Lexer or nil if none was found.
def self.find_by_name(name)
@name_index[name]
end
# Public: Look up Lexer by one of its aliases.
#
# name - A String alias of the Lexer
#
# Examples
#
# Lexer.find_by_alias('rb')
# # => #<Lexer name="Ruby">
#
# Returns the Lexer or nil if none was found.
def self.find_by_alias(name)
@alias_index[name]
end
# Public: Look up Lexer by one of it's mime types.
#
# type - A mime type String.
#
# Examples
#
# Lexer.find_by_mimetype('application/x-ruby')
# # => #<Lexer name="Ruby">
#
# Returns the Lexer or nil if none was found.
def self.find_by_mimetype(type)
@mimetypes_index[type]
end
# Public: Return a alias of the Lexer to pass to Pygments.
#
# The alias we choose is arbitrary.
#
# Returns the alias String
def to_s
aliases.first
end
# Public: Highlight syntax of text
#
# text - String of code to be highlighted
#
# Returns html String
def colorize(text)
Pygments.highlight(text, :lexer => aliases.first, :options => {:stripnl => false})
end
# Public: Highlight syntax of text without the outer highlight div
# wrapper.
#
# text - String of code to be highlighted
#
# Returns html String
def colorize_without_wrapper(text)
if text = colorize(text)
text[%r{<div class="highlight"><pre>(.*?)</pre>\s*</div>}m, 1]
else
''
end
end
def ==(other)
eql?(other)
end
def eql?(other)
equal?(other)
end
# Load lexers from lexers.yml
#
# `bin/pygments-lexers` dumps a YAML list of all the available
# Pygments lexers.
YAML.load_file(File.expand_path("../lexers.yml", __FILE__)).each do |name, attrs|
Lexer.create(name, attrs)
end
end
end

1652
lib/linguist/lexers.yml
View File

File diff suppressed because it is too large Load Diff

3
lib/linguist/pathname.rb
View File

@@ -1,5 +1,6 @@
require 'linguist/language'
require 'linguist/mime'
require 'pygments'
module Linguist
# Similar to ::Pathname, Linguist::Pathname wraps a path string and
@@ -61,7 +62,7 @@ module Linguist
#
# Returns a Lexer.
def lexer
language ? language.lexer : Lexer['Text only']
language ? language.lexer : Pygments::Lexer.find_by_name('Text only')
end
# Public: Get the mime type

3
test/test_blob.rb
View File

@@ -2,10 +2,13 @@ require 'linguist/file_blob'
require 'test/unit'
require 'mime/types'
require 'pygments'
class TestBlob < Test::Unit::TestCase
include Linguist
Lexer = Pygments::Lexer
def fixtures_path
File.expand_path("../fixtures", __FILE__)
end

25
test/test_language.rb
View File

@@ -1,10 +1,13 @@
require 'linguist/language'
require 'test/unit'
require 'pygments'
class TestLanguage < Test::Unit::TestCase
include Linguist
Lexer = Pygments::Lexer
def test_ambiguous_extensions
assert Language.ambiguous?('.h')
assert_equal Language['C'], Language.find_by_extension('h')
@@ -311,26 +314,4 @@ class TestLanguage < Test::Unit::TestCase
</div>
HTML
end
def test_colorize_without_wrapper
assert_equal <<-HTML, Language['Text'].colorize_without_wrapper("Hello")
Hello
HTML
assert_equal <<-HTML, Language['Ruby'].colorize_without_wrapper("def foo\n 'foo'\nend\n")
<span class="k">def</span> <span class="nf">foo</span>
<span class="s1">&#39;foo&#39;</span>
<span class="k">end</span>
HTML
end
def test_colorize_doesnt_strip_newlines
assert_equal <<-HTML, Language['Ruby'].colorize_without_wrapper("\n\n# Foo\ndef 'foo'\nend\n")
<span class="c1"># Foo</span>
<span class="k">def</span> <span class="s1">&#39;foo&#39;</span>
<span class="k">end</span>
HTML
end
end

91
test/test_lexer.rb
View File

@@ -1,91 +0,0 @@
require 'linguist/lexer'
require 'test/unit'
class TestLexer < Test::Unit::TestCase
include Linguist
def test_find_by_name
assert_equal Lexer['Ruby'], Lexer.find_by_name('Ruby')
end
def test_find_all_by_name
Lexer.all.each do |lexer|
assert_equal lexer, Lexer.find_by_name(lexer.name)
assert_equal lexer, Lexer[lexer.name]
end
end
def test_find_by_alias
assert_equal Lexer['Ruby'], Lexer.find_by_alias('rb')
assert_equal Lexer['Ruby'], Lexer.find_by_alias('ruby')
end
def test_find_all_by_alias
Lexer.all.each do |lexer|
lexer.aliases.each do |name|
assert_equal lexer, Lexer.find_by_alias(name)
assert_equal lexer, Lexer[name]
end
end
end
def test_find_by_mimetype
assert_equal Lexer['Ruby'], Lexer.find_by_mimetype('text/x-ruby')
assert_equal Lexer['Ruby'], Lexer.find_by_mimetype('application/x-ruby')
end
def test_find_all_by_mimetype
Lexer.all.each do |lexer|
lexer.mimetypes.each do |type|
assert_equal lexer, Lexer.find_by_mimetype(type)
end
end
end
def test_name
assert_equal 'Ruby', Lexer['Ruby'].name
assert_equal 'Python', Lexer['Python'].name
assert_equal 'Perl', Lexer['Perl'].name
end
def test_aliases
assert_equal ['rb', 'ruby', 'duby'], Lexer['Ruby'].aliases
assert_equal ['python', 'py'], Lexer['Python'].aliases
assert_equal ['perl', 'pl'], Lexer['Perl'].aliases
end
def test_eql
assert Lexer['Ruby'].eql?(Lexer['Ruby'])
assert !Lexer['Ruby'].eql?(Lexer['Python'])
assert !Lexer['Ruby'].eql?(Lexer.new('Ruby'))
end
def test_colorize
assert_equal <<-HTML, Lexer['Text only'].colorize("Hello")
<div class="highlight"><pre>Hello
</pre>
</div>
HTML
assert_equal <<-HTML, Lexer['Ruby'].colorize("def foo\n 'foo'\nend\n")
<div class="highlight"><pre><span class="k">def</span> <span class="nf">foo</span>
<span class="s1">&#39;foo&#39;</span>
<span class="k">end</span>
</pre>
</div>
HTML
end
def test_colorize_without_wrapper
assert_equal <<-HTML, Lexer['Text only'].colorize_without_wrapper("Hello")
Hello
HTML
assert_equal <<-HTML, Lexer['Ruby'].colorize_without_wrapper("def foo\n 'foo'\nend\n")
<span class="k">def</span> <span class="nf">foo</span>
<span class="s1">&#39;foo&#39;</span>
<span class="k">end</span>
HTML
end
end

3
test/test_pathname.rb
View File

@@ -1,10 +1,13 @@
require 'linguist/pathname'
require 'test/unit'
require 'pygments'
class TestPathname < Test::Unit::TestCase
include Linguist
Lexer = Pygments::Lexer
def test_to_s
assert_equal "file.rb", Pathname.new("file.rb").to_s
end