Replace Lexer with Pygments::Lexer

Gemfile

@@ -1,2 +1,4 @@
 source :rubygems
 gemspec
+
+gem 'pygments.rb', :git => 'git://github.com/tmm1/pygments.rb.git'

Rakefile

@@ -1,4 +1,3 @@
-require 'rake/clean'
 require 'rake/testtask'
 
 task :default => :test
@@ -6,16 +5,3 @@ task :default => :test
 Rake::TestTask.new do |t|
   t.warning = true
 end
-
-CLOBBER.include 'lib/linguist/lexers.yml'
-
-file 'lib/linguist/lexers.yml' do |f|
-  # GitHub vendored pygments path
-  #   vendor/python/pygments
-  path = File.expand_path('../../../python/pygments', __FILE__)
-  ENV['PYTHONPATH'] = path if File.directory?(path)
-
-  sh "python ./bin/pygments-lexers > #{f.name}"
-end
-
-task :lexers => 'lib/linguist/lexers.yml'

bin/pygments-lexers

@@ -1,27 +0,0 @@
-#!/usr/bin/env python
-
-from pygments import __version__
-from pygments.lexers import get_all_lexers
-
-print "# AUTOMATICALLY GENERATED BY bin/pygments-lexers"
-print "# DO NOT MODIFIY. YOUR PULL WILL BE REJECTED."
-print "# Pygments version %s" % __version__
-print "---"
-
-for name, aliases, filenames, mimetypes in get_all_lexers():
-    print "%s:" % name
-
-    if any(aliases):
-        print "  aliases:"
-        for alias in aliases:
-            print "  - '%s'" % alias
-
-    if any(filenames):
-        print "  filenames:"
-        for filename in filenames:
-            print "  - '%s'" % filename
-
-    if any(mimetypes):
-        print "  mimetypes:"
-        for mimetype in mimetypes:
-            print "  - '%s'" % mimetype

lib/linguist/blob_helper.rb

@@ -3,6 +3,7 @@ require 'linguist/mime'
 require 'linguist/pathname'
 
 require 'escape_utils'
+require 'pygments'
 require 'yaml'
 
 module Linguist
@@ -343,7 +344,7 @@ module Linguist
     #
     # Returns a Lexer.
     def lexer
-      language ? language.lexer : Lexer['Text only']
+      language ? language.lexer : Pygments::Lexer.find_by_name('Text only')
     end
 
     # Internal: Disambiguates between multiple language extensions.
@@ -512,19 +513,27 @@ module Linguist
 
     # Public: Highlight syntax of blob
     #
+    # options - A Hash of options (defaults to {})
+    #
     # Returns html String
-    def colorize
+    def colorize(options = {})
       return if !text? || large?
-      lexer.colorize(data)
+      lexer.highlight(data, options)
     end
 
     # Public: Highlight syntax of blob without the outer highlight div
     # wrapper.
     #
+    # options - A Hash of options (defaults to {})
+    #
     # Returns html String
-    def colorize_without_wrapper
+    def colorize_without_wrapper(options = {})
       return if !text? || large?
-      lexer.colorize_without_wrapper(data)
+      if text = lexer.highlight(data, options)
+        text[%r{<div class="highlight"><pre>(.*?)</pre>\s*</div>}m, 1]
+      else
+        ''
+      end
     end
 
     Language.overridden_extensions.each do |extension|

lib/linguist/language.rb

@@ -1,6 +1,5 @@
-require 'linguist/lexer'
-
 require 'yaml'
+require 'pygments'
 
 module Linguist
   # Language names that are recognizable by GitHub. Defined languages
@@ -211,7 +210,7 @@ module Linguist
       @aliases = [default_alias_name] + (attributes[:aliases] || [])
 
       # Lookup Lexer object
-      @lexer = Lexer.find_by_name(attributes[:lexer] || name) ||
+      @lexer = Pygments::Lexer.find_by_name(attributes[:lexer] || name) ||
         raise(ArgumentError, "#{@name} is missing lexer")
 
       # Set legacy search term
@@ -370,21 +369,12 @@ module Linguist
 
     # Public: Highlight syntax of text
     #
-    # text - String of code to be highlighted
+    # text    - String of code to be highlighted
+    # options - A Hash of options (defaults to {})
     #
     # Returns html String
-    def colorize(text)
+    def colorize(text, options = {})
-      lexer.colorize(text)
+      lexer.highlight(text, options = {})
-    end
-
-    # Public: Highlight syntax of text without the outer highlight div
-    # wrapper.
-    #
-    # text - String of code to be highlighted
-    #
-    # Returns html String
-    def colorize_without_wrapper(text)
-      lexer.colorize_without_wrapper(text)
     end
 
     # Public: Return name as String representation

lib/linguist/lexer.rb

@@ -1,167 +0,0 @@
-require 'pygments'
-require 'yaml'
-
-module Linguist
-  # Mirror of Pygments Lexer structure.
-  #
-  # name      - Proper lexer name (JavaScript, Ruby, Python)
-  # aliases   - Aliases for lookup (js, javascript)
-  # filenames - Filename globs (*.js)
-  # mimetypes - Mime types (application/javascript)
-  class Lexer < Struct.new(:name, :aliases, :filenames, :mimetypes)
-    @lexers          = []
-    @index           = {}
-    @name_index      = {}
-    @alias_index     = {}
-    @mimetypes_index = {}
-
-    # Internal: Create a new Lexer object
-    #
-    # name  - Name of Lexer
-    # attrs - A hash of attributes
-    #
-    # Returns a Lexer object
-    def self.create(name, attrs)
-      name      = name
-      aliases   = attrs['aliases']   || []
-      filenames = attrs['filenames'] || []
-      mimetypes = attrs['mimetypes'] || []
-
-      @lexers << lexer = new(name, aliases, filenames, mimetypes)
-
-      # All Lexer names should be unique. Warn if there is a duplicate.
-      if @name_index.key?(lexer.name)
-        warn "Duplicate lexer name: #{lexer.name}"
-      end
-
-      @index[lexer.name] = @name_index[lexer.name] = lexer
-
-      lexer.aliases.each do |name|
-        # All Lexer aliases should be unique. Warn if there is a duplicate.
-        if @alias_index.key?(name)
-          warn "Duplicate alias: #{name}"
-        end
-
-        @index[name] = @alias_index[name] = lexer
-      end
-
-      lexer.mimetypes.each do |type|
-        # All Lexer mimetypes should be unique. Warn if there is a duplicate.
-        if @mimetypes_index.key?(name)
-          warn "Duplicate mimetype: #{name}"
-        end
-
-        @mimetypes_index[type] = lexer
-      end
-    end
-
-    # Public: Get all Lexers
-    #
-    # Returns an Array of Lexers
-    def self.all
-      @lexers
-    end
-
-    # Public: Look up Lexer by name or alias.
-    #
-    # name - A String name or alias
-    #
-    #   Lexer['Ruby']
-    #   => #<Lexer name="Ruby">
-    #
-    # Returns the Lexer or nil if none was found.
-    def self.[](name)
-      @index[name]
-    end
-
-    # Public: Look up Lexer by its proper name.
-    #
-    # name - The String name of the Lexer
-    #
-    # Examples
-    #
-    #   Lexer.find_by_name('Ruby')
-    #   # => #<Lexer name="Ruby">
-    #
-    # Returns the Lexer or nil if none was found.
-    def self.find_by_name(name)
-      @name_index[name]
-    end
-
-    # Public: Look up Lexer by one of its aliases.
-    #
-    # name - A String alias of the Lexer
-    #
-    # Examples
-    #
-    #   Lexer.find_by_alias('rb')
-    #   # => #<Lexer name="Ruby">
-    #
-    # Returns the Lexer or nil if none was found.
-    def self.find_by_alias(name)
-      @alias_index[name]
-    end
-
-    # Public: Look up Lexer by one of it's mime types.
-    #
-    # type - A mime type String.
-    #
-    # Examples
-    #
-    #  Lexer.find_by_mimetype('application/x-ruby')
-    #  # => #<Lexer name="Ruby">
-    #
-    # Returns the Lexer or nil if none was found.
-    def self.find_by_mimetype(type)
-      @mimetypes_index[type]
-    end
-
-    # Public: Return a alias of the Lexer to pass to Pygments.
-    #
-    # The alias we choose is arbitrary.
-    #
-    # Returns the alias String
-    def to_s
-      aliases.first
-    end
-
-    # Public: Highlight syntax of text
-    #
-    # text - String of code to be highlighted
-    #
-    # Returns html String
-    def colorize(text)
-      Pygments.highlight(text, :lexer => aliases.first, :options => {:stripnl => false})
-    end
-
-    # Public: Highlight syntax of text without the outer highlight div
-    # wrapper.
-    #
-    # text - String of code to be highlighted
-    #
-    # Returns html String
-    def colorize_without_wrapper(text)
-      if text = colorize(text)
-        text[%r{<div class="highlight"><pre>(.*?)</pre>\s*</div>}m, 1]
-      else
-        ''
-      end
-    end
-
-    def ==(other)
-      eql?(other)
-    end
-
-    def eql?(other)
-      equal?(other)
-    end
-
-    # Load lexers from lexers.yml
-    #
-    # `bin/pygments-lexers` dumps a YAML list of all the available
-    # Pygments lexers.
-    YAML.load_file(File.expand_path("../lexers.yml", __FILE__)).each do |name, attrs|
-      Lexer.create(name, attrs)
-    end
-  end
-end

lib/linguist/pathname.rb

@@ -1,5 +1,6 @@
 require 'linguist/language'
 require 'linguist/mime'
+require 'pygments'
 
 module Linguist
   # Similar to ::Pathname, Linguist::Pathname wraps a path string and
@@ -61,7 +62,7 @@ module Linguist
     #
     # Returns a Lexer.
     def lexer
-      language ? language.lexer : Lexer['Text only']
+      language ? language.lexer : Pygments::Lexer.find_by_name('Text only')
     end
 
     # Public: Get the mime type

test/test_blob.rb

@@ -2,10 +2,13 @@ require 'linguist/file_blob'
 
 require 'test/unit'
 require 'mime/types'
+require 'pygments'
 
 class TestBlob < Test::Unit::TestCase
   include Linguist
 
+  Lexer = Pygments::Lexer
+
   def fixtures_path
     File.expand_path("../fixtures", __FILE__)
   end

test/test_language.rb

@@ -1,10 +1,13 @@
 require 'linguist/language'
 
 require 'test/unit'
+require 'pygments'
 
 class TestLanguage < Test::Unit::TestCase
   include Linguist
 
+  Lexer = Pygments::Lexer
+
   def test_ambiguous_extensions
     assert Language.ambiguous?('.h')
     assert_equal Language['C'], Language.find_by_extension('h')
@@ -311,26 +314,4 @@ class TestLanguage < Test::Unit::TestCase
 </div>
     HTML
   end
-
-  def test_colorize_without_wrapper
-    assert_equal <<-HTML, Language['Text'].colorize_without_wrapper("Hello")
-Hello
-    HTML
-
-    assert_equal <<-HTML, Language['Ruby'].colorize_without_wrapper("def foo\n  'foo'\nend\n")
-<span class="k">def</span> <span class="nf">foo</span>
-  <span class="s1">&#39;foo&#39;</span>
-<span class="k">end</span>
-    HTML
-  end
-
-  def test_colorize_doesnt_strip_newlines
-    assert_equal <<-HTML, Language['Ruby'].colorize_without_wrapper("\n\n# Foo\ndef 'foo'\nend\n")
-
-
-<span class="c1"># Foo</span>
-<span class="k">def</span> <span class="s1">&#39;foo&#39;</span>
-<span class="k">end</span>
-    HTML
-  end
 end

test/test_lexer.rb

@@ -1,91 +0,0 @@
-require 'linguist/lexer'
-
-require 'test/unit'
-
-class TestLexer < Test::Unit::TestCase
-  include Linguist
-
-  def test_find_by_name
-    assert_equal Lexer['Ruby'], Lexer.find_by_name('Ruby')
-  end
-
-  def test_find_all_by_name
-    Lexer.all.each do |lexer|
-      assert_equal lexer, Lexer.find_by_name(lexer.name)
-      assert_equal lexer, Lexer[lexer.name]
-    end
-  end
-
-  def test_find_by_alias
-    assert_equal Lexer['Ruby'], Lexer.find_by_alias('rb')
-    assert_equal Lexer['Ruby'], Lexer.find_by_alias('ruby')
-  end
-
-  def test_find_all_by_alias
-    Lexer.all.each do |lexer|
-      lexer.aliases.each do |name|
-        assert_equal lexer, Lexer.find_by_alias(name)
-        assert_equal lexer, Lexer[name]
-      end
-    end
-  end
-
-  def test_find_by_mimetype
-    assert_equal Lexer['Ruby'], Lexer.find_by_mimetype('text/x-ruby')
-    assert_equal Lexer['Ruby'], Lexer.find_by_mimetype('application/x-ruby')
-  end
-
-  def test_find_all_by_mimetype
-    Lexer.all.each do |lexer|
-      lexer.mimetypes.each do |type|
-        assert_equal lexer, Lexer.find_by_mimetype(type)
-      end
-    end
-  end
-
-  def test_name
-    assert_equal 'Ruby',   Lexer['Ruby'].name
-    assert_equal 'Python', Lexer['Python'].name
-    assert_equal 'Perl',   Lexer['Perl'].name
-  end
-
-  def test_aliases
-    assert_equal ['rb', 'ruby', 'duby'], Lexer['Ruby'].aliases
-    assert_equal ['python', 'py'],       Lexer['Python'].aliases
-    assert_equal ['perl', 'pl'],         Lexer['Perl'].aliases
-  end
-
-  def test_eql
-    assert Lexer['Ruby'].eql?(Lexer['Ruby'])
-    assert !Lexer['Ruby'].eql?(Lexer['Python'])
-    assert !Lexer['Ruby'].eql?(Lexer.new('Ruby'))
-  end
-
-  def test_colorize
-    assert_equal <<-HTML, Lexer['Text only'].colorize("Hello")
-<div class="highlight"><pre>Hello
-</pre>
-</div>
-    HTML
-
-    assert_equal <<-HTML, Lexer['Ruby'].colorize("def foo\n  'foo'\nend\n")
-<div class="highlight"><pre><span class="k">def</span> <span class="nf">foo</span>
-  <span class="s1">&#39;foo&#39;</span>
-<span class="k">end</span>
-</pre>
-</div>
-    HTML
-  end
-
-  def test_colorize_without_wrapper
-    assert_equal <<-HTML, Lexer['Text only'].colorize_without_wrapper("Hello")
-Hello
-    HTML
-
-    assert_equal <<-HTML, Lexer['Ruby'].colorize_without_wrapper("def foo\n  'foo'\nend\n")
-<span class="k">def</span> <span class="nf">foo</span>
-  <span class="s1">&#39;foo&#39;</span>
-<span class="k">end</span>
-    HTML
-  end
-end

test/test_pathname.rb

@@ -1,10 +1,13 @@
 require 'linguist/pathname'
 
 require 'test/unit'
+require 'pygments'
 
 class TestPathname < Test::Unit::TestCase
   include Linguist
 
+  Lexer = Pygments::Lexer
+
   def test_to_s
     assert_equal "file.rb", Pathname.new("file.rb").to_s
   end