From eccea656410f8dcd0dbe0b98999e39337c4141e4 Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Thu, 20 Nov 2014 11:14:05 -0500 Subject: [PATCH 01/10] Fix for interpreters not getting add to samples.json --- lib/linguist/samples.rb | 8 +++++--- test/test_samples.rb | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/linguist/samples.rb b/lib/linguist/samples.rb index 82c011b1..ca91dc15 100644 --- a/lib/linguist/samples.rb +++ b/lib/linguist/samples.rb @@ -52,14 +52,16 @@ module Linguist }) end else + path = File.join(dirname, filename) + if File.extname(filename) == "" - raise "#{File.join(dirname, filename)} is missing an extension, maybe it belongs in filenames/ subdir" + raise "#{path} is missing an extension, maybe it belongs in filenames/ subdir" end yield({ - :path => File.join(dirname, filename), + :path => path, :language => category, - :interpreter => File.exist?(filename) ? Linguist.interpreter_from_shebang(File.read(filename)) : nil, + :interpreter => Linguist.interpreter_from_shebang(File.read(path)), :extname => File.extname(filename) }) end diff --git a/test/test_samples.rb b/test/test_samples.rb index 929a7a00..69b3cde3 100644 --- a/test/test_samples.rb +++ b/test/test_samples.rb @@ -34,6 +34,7 @@ class TestSamples < Test::Unit::TestCase assert_equal data['languages_total'], data['languages'].inject(0) { |n, (_, c)| n += c } assert_equal data['tokens_total'], data['language_tokens'].inject(0) { |n, (_, c)| n += c } assert_equal data['tokens_total'], data['tokens'].inject(0) { |n, (_, ts)| n += ts.inject(0) { |m, (_, c)| m += c } } + assert !data["interpreters"].empty? end # Check that there aren't samples with extensions that aren't explicitly defined in languages.yml From 2670e2b035fbb68a793005b80adf588747c1e0ce Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Thu, 20 Nov 2014 11:21:52 -0500 Subject: [PATCH 02/10] Test that interpreters are defined in languages.yml --- test/test_samples.rb | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/test/test_samples.rb b/test/test_samples.rb index 69b3cde3..b36cf363 100644 --- a/test/test_samples.rb +++ b/test/test_samples.rb @@ -37,21 +37,26 @@ class TestSamples < Test::Unit::TestCase assert !data["interpreters"].empty? end - # Check that there aren't samples with extensions that aren't explicitly defined in languages.yml - def test_parity - extensions = Samples.cache['extnames'] - languages_yml = File.expand_path("../../lib/linguist/languages.yml", __FILE__) - languages = YAML.load_file(languages_yml) - - languages.each do |name, options| + # Check that there aren't samples with extensions or interpreters that + # aren't explicitly defined in languages.yml + languages_yml = File.expand_path("../../lib/linguist/languages.yml", __FILE__) + YAML.load_file(languages_yml).each do |name, options| + define_method "test_samples_have_parity_with_languages_yml_for_#{name}" do options['extensions'] ||= [] - - if extnames = extensions[name] + if extnames = Samples.cache['extnames'][name] extnames.each do |extname| next if extname == '.script!' assert options['extensions'].include?(extname), "#{name} has a sample with extension (#{extname}) that isn't explicitly defined in languages.yml" end end + + options['interpreters'] ||= [] + if interpreters = Samples.cache['interpreters'][name] + interpreters.each do |interpreter| + # next if extname == '.script!' + assert options['interpreters'].include?(interpreter), "#{name} has a sample with an interpreter (#{interpreter}) that isn't explicitly defined in languages.yml" + end + end end end From 870feb8592be57e929ce35b7356811386a43f51c Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Thu, 20 Nov 2014 11:27:54 -0500 Subject: [PATCH 03/10] Add missing interpreters --- lib/linguist/languages.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/linguist/languages.yml b/lib/linguist/languages.yml index 4d36d8d0..307bd24d 100644 --- a/lib/linguist/languages.yml +++ b/lib/linguist/languages.yml @@ -717,6 +717,8 @@ Erlang: - .es - .escript - .hrl + interpreters: + - escript F#: type: programming @@ -907,6 +909,8 @@ Gnuplot: - .gnuplot - .plot - .plt + interpreters: + - gnuplot Go: type: programming @@ -1161,6 +1165,8 @@ Ioke: color: "#078193" extensions: - .ik + interpreters: + - ioke Isabelle: type: programming @@ -1655,6 +1661,8 @@ Nu: filenames: - Nukefile tm_scope: source.scheme + interpreters: + - nush NumPy: group: Python @@ -1831,6 +1839,8 @@ Parrot Assembly: - pasm extensions: - .pasm + interpreters: + - parrot Parrot Internal Representation: group: Parrot @@ -2023,6 +2033,8 @@ QMake: extensions: - .pro - .pri + interpreters: + - qmake R: type: programming @@ -2172,6 +2184,7 @@ Ruby: - .watchr interpreters: - ruby + - macruby filenames: - .pryrc - Appraisals @@ -2257,6 +2270,8 @@ Scala: - .scala - .sbt - .sc + interpreters: + - scala Scaml: group: HTML From 56bfde998b1f10895a31b9b3d5097ad7962fc47f Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Thu, 20 Nov 2014 12:28:30 -0500 Subject: [PATCH 04/10] Only strip minor version off of interpreters This used to turn `python2.4` into `python`, which causes trouble with `perl6`, which is a different language definition. --- lib/linguist/languages.yml | 1 + lib/linguist/samples.rb | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/linguist/languages.yml b/lib/linguist/languages.yml index 307bd24d..7e8aecc6 100644 --- a/lib/linguist/languages.yml +++ b/lib/linguist/languages.yml @@ -2013,6 +2013,7 @@ Python: - wscript interpreters: - python + - python2 Python traceback: type: data diff --git a/lib/linguist/samples.rb b/lib/linguist/samples.rb index ca91dc15..2d72296a 100644 --- a/lib/linguist/samples.rb +++ b/lib/linguist/samples.rb @@ -133,10 +133,8 @@ module Linguist script = script == 'env' ? tokens[1] : script - # "python2.6" -> "python" - if script =~ /((?:\d+\.?)+)/ - script.sub! $1, '' - end + # "python2.6" -> "python2" + script.sub! $1, '' if script =~ /(\.\d+)$/ # Check for multiline shebang hacks that call `exec` if script == 'sh' && From 45384bd49815ab8c75bd60d38251e52b95ecce61 Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Thu, 20 Nov 2014 12:29:16 -0500 Subject: [PATCH 05/10] More missing interpreters --- lib/linguist/languages.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/linguist/languages.yml b/lib/linguist/languages.yml index 7e8aecc6..810f6685 100644 --- a/lib/linguist/languages.yml +++ b/lib/linguist/languages.yml @@ -1850,6 +1850,8 @@ Parrot Internal Representation: - pir extensions: - .pir + interpreters: + - parrot Pascal: type: programming @@ -1892,6 +1894,8 @@ Perl6: - .p6m - .pl6 - .pm6 + interpreters: + - perl6 PigLatin: type: programming @@ -1952,6 +1956,8 @@ Prolog: - .ecl - .pro - .prolog + interpreters: + - swipl Propeller Spin: type: programming From 9823af0cb45291c2ccf540dcf2567e4c142e41dd Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Thu, 20 Nov 2014 12:50:35 -0500 Subject: [PATCH 06/10] Fix for shebang with relative bin `#!/usr/bin/env bin/linguist` is a valid shebang --- lib/linguist/languages.yml | 2 ++ lib/linguist/samples.rb | 7 +++++-- test/test_samples.rb | 5 +++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/linguist/languages.yml b/lib/linguist/languages.yml index 810f6685..db9ece79 100644 --- a/lib/linguist/languages.yml +++ b/lib/linguist/languages.yml @@ -544,6 +544,8 @@ Crystal: - .cr ace_mode: ruby tm_scope: source.ruby + interpreters: + - crystal Cucumber: extensions: diff --git a/lib/linguist/samples.rb b/lib/linguist/samples.rb index 2d72296a..001204b5 100644 --- a/lib/linguist/samples.rb +++ b/lib/linguist/samples.rb @@ -133,6 +133,9 @@ module Linguist script = script == 'env' ? tokens[1] : script + # If script has an invalid shebang, we might get here + return unless script + # "python2.6" -> "python2" script.sub! $1, '' if script =~ /(\.\d+)$/ @@ -141,8 +144,8 @@ module Linguist lines[0...5].any? { |l| l.match(/exec (\w+).+\$0.+\$@/) } script = $1 end - - script + + File.basename(script) else nil end diff --git a/test/test_samples.rb b/test/test_samples.rb index b36cf363..cf7a276b 100644 --- a/test/test_samples.rb +++ b/test/test_samples.rb @@ -85,4 +85,9 @@ class TestSamples < Test::Unit::TestCase end end end + + def test_shebang + assert_equal "crystal", Linguist.interpreter_from_shebang("#!/usr/bin/env bin/crystial") + assert_equal "python2", Linguist.interpreter_from_shebang("#!/usr/bin/python2.4") + end end From 0651568bfbfc81da372db666589c28a7408ca104 Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Wed, 26 Nov 2014 15:34:03 -0500 Subject: [PATCH 07/10] Remove old wrong_shebang.rb sample This was added in a69118bd1738bb2d3a66cf5bb026a69a0d535093, but that test has since been removed. --- samples/Ruby/wrong_shebang.rb | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 samples/Ruby/wrong_shebang.rb diff --git a/samples/Ruby/wrong_shebang.rb b/samples/Ruby/wrong_shebang.rb deleted file mode 100644 index 22b4804a..00000000 --- a/samples/Ruby/wrong_shebang.rb +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -puts "Not Python" From b3a49ce62761d8daed12fb2c82dec6d66fc60bd0 Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Wed, 26 Nov 2014 15:40:33 -0500 Subject: [PATCH 08/10] add python3 interpreter --- lib/linguist/languages.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/linguist/languages.yml b/lib/linguist/languages.yml index 02ee451d..02094a51 100644 --- a/lib/linguist/languages.yml +++ b/lib/linguist/languages.yml @@ -2087,6 +2087,7 @@ Python: interpreters: - python - python2 + - python3 Python traceback: type: data From 06c0cb916bb872f49fdee68f5ec694fc6464898c Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Wed, 26 Nov 2014 15:40:40 -0500 Subject: [PATCH 09/10] add rake interpreter --- lib/linguist/languages.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/linguist/languages.yml b/lib/linguist/languages.yml index 02094a51..c80fdb2b 100644 --- a/lib/linguist/languages.yml +++ b/lib/linguist/languages.yml @@ -2265,6 +2265,7 @@ Ruby: interpreters: - ruby - macruby + - rake filenames: - .pryrc - Appraisals From 5b41ab4774b93a76bd6e1c719d17844f4f90120c Mon Sep 17 00:00:00 2001 From: Brandon Keepers Date: Wed, 26 Nov 2014 15:40:51 -0500 Subject: [PATCH 10/10] Fix typo in test --- test/test_samples.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_samples.rb b/test/test_samples.rb index 30558556..06ede379 100644 --- a/test/test_samples.rb +++ b/test/test_samples.rb @@ -84,7 +84,7 @@ class TestSamples < Test::Unit::TestCase end def test_shebang - assert_equal "crystal", Linguist.interpreter_from_shebang("#!/usr/bin/env bin/crystial") + assert_equal "crystal", Linguist.interpreter_from_shebang("#!/usr/bin/env bin/crystal") assert_equal "python2", Linguist.interpreter_from_shebang("#!/usr/bin/python2.4") end end