Revert "Replace the tokenizer with a flex-based scanner (#3846)"

This reverts commit 99eaf5faf9.

.gitmodules

@@ -169,9 +169,6 @@
 [submodule "vendor/grammars/Agda.tmbundle"]
 	path = vendor/grammars/Agda.tmbundle
 	url = https://github.com/mokus0/Agda.tmbundle
-[submodule "vendor/grammars/Julia.tmbundle"]
-	path = vendor/grammars/Julia.tmbundle
-	url = https://github.com/JuliaEditorSupport/Julia.tmbundle
 [submodule "vendor/grammars/ooc.tmbundle"]
 	path = vendor/grammars/ooc.tmbundle
 	url = https://github.com/nilium/ooc.tmbundle
@@ -883,3 +880,6 @@
 [submodule "vendor/grammars/wdl-sublime-syntax-highlighter"]
 	path = vendor/grammars/wdl-sublime-syntax-highlighter
 	url = https://github.com/broadinstitute/wdl-sublime-syntax-highlighter
+[submodule "vendor/grammars/atom-language-julia"]
+	path = vendor/grammars/atom-language-julia
+	url = https://github.com/JuliaEditorSupport/atom-language-julia

.travis.yml

@@ -19,10 +19,6 @@ rvm:
   - 2.3.3
   - 2.4.0
 
-matrix:
-  allow_failures:
-  - rvm: 2.4.0
-
 notifications:
   disabled: true
 

grammars.yml

@@ -45,8 +45,6 @@ vendor/grammars/Isabelle.tmbundle:
 - source.isabelle.theory
 vendor/grammars/JSyntax:
 - source.j
-vendor/grammars/Julia.tmbundle:
-- source.julia
 vendor/grammars/Lean.tmbundle:
 - source.lean
 vendor/grammars/LiveScript.tmbundle:
@@ -192,6 +190,9 @@ vendor/grammars/atom-language-1c-bsl:
 vendor/grammars/atom-language-clean:
 - source.clean
 - text.restructuredtext.clean
+vendor/grammars/atom-language-julia:
+- source.julia
+- source.julia.console
 vendor/grammars/atom-language-p4:
 - source.p4
 vendor/grammars/atom-language-perl6:

lib/linguist/heuristics.rb

@@ -358,23 +358,15 @@ module Linguist
     end
 
     disambiguate ".pm" do |data|
-      if /^\s*(?:use\s+v6\s*;|(?:\bmy\s+)?class|module)\b/.match(data)
-        Language["Perl 6"]
-      elsif /\buse\s+(?:strict\b|v?5\.)/.match(data)
+      if /\buse\s+(?:strict\b|v?5\.)/.match(data)
         Language["Perl"]
+      elsif /^\s*(?:use\s+v6\s*;|(?:\bmy\s+)?class|module)\b/.match(data)
+        Language["Perl 6"]
       elsif /^\s*\/\* XPM \*\//.match(data)
         Language["XPM"]
       end
     end
 
-    disambiguate ".pod", "Pod", "Perl" do |data|
-      if /^=\w+\b/.match(data)
-        Language["Pod"]
-      else
-        Language["Perl"]
-      end
-    end
-
     disambiguate ".pro" do |data|
       if /^[^#]+:-/.match(data)
         Language["Prolog"]
@@ -476,7 +468,7 @@ module Linguist
     end
 
     disambiguate ".ts" do |data|
-      if data.include?("<TS")
+      if /<TS\b/.match(data)
         Language["XML"]
       else
         Language["TypeScript"]

lib/linguist/languages.yml

@@ -1144,6 +1144,15 @@ Ecere Projects:
   codemirror_mode: javascript
   codemirror_mime_type: application/json
   language_id: 98
+Edje Data Collection:
+  type: data
+  extensions:
+   - ".edc"
+  tm_scope: source.json
+  ace_mode: json
+  codemirror_mode: javascript
+  codemirror_mime_type: application/json
+  language_id: 342840478
 Eiffel:
   type: programming
   color: "#946d57"
@@ -1921,6 +1930,7 @@ IRC log:
   language_id: 164
 Idris:
   type: programming
+  color: "#b30000"
   extensions:
   - ".idr"
   - ".lidr"
@@ -3313,7 +3323,6 @@ Perl:
   - ".ph"
   - ".plx"
   - ".pm"
-  - ".pod"
   - ".psgi"
   - ".t"
   filenames:

lib/linguist/vendor.yml

@@ -81,6 +81,9 @@
 # Animate.css
 - (^|/)animate\.(css|less|scss|styl)$
 
+# Select2
+- (^|/)select2/.*\.(css|scss|js)$
+
 # Vendored dependencies
 - third[-_]?party/
 - 3rd[-_]?party/
@@ -119,6 +122,15 @@
 # jQuery File Upload
 - (^|/)jquery\.fileupload(-\w+)?\.js$
 
+# jQuery dataTables
+- jquery.dataTables.js
+
+# bootboxjs
+- bootbox.js
+
+# pdf-worker
+- pdf.worker.js
+
 # Slick
 - (^|/)slick\.\w+.js$
 

lib/linguist/version.rb

@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "5.3.0"
+  VERSION = "5.3.2"
 end

samples/TypeScript/cache.ts

@@ -0,0 +1,102 @@
+import { DocumentNode } from 'graphql';
+import { getFragmentQueryDocument } from 'apollo-utilities';
+
+import { DataProxy, Cache } from './types';
+
+export type Transaction<T> = (c: ApolloCache<T>) => void;
+
+export abstract class ApolloCache<TSerialized> implements DataProxy {
+  // required to implement
+  // core API
+  public abstract read<T>(query: Cache.ReadOptions): T;
+  public abstract write(write: Cache.WriteOptions): void;
+  public abstract diff<T>(query: Cache.DiffOptions): Cache.DiffResult<T>;
+  public abstract watch(watch: Cache.WatchOptions): () => void;
+  public abstract evict(query: Cache.EvictOptions): Cache.EvictionResult;
+  public abstract reset(): Promise<void>;
+
+  // intializer / offline / ssr API
+  /**
+   * Replaces existing state in the cache (if any) with the values expressed by
+   * `serializedState`.
+   *
+   * Called when hydrating a cache (server side rendering, or offline storage),
+   * and also (potentially) during hot reloads.
+   */
+  public abstract restore(
+    serializedState: TSerialized,
+  ): ApolloCache<TSerialized>;
+
+  /**
+   * Exposes the cache's complete state, in a serializable format for later restoration.
+   */
+  public abstract extract(optimistic: boolean): TSerialized;
+
+  // optimistic API
+  public abstract removeOptimistic(id: string): void;
+
+  // transactional API
+  public abstract performTransaction(
+    transaction: Transaction<TSerialized>,
+  ): void;
+  public abstract recordOptimisticTransaction(
+    transaction: Transaction<TSerialized>,
+    id: string,
+  ): void;
+
+  // optional API
+  public transformDocument(document: DocumentNode): DocumentNode {
+    return document;
+  }
+  // experimental
+  public transformForLink(document: DocumentNode): DocumentNode {
+    return document;
+  }
+
+  // DataProxy API
+  /**
+   *
+   * @param options
+   * @param optimistic
+   */
+  public readQuery<QueryType>(
+    options: DataProxy.Query,
+    optimistic: boolean = false,
+  ): QueryType {
+    return this.read({
+      query: options.query,
+      variables: options.variables,
+      optimistic,
+    });
+  }
+
+  public readFragment<FragmentType>(
+    options: DataProxy.Fragment,
+    optimistic: boolean = false,
+  ): FragmentType | null {
+    return this.read({
+      query: getFragmentQueryDocument(options.fragment, options.fragmentName),
+      variables: options.variables,
+      rootId: options.id,
+      optimistic,
+    });
+  }
+
+  public writeQuery(options: Cache.WriteQueryOptions): void {
+    this.write({
+      dataId: 'ROOT_QUERY',
+      result: options.data,
+      query: options.query,
+      variables: options.variables,
+    });
+  }
+
+  public writeFragment(options: Cache.WriteFragmentOptions): void {
+    this.write({
+      dataId: options.id,
+      result: options.data,
+      variables: options.variables,
+      query: getFragmentQueryDocument(options.fragment, options.fragmentName),
+    });
+  }
+}

test/fixtures/Perl/Module.pm

@@ -0,0 +1,8 @@
+use 5.006;
+use strict;
+
+=head1
+
+module
+
+=cut

test/test_grammars.rb

@@ -23,7 +23,6 @@ class TestGrammars < Minitest::Test
     "8653305b358375d0fced85dc24793b99919b11ef", # language-shellscript
     "9f0c0b0926a18f5038e455e8df60221125fc3111", # elixir-tmbundle
     "a4dadb2374282098c5b8b14df308906f5347d79a", # mako-tmbundle
-    "b9b24778619dce325b651f0d77cbc72e7ae0b0a3", # Julia.tmbundle
     "e06722add999e7428048abcc067cd85f1f7ca71c", # r.tmbundle
     "50b14a0e3f03d7ca754dac42ffb33302b5882b78", # smalltalk-tmbundle
     "eafbc4a2f283752858e6908907f3c0c90188785b", # gap-tmbundle
@@ -44,6 +43,7 @@ class TestGrammars < Minitest::Test
     "9dafd4e2a79cb13a6793b93877a254bc4d351e74", # sublime-text-ox
     "8e111741d97ba2e27b3d18a309d426b4a37e604f", # sublime-varnish
     "23d2538e33ce62d58abda2c039364b92f64ea6bc", # sublime-angelscript
+    "53714285caad3c480ebd248c490509695d10404b", # atom-language-julia
   ].freeze
 
   # List of allowed SPDX license names

test/test_heuristics.rb

@@ -1,6 +1,6 @@
 require_relative "./helper"
 
-class TestHeuristcs < Minitest::Test
+class TestHeuristics < Minitest::Test
   include Linguist
 
   def fixture(name)
@@ -237,14 +237,6 @@ class TestHeuristcs < Minitest::Test
     })
   end
 
-  # Candidate languages = ["Pod", "Perl"]
-  def test_pod_by_heuristics
-    assert_heuristics({
-      "Perl" => all_fixtures("Perl", "*.pod"),
-      "Pod" => all_fixtures("Pod", "*.pod")
-    })
-  end
-
   # Candidate languages = ["IDL", "Prolog", "QMake", "INI"]
   def test_pro_by_heuristics
     assert_heuristics({

vendor/README.md

@@ -101,6 +101,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **eC:** [ecere/ec.tmbundle](https://github.com/ecere/ec.tmbundle)
 - **Ecere Projects:** [textmate/json.tmbundle](https://github.com/textmate/json.tmbundle)
 - **ECLiPSe:** [alnkpa/sublimeprolog](https://github.com/alnkpa/sublimeprolog)
+- **Edje Data Collection:** [textmate/json.tmbundle](https://github.com/textmate/json.tmbundle)
 - **edn:** [atom/language-clojure](https://github.com/atom/language-clojure)
 - **Eiffel:** [textmate/eiffel.tmbundle](https://github.com/textmate/eiffel.tmbundle)
 - **EJS:** [gregory-m/ejs-tmbundle](https://github.com/gregory-m/ejs-tmbundle)
@@ -181,7 +182,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **JSONiq:** [wcandillon/language-jsoniq](https://github.com/wcandillon/language-jsoniq)
 - **JSONLD:** [atom/language-javascript](https://github.com/atom/language-javascript)
 - **JSX:** [github-linguist/language-babel](https://github.com/github-linguist/language-babel)
-- **Julia:** [JuliaEditorSupport/Julia.tmbundle](https://github.com/JuliaEditorSupport/Julia.tmbundle)
+- **Julia:** [JuliaEditorSupport/atom-language-julia](https://github.com/JuliaEditorSupport/atom-language-julia)
 - **Jupyter Notebook:** [textmate/json.tmbundle](https://github.com/textmate/json.tmbundle)
 - **KiCad Layout:** [Alhadis/language-pcb](https://github.com/Alhadis/language-pcb)
 - **KiCad Legacy Layout:** [Alhadis/language-pcb](https://github.com/Alhadis/language-pcb)

vendor/licenses/grammar/Julia.tmbundle.txt

@@ -1,27 +0,0 @@
----
-type: grammar
-name: Julia.tmbundle
-license: mit
----
-Copyright (c) 2012-2014 Stefan Karpinski, Elliot Saba, Dirk Gadsden,
-Adam Strzelecki, Jonathan Malmaud and other contributors:
-
-https://github.com/JuliaEditorSupport/Julia.tmbundle/contributors
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

vendor/licenses/grammar/atom-language-julia.txt

@@ -0,0 +1,27 @@
+---
+type: grammar
+name: atom-language-julia
+license: mit
+---
+The atom-language-julia package is licensed under the MIT "Expat" License:
+
+> Copyright (c) 2015
+>
+> Permission is hereby granted, free of charge, to any person obtaining
+> a copy of this software and associated documentation files (the
+> "Software"), to deal in the Software without restriction, including
+> without limitation the rights to use, copy, modify, merge, publish,
+> distribute, sublicense, and/or sell copies of the Software, and to
+> permit persons to whom the Software is furnished to do so, subject to
+> the following conditions:
+>
+> The above copyright notice and this permission notice shall be
+> included in all copies or substantial portions of the Software.
+>
+> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+> IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+> CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+> TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+> SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.