Lexer crash fix (#3900)

* input may return 0 for EOF

Stops overruns into fread from nothing.

* remove two trailing contexts

* fix up sgml tokens

.gitmodules

@@ -169,9 +169,6 @@
 [submodule "vendor/grammars/Agda.tmbundle"]
 	path = vendor/grammars/Agda.tmbundle
 	url = https://github.com/mokus0/Agda.tmbundle
-[submodule "vendor/grammars/Julia.tmbundle"]
-	path = vendor/grammars/Julia.tmbundle
-	url = https://github.com/JuliaEditorSupport/Julia.tmbundle
 [submodule "vendor/grammars/ooc.tmbundle"]
 	path = vendor/grammars/ooc.tmbundle
 	url = https://github.com/nilium/ooc.tmbundle
@@ -883,3 +880,6 @@
 [submodule "vendor/grammars/wdl-sublime-syntax-highlighter"]
 	path = vendor/grammars/wdl-sublime-syntax-highlighter
 	url = https://github.com/broadinstitute/wdl-sublime-syntax-highlighter
+[submodule "vendor/grammars/atom-language-julia"]
+	path = vendor/grammars/atom-language-julia
+	url = https://github.com/JuliaEditorSupport/atom-language-julia

Rakefile

@@ -47,15 +47,10 @@ task :samples => :compile do
   File.write 'lib/linguist/samples.json', json
 end
 
-FLEX_MIN_VER = [2, 5, 39]
 task :flex do
-  if `flex -V` !~ /^flex (\d+)\.(\d+)\.(\d+)/
+  if `flex -V` !~ /^flex \d+\.\d+\.\d+/
     fail "flex not detected"
   end
-  maj, min, rev = $1.to_i, $2.to_i, $3.to_i
-  if maj < FLEX_MIN_VER[0] || (maj == FLEX_MIN_VER[0] && (min < FLEX_MIN_VER[1] || (min == FLEX_MIN_VER[1] && rev < FLEX_MIN_VER[2])))
-    fail "building linguist's lexer requires at least flex #{FLEX_MIN_VER.join(".")}"
-  end
   system "cd ext/linguist && flex tokenizer.l"
 end
 

ext/linguist/lex.linguist_yy.h

@@ -11,7 +11,7 @@
 #define FLEX_SCANNER
 #define YY_FLEX_MAJOR_VERSION 2
 #define YY_FLEX_MINOR_VERSION 5
-#define YY_FLEX_SUBMINOR_VERSION 39
+#define YY_FLEX_SUBMINOR_VERSION 35
 #if YY_FLEX_SUBMINOR_VERSION > 0
 #define FLEX_BETA
 #endif
@@ -49,6 +49,7 @@ typedef int16_t flex_int16_t;
 typedef uint16_t flex_uint16_t;
 typedef int32_t flex_int32_t;
 typedef uint32_t flex_uint32_t;
+typedef uint64_t flex_uint64_t;
 #else
 typedef signed char flex_int8_t;
 typedef short int flex_int16_t;
@@ -56,6 +57,7 @@ typedef int flex_int32_t;
 typedef unsigned char flex_uint8_t; 
 typedef unsigned short int flex_uint16_t;
 typedef unsigned int flex_uint32_t;
+#endif /* ! C99 */
 
 /* Limits of integral types. */
 #ifndef INT8_MIN
@@ -86,8 +88,6 @@ typedef unsigned int flex_uint32_t;
 #define UINT32_MAX             (4294967295U)
 #endif
 
-#endif /* ! C99 */
-
 #endif /* ! FLEXINT_H */
 
 #ifdef __cplusplus
@@ -130,15 +130,7 @@ typedef void* yyscan_t;
 
 /* Size of default input buffer. */
 #ifndef YY_BUF_SIZE
-#ifdef __ia64__
-/* On IA-64, the buffer size is 16k, not 8k.
- * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case.
- * Ditto for the __ia64__ case accordingly.
- */
-#define YY_BUF_SIZE 32768
-#else
 #define YY_BUF_SIZE 16384
-#endif /* __ia64__ */
 #endif
 
 #ifndef YY_TYPEDEF_YY_BUFFER_STATE
@@ -277,10 +269,6 @@ int linguist_yyget_lineno (yyscan_t yyscanner );
 
 void linguist_yyset_lineno (int line_number ,yyscan_t yyscanner );
 
-int linguist_yyget_column  (yyscan_t yyscanner );
-
-void linguist_yyset_column (int column_no ,yyscan_t yyscanner );
-
 /* Macros after this point can all be overridden by user definitions in
  * section 1.
  */
@@ -307,12 +295,7 @@ static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner);
 
 /* Amount of stuff to slurp up with each read. */
 #ifndef YY_READ_BUF_SIZE
-#ifdef __ia64__
-/* On IA-64, the buffer size is 16k, not 8k */
-#define YY_READ_BUF_SIZE 16384
-#else
 #define YY_READ_BUF_SIZE 8192
-#endif /* __ia64__ */
 #endif
 
 /* Number of entries by which start-condition stack grows. */
@@ -345,9 +328,9 @@ extern int linguist_yylex (yyscan_t yyscanner);
 #undef YY_DECL
 #endif
 
-#line 117 "tokenizer.l"
+#line 118 "tokenizer.l"
 
 
-#line 352 "lex.linguist_yy.h"
+#line 335 "lex.linguist_yy.h"
 #undef linguist_yyIN_HEADER
 #endif /* linguist_yyHEADER_H */

ext/linguist/tokenizer.l

@@ -9,25 +9,25 @@
 
 #define eat_until_eol() do { \
     int c; \
-    while ((c = input(yyscanner)) != '\n' && c != EOF); \
+    while ((c = input(yyscanner)) != '\n' && c != EOF && c); \
-    if (c == EOF) \
+    if (c == EOF || !c) \
-      yyterminate(); \
+      return 0; \
   } while (0)
 
 #define eat_until_unescaped(q) do { \
     int c; \
-    while ((c = input(yyscanner)) != EOF) { \
+    while ((c = input(yyscanner)) != EOF && c) { \
       if (c == '\n') \
         break; \
       if (c == '\\') { \
         c = input(yyscanner); \
-        if (c == EOF) \
+        if (c == EOF || !c) \
-          yyterminate(); \
+          return 0; \
       } else if (c == q) \
         break; \
     } \
-    if (c == EOF) \
+    if (c == EOF || !c) \
-      yyterminate(); \
+      return 0; \
   } while (0)
 
 %}
@@ -84,7 +84,7 @@
 \"                                { eat_until_unescaped('"'); }
 '                                 { eat_until_unescaped('\''); }
 (0x[0-9a-fA-F]([0-9a-fA-F]|\.)*|[0-9]([0-9]|\.)*)([uU][lL]{0,2}|([eE][-+][0-9]*)?[fFlL]*) { /* nothing */ }
-\<[^ \t\n\r<>]+/>|" "[^<>\n]{0,2048}>               {
+\<[[:alnum:]_!./?-]+              {
     if (strcmp(yytext, "<!--") == 0) {
      BEGIN(xml_comment);
     } else {
@@ -93,8 +93,8 @@
       return 1;
     }
   }
-<sgml>[[:alnum:]_]+=/\"           { feed_token(strdup(yytext), REGULAR_TOKEN); input(yyscanner); eat_until_unescaped('"'); return 1; }
+<sgml>[[:alnum:]_]+=\"            { feed_token(strndup(yytext, strlen(yytext) - 1), REGULAR_TOKEN); eat_until_unescaped('"'); return 1; }
-<sgml>[[:alnum:]_]+=/'            { feed_token(strdup(yytext), REGULAR_TOKEN); input(yyscanner); eat_until_unescaped('\''); return 1; }
+<sgml>[[:alnum:]_]+='             { feed_token(strndup(yytext, strlen(yytext) - 1), REGULAR_TOKEN); eat_until_unescaped('\''); return 1; }
 <sgml>[[:alnum:]_]+=[[:alnum:]_]* { feed_token(strdup(yytext), REGULAR_TOKEN); *(strchr(yyextra->token, '=') + 1) = 0; return 1; }
 <sgml>[[:alnum:]_]+               { feed_token(strdup(yytext), REGULAR_TOKEN); return 1; }
 <sgml>\>                          { BEGIN(INITIAL); }

grammars.yml

@@ -45,8 +45,6 @@ vendor/grammars/Isabelle.tmbundle:
 - source.isabelle.theory
 vendor/grammars/JSyntax:
 - source.j
-vendor/grammars/Julia.tmbundle:
-- source.julia
 vendor/grammars/Lean.tmbundle:
 - source.lean
 vendor/grammars/LiveScript.tmbundle:
@@ -192,6 +190,9 @@ vendor/grammars/atom-language-1c-bsl:
 vendor/grammars/atom-language-clean:
 - source.clean
 - text.restructuredtext.clean
+vendor/grammars/atom-language-julia:
+- source.julia
+- source.julia.console
 vendor/grammars/atom-language-p4:
 - source.p4
 vendor/grammars/atom-language-perl6:

lib/linguist/heuristics.rb

@@ -477,7 +477,7 @@ module Linguist
     end
 
     disambiguate ".ts" do |data|
-      if data.include?("<TS")
+      if /<TS\b/.match(data)
         Language["XML"]
       else
         Language["TypeScript"]

samples/TypeScript/cache.ts

@@ -0,0 +1,102 @@
+import { DocumentNode } from 'graphql';
+import { getFragmentQueryDocument } from 'apollo-utilities';
+
+import { DataProxy, Cache } from './types';
+
+export type Transaction<T> = (c: ApolloCache<T>) => void;
+
+export abstract class ApolloCache<TSerialized> implements DataProxy {
+  // required to implement
+  // core API
+  public abstract read<T>(query: Cache.ReadOptions): T;
+  public abstract write(write: Cache.WriteOptions): void;
+  public abstract diff<T>(query: Cache.DiffOptions): Cache.DiffResult<T>;
+  public abstract watch(watch: Cache.WatchOptions): () => void;
+  public abstract evict(query: Cache.EvictOptions): Cache.EvictionResult;
+  public abstract reset(): Promise<void>;
+
+  // intializer / offline / ssr API
+  /**
+   * Replaces existing state in the cache (if any) with the values expressed by
+   * `serializedState`.
+   *
+   * Called when hydrating a cache (server side rendering, or offline storage),
+   * and also (potentially) during hot reloads.
+   */
+  public abstract restore(
+    serializedState: TSerialized,
+  ): ApolloCache<TSerialized>;
+
+  /**
+   * Exposes the cache's complete state, in a serializable format for later restoration.
+   */
+  public abstract extract(optimistic: boolean): TSerialized;
+
+  // optimistic API
+  public abstract removeOptimistic(id: string): void;
+
+  // transactional API
+  public abstract performTransaction(
+    transaction: Transaction<TSerialized>,
+  ): void;
+  public abstract recordOptimisticTransaction(
+    transaction: Transaction<TSerialized>,
+    id: string,
+  ): void;
+
+  // optional API
+  public transformDocument(document: DocumentNode): DocumentNode {
+    return document;
+  }
+  // experimental
+  public transformForLink(document: DocumentNode): DocumentNode {
+    return document;
+  }
+
+  // DataProxy API
+  /**
+   *
+   * @param options
+   * @param optimistic
+   */
+  public readQuery<QueryType>(
+    options: DataProxy.Query,
+    optimistic: boolean = false,
+  ): QueryType {
+    return this.read({
+      query: options.query,
+      variables: options.variables,
+      optimistic,
+    });
+  }
+
+  public readFragment<FragmentType>(
+    options: DataProxy.Fragment,
+    optimistic: boolean = false,
+  ): FragmentType | null {
+    return this.read({
+      query: getFragmentQueryDocument(options.fragment, options.fragmentName),
+      variables: options.variables,
+      rootId: options.id,
+      optimistic,
+    });
+  }
+
+  public writeQuery(options: Cache.WriteQueryOptions): void {
+    this.write({
+      dataId: 'ROOT_QUERY',
+      result: options.data,
+      query: options.query,
+      variables: options.variables,
+    });
+  }
+
+  public writeFragment(options: Cache.WriteFragmentOptions): void {
+    this.write({
+      dataId: options.id,
+      result: options.data,
+      variables: options.variables,
+      query: getFragmentQueryDocument(options.fragment, options.fragmentName),
+    });
+  }
+}

test/test_grammars.rb

@@ -23,7 +23,6 @@ class TestGrammars < Minitest::Test
     "8653305b358375d0fced85dc24793b99919b11ef", # language-shellscript
     "9f0c0b0926a18f5038e455e8df60221125fc3111", # elixir-tmbundle
     "a4dadb2374282098c5b8b14df308906f5347d79a", # mako-tmbundle
-    "b9b24778619dce325b651f0d77cbc72e7ae0b0a3", # Julia.tmbundle
     "e06722add999e7428048abcc067cd85f1f7ca71c", # r.tmbundle
     "50b14a0e3f03d7ca754dac42ffb33302b5882b78", # smalltalk-tmbundle
     "eafbc4a2f283752858e6908907f3c0c90188785b", # gap-tmbundle
@@ -44,6 +43,7 @@ class TestGrammars < Minitest::Test
     "9dafd4e2a79cb13a6793b93877a254bc4d351e74", # sublime-text-ox
     "8e111741d97ba2e27b3d18a309d426b4a37e604f", # sublime-varnish
     "23d2538e33ce62d58abda2c039364b92f64ea6bc", # sublime-angelscript
+    "53714285caad3c480ebd248c490509695d10404b", # atom-language-julia
   ].freeze
 
   # List of allowed SPDX license names

vendor/README.md

@@ -182,7 +182,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting
 - **JSONiq:** [wcandillon/language-jsoniq](https://github.com/wcandillon/language-jsoniq)
 - **JSONLD:** [atom/language-javascript](https://github.com/atom/language-javascript)
 - **JSX:** [github-linguist/language-babel](https://github.com/github-linguist/language-babel)
-- **Julia:** [JuliaEditorSupport/Julia.tmbundle](https://github.com/JuliaEditorSupport/Julia.tmbundle)
+- **Julia:** [JuliaEditorSupport/atom-language-julia](https://github.com/JuliaEditorSupport/atom-language-julia)
 - **Jupyter Notebook:** [textmate/json.tmbundle](https://github.com/textmate/json.tmbundle)
 - **KiCad Layout:** [Alhadis/language-pcb](https://github.com/Alhadis/language-pcb)
 - **KiCad Legacy Layout:** [Alhadis/language-pcb](https://github.com/Alhadis/language-pcb)

vendor/licenses/grammar/Julia.tmbundle.txt

@@ -1,27 +0,0 @@
----
-type: grammar
-name: Julia.tmbundle
-license: mit
----
-Copyright (c) 2012-2014 Stefan Karpinski, Elliot Saba, Dirk Gadsden,
-Adam Strzelecki, Jonathan Malmaud and other contributors:
-
-https://github.com/JuliaEditorSupport/Julia.tmbundle/contributors
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

vendor/licenses/grammar/atom-language-julia.txt

@@ -0,0 +1,27 @@
+---
+type: grammar
+name: atom-language-julia
+license: mit
+---
+The atom-language-julia package is licensed under the MIT "Expat" License:
+
+> Copyright (c) 2015
+>
+> Permission is hereby granted, free of charge, to any person obtaining
+> a copy of this software and associated documentation files (the
+> "Software"), to deal in the Software without restriction, including
+> without limitation the rights to use, copy, modify, merge, publish,
+> distribute, sublicense, and/or sell copies of the Software, and to
+> permit persons to whom the Software is furnished to do so, subject to
+> the following conditions:
+>
+> The above copyright notice and this permission notice shall be
+> included in all copies or substantial portions of the Software.
+>
+> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+> IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+> CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+> TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+> SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.