Js support encoding in file stream + packaging issues (#4112)

ericvergnaud · web-flow · commit fbd4452fff79 · 2023-02-13T14:32:52.000-08:00
* export Recognizer

Signed-off-by: Eric Vergnaud &lt;eric.vergnaud@wanadoo.fr&gt;

* Fix packaging issue

* Bump beta version

* support encoding in FileStream sync

* fix prototype

* fix prototype

* more exports

* bump beta version number

* fix packaging issues

* fix failing tests

---------

Signed-off-by: Eric Vergnaud &lt;eric.vergnaud@wanadoo.fr&gt;
diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.js.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.js.stg
@@ -1,4 +1,4 @@
-import antlr4 from 'file://<runtimePath>/src/antlr4/index.js'
+import antlr4 from 'file://<runtimePath>/src/antlr4/index.node.js'
 import <lexerName> from './<lexerName>.js';
 <if(parserName)>
 import <parserName> from './<parserName>.js';
@@ -23,7 +23,7 @@ class TreeShapeListener extends antlr4.tree.ParseTreeListener {
 <endif>
 
 function main(argv) {
-    var input = new antlr4.FileStream(argv[2], true);
+    var input = new antlr4.FileStream(argv[2], "utf-8", true);
     var lexer = new <lexerName>(input);
     var stream = new antlr4.CommonTokenStream(lexer);
 <if(parserName)>
diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.ts.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.ts.stg
@@ -33,7 +33,7 @@ class TreeShapeListener extends ParseTreeListener {
 <endif>
 
 function main(argv: string[]): void {
-    const input = new FileStream(argv[2], true);
+    const input = new FileStream(argv[2], "utf-8", true);
     const lexer = new <lexerName>(input);
     const stream = new CommonTokenStream(lexer);
 <if(parserName)>
diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/NodeRunner.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/NodeRunner.java
@@ -36,7 +36,7 @@ public String getLanguage() {
 
 	private final static String normalizedRuntimePath = getRuntimePath("JavaScript").replace('\\', '/');
 	private final static String newImportAntlrString =
-			"import antlr4 from 'file://" + normalizedRuntimePath + "/src/antlr4/index.js'";
+			"import antlr4 from 'file://" + normalizedRuntimePath + "/src/antlr4/index.node.js'";
 
 	@Override
 	protected CompiledState compile(RunOptions runOptions, GeneratedState generatedState) {
diff --git a/runtime/JavaScript/package.json b/runtime/JavaScript/package.json
@@ -1,9 +1,10 @@
 {
   "name": "antlr4",
-  "version": "4.12.0-beta.5",
+  "version": "4.12.0-beta.10",
   "type": "module",
   "description": "JavaScript runtime for ANTLR4",
-  "main": "dist/antlr4.js",
+  "browser": "dist/antlr4.web.js",
+  "main": "dist/antlr4.node.js",
   "types": "src/antlr4/index.d.ts",
   "repository": "antlr/antlr4.git",
   "keywords": [
@@ -43,8 +44,5 @@
   },
   "engines": {
     "node": ">=16"
-  },
-  "browser": {
-    "fs": false
   }
 }
diff --git a/runtime/JavaScript/src/antlr4/CharStreams.js b/runtime/JavaScript/src/antlr4/CharStreams.js
@@ -3,8 +3,8 @@
  * can be found in the LICENSE.txt file in the project root.
  */
 
-import InputStream from './InputStream.js';
-import fs from "fs";
+import CharStream from "./CharStream.js";
+import FileStream from "./FileStream.js";
 
 /**
  * Utility functions to create InputStreams from various sources.
@@ -16,7 +16,7 @@ import fs from "fs";
 export default {
   // Creates an InputStream from a string.
   fromString: function(str) {
-    return new InputStream(str, true);
+    return new CharStream(str, true);
   },
 
   /**
@@ -30,7 +30,7 @@ export default {
   fromBlob: function(blob, encoding, onLoad, onError) {
     const reader = new window.FileReader();
     reader.onload = function(e) {
-      const is = new InputStream(e.target.result, true);
+      const is = new CharStream(e.target.result, true);
       onLoad(is);
     };
     reader.onerror = onError;
@@ -43,7 +43,7 @@ export default {
    * encoding is null).
    */
   fromBuffer: function(buffer, encoding) {
-    return new InputStream(buffer.toString(encoding), true);
+    return new CharStream(buffer.toString(encoding), true);
   },
 
   /** Asynchronously creates an InputStream from a file on disk given
@@ -53,13 +53,7 @@ export default {
    * Invokes callback(error, result) on completion.
    */
   fromPath: function(path, encoding, callback) {
-    fs.readFile(path, encoding, function(err, data) {
-      let is = null;
-      if (data !== null) {
-        is = new InputStream(data, true);
-      }
-      callback(err, is);
-    });
+    FileStream.fromPath(path, encoding, callback);
   },
 
   /**
@@ -68,7 +62,6 @@ export default {
    * 'utf8' if encoding is null).
    */
   fromPathSync: function(path, encoding) {
-    const data = fs.readFileSync(path, encoding);
-    return new InputStream(data, true);
+    return new FileStream(path, encoding);
   }
 };
diff --git a/runtime/JavaScript/src/antlr4/FileStream.d.ts b/runtime/JavaScript/src/antlr4/FileStream.d.ts
@@ -4,7 +4,6 @@ export declare class FileStream extends CharStream {
 
     fileName: string;
 
-    constructor(fileName: string);
-    constructor(fileName: string, decodeToUnicodeCodePoints: boolean);
+    constructor(fileName: string, encoding?: string, decodeToUnicodeCodePoints?: boolean);
 
 }
diff --git a/runtime/JavaScript/src/antlr4/FileStream.js b/runtime/JavaScript/src/antlr4/FileStream.js
@@ -4,22 +4,36 @@
  */
 
 import InputStream from './InputStream.js';
+import CharStream from './CharStream.js';
 const isNode =
 	typeof process !== "undefined" &&
 	process.versions != null &&
 	process.versions.node != null;
-// use eval to fool webpack and mocha
-const fs = isNode ? await eval("import('fs')") : null;
+import fs from 'fs';
 
 /**
  * This is an InputStream that is loaded from a file all at once
  * when you construct the object.
  */
 export default class FileStream extends InputStream {
-	constructor(fileName, decodeToUnicodeCodePoints) {
+
+	static fromPath(path, encoding, callback) {
+		if(!isNode)
+			throw new Error("FileStream is only available when running in Node!");
+		fs.readFile(path, encoding, function(err, data) {
+			let is = null;
+			if (data !== null) {
+				is = new CharStream(data, true);
+			}
+			callback(err, is);
+		});
+
+	}
+
+	constructor(fileName, encoding, decodeToUnicodeCodePoints) {
 		if(!isNode)
 			throw new Error("FileStream is only available when running in Node!");
-		const data = fs.readFileSync(fileName, "utf8");
+		const data = fs.readFileSync(fileName, encoding || "utf-8" );
 		super(data, decodeToUnicodeCodePoints);
 		this.fileName = fileName;
 	}
diff --git a/runtime/JavaScript/src/antlr4/Lexer.d.ts b/runtime/JavaScript/src/antlr4/Lexer.d.ts
@@ -18,6 +18,7 @@ export declare class Lexer extends Recognizer<number> {
     _type: number;
 
     constructor(input: CharStream);
-    nextToken() : Token;
-    emit() : Token;
+    nextToken(): Token;
+    emit(): Token;
+    reset(): void;
 }
diff --git a/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.d.ts b/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.d.ts
@@ -14,6 +14,7 @@ export declare class ParserATNSimulator extends ATNSimulator {
     decisionToDFA: DFA[];
     atn: ATN;
     debug?: boolean;
+    trace_atn_sim?: boolean;
 
     constructor(recog: Recognizer<Token>, atn: ATN, decisionToDFA: DFA[], sharedContextCache: PredictionContextCache);
     adaptivePredict(input: TokenStream, decision: number, outerContext: ParserRuleContext) : number;
diff --git a/runtime/JavaScript/src/antlr4/index.d.ts b/runtime/JavaScript/src/antlr4/index.d.ts
@@ -5,6 +5,7 @@ export * from "./CharStreams";
 export * from "./TokenStream";
 export * from "./BufferedTokenStream";
 export * from "./CommonTokenStream";
+export * from "./Recognizer";
 export * from "./Lexer";
 export * from "./Parser";
 export * from './Token';
diff --git a/runtime/JavaScript/src/antlr4/index.node.js b/runtime/JavaScript/src/antlr4/index.node.js
diff --git a/runtime/JavaScript/src/antlr4/index.web.js b/runtime/JavaScript/src/antlr4/index.web.js
@@ -0,0 +1,64 @@
+/* Copyright (c) 2012-2022 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+import { default as atn } from './atn/index.js';
+import { default as dfa } from './dfa/index.js';
+import { default as context } from './context/index.js';
+import { default as misc } from './misc/index.js';
+import { default as tree } from './tree/index.js';
+import { default as error } from './error/index.js';
+import { default as CharStreams } from './CharStreams.js';
+import { default as Utils } from './utils/index.js';
+
+import Token from './Token.js';
+import CommonToken from './CommonToken.js';
+import InputStream from './InputStream.js';
+import CharStream from './InputStream.js';
+import CommonTokenStream from './CommonTokenStream.js';
+import Lexer from './Lexer.js';
+import Parser from './Parser.js';
+
+import RuleContext from './context/RuleContext.js';
+import ParserRuleContext from './context/ParserRuleContext.js';
+import ATN from './atn/ATN.js';
+import PredictionMode from './atn/PredictionMode.js';
+import LL1Analyzer from './atn/LL1Analyzer.js';
+import ATNDeserializer from './atn/ATNDeserializer.js';
+import LexerATNSimulator from './atn/LexerATNSimulator.js';
+import ParserATNSimulator from './atn/ParserATNSimulator.js';
+import PredictionContextCache from './atn/PredictionContextCache.js';
+import DFA from "./dfa/DFA.js";
+import RecognitionException from "./error/RecognitionException.js";
+import FailedPredicateException from "./error/FailedPredicateException.js";
+import NoViableAltException from "./error/NoViableAltException.js";
+import BailErrorStrategy from "./error/BailErrorStrategy.js";
+import Interval from './misc/Interval.js';
+import IntervalSet from './misc/IntervalSet.js';
+import ParseTreeListener from "./tree/ParseTreeListener.js";
+import ParseTreeVisitor from "./tree/ParseTreeVisitor.js";
+import ParseTreeWalker from "./tree/ParseTreeWalker.js";
+import ErrorListener from "./error/ErrorListener.js"
+import DiagnosticErrorListener from "./error/DiagnosticErrorListener.js"
+import RuleNode from "./tree/RuleNode.js"
+import TerminalNode from "./tree/TerminalNode.js"
+import arrayToString from "./utils/arrayToString.js"
+
+export default {
+    atn, dfa, context, misc, tree, error, Token, CommonToken, CharStreams, CharStream, InputStream, CommonTokenStream, Lexer, Parser,
+    ParserRuleContext, Interval, IntervalSet, LL1Analyzer, Utils
+}
+
+export {
+    Token, CommonToken, CharStreams, CharStream, InputStream, CommonTokenStream, Lexer, Parser,
+    RuleNode, TerminalNode, ParseTreeWalker, RuleContext, ParserRuleContext, Interval, IntervalSet,
+    PredictionMode, LL1Analyzer, ParseTreeListener, ParseTreeVisitor, ATN, ATNDeserializer, PredictionContextCache, LexerATNSimulator, ParserATNSimulator, DFA,
+    RecognitionException, NoViableAltException, FailedPredicateException, ErrorListener, DiagnosticErrorListener, BailErrorStrategy,
+    arrayToString
+}
+
+/* eslint no-unused-vars: [ "off"] */
+// need to import unused to force loading
+import StringHashCode from './utils/stringHashCode.js';
+import CodePointAt from './polyfills/codepointat.js';
+import FromCodePoint from './polyfills/fromcodepoint.js';
diff --git a/runtime/JavaScript/webpack.config.cjs b/runtime/JavaScript/webpack.config.cjs
diff --git a/runtime/JavaScript/webpack.config.js b/runtime/JavaScript/webpack.config.js
@@ -0,0 +1,72 @@
+import path from 'path';
+import ESLintPlugin from 'eslint-webpack-plugin';
+import {fileURLToPath} from "url";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const nodeConfig = {
+    mode: "production",
+    entry: './src/antlr4/index.node.js',
+    output: {
+        path: path.resolve(__dirname, 'dist'),
+        filename: 'antlr4.node.js',
+        chunkFormat: "module",
+        library: {
+            type: "module"
+        }
+    },
+    resolve: {
+        extensions: [ '.js']
+    },
+    target: "node",
+        module: {
+        rules: [{
+            test: /\.js$/,
+            exclude: /node_modules/,
+            use: [ 'babel-loader' ]
+        }]
+    },
+    plugins: [ new ESLintPlugin() ],
+        experiments: {
+            outputModule: true
+    },
+    devtool: "source-map"
+};
+
+const webConfig = {
+    mode: "production",
+    entry: './src/antlr4/index.web.js',
+    output: {
+        path: path.resolve(__dirname, 'dist'),
+        filename: 'antlr4.web.js',
+        library: {
+            type: "module"
+        }
+    },
+    resolve: {
+        extensions: [ '.js'],
+        fallback: {
+            fs: false
+        }
+    },
+    target: "web",
+    module: {
+        rules: [{
+            test: /\.js$/,
+            exclude: [ /node_modules/, path.resolve(__dirname, "src/FileStream.js") ],
+            use: [ 'babel-loader' ]
+        }]
+    },
+    performance: {
+        maxAssetSize: 512000,
+        maxEntrypointSize: 512000
+    },
+    plugins: [ new ESLintPlugin() ],
+    experiments: {
+        outputModule: true
+    },
+    devtool: "source-map"
+};
+
+export default [ nodeConfig, webConfig ];

Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,6 @@ export declare class FileStream extends CharStream {`
`4`	`4`
`5`	`5`	`fileName: string;`
`6`	`6`
`7`		`- constructor(fileName: string);`
`8`		`- constructor(fileName: string, decodeToUnicodeCodePoints: boolean);`
	`7`	`+ constructor(fileName: string, encoding?: string, decodeToUnicodeCodePoints?: boolean);`
`9`	`8`
`10`	`9`	`}`
Original file line number	Diff line number	Diff line change
`@@ -18,6 +18,7 @@ export declare class Lexer extends Recognizer<number> {`
`18`	`18`	`_type: number;`
`19`	`19`
`20`	`20`	`constructor(input: CharStream);`
`21`		`- nextToken() : Token;`
`22`		`- emit() : Token;`
	`21`	`+ nextToken(): Token;`
	`22`	`+ emit(): Token;`
	`23`	`+ reset(): void;`
`23`	`24`	`}`