feat: add --external and --loader support for nodejs_npm_esbuild workflow (#353)

Author: chris-armstrong

aws_lambda_builders/workflows/nodejs_npm/utils.py

@@ -53,3 +53,6 @@ def is_windows(self):
     def parse_json(self, path):
         with open(path) as json_file:
             return json.load(json_file)
+
+    def check_output(self, path):
+        return subprocess.check_output(["node", path])

aws_lambda_builders/workflows/nodejs_npm_esbuild/actions.py

@@ -54,6 +54,9 @@ def __init__(
 
         :type skip_deps: bool
         :param skip_deps: if dependencies should be omitted from bundling
+
+        :type bundler_config: Dict[str,Any]
+        :param bundler_config: the bundler configuration
         """
         super(EsbuildBundleAction, self).__init__()
         self.scratch_dir = scratch_dir
@@ -71,38 +74,29 @@ def execute(self):
         :raises lambda_builders.actions.ActionFailedError: when esbuild packaging fails
         """
 
-        if self.ENTRY_POINTS not in self.bundler_config:
-            raise ActionFailedError(f"{self.ENTRY_POINTS} not set ({self.bundler_config})")
-
-        entry_points = self.bundler_config[self.ENTRY_POINTS]
-
-        if not isinstance(entry_points, list):
-            raise ActionFailedError(f"{self.ENTRY_POINTS} must be a list ({self.bundler_config})")
-
-        if not entry_points:
-            raise ActionFailedError(f"{self.ENTRY_POINTS} must not be empty ({self.bundler_config})")
-
-        entry_paths = [self.osutils.joinpath(self.scratch_dir, entry_point) for entry_point in entry_points]
-
-        LOG.debug("NODEJS building %s using esbuild to %s", entry_paths, self.artifacts_dir)
-
-        explicit_entry_points = []
-        for entry_path, entry_point in zip(entry_paths, entry_points):
-            explicit_entry_points.append(self._get_explicit_file_type(entry_point, entry_path))
+        explicit_entry_points = self._construct_esbuild_entry_points()
 
         args = explicit_entry_points + ["--bundle", "--platform=node", "--format=cjs"]
         minify = self.bundler_config.get("minify", True)
         sourcemap = self.bundler_config.get("sourcemap", True)
         target = self.bundler_config.get("target", "es2020")
+        external = self.bundler_config.get("external", [])
+        loader = self.bundler_config.get("loader", [])
         if minify:
             args.append("--minify")
         if sourcemap:
             args.append("--sourcemap")
+        if external:
+            args.extend(map(lambda x: f"--external:{x}", external))
+        if loader:
+            args.extend(map(lambda x: f"--loader:{x}", loader))
+
         args.append("--target={}".format(target))
         args.append("--outdir={}".format(self.artifacts_dir))
 
         if self.skip_deps:
             LOG.info("Running custom esbuild using Node.js")
+            # Don't pass externals because the esbuild.js template makes everything external
             script = EsbuildBundleAction._get_node_esbuild_template(
                 explicit_entry_points, target, self.artifacts_dir, minify, sourcemap
             )
@@ -132,6 +126,30 @@ def _run_external_esbuild_in_nodejs(self, script):
             except EsbuildExecutionError as ex:
                 raise ActionFailedError(str(ex))
 
+    def _construct_esbuild_entry_points(self):
+        """
+        Construct the list of explicit entry points
+        """
+        if self.ENTRY_POINTS not in self.bundler_config:
+            raise ActionFailedError(f"{self.ENTRY_POINTS} not set ({self.bundler_config})")
+
+        entry_points = self.bundler_config[self.ENTRY_POINTS]
+
+        if not isinstance(entry_points, list):
+            raise ActionFailedError(f"{self.ENTRY_POINTS} must be a list ({self.bundler_config})")
+
+        if not entry_points:
+            raise ActionFailedError(f"{self.ENTRY_POINTS} must not be empty ({self.bundler_config})")
+
+        entry_paths = [self.osutils.joinpath(self.scratch_dir, entry_point) for entry_point in entry_points]
+
+        LOG.debug("NODEJS building %s using esbuild to %s", entry_paths, self.artifacts_dir)
+
+        explicit_entry_points = []
+        for entry_path, entry_point in zip(entry_paths, entry_points):
+            explicit_entry_points.append(self._get_explicit_file_type(entry_point, entry_path))
+        return explicit_entry_points
+
     @staticmethod
     def _get_node_esbuild_template(entry_points, target, out_dir, minify, sourcemap):
         """

tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py

@@ -287,3 +287,66 @@ def test_builds_project_without_combine_dependencies(self):
         expected_dependencies_files = {"node_modules"}
         output_dependencies_files = set(os.listdir(os.path.join(self.dependencies_dir)))
         self.assertNotIn(expected_dependencies_files, output_dependencies_files)
+
+    def test_builds_javascript_project_with_external(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps-esbuild-externals")
+
+        options = {"entry_points": ["included.js"], "external": ["minimal-request-promise"]}
+
+        self.builder.build(
+            source_dir,
+            self.artifacts_dir,
+            self.scratch_dir,
+            os.path.join(source_dir, "package.json"),
+            runtime=self.runtime,
+            options=options,
+            experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD],
+        )
+
+        expected_files = {"included.js", "included.js.map"}
+        output_files = set(os.listdir(self.artifacts_dir))
+        self.assertEqual(expected_files, output_files)
+        with open(str(os.path.join(self.artifacts_dir, "included.js"))) as f:
+            js_file = f.read()
+            # Check that the module has been require() instead of bundled
+            self.assertIn('require("minimal-request-promise")', js_file)
+
+    def test_builds_javascript_project_with_loader(self):
+        osutils = OSUtils()
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps-esbuild-loader")
+
+        options = {"entry_points": ["included.js"], "loader": [".reference=json"]}
+
+        self.builder.build(
+            source_dir,
+            self.artifacts_dir,
+            self.scratch_dir,
+            os.path.join(source_dir, "package.json"),
+            runtime=self.runtime,
+            options=options,
+            experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD],
+        )
+
+        expected_files = {"included.js", "included.js.map"}
+        output_files = set(os.listdir(self.artifacts_dir))
+        self.assertEqual(expected_files, output_files)
+
+        included_js_path = os.path.join(self.artifacts_dir, "included.js")
+
+        # check that the .reference file is correctly bundled as code by running the result
+        self.assertEqual(
+            osutils.check_output(included_js_path),
+            str.encode(
+                "===\n"
+                "The Muses\n"
+                "===\n"
+                "\n"
+                "\tcalliope: eloquence and heroic poetry\n"
+                "\terato: lyric or erotic poetry\n"
+                "\tmelpomene: tragedy\n"
+                "\tpolymnia: sacred poetry\n"
+                "\tterpsichore: dance\n"
+                "\tthalia: comedy\n"
+                "\turania: astronomy and astrology"
+            ),
+        )

tests/integration/workflows/nodejs_npm_esbuild/testdata/no-deps-esbuild-loader/.gitignore

@@ -0,0 +1 @@
+package-lock.json

tests/integration/workflows/nodejs_npm_esbuild/testdata/no-deps-esbuild-loader/excluded.js

@@ -0,0 +1,2 @@
+//excluded
+const x = 1;

tests/integration/workflows/nodejs_npm_esbuild/testdata/no-deps-esbuild-loader/included.js

@@ -0,0 +1,8 @@
+//included
+import { muses } from './reference.reference';
+
+process.stdout.write("===\nThe Muses\n===\n\n");
+process.stdout.write(muses.map(muse => `\t${muse.name}: ${muse.description}`).join("\n"));
+
+const x = 1;
+module.exports = x;

tests/integration/workflows/nodejs_npm_esbuild/testdata/no-deps-esbuild-loader/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "nodeps-esbuild",
+  "version": "1.0.0",
+  "description": "",
+  "keywords": [],
+  "author": "",
+  "license": "APACHE2.0",
+  "devDependencies": {
+    "esbuild": "^0.14.36"
+  }
+}

tests/integration/workflows/nodejs_npm_esbuild/testdata/no-deps-esbuild-loader/reference.reference

@@ -0,0 +1,11 @@
+{
+  "muses": [
+    { "name": "calliope", "description": "eloquence and heroic poetry" },
+    { "name": "erato", "description": "lyric or erotic poetry" },
+    { "name": "melpomene", "description": "tragedy" },
+    { "name": "polymnia", "description": "sacred poetry" },
+    { "name": "terpsichore", "description": "dance" },
+    { "name": "thalia", "description": "comedy" },
+    { "name": "urania", "description": "astronomy and astrology" }
+  ]
+}

tests/integration/workflows/nodejs_npm_esbuild/testdata/no-deps-esbuild/package.json

@@ -4,5 +4,8 @@
   "description": "",
   "keywords": [],
   "author": "",
-  "license": "APACHE2.0"
+  "license": "APACHE2.0",
+  "devDependencies": {
+    "esbuild": "^0.14.36"
+  }
 }

tests/integration/workflows/nodejs_npm_esbuild/testdata/with-deps-esbuild-externals/excluded.js

@@ -0,0 +1,2 @@
+//excluded
+const x = 1;