[WIP] support for building nodejs_npm functions

aws_lambda_builders/workflows/__init__.py

@@ -3,3 +3,4 @@
 """
 
 import aws_lambda_builders.workflows.python_pip
+import aws_lambda_builders.workflows.nodejs_npm

aws_lambda_builders/workflows/nodejs_npm/__init__.py

@@ -0,0 +1,5 @@
+"""
+Builds NodeJS Lambda functions using NPM dependency manager
+"""
+
+from .workflow import NodejsNpmWorkflow

aws_lambda_builders/workflows/nodejs_npm/actions.py

@@ -0,0 +1,91 @@
+"""
+Action to resolve NodeJS dependencies using NPM
+"""
+
+import logging
+from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError
+from aws_lambda_builders.workflows.python_pip.utils import OSUtils
+from .npm import SubprocessNpm, NpmError
+
+LOG = logging.getLogger(__name__)
+
+
+class NodejsNpmPackAction(BaseAction):
+
+    NAME = 'CopySource'
+    DESCRIPTION = "Packaging source using NPM"
+    PURPOSE = Purpose.COPY_SOURCE
+
+    def __init__(self, artifacts_dir, scratch_dir, manifest_path, runtime, osutils=None, subprocess_npm=None):
+        self.artifacts_dir = artifacts_dir
+        self.manifest_path = manifest_path
+        self.scratch_dir = scratch_dir
+        self.runtime = runtime
+
+        self.osutils = osutils
+        if osutils is None:
+            self.osutils = OSUtils()
+
+        self.subprocess_npm = subprocess_npm
+
+        if self.subprocess_npm is None:
+            self.subprocess_npm = SubprocessNpm(self.osutils)
+
+    def execute(self):
+        try:
+            if not self.osutils.file_exists(self.manifest_path):
+                raise ActionFailedError('package.json not found in: %s' % self.manifest_path)
+
+            package_path = "file:{}".format(self.osutils.abspath(self.osutils.dirname(self.manifest_path)))
+
+            LOG.debug("NODEJS packaging %s to %s", package_path, self.scratch_dir)
+
+            tarfile_name = self.subprocess_npm.main(['pack', '-q', package_path], cwd=self.scratch_dir)
+
+            LOG.debug("NODEJS packed to %s", tarfile_name)
+
+            tarfile_path = self.osutils.joinpath(self.scratch_dir, tarfile_name)
+
+            self.osutils.extract_tarfile(tarfile_path, tarfile_path + '-unpacked')
+
+            self.osutils.copytree(self.osutils.joinpath(tarfile_path + '-unpacked', 'package'), self.artifacts_dir)
+
+        except NpmError as ex:
+            raise ActionFailedError(str(ex))
+
+
+class NodejsNpmInstallAction(BaseAction):
+
+    NAME = 'ResolveDependencies'
+    DESCRIPTION = "Installing dependencies from NPM"
+    PURPOSE = Purpose.RESOLVE_DEPENDENCIES
+
+    def __init__(self, artifacts_dir, scratch_dir, manifest_path, runtime, osutils=None, subprocess_npm=None):
+        self.artifacts_dir = artifacts_dir
+        self.manifest_path = manifest_path
+        self.scratch_dir = scratch_dir
+        self.runtime = runtime
+
+        self.osutils = osutils
+        if osutils is None:
+            self.osutils = OSUtils()
+
+        self.subprocess_npm = subprocess_npm
+
+        if self.subprocess_npm is None:
+            self.subprocess_npm = SubprocessNpm(self.osutils)
+
+    def execute(self):
+        try:
+            LOG.debug("NODEJS installing in: %s from: %s", self.artifacts_dir, self.manifest_path)
+
+            if not self.osutils.file_exists(self.osutils.joinpath(self.artifacts_dir, 'package.json')):
+                raise ActionFailedError('package.json not found in: %s' % self.artifacts_dir)
+
+            self.subprocess_npm.main(
+                    ['install', '-q', '--no-audit', '--no-save', '--production'],
+                    cwd=self.artifacts_dir
+            )
+
+        except NpmError as ex:
+            raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/nodejs_npm/npm.py

@@ -0,0 +1,53 @@
+"""
+Wrapper around calling npm through a subprocess.
+"""
+
+import logging
+
+from aws_lambda_builders.workflows.python_pip.utils import OSUtils
+
+LOG = logging.getLogger(__name__)
+
+
+class NpmError(Exception):
+    pass
+
+
+class NpmExecutionError(NpmError):
+    def __init__(self, err):
+        super(NpmExecutionError, self).__init__(
+            'NPM failed: %s' % err)
+
+
+class SubprocessNpm(object):
+
+    def __init__(self, osutils=None, npm_exe=None):
+        if osutils is None:
+            osutils = OSUtils()
+        self.osutils = osutils
+
+        if npm_exe is None:
+            npm_exe = 'npm'
+
+        self.npm_exe = npm_exe
+
+    def main(self, args, cwd=None):
+
+        if not isinstance(args, list):
+            raise NpmExecutionError('args must be a list')
+
+        invoke_npm = [self.npm_exe] + args
+
+        LOG.debug("executing NPM: %s", invoke_npm)
+
+        p = self.osutils.popen(invoke_npm,
+                               stdout=self.osutils.pipe,
+                               stderr=self.osutils.pipe,
+                               cwd=cwd)
+
+        out, err = p.communicate()
+
+        if p.returncode != 0:
+            raise NpmExecutionError(err.decode('utf8').strip())
+
+        return out.decode('utf8').strip()

aws_lambda_builders/workflows/nodejs_npm/workflow.py

@@ -0,0 +1,35 @@
+"""
+NodeJS NPM Workflow
+"""
+
+from aws_lambda_builders.workflow import BaseWorkflow, Capability
+
+from .actions import NodejsNpmPackAction, NodejsNpmInstallAction
+
+
+class NodejsNpmWorkflow(BaseWorkflow):
+
+    NAME = "NodejsNpmBuilder"
+
+    CAPABILITY = Capability(language="nodejs",
+                            dependency_manager="npm",
+                            application_framework=None)
+
+    def __init__(self,
+                 source_dir,
+                 artifacts_dir,
+                 scratch_dir,
+                 manifest_path,
+                 runtime=None, **kwargs):
+
+        super(NodejsNpmWorkflow, self).__init__(source_dir,
+                                                artifacts_dir,
+                                                scratch_dir,
+                                                manifest_path,
+                                                runtime=runtime,
+                                                **kwargs)
+
+        self.actions = [
+            NodejsNpmPackAction(artifacts_dir, scratch_dir, manifest_path, runtime),
+            NodejsNpmInstallAction(artifacts_dir, scratch_dir, manifest_path, runtime)
+        ]

aws_lambda_builders/workflows/python_pip/utils.py

@@ -76,8 +76,8 @@ def tempdir(self):
         finally:
             shutil.rmtree(tempdir)
 
-    def popen(self, command, stdout=None, stderr=None, env=None):
-        p = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env)
+    def popen(self, command, stdout=None, stderr=None, env=None, cwd=None):
+        p = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env, cwd=cwd)
         return p
 
     def mtime(self, path):
@@ -86,3 +86,9 @@ def mtime(self, path):
     @property
     def pipe(self):
         return subprocess.PIPE
+
+    def dirname(self, path):
+        return os.path.dirname(path)
+
+    def abspath(self, path):
+        return os.path.abspath(path)

tests/functional/testdata/cwd.py

@@ -0,0 +1,3 @@
+import os
+
+print(os.getcwd())

tests/functional/workflows/python_pip/test_utils.py

@@ -2,6 +2,10 @@
 
 import pytest
 
+import sys
+
+import os
+
 from aws_lambda_builders.workflows.python_pip import utils
 
 
@@ -20,3 +24,31 @@ def test_can_read_unicode(self, tmpdir, osutils):
         content = osutils.get_file_contents(filename, binary=False,
                                             encoding='utf-16')
         assert content == checkmark
+
+    def test_dirname_returns_directory_for_path(self, tmpdir, osutils):
+        dirname = osutils.dirname(sys.executable)
+
+        assert dirname == os.path.dirname(sys.executable)
+
+    def test_abspath_returns_absolute_path(self, tmpdir, osutils):
+
+        result = osutils.abspath('.')
+
+        assert os.path.isabs(result)
+
+        assert result == os.path.abspath('.')
+
+    def test_popen_can_accept_cwd(self, tmpdir, osutils):
+
+        testdata_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata')
+
+        p = osutils.popen([sys.executable, 'cwd.py'],
+                          stdout=osutils.pipe,
+                          stderr=osutils.pipe,
+                          cwd=testdata_dir)
+
+        out, err = p.communicate()
+
+        assert p.returncode == 0
+
+        assert out.decode('utf8').strip() == os.path.abspath(testdata_dir)

tests/integration/workflows/nodejs_npm/test_nodejs_npm.py

@@ -0,0 +1,79 @@
+
+import os
+import shutil
+import tempfile
+from unittest import TestCase
+
+from aws_lambda_builders.builder import LambdaBuilder
+from aws_lambda_builders.exceptions import WorkflowFailedError
+
+
+class TestNodejsNpmWorkflow(TestCase):
+    """
+    Verifies that `nodejs_npm` workflow works by building a Lambda using NPM
+    """
+
+    TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata")
+
+    def setUp(self):
+        self.artifacts_dir = tempfile.mkdtemp()
+        self.scratch_dir = tempfile.mkdtemp()
+
+        self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
+
+        self.builder = LambdaBuilder(language="nodejs",
+                                     dependency_manager="npm",
+                                     application_framework=None)
+        self.runtime = "nodejs8.10"
+
+    def tearDown(self):
+        shutil.rmtree(self.artifacts_dir)
+        shutil.rmtree(self.scratch_dir)
+
+    def test_builds_project_without_dependencies(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
+
+        self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                           os.path.join(source_dir, "package.json"),
+                           runtime=self.runtime)
+
+        expected_files = {"package.json", "included.js"}
+        output_files = set(os.listdir(self.artifacts_dir))
+        self.assertEquals(expected_files, output_files)
+
+    def test_builds_project_with_remote_dependencies(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps")
+
+        self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                           os.path.join(source_dir, "package.json"),
+                           runtime=self.runtime)
+
+        expected_files = {"package.json", "included.js", "node_modules"}
+        output_files = set(os.listdir(self.artifacts_dir))
+        self.assertEquals(expected_files, output_files)
+
+        expected_modules = {"minimal-request-promise"}
+        output_modules = set(os.listdir(os.path.join(self.artifacts_dir, "node_modules")))
+        self.assertEquals(expected_modules, output_modules)
+
+    def test_fails_if_npm_cannot_resolve_dependencies(self):
+
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-deps")
+
+        with self.assertRaises(WorkflowFailedError) as ctx:
+            self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                               os.path.join(source_dir, "package.json"),
+                               runtime=self.runtime)
+
+        self.assertIn("No matching version found for [email protected]_EXISTENT", str(ctx.exception))
+
+    def test_fails_if_package_json_is_broken(self):
+
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-package")
+
+        with self.assertRaises(WorkflowFailedError) as ctx:
+            self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir,
+                               os.path.join(source_dir, "package.json"),
+                               runtime=self.runtime)
+
+        self.assertIn("Unexpected end of JSON input", str(ctx.exception))

tests/integration/workflows/nodejs_npm/testdata/broken-deps/excluded.js

@@ -0,0 +1,2 @@
+//excluded
+const x = 1;

tests/integration/workflows/nodejs_npm/testdata/broken-deps/included.js

@@ -0,0 +1,2 @@
+//included
+const x = 1;

tests/integration/workflows/nodejs_npm/testdata/broken-deps/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "brokendeps",
+  "version": "1.0.0",
+  "description": "",
+  "files": ["included.js"],
+  "keywords": [],
+  "author": "",
+  "license": "APACHE2.0",
+  "dependencies": {
+    "minimal-request-promise": "0.0.0-NON_EXISTENT" 
+  }
+}

tests/integration/workflows/nodejs_npm/testdata/broken-package/excluded.js

@@ -0,0 +1,2 @@
+//excluded
+const x = 1;

tests/integration/workflows/nodejs_npm/testdata/broken-package/included.js

@@ -0,0 +1,2 @@
+//included
+const x = 1;

tests/integration/workflows/nodejs_npm/testdata/broken-package/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "broken-package-json",
+  "version": "1.0.0",
+  "description": "",
+  "author": "",
+  "license": "APACHE2.0",
+  "dependencies": {
+
+}

tests/integration/workflows/nodejs_npm/testdata/no-deps/excluded.js

@@ -0,0 +1,2 @@
+//excluded
+const x = 1;

tests/integration/workflows/nodejs_npm/testdata/no-deps/included.js

@@ -0,0 +1,2 @@
+//included
+const x = 1;

tests/integration/workflows/nodejs_npm/testdata/no-deps/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "nodeps",
+  "version": "1.0.0",
+  "description": "",
+  "files": ["included.js"],
+  "keywords": [],
+  "author": "",
+  "license": "APACHE2.0"
+}

tests/integration/workflows/nodejs_npm/testdata/npm-deps/excluded.js

@@ -0,0 +1,2 @@
+//excluded
+const x = 1;

tests/integration/workflows/nodejs_npm/testdata/npm-deps/included.js

@@ -0,0 +1,2 @@
+//included
+const x = 1;