From 2686da9df57b4dbc4ff4f7738a53a8966b8cb2f7 Mon Sep 17 00:00:00 2001 From: Daniel Mil <84205762+mildaniel@users.noreply.github.com> Date: Wed, 9 Jun 2021 16:27:31 -0700 Subject: [PATCH 01/19] feat: Allow Python runtime to build without requiring a manifest (#243) * Allow Python to continue build without requirements.txt * Allow missing requirements.txt file for Python builds * Ruby optional Gemfile and test * Style changes * Add unit tests and additional comments * Fix assertLogs() not compatible with Python2.7 * Fix assertion string * Remove unused exception * Integ. test make sure no artifacts dir has no additional files after build * Kick off build * Check for manifest and skip package actions if not found * Revert Ruby changes until further discussion is had. Make Python workflow more readable. * Remove unused import * Whitespace fix --- .../workflows/python_pip/packager.py | 8 +++--- .../workflows/python_pip/workflow.py | 25 +++++++++++++++---- .../workflows/python_pip/test_python_pip.py | 25 +++++++++---------- .../workflows/python_pip/test_packager.py | 1 - .../workflows/python_pip/test_workflow.py | 20 ++++++++++++++- 5 files changed, 56 insertions(+), 23 deletions(-) diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index c427a958b..34058602b 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -35,6 +35,11 @@ class InvalidSourceDistributionNameError(PackagerError): class RequirementsFileNotFoundError(PackagerError): + """ + Exceptions is no longer raised. + Keeping it here because this exception is 'public' and could still be used by a customer. + """ + def __init__(self, requirements_path): super(RequirementsFileNotFoundError, self).__init__("Requirements file not found: %s" % requirements_path) @@ -131,9 +136,6 @@ def build_dependencies(self, artifacts_dir_path, scratch_dir_path, requirements_ # by finding/creating a virtualenv of the correct version and when # pip is called set the appropriate env vars. - if not self.osutils.file_exists(requirements_path): - raise RequirementsFileNotFoundError(requirements_path) - self._dependency_builder.build_site_packages(requirements_path, artifacts_dir_path, scratch_dir_path) diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index 2ed5c0976..c682df8cf 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -1,11 +1,16 @@ """ Python PIP Workflow """ +import logging + from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator from .actions import PythonPipBuildAction +from .utils import OSUtils + +LOG = logging.getLogger(__name__) class PythonPipWorkflow(BaseWorkflow): @@ -59,16 +64,26 @@ class PythonPipWorkflow(BaseWorkflow): ".idea", ) - def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, **kwargs): + def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): super(PythonPipWorkflow, self).__init__( source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=runtime, **kwargs ) - self.actions = [ - PythonPipBuildAction(artifacts_dir, scratch_dir, manifest_path, runtime, binaries=self.binaries), - CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), - ] + if osutils is None: + osutils = OSUtils() + + if osutils.file_exists(manifest_path): + # If a requirements.txt exists, run pip builder before copy action. + self.actions = [ + PythonPipBuildAction(artifacts_dir, scratch_dir, manifest_path, runtime, binaries=self.binaries), + CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), + ] + else: + LOG.warning("requirements.txt file not found. Continuing the build without dependencies.") + self.actions = [ + CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), + ] def get_validators(self): return [PythonRuntimeValidator(runtime=self.runtime)] diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 745e560d7..265b4513f 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -3,9 +3,13 @@ import sys import tempfile from unittest import TestCase +import mock from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowFailedError +import logging + +logger = logging.getLogger("aws_lambda_builders.workflows.python_pip.workflow") class TestPythonPipWorkflow(TestCase): @@ -88,9 +92,8 @@ def test_must_fail_to_resolve_dependencies(self): ) or "Invalid requirement: u'adfasf=1.2.3'" in str(ctx.exception) self.assertTrue(message_in_exception) - def test_must_fail_if_requirements_not_found(self): - - with self.assertRaises(WorkflowFailedError) as ctx: + def test_must_log_warning_if_requirements_not_found(self): + with mock.patch.object(logger, "warning") as mock_warning: self.builder.build( self.source_dir, self.artifacts_dir, @@ -98,13 +101,9 @@ def test_must_fail_if_requirements_not_found(self): os.path.join("non", "existent", "manifest"), runtime=self.runtime, ) - - self.builder.build( - self.source_dir, - self.artifacts_dir, - self.scratch_dir, - os.path.join("non", "existent", "manifest"), - runtime=self.runtime, - ) - - self.assertIn("Requirements file not found", str(ctx.exception)) + expected_files = self.test_data_files + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + mock_warning.assert_called_once_with( + "requirements.txt file not found. Continuing the build without dependencies." + ) diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 032d0b95f..af3a4c270 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -113,7 +113,6 @@ def test_can_call_dependency_builder(self, osutils): mock_dep_builder.build_site_packages.assert_called_once_with( "path/to/requirements.txt", "artifacts/path/", "scratch_dir/path/" ) - osutils_mock.file_exists.assert_called_once_with("path/to/requirements.txt") class TestPackage(object): diff --git a/tests/unit/workflows/python_pip/test_workflow.py b/tests/unit/workflows/python_pip/test_workflow.py index b4adbc925..01e960543 100644 --- a/tests/unit/workflows/python_pip/test_workflow.py +++ b/tests/unit/workflows/python_pip/test_workflow.py @@ -1,19 +1,37 @@ +import mock + from unittest import TestCase from aws_lambda_builders.actions import CopySourceAction +from aws_lambda_builders.workflows.python_pip.utils import OSUtils from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator from aws_lambda_builders.workflows.python_pip.workflow import PythonPipBuildAction, PythonPipWorkflow class TestPythonPipWorkflow(TestCase): def setUp(self): - self.workflow = PythonPipWorkflow("source", "artifacts", "scratch_dir", "manifest", runtime="python3.7") + self.osutils = OSUtils() def test_workflow_sets_up_actions(self): + osutils_mock = mock.Mock(spec=self.osutils) + osutils_mock.file_exists.return_value = True + self.workflow = PythonPipWorkflow( + "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=osutils_mock + ) self.assertEqual(len(self.workflow.actions), 2) self.assertIsInstance(self.workflow.actions[0], PythonPipBuildAction) self.assertIsInstance(self.workflow.actions[1], CopySourceAction) + def test_workflow_sets_up_actions_without_requirements(self): + osutils_mock = mock.Mock(spec=self.osutils) + osutils_mock.file_exists.return_value = False + self.workflow = PythonPipWorkflow( + "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=osutils_mock + ) + self.assertEqual(len(self.workflow.actions), 1) + self.assertIsInstance(self.workflow.actions[0], CopySourceAction) + def test_workflow_validator(self): + self.workflow = PythonPipWorkflow("source", "artifacts", "scratch_dir", "manifest", runtime="python3.7") for validator in self.workflow.get_validators(): self.assertTrue(isinstance(validator, PythonRuntimeValidator)) From c0164ca2528a73d65f69059663f2d51b83504fea Mon Sep 17 00:00:00 2001 From: Daniel Mil <84205762+mildaniel@users.noreply.github.com> Date: Tue, 15 Jun 2021 16:52:21 -0700 Subject: [PATCH 02/19] feat: Allow Ruby runtime to build without requiring a manifest (#245) * Allow Python to continue build without requirements.txt * Allow missing requirements.txt file for Python builds * Ruby optional Gemfile and test * Style changes * Add unit tests and additional comments * Fix assertLogs() not compatible with Python2.7 * Fix assertion string * Remove unused exception * Integ. test make sure no artifacts dir has no additional files after build * Kick off build * Check for manifest and skip package actions if not found * Revert Ruby changes until further discussion is had. Make Python workflow more readable. * Remove unused import * Whitespace fix * Readability changes for Ruby workflow * Remove magic number. Add link to Bundler error codes * Moved var declaration --- .../workflows/ruby_bundler/bundler.py | 18 +++++++++++-- .../workflows/ruby_bundler/utils.py | 10 ++++++++ .../workflows/ruby_bundler/test_ruby_utils.py | 25 +++++++++++++++++++ .../workflows/ruby_bundler/test_ruby.py | 20 +++++++++++++++ .../testdata/excludes-gemfile/handler.rb | 3 +++ .../workflows/ruby_bundler/test_bundler.py | 18 +++++++++++++ 6 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 tests/integration/workflows/ruby_bundler/testdata/excludes-gemfile/handler.rb diff --git a/aws_lambda_builders/workflows/ruby_bundler/bundler.py b/aws_lambda_builders/workflows/ruby_bundler/bundler.py index fecee69a9..680310822 100644 --- a/aws_lambda_builders/workflows/ruby_bundler/bundler.py +++ b/aws_lambda_builders/workflows/ruby_bundler/bundler.py @@ -6,6 +6,12 @@ LOG = logging.getLogger(__name__) +""" +Bundler error codes can be found here: +https://github.com/rubygems/bundler/blob/3f0638c6c8d340c2f2405ecb84eb3b39c433e36e/lib/bundler/errors.rb#L36 +""" +GEMFILE_NOT_FOUND = 10 + class BundlerExecutionError(Exception): """ @@ -51,7 +57,15 @@ def run(self, args, cwd=None): out, _ = p.communicate() if p.returncode != 0: - # Bundler has relevant information in stdout, not stderr. - raise BundlerExecutionError(message=out.decode("utf8").strip()) + if p.returncode == GEMFILE_NOT_FOUND: + LOG.warning("Gemfile not found. Continuing the build without dependencies.") + + # Clean up '.bundle' dir that gets generated before the build fails + check_dir = self.osutils.get_bundle_dir(cwd) + if self.osutils.directory_exists(check_dir): + self.osutils.remove_directory(check_dir) + else: + # Bundler has relevant information in stdout, not stderr. + raise BundlerExecutionError(message=out.decode("utf8").strip()) return out.decode("utf8").strip() diff --git a/aws_lambda_builders/workflows/ruby_bundler/utils.py b/aws_lambda_builders/workflows/ruby_bundler/utils.py index 26c4339f3..a3f36439e 100644 --- a/aws_lambda_builders/workflows/ruby_bundler/utils.py +++ b/aws_lambda_builders/workflows/ruby_bundler/utils.py @@ -6,6 +6,7 @@ import platform import tarfile import subprocess +import shutil class OSUtils(object): @@ -38,3 +39,12 @@ def abspath(self, path): def is_windows(self): return platform.system().lower() == "windows" + + def directory_exists(self, dirpath): + return os.path.exists(dirpath) and os.path.isdir(dirpath) + + def remove_directory(self, dirpath): + shutil.rmtree(dirpath) + + def get_bundle_dir(self, cwd): + return os.path.join(cwd, ".bundle") diff --git a/tests/functional/workflows/ruby_bundler/test_ruby_utils.py b/tests/functional/workflows/ruby_bundler/test_ruby_utils.py index a13eeb41f..f2e6e4d35 100644 --- a/tests/functional/workflows/ruby_bundler/test_ruby_utils.py +++ b/tests/functional/workflows/ruby_bundler/test_ruby_utils.py @@ -48,3 +48,28 @@ def test_popen_can_accept_cwd(self): out, err = p.communicate() self.assertEqual(p.returncode, 0) self.assertEqual(out.decode("utf8").strip(), os.path.abspath(testdata_dir)) + + def test_returns_true_if_directory_exists(self): + testdata_dir = os.path.dirname(__file__) + out = self.osutils.directory_exists(testdata_dir) + self.assertTrue(out) + + def test_returns_false_if_directory_not_found(self): + testdata_dir = os.path.join(os.path.dirname(__file__), "test") + out = self.osutils.directory_exists(testdata_dir) + self.assertFalse(out) + + def test_returns_bundle_directory(self): + testdata_dir = os.path.dirname(__file__) + out = self.osutils.get_bundle_dir(testdata_dir) + self.assertEqual(out, os.path.join(os.path.dirname(__file__), ".bundle")) + + def test_removes_directory_if_exists(self): + test_dir = tempfile.mkdtemp() + bundle_dir = os.path.join(test_dir, ".bundle") + expected_files = set(os.listdir(test_dir)) + os.mkdir(bundle_dir) + self.osutils.remove_directory(bundle_dir) + actual_files = set(os.listdir(test_dir)) + shutil.rmtree(test_dir) + self.assertEqual(actual_files, expected_files) diff --git a/tests/integration/workflows/ruby_bundler/test_ruby.py b/tests/integration/workflows/ruby_bundler/test_ruby.py index 9d392d811..d17f6c380 100644 --- a/tests/integration/workflows/ruby_bundler/test_ruby.py +++ b/tests/integration/workflows/ruby_bundler/test_ruby.py @@ -7,6 +7,11 @@ from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowFailedError +import mock +import logging + +logger = logging.getLogger("aws_lambda_builders.workflows.ruby_bundler.bundler") + class TestRubyWorkflow(TestCase): """ @@ -64,3 +69,18 @@ def test_fails_if_bundler_cannot_resolve_dependencies(self): runtime=self.runtime, ) self.assertIn("RubyBundlerBuilder:RubyBundle - Bundler Failed: ", str(ctx.exception)) + + def test_must_log_warning_if_gemfile_not_found(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "excludes-gemfile") + with mock.patch.object(logger, "warning") as mock_warning: + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join("non", "existent", "manifest"), + runtime=self.runtime, + ) + expected_files = {"handler.rb"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + mock_warning.assert_called_with("Gemfile not found. Continuing the build without dependencies.") diff --git a/tests/integration/workflows/ruby_bundler/testdata/excludes-gemfile/handler.rb b/tests/integration/workflows/ruby_bundler/testdata/excludes-gemfile/handler.rb new file mode 100644 index 000000000..0093e4da1 --- /dev/null +++ b/tests/integration/workflows/ruby_bundler/testdata/excludes-gemfile/handler.rb @@ -0,0 +1,3 @@ +def handle(event:,context:) + "Hello!" +end diff --git a/tests/unit/workflows/ruby_bundler/test_bundler.py b/tests/unit/workflows/ruby_bundler/test_bundler.py index fb8ab4612..ef121f7d1 100644 --- a/tests/unit/workflows/ruby_bundler/test_bundler.py +++ b/tests/unit/workflows/ruby_bundler/test_bundler.py @@ -3,6 +3,11 @@ from aws_lambda_builders.workflows.ruby_bundler.bundler import SubprocessBundler, BundlerExecutionError +import mock +import logging + +logger = logging.getLogger("aws_lambda_builders.workflows.ruby_bundler.bundler") + class FakePopen: def __init__(self, out=b"out", err=b"err", retcode=0): @@ -56,6 +61,19 @@ def test_returns_popen_out_decoded_if_retcode_is_0(self): result = self.under_test.run(["install", "--without", "development", "test"]) self.assertEqual(result, "some encoded text") + def test_logs_warning_when_gemfile_missing(self): + self.popen.returncode = 10 + with mock.patch.object(logger, "warning") as mock_warning: + self.under_test.run(["install", "--without", "development", "test"]) + mock_warning.assert_called_once_with("Gemfile not found. Continuing the build without dependencies.") + + def test_bundle_file_removed_if_generated(self): + self.popen.returncode = 10 + self.osutils.directory_exists.return_value = True + self.under_test.run(["install", "--without", "development", "test"]) + self.osutils.get_bundle_dir.assert_called_once() + self.osutils.remove_directory.assert_called_once() + def test_raises_BundlerExecutionError_with_err_text_if_retcode_is_not_0(self): self.popen.returncode = 1 self.popen.out = b"some error text\n\n" From ae5170873e03798c82d993fafd2f2d1547a5f878 Mon Sep 17 00:00:00 2001 From: Giorgio Azzinnaro Date: Sat, 19 Jun 2021 00:04:50 +0200 Subject: [PATCH 03/19] docs: Guidance on integrating with Lambda Builders (#242) --- DESIGN.md | 52 ++++++++++++++++++++++++++++++++++++++-------------- README.md | 10 ++++++++++ 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 708937948..6bfa0af57 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -8,7 +8,7 @@ Lambda Builders is a separate project that contains scripts to build Lambda functions, given a source location. It was -built as part of SAM CLI `sam build` command.Read https://github.com/awslabs/aws-sam-cli/pull/743 for design document +built as part of SAM CLI `sam build` command. Read https://github.com/awslabs/aws-sam-cli/pull/743 for design document explaining how Lambda Builders work in context of SAM CLI. This project has certain attributes that make it unique: @@ -60,14 +60,28 @@ to make use of the default implementation. It helps reduce the variance in behav customers with a standard expectation. #### Command Line Interface (Internal) -This library provides a wrapper CLI interface for convenience. This interface is not supported at the moment. So we -don't provide any guarantees of back compatibility. +This library provides a wrapper CLI interface for convenience. This interface **is not supported** at the moment. So we +don't provide any guarantees of back compatibility. It is a very thin wrapper over the library. It is meant to integrate with tools written in other programming languages that can't import Python libraries directly. The CLI provides -a JSON-RPC interface over stdin/stdout to invoke the builder and get response. +[a JSON-RPC 2.0 interface](https://www.jsonrpc.org/specification) +over stdin/stdout to invoke the builder and get a response. -**Request Format** +The CLI should be installed and available on the path: + +```shell +pip install aws-lambda-builders +``` + +Each execution of `aws-lambda-builders` handles one JSON-RPC request. +Provide the whole body of the request via stdin, terminated by `EOF`. + +Currently, the only exposed method is `LambdaBuilder.build`. +It closely maps to the +[Python method `LambdaBuilder.build` in `aws_lambda_builders/builder.py`](aws_lambda_builders/builder.py). + +#### Request Format ```json { @@ -75,7 +89,7 @@ a JSON-RPC interface over stdin/stdout to invoke the builder and get response. "method": "LambdaBuilder.build", "id": 1, "params": { - "__protocol_version": "0.1", // Expected version of RPC protocol + "__protocol_version": "0.3", // expected version of RPC protocol - from aws_lambda_builders/__main__.py "capability": { "language": "", "dependency_manager": "", @@ -86,13 +100,13 @@ a JSON-RPC interface over stdin/stdout to invoke the builder and get response. "scratch_dir": "/path/to/tmp", "manifest_path": "/path/to/manifest.json", "runtime": "Function's runtime ex: nodejs8.10", - "optimizations": {}, // not supported - "options": {} // not supported + "optimizations": {}, // not supported + "options": {} // depending on the workflow } } ``` -**Successful Response Format** +#### Successful Response Format ```json { @@ -102,7 +116,7 @@ a JSON-RPC interface over stdin/stdout to invoke the builder and get response. } ``` -**Error Response Format** +#### Error Response Format ```json { @@ -116,7 +130,7 @@ a JSON-RPC interface over stdin/stdout to invoke the builder and get response. } ``` -**Error Codes**: +#### Error Codes Error codes returned by the application are similar to HTTP Status Codes. @@ -125,6 +139,19 @@ Error codes returned by the application are similar to HTTP Status Codes. - 505 - RPC Protocol unsupported - -32601 - Method unsupported (standard JSON-RPC protocol error code) +#### Params + +##### `capability` +The 3-tuple `capability` is used to identify different workflows. +As of today, `application_framework` is unused and may be ignored. + +##### `options` +The parameter `options` should be configured depending on the selected workflow/capability. + +For more detail around the capabilities and options, +check out the corresponding _design document_ and `workflow.py` for +[the workflows you're interested in](aws_lambda_builders/workflows). + ### Project Meta #### Directory Structure This project's directories are laid as follows: @@ -173,6 +200,3 @@ And essentially drop into the builders package (or maybe we can have a notion of - **builder**: The entire project is called builder, because it can build Lambda functions - **workflows**: Building for each language+framework combination is defined using a workflow. - **actions**: A workflow is implemented as a chain of actions. - - - diff --git a/README.md b/README.md index bf1e12779..c3f820578 100644 --- a/README.md +++ b/README.md @@ -18,4 +18,14 @@ Lambda Builders currently contains the following workflows Lambda Builders is the brains behind the `sam build` command from [AWS SAM CLI](https://github.com/awslabs/aws-sam-cli) +### Integrating with Lambda Builders + +Lambda Builders is a Python library. +It additionally exposes a JSON-RPC 2.0 interface to use from other languages. + +If you intend to integrate with Lambda Builders, +check out [this section of the DESIGN DOCUMENT](DESIGN.md#builders-library). + +### Contributing + If you are a developer and interested in contributing, read the [DESIGN DOCUMENT](./DESIGN.md) to understand how this works. From 86a9aafc9408d0f3a9a6592abe5fa47dd6595df3 Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan <3770774+sriram-mv@users.noreply.github.com> Date: Fri, 18 Jun 2021 15:05:35 -0700 Subject: [PATCH 04/19] fix: README - showcase Makefile support (#247) --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c3f820578..cfd4f8105 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ Lambda Builders currently contains the following workflows * Go with Dep * Go with Mod +In Addition to above workflows, AWS Lambda Builders also supports *Custom Workflows* through a Makefile. + Lambda Builders is the brains behind the `sam build` command from [AWS SAM CLI](https://github.com/awslabs/aws-sam-cli) ### Integrating with Lambda Builders From 5e3ac41bd1b927dd87aca422772c256e1a2c9ee5 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Fri, 18 Jun 2021 15:05:44 -0700 Subject: [PATCH 05/19] Update VS2017 to VS2019 (#244) --- .appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 61a5b8032..b685a9e75 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,12 +1,12 @@ version: 1.0.{build} image: - - Visual Studio 2017 + - Visual Studio 2019 - Ubuntu environment: GOVERSION: 1.11 GRADLE_OPTS: -Dorg.gradle.daemon=false - nodejs_version: "8.10.0" + nodejs_version: "10.10.0" matrix: @@ -49,7 +49,7 @@ for: install: # To run Nodejs workflow integ tests - - ps: Install-Product node 8.10 + - ps: Install-Product node $env:nodejs_version - "set PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%" - "%PYTHON%\\python.exe -m pip install -r requirements/dev.txt" From ec479878911e3fede9d20f2ee381428156d71118 Mon Sep 17 00:00:00 2001 From: Daniel Mil <84205762+mildaniel@users.noreply.github.com> Date: Thu, 8 Jul 2021 10:19:24 -0700 Subject: [PATCH 06/19] fix: Pip not resolving local packages (#250) * Fix local packages not being built * Add int. test to catch future local dependency issues * Specify test requirements path from cwd * Removed redundant/superset pattern * Document the pip regex pattern change * Updated integ to match use case * Tests to check backward comp. --- .../workflows/python_pip/packager.py | 17 ++++++++--- tests/integration/workflows/__init__.py | 0 .../workflows/python_pip/__init__.py | 0 .../workflows/python_pip/test_python_pip.py | 29 ++++++++++++++++++- .../testdata/local-dependencies/setup.cfg | 16 ++++++++++ .../testdata/local-dependencies/setup.py | 3 ++ .../src/local_package/__init__.py | 0 .../src/local_package/entrypoint.py | 2 ++ .../workflows/python_pip/test_packager.py | 13 +++++---- 9 files changed, 69 insertions(+), 11 deletions(-) create mode 100644 tests/integration/workflows/__init__.py create mode 100644 tests/integration/workflows/python_pip/__init__.py create mode 100644 tests/integration/workflows/python_pip/testdata/local-dependencies/setup.cfg create mode 100644 tests/integration/workflows/python_pip/testdata/local-dependencies/setup.py create mode 100644 tests/integration/workflows/python_pip/testdata/local-dependencies/src/local_package/__init__.py create mode 100644 tests/integration/workflows/python_pip/testdata/local-dependencies/src/local_package/entrypoint.py diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 34058602b..8cebd15ee 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -596,7 +596,10 @@ def main(self, args, env_vars=None, shim=None): class PipRunner(object): """Wrapper around pip calls used by chalice.""" - _LINK_IS_DIR_PATTERN = "Processing (.+?)\n Link is a directory, ignoring download_dir" + # Update regex pattern to correspond with the updated output from pip + # Specific commit: + # https://github.com/pypa/pip/commit/b28e2c4928cc62d90b738a4613886fb1e2ad6a81#diff-5225c8e359020adb25dfc8c7a505950fd649c6c5775789c6f6517f7913f94542L529 + _LINK_IS_DIR_PATTERNS = ["Processing (.+?)\n"] def __init__(self, python_exe, pip, osutils=None): if osutils is None: @@ -644,10 +647,16 @@ def download_all_dependencies(self, requirements_filename, directory): package_name = match.group(1) raise NoSuchPackageError(str(package_name)) raise PackageDownloadError(error) + + # Extract local packages from pip output. + # Iterate over possible pip outputs depending on pip version. stdout = out.decode() - matches = re.finditer(self._LINK_IS_DIR_PATTERN, stdout) - for match in matches: - wheel_package_path = str(match.group(1)) + wheel_package_paths = set() + for pattern in self._LINK_IS_DIR_PATTERNS: + for match in re.finditer(pattern, stdout): + wheel_package_paths.add(str(match.group(1))) + + for wheel_package_path in wheel_package_paths: # Looks odd we do not check on the error status of building the # wheel here. We can assume this is a valid package path since # we already passed the pip download stage. This stage would have diff --git a/tests/integration/workflows/__init__.py b/tests/integration/workflows/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/workflows/python_pip/__init__.py b/tests/integration/workflows/python_pip/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 265b4513f..19529779a 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -27,7 +27,13 @@ def setUp(self): self.manifest_path_valid = os.path.join(self.TEST_DATA_FOLDER, "requirements-numpy.txt") self.manifest_path_invalid = os.path.join(self.TEST_DATA_FOLDER, "requirements-invalid.txt") - self.test_data_files = {"__init__.py", "main.py", "requirements-invalid.txt", "requirements-numpy.txt"} + self.test_data_files = { + "__init__.py", + "main.py", + "requirements-invalid.txt", + "requirements-numpy.txt", + "local-dependencies", + } self.builder = LambdaBuilder(language="python", dependency_manager="pip", application_framework=None) self.runtime = "{language}{major}.{minor}".format( @@ -78,6 +84,27 @@ def test_runtime_validate_python_project_fail_open_unsupported_runtime(self): self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime="python2.8" ) + def test_must_resolve_local_dependency(self): + source_dir = os.path.join(self.source_dir, "local-dependencies") + manifest = os.path.join(source_dir, "requirements.txt") + path_to_package = os.path.join(self.source_dir, "local-dependencies") + # pip resolves dependencies in requirements files relative to the current working directory + # need to make sure the correct path is used in the requirements file locally and in CI + with open(manifest, "w") as f: + f.write(str(path_to_package)) + self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest, runtime=self.runtime) + expected_files = { + "local_package", + "local_package-0.0.0.dist-info", + "requests", + "requests-2.23.0.dist-info", + "setup.py", + "requirements.txt", + } + output_files = set(os.listdir(self.artifacts_dir)) + for f in expected_files: + self.assertIn(f, output_files) + def test_must_fail_to_resolve_dependencies(self): with self.assertRaises(WorkflowFailedError) as ctx: diff --git a/tests/integration/workflows/python_pip/testdata/local-dependencies/setup.cfg b/tests/integration/workflows/python_pip/testdata/local-dependencies/setup.cfg new file mode 100644 index 000000000..86534d155 --- /dev/null +++ b/tests/integration/workflows/python_pip/testdata/local-dependencies/setup.cfg @@ -0,0 +1,16 @@ +[metadata] +name = local_package +description = "Use src/ setup of a python project to demonstrate local package resolution" +version = 0.0.0 + +[options] +zip_safe = False +packages = find: +package_dir = + =src +install_requires = + requests==2.23.0 +include_package_data = True + +[options.packages.find] +where = src diff --git a/tests/integration/workflows/python_pip/testdata/local-dependencies/setup.py b/tests/integration/workflows/python_pip/testdata/local-dependencies/setup.py new file mode 100644 index 000000000..606849326 --- /dev/null +++ b/tests/integration/workflows/python_pip/testdata/local-dependencies/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() diff --git a/tests/integration/workflows/python_pip/testdata/local-dependencies/src/local_package/__init__.py b/tests/integration/workflows/python_pip/testdata/local-dependencies/src/local_package/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/workflows/python_pip/testdata/local-dependencies/src/local_package/entrypoint.py b/tests/integration/workflows/python_pip/testdata/local-dependencies/src/local_package/entrypoint.py new file mode 100644 index 000000000..e57ab293d --- /dev/null +++ b/tests/integration/workflows/python_pip/testdata/local-dependencies/src/local_package/entrypoint.py @@ -0,0 +1,2 @@ +def handler(event, ctx): + pass diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index af3a4c270..236b4655f 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -253,7 +253,7 @@ def test_download_wheels_no_wheels(self, pip_factory): def test_does_find_local_directory(self, pip_factory): pip, runner = pip_factory() - pip.add_return((0, (b"Processing ../local-dir\n" b" Link is a directory," b" ignoring download_dir"), b"")) + pip.add_return((0, b"Processing ../local-dir\n", b"")) runner.download_all_dependencies("requirements.txt", "directory") assert len(pip.calls) == 2 assert pip.calls[1].args == ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir"] @@ -265,20 +265,21 @@ def test_does_find_multiple_local_directories(self, pip_factory): 0, ( b"Processing ../local-dir-1\n" - b" Link is a directory," - b" ignoring download_dir" b"\nsome pip output...\n" b"Processing ../local-dir-2\n" b" Link is a directory," b" ignoring download_dir" + b"Processing ../local-dir-3\n" ), b"", ) ) runner.download_all_dependencies("requirements.txt", "directory") - assert len(pip.calls) == 3 - assert pip.calls[1].args == ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-1"] - assert pip.calls[2].args == ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-2"] + pip_calls = [call.args for call in pip.calls] + assert len(pip.calls) == 4 + assert ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-1"] in pip_calls + assert ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-2"] in pip_calls + assert ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-3"] in pip_calls def test_raise_no_such_package_error(self, pip_factory): pip, runner = pip_factory() From 53eabd226f6f368ef95b79aafe0ae1fed8145631 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 20 Jul 2021 09:00:19 -0700 Subject: [PATCH 07/19] chore: bump version to 1.5.0 (#254) --- aws_lambda_builders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index 84da0b77a..7637cb1ae 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = "1.4.0" +__version__ = "1.5.0" RPC_PROTOCOL_VERSION = "0.3" From 9ccd9191e5f767ecccefff254e6dfc99ac52979c Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Mon, 16 Aug 2021 10:44:15 -0700 Subject: [PATCH 08/19] runtime: Add python3.9 support (#260) * Adding python39 support (#14) * Adding python39 support * changing Visual Studio image from 2017 to Visual Studio 2019 * update Appveyor to solve make pr failure * update packager.py * Update test case * skip new test case which is not tests in windows * update skipif part * black reformat Co-authored-by: jonife <79116465+jonife@users.noreply.github.com> --- .appveyor.yml | 9 ++++++++- aws_lambda_builders/workflows/python_pip/packager.py | 9 ++++++++- aws_lambda_builders/workflows/python_pip/validator.py | 2 +- .../integration/workflows/python_pip/test_python_pip.py | 6 +++++- tests/unit/workflows/python_pip/test_packager.py | 3 +++ tests/unit/workflows/python_pip/test_validator.py | 2 +- 6 files changed, 26 insertions(+), 5 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index b685a9e75..3646ca43b 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -34,6 +34,13 @@ environment: LINE_COVERAGE: '72' NEW_FLAKE8: 1 JAVA_HOME: "C:\\Program Files\\Java\\jdk11" + - PYTHON: "C:\\Python39-x64" + PYTHON_VERSION: '3.9' + PYTHON_ARCH: '64' + LINE_COVERAGE: '72' + NEW_FLAKE8: 1 + JAVA_HOME: "C:\\Program Files\\Java\\jdk11" + build: off @@ -42,7 +49,7 @@ for: - matrix: only: - - image: Visual Studio 2017 + - image: Visual Studio 2019 environment: GOPATH: c:\gopath diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 8cebd15ee..68c2fc892 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -72,7 +72,13 @@ def __init__(self, version): def get_lambda_abi(runtime): - supported = {"python2.7": "cp27mu", "python3.6": "cp36m", "python3.7": "cp37m", "python3.8": "cp38"} + supported = { + "python2.7": "cp27mu", + "python3.6": "cp36m", + "python3.7": "cp37m", + "python3.8": "cp38", + "python3.9": "cp39", + } if runtime not in supported: raise UnsupportedPythonVersion(runtime) @@ -164,6 +170,7 @@ class DependencyBuilder(object): "cp36m": (2, 17), "cp37m": (2, 17), "cp38": (2, 26), + "cp39": (2, 26), } # Fallback version if we're on an unknown python version # not in _RUNTIME_GLIBC. diff --git a/aws_lambda_builders/workflows/python_pip/validator.py b/aws_lambda_builders/workflows/python_pip/validator.py index 0032181a1..d31a929bc 100644 --- a/aws_lambda_builders/workflows/python_pip/validator.py +++ b/aws_lambda_builders/workflows/python_pip/validator.py @@ -13,7 +13,7 @@ class PythonRuntimeValidator(object): - SUPPORTED_RUNTIMES = {"python2.7", "python3.6", "python3.7", "python3.8"} + SUPPORTED_RUNTIMES = {"python2.7", "python3.6", "python3.7", "python3.8", "python3.9"} def __init__(self, runtime): self.language = "python" diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 19529779a..11ab098d3 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -1,8 +1,9 @@ import os import shutil import sys +import platform import tempfile -from unittest import TestCase +from unittest import TestCase, skipIf import mock from aws_lambda_builders.builder import LambdaBuilder @@ -10,6 +11,7 @@ import logging logger = logging.getLogger("aws_lambda_builders.workflows.python_pip.workflow") +IS_WINDOWS = platform.system().lower() == "windows" class TestPythonPipWorkflow(TestCase): @@ -44,6 +46,7 @@ def setUp(self): "python3.7": "python2.7", "python2.7": "python3.8", "python3.8": "python2.7", + "python3.9": "python2.7", } def tearDown(self): @@ -84,6 +87,7 @@ def test_runtime_validate_python_project_fail_open_unsupported_runtime(self): self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime="python2.8" ) + @skipIf(IS_WINDOWS, "Skip in windows tests") def test_must_resolve_local_dependency(self): source_dir = os.path.join(self.source_dir, "local-dependencies") manifest = os.path.join(source_dir, "requirements.txt") diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 236b4655f..6fa61a7de 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -101,6 +101,9 @@ def test_get_lambda_abi_python37(self): def test_get_lambda_abi_python38(self): assert "cp38" == get_lambda_abi("python3.8") + def test_get_lambda_abi_python39(self): + assert "cp39" == get_lambda_abi("python3.9") + class TestPythonPipDependencyBuilder(object): def test_can_call_dependency_builder(self, osutils): diff --git a/tests/unit/workflows/python_pip/test_validator.py b/tests/unit/workflows/python_pip/test_validator.py index 94682b0b8..d1db8a17a 100644 --- a/tests/unit/workflows/python_pip/test_validator.py +++ b/tests/unit/workflows/python_pip/test_validator.py @@ -19,7 +19,7 @@ class TestPythonRuntimeValidator(TestCase): def setUp(self): self.validator = PythonRuntimeValidator(runtime="python3.7") - @parameterized.expand(["python2.7", "python3.6", "python3.7", "python3.8"]) + @parameterized.expand(["python2.7", "python3.6", "python3.7", "python3.8", "python3.9"]) def test_supported_runtimes(self, runtime): validator = PythonRuntimeValidator(runtime=runtime) self.assertTrue(validator.has_runtime()) From 0ca24df60f0f44cc8f8f096e6af133130a406efd Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Mon, 16 Aug 2021 10:55:40 -0700 Subject: [PATCH 09/19] chore: bump aws-lambda-builder version to 1.6.0 (#261) --- aws_lambda_builders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index 7637cb1ae..952e5f9a3 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = "1.5.0" +__version__ = "1.6.0" RPC_PROTOCOL_VERSION = "0.3" From df4d474ed741de50285f05febb28685fcffb5e91 Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan <3770774+sriram-mv@users.noreply.github.com> Date: Wed, 25 Aug 2021 09:44:39 -0700 Subject: [PATCH 10/19] fix: improve support for pep517 builds (#265) * A pep517 build can declare build dependencies. Pip will then know to install these dependencies before trying to build a wheel file. * When creating a build environment, it's only guaranteed to last for the duration of the build process. It's not accessible once a pip command finishes running. * When we try to retrieve the version of a package we run a "modified" form of "python setup.py egg_info". * The problem with this is that we're not using the build environment that has all the build dependencies installed (it's already gone), so if setup.py imports a module (e.g. cython) because it expects it to be there because it declared it as a build dependency the egg_info command will fail. * We don't check the RC or have a fallback case if we can't generate egg info. * We fail with an indecipherable IndexError. We now have a fallback where if we can't import/run the setup.py file, we assume the PKG-INFO file should be in the top level directory of the sdist so we check if it's there, and if so we use that file. --- .../workflows/python_pip/packager.py | 27 ++++++++++++--- .../workflows/python_pip/utils.py | 4 +++ .../workflows/python_pip/test_packager.py | 33 +++++++++++++++++-- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 68c2fc892..0e884c8da 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -64,6 +64,14 @@ class PackageDownloadError(PackagerError): pass +class UnsupportedPackageError(Exception): + """Unable to parse package metadata.""" + + def __init__(self, package_name): + # type: (str) -> None + super(UnsupportedPackageError, self).__init__("Unable to retrieve name/version for package: %s" % package_name) + + class UnsupportedPythonVersion(PackagerError): """Generic networking error during a package download.""" @@ -538,7 +546,7 @@ def _parse_pkg_info_file(self, filepath): parser.feed(data) return parser.close() - def _generate_egg_info(self, package_dir): + def _get_pkg_info_filepath(self, package_dir): setup_py = self._osutils.joinpath(package_dir, "setup.py") script = self._SETUPTOOLS_SHIM % setup_py @@ -548,9 +556,20 @@ def _generate_egg_info(self, package_dir): p = subprocess.Popen( cmd, cwd=package_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self._osutils.original_environ() ) - p.communicate() + _, stderr = p.communicate() info_contents = self._osutils.get_directory_contents(egg_info_dir) - pkg_info_path = self._osutils.joinpath(egg_info_dir, info_contents[0], "PKG-INFO") + if p.returncode != 0: + LOG.debug("Non zero rc (%s) from the setup.py egg_info command: %s", p.returncode, stderr) + if info_contents: + pkg_info_path = self._osutils.joinpath(egg_info_dir, info_contents[0], "PKG-INFO") + else: + # This might be a pep 517 package in which case this PKG-INFO file + # should be available right in the top level directory of the sdist + # in the case where the egg_info command fails. + LOG.debug("Using fallback location for PKG-INFO file in package directory: %s", package_dir) + pkg_info_path = self._osutils.joinpath(package_dir, "PKG-INFO") + if not self._osutils.file_exists(pkg_info_path): + raise UnsupportedPackageError(self._osutils.basename(package_dir)) return pkg_info_path def _unpack_sdist_into_dir(self, sdist_path, unpack_dir): @@ -567,7 +586,7 @@ def _unpack_sdist_into_dir(self, sdist_path, unpack_dir): def get_package_name_and_version(self, sdist_path): with self._osutils.tempdir() as tempdir: package_dir = self._unpack_sdist_into_dir(sdist_path, tempdir) - pkg_info_filepath = self._generate_egg_info(package_dir) + pkg_info_filepath = self._get_pkg_info_filepath(package_dir) metadata = self._parse_pkg_info_file(pkg_info_filepath) name = metadata["Name"] version = metadata["Version"] diff --git a/aws_lambda_builders/workflows/python_pip/utils.py b/aws_lambda_builders/workflows/python_pip/utils.py index eab58bda4..19ee7656d 100644 --- a/aws_lambda_builders/workflows/python_pip/utils.py +++ b/aws_lambda_builders/workflows/python_pip/utils.py @@ -102,3 +102,7 @@ def mtime(self, path): @property def pipe(self): return subprocess.PIPE + + def basename(self, path): + # type: (str) -> str + return os.path.basename(path) diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index 7c00aa66c..2b627fac0 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -8,7 +8,7 @@ import pytest import mock -from aws_lambda_builders.workflows.python_pip.packager import PipRunner +from aws_lambda_builders.workflows.python_pip.packager import PipRunner, UnsupportedPackageError from aws_lambda_builders.workflows.python_pip.packager import DependencyBuilder from aws_lambda_builders.workflows.python_pip.packager import Package from aws_lambda_builders.workflows.python_pip.packager import MissingDependencyError @@ -858,18 +858,24 @@ class TestSdistMetadataFetcher(object): _SETUP_PY = "%s\n" "setup(\n" ' name="%s",\n' ' version="%s"\n' ")\n" _VALID_TAR_FORMATS = ["tar.gz", "tar.bz2"] - def _write_fake_sdist(self, setup_py, directory, ext): + def _write_fake_sdist(self, setup_py, directory, ext, pkg_info_contents=None): filename = "sdist.%s" % ext path = "%s/%s" % (directory, filename) if ext == "zip": with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as z: z.writestr("sdist/setup.py", setup_py) + if pkg_info_contents is not None: + z.writestr("sdist/PKG-INFO", pkg_info_contents) elif ext in self._VALID_TAR_FORMATS: compression_format = ext.split(".")[1] with tarfile.open(path, "w:%s" % compression_format) as tar: tarinfo = tarfile.TarInfo("sdist/setup.py") tarinfo.size = len(setup_py) tar.addfile(tarinfo, io.BytesIO(setup_py.encode())) + if pkg_info_contents is not None: + tarinfo = tarfile.TarInfo("sdist/PKG-INFO") + tarinfo.size = len(pkg_info_contents) + tar.addfile(tarinfo, io.BytesIO(pkg_info_contents.encode())) else: open(path, "a").close() filepath = os.path.join(directory, filename) @@ -967,6 +973,29 @@ def test_bad_format(self, osutils, sdist_reader): with pytest.raises(InvalidSourceDistributionNameError): name, version = sdist_reader.get_package_name_and_version(filepath) + def test_cant_get_egg_info_filename(self, osutils, sdist_reader): + # In this scenario the setup.py file will fail with an import + # error so we should verify we try a fallback to look for + # PKG-INFO. + bad_setup_py = self._SETUP_PY % ( + "import some_build_dependency", + "foo", + "1.0", + ) + pkg_info_file = "Name: foo\n" "Version: 1.0\n" + with osutils.tempdir() as tempdir: + filepath = self._write_fake_sdist(bad_setup_py, tempdir, "zip", pkg_info_file) + name, version = sdist_reader.get_package_name_and_version(filepath) + assert name == "foo" + assert version == "1.0" + + def test_pkg_info_fallback_fails_raises_error(self, osutils, sdist_reader): + setup_py = self._SETUP_PY % ("import build_time_dependency", "foo", "1.0") + with osutils.tempdir() as tempdir: + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz") + with pytest.raises(UnsupportedPackageError): + sdist_reader.get_package_name_and_version(filepath) + class TestPackage(object): def test_same_pkg_sdist_and_wheel_collide(self, osutils, sdist_builder): From 262d02ff81f052eb316f3e6db91215bdd6f8ca74 Mon Sep 17 00:00:00 2001 From: Cosh_ Date: Wed, 25 Aug 2021 10:06:08 -0700 Subject: [PATCH 11/19] Fixed Unit Test Requiring "test" Binary (#266) --- tests/unit/test_workflow.py | 63 +++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/tests/unit/test_workflow.py b/tests/unit/test_workflow.py index 390297e49..149a82fe7 100644 --- a/tests/unit/test_workflow.py +++ b/tests/unit/test_workflow.py @@ -176,6 +176,21 @@ def setUp(self): options={"c": "d"}, ) + def mock_binaries(self): + self.validator_mock = Mock() + self.validator_mock.validate = Mock() + self.validator_mock.validate.return_value = "/usr/bin/binary" + self.resolver_mock = Mock() + self.resolver_mock.exec_paths = ["/usr/bin/binary"] + self.binaries_mock = Mock() + self.binaries_mock.return_value = [] + + self.work.get_validators = lambda: self.validator_mock + self.work.get_resolvers = lambda: self.resolver_mock + self.work.binaries = { + "binary": BinaryPath(resolver=self.resolver_mock, validator=self.validator_mock, binary="binary") + } + def test_get_binaries(self): self.assertIsNotNone(self.work.binaries) for binary, binary_path in self.work.binaries.items(): @@ -187,63 +202,39 @@ def test_get_validator(self): self.assertTrue(isinstance(validator, RuntimeValidator)) def test_must_execute_actions_in_sequence(self): + self.mock_binaries() action_mock = Mock() - validator_mock = Mock() - validator_mock.validate = Mock() - validator_mock.validate.return_value = "/usr/bin/binary" - resolver_mock = Mock() - resolver_mock.exec_paths = ["/usr/bin/binary"] - binaries_mock = Mock() - binaries_mock.return_value = [] - - self.work.get_validators = lambda: validator_mock - self.work.get_resolvers = lambda: resolver_mock self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] - self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")} self.work.run() self.assertEqual( action_mock.method_calls, [call.action1.execute(), call.action2.execute(), call.action3.execute()] ) - self.assertTrue(validator_mock.validate.call_count, 1) + self.assertTrue(self.validator_mock.validate.call_count, 1) def test_must_fail_workflow_binary_resolution_failure(self): + self.mock_binaries() action_mock = Mock() - validator_mock = Mock() - validator_mock.validate = Mock() - validator_mock.validate.return_value = None - resolver_mock = Mock() - resolver_mock.exec_paths = MagicMock(side_effect=ValueError("Binary could not be resolved")) - binaries_mock = Mock() - binaries_mock.return_value = [] - - self.work.get_validators = lambda: validator_mock - self.work.get_resolvers = lambda: resolver_mock + self.resolver_mock.exec_paths = MagicMock(side_effect=ValueError("Binary could not be resolved")) + self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] - self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")} with self.assertRaises(WorkflowFailedError) as ex: self.work.run() def test_must_fail_workflow_binary_validation_failure(self): - action_mock = Mock() - validator_mock = Mock() - validator_mock.validate = Mock() - validator_mock.validate = MagicMock( + self.mock_binaries() + self.validator_mock.validate = MagicMock( side_effect=MisMatchRuntimeError(language="test", required_runtime="test1", runtime_path="/usr/bin/binary") ) - resolver_mock = Mock() - resolver_mock.exec_paths = ["/usr/bin/binary"] - binaries_mock = Mock() - binaries_mock.return_value = [] - self.work.get_validators = lambda: validator_mock - self.work.get_resolvers = lambda: resolver_mock + action_mock = Mock() self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] - self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")} with self.assertRaises(WorkflowFailedError) as ex: self.work.run() def test_must_raise_with_no_actions(self): + self.mock_binaries() + self.work.actions = [] with self.assertRaises(WorkflowFailedError) as ctx: @@ -252,6 +243,7 @@ def test_must_raise_with_no_actions(self): self.assertIn("Workflow does not have any actions registered", str(ctx.exception)) def test_must_raise_if_action_failed(self): + self.mock_binaries() action_mock = Mock() self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] @@ -264,6 +256,7 @@ def test_must_raise_if_action_failed(self): self.assertIn("testfailure", str(ctx.exception)) def test_must_raise_if_action_crashed(self): + self.mock_binaries() action_mock = Mock() self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] @@ -290,6 +283,8 @@ def test_supply_executable_path(self): options={"c": "d"}, ) self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] + self.mock_binaries() + self.work.run() From 880a9fd2dbbd572745f921472e71418a4d3011de Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan <3770774+sriram-mv@users.noreply.github.com> Date: Wed, 25 Aug 2021 13:42:15 -0700 Subject: [PATCH 12/19] fix(go version parts): remove alphabets from the version for validation (#259) * fix(go version parts): remove alphabets from the version for validation - Go versions like 1.12rc1 or 1.16beta1 are supported. - Test added. * fix: use regex for go versions --- .../workflows/go_modules/validator.py | 25 +++++++++++------ .../workflows/go_modules/test_validator.py | 28 ++++++++++++++----- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/aws_lambda_builders/workflows/go_modules/validator.py b/aws_lambda_builders/workflows/go_modules/validator.py index fbb2e6877..6ac325b68 100644 --- a/aws_lambda_builders/workflows/go_modules/validator.py +++ b/aws_lambda_builders/workflows/go_modules/validator.py @@ -3,6 +3,7 @@ """ import logging +import re import os import subprocess @@ -12,9 +13,9 @@ class GoRuntimeValidator(object): - LANGUAGE = "go" SUPPORTED_RUNTIMES = {"go1.x"} + GO_VERSION_REGEX = re.compile("go(\\d)\\.(x|\\d+)") def __init__(self, runtime): self.runtime = runtime @@ -28,6 +29,15 @@ def has_runtime(self): """ return self.runtime in self.SUPPORTED_RUNTIMES + @staticmethod + def get_go_versions(version_string): + parts = GoRuntimeValidator.GO_VERSION_REGEX.findall(version_string) + try: + # NOTE(sriram-mv): The version parts need to be a list with a major and minor version. + return int(parts[0][0]), int(parts[0][1]) + except IndexError: + return 0, 0 + def validate(self, runtime_path): """ Checks if the language supplied matches the required lambda runtime @@ -42,16 +52,13 @@ def validate(self, runtime_path): min_expected_minor_version = 11 if expected_major_version == 1 else 0 p = subprocess.Popen([runtime_path, "version"], cwd=os.getcwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, _ = p.communicate() + version_string, _ = p.communicate() if p.returncode == 0: - out_parts = out.decode().split() - if len(out_parts) >= 3: - version_parts = [int(x.replace("rc", "")) for x in out_parts[2].replace(self.LANGUAGE, "").split(".")] - if len(version_parts) >= 2: - if version_parts[0] == expected_major_version and version_parts[1] >= min_expected_minor_version: - self._valid_runtime_path = runtime_path - return self._valid_runtime_path + major_version, minor_version = GoRuntimeValidator.get_go_versions(version_string.decode()) + if major_version == expected_major_version and minor_version >= min_expected_minor_version: + self._valid_runtime_path = runtime_path + return self._valid_runtime_path # otherwise, raise mismatch exception raise MisMatchRuntimeError(language=self.LANGUAGE, required_runtime=self.runtime, runtime_path=runtime_path) diff --git a/tests/unit/workflows/go_modules/test_validator.py b/tests/unit/workflows/go_modules/test_validator.py index 2f0311925..221fea415 100644 --- a/tests/unit/workflows/go_modules/test_validator.py +++ b/tests/unit/workflows/go_modules/test_validator.py @@ -14,7 +14,7 @@ def __init__(self, returncode, out=b"", err=b""): self.err = err def communicate(self): - return (self.out, self.err) + return self.out, self.err class TestGoRuntimeValidator(TestCase): @@ -30,36 +30,50 @@ def test_runtime_validate_unsupported_language_fail_open(self): validator = GoRuntimeValidator(runtime="go2.x") validator.validate(runtime_path="/usr/bin/go2") - @parameterized.expand([(b"go version go1.11.2 test",), (b"go version go1.11rc.2 test",)]) + @parameterized.expand( + [ + ("go1.11.2", (1, 11)), + ("go1.11rc.2", (1, 11)), + ("go1.16beta1", (1, 16)), + ("go%$", (0, 0)), + ("unknown", (0, 0)), + ] + ) + def test_get_go_versions(self, version_string, version_parts): + self.assertEqual(self.validator.get_go_versions(version_string), version_parts) + + @parameterized.expand( + [(b"go version go1.11.2 test",), (b"go version go1.11rc.2 test",), (b"go version go1.16beta1 test",)] + ) def test_runtime_validate_supported_version_runtime(self, go_version_output): with mock.patch("subprocess.Popen") as mock_subprocess: mock_subprocess.return_value = MockSubProcess(0, out=go_version_output) self.validator.validate(runtime_path="/usr/bin/go") - self.assertTrue(mock_subprocess.call_count, 1) + self.assertEqual(mock_subprocess.call_count, 1) def test_runtime_validate_supported_higher_than_min_version_runtime(self): with mock.patch("subprocess.Popen") as mock_subprocess: mock_subprocess.return_value = MockSubProcess(0, out=b"go version go1.12 test") self.validator.validate(runtime_path="/usr/bin/go") - self.assertTrue(mock_subprocess.call_count, 1) + self.assertEqual(mock_subprocess.call_count, 1) def test_runtime_validate_mismatch_nonzero_exit(self): with mock.patch("subprocess.Popen") as mock_subprocess: mock_subprocess.return_value = MockSubProcess(1) with self.assertRaises(MisMatchRuntimeError): self.validator.validate(runtime_path="/usr/bin/go") - self.assertTrue(mock_subprocess.call_count, 1) + self.assertEqual(mock_subprocess.call_count, 1) def test_runtime_validate_mismatch_invalid_version(self): with mock.patch("subprocess.Popen") as mock_subprocess: mock_subprocess.return_value = MockSubProcess(0, out=b"go version") with self.assertRaises(MisMatchRuntimeError): self.validator.validate(runtime_path="/usr/bin/go") - self.assertTrue(mock_subprocess.call_count, 1) + self.assertEqual(mock_subprocess.call_count, 1) def test_runtime_validate_mismatch_minor_version(self): with mock.patch("subprocess.Popen") as mock_subprocess: mock_subprocess.return_value = MockSubProcess(0, out=b"go version go1.10.2 test") with self.assertRaises(MisMatchRuntimeError): self.validator.validate(runtime_path="/usr/bin/go") - self.assertTrue(mock_subprocess.call_count, 1) + self.assertEqual(mock_subprocess.call_count, 1) From 073e26e750f2b23b0f35bccc804091d200721548 Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan <3770774+sriram-mv@users.noreply.github.com> Date: Mon, 30 Aug 2021 11:15:02 -0700 Subject: [PATCH 13/19] chore: aws lambda builders version set to 1.7.0 (#269) --- aws_lambda_builders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index 952e5f9a3..cb6f432a3 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = "1.6.0" +__version__ = "1.7.0" RPC_PROTOCOL_VERSION = "0.3" From 41804217a18ff47cce5b62eff5cf884a7f81a0c5 Mon Sep 17 00:00:00 2001 From: Mathieu Grandis <73313235+mgrandis@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:59:42 -0700 Subject: [PATCH 14/19] test: Temporarily disabling two Python integration tests failing on Windows (#282) --- tests/integration/workflows/python_pip/test_python_pip.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 11ab098d3..8ea5b842a 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -53,6 +53,10 @@ def tearDown(self): shutil.rmtree(self.artifacts_dir) shutil.rmtree(self.scratch_dir) + # Temporarily skipping this test in Windows + # Fails and we are not sure why: pip version/multiple Python versions in path/os/pypa issue? + # TODO: Revisit when we deprecate Python2 + @skipIf(IS_WINDOWS, "Skip in windows tests") def test_must_build_python_project(self): self.builder.build( self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime=self.runtime @@ -67,6 +71,10 @@ def test_must_build_python_project(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + # Temporarily skipping this test in Windows + # Fails and we are not sure why: pip version/multiple Python versions in path/os/pypa issue? + # TODO: Revisit when we deprecate Python2 + @skipIf(IS_WINDOWS, "Skip in windows tests") def test_mismatch_runtime_python_project(self): # NOTE : Build still works if other versions of python are accessible on the path. eg: /usr/bin/python2.7 # is still accessible within a python 3 virtualenv. From bc95771423d69aed4cb9fe9f60fd061a8b8d537e Mon Sep 17 00:00:00 2001 From: Daniel Mil <84205762+mildaniel@users.noreply.github.com> Date: Mon, 27 Sep 2021 16:16:43 -0700 Subject: [PATCH 15/19] feat: Allow Node.js projects to be built without requiring a package.json (#284) * Allow nodejs to build without requiring a manifest * Fix existing tests * Remove expected .package-lock * Define actions in the scope they're used --- .../workflows/nodejs_npm/workflow.py | 38 +++++++++++-------- .../workflows/nodejs_npm/test_nodejs_npm.py | 22 +++++++++++ .../nodejs_npm/testdata/no-manifest/app.js | 2 + .../workflows/nodejs_npm/test_workflow.py | 19 +++++++++- 4 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 tests/integration/workflows/nodejs_npm/testdata/no-manifest/app.js diff --git a/aws_lambda_builders/workflows/nodejs_npm/workflow.py b/aws_lambda_builders/workflows/nodejs_npm/workflow.py index b48ea4430..9880a089c 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/workflow.py +++ b/aws_lambda_builders/workflows/nodejs_npm/workflow.py @@ -1,6 +1,8 @@ """ NodeJS NPM Workflow """ +import logging + from aws_lambda_builders.path_resolver import PathResolver from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.actions import CopySourceAction @@ -8,6 +10,8 @@ from .utils import OSUtils from .npm import SubprocessNpm +LOG = logging.getLogger(__name__) + class NodejsNpmWorkflow(BaseWorkflow): @@ -36,21 +40,25 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim tar_dest_dir = osutils.joinpath(scratch_dir, "unpacked") tar_package_dir = osutils.joinpath(tar_dest_dir, "package") - npm_pack = NodejsNpmPackAction( - tar_dest_dir, scratch_dir, manifest_path, osutils=osutils, subprocess_npm=subprocess_npm - ) - - npm_install = NodejsNpmInstallAction(artifacts_dir, subprocess_npm=subprocess_npm) - - npm_copy_npmrc = NodejsNpmrcCopyAction(tar_package_dir, source_dir, osutils=osutils) - - self.actions = [ - npm_pack, - npm_copy_npmrc, - CopySourceAction(tar_package_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), - npm_install, - NodejsNpmrcCleanUpAction(artifacts_dir, osutils=osutils), - ] + if osutils.file_exists(manifest_path): + npm_pack = NodejsNpmPackAction( + tar_dest_dir, scratch_dir, manifest_path, osutils=osutils, subprocess_npm=subprocess_npm + ) + + npm_install = NodejsNpmInstallAction(artifacts_dir, subprocess_npm=subprocess_npm) + + npm_copy_npmrc = NodejsNpmrcCopyAction(tar_package_dir, source_dir, osutils=osutils) + + self.actions = [ + npm_pack, + npm_copy_npmrc, + CopySourceAction(tar_package_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), + npm_install, + NodejsNpmrcCleanUpAction(artifacts_dir, osutils=osutils), + ] + else: + LOG.warning("package.json file not found. Continuing the build without dependencies.") + self.actions = [CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)] def get_resolvers(self): """ diff --git a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py index 83a8de6e0..f6bb38c4e 100644 --- a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py +++ b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py @@ -1,3 +1,6 @@ +import logging +import mock + import os import shutil import tempfile @@ -7,6 +10,8 @@ from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowFailedError +logger = logging.getLogger("aws_lambda_builders.workflows.nodejs_npm.workflow") + class TestNodejsNpmWorkflow(TestCase): """ @@ -43,6 +48,23 @@ def test_builds_project_without_dependencies(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + def test_builds_project_without_manifest(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-manifest") + + with mock.patch.object(logger, "warning") as mock_warning: + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + ) + + expected_files = {"app.js"} + output_files = set(os.listdir(self.artifacts_dir)) + mock_warning.assert_called_once_with("package.json file not found. Continuing the build without dependencies.") + self.assertEqual(expected_files, output_files) + def test_builds_project_and_excludes_hidden_aws_sam(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "excluded-files") diff --git a/tests/integration/workflows/nodejs_npm/testdata/no-manifest/app.js b/tests/integration/workflows/nodejs_npm/testdata/no-manifest/app.js new file mode 100644 index 000000000..67021f79c --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/no-manifest/app.js @@ -0,0 +1,2 @@ +const HELLO_WORLD = "Hello world!" +console.log(HELLO_WORLD) \ No newline at end of file diff --git a/tests/unit/workflows/nodejs_npm/test_workflow.py b/tests/unit/workflows/nodejs_npm/test_workflow.py index 3dac6d791..f63565203 100644 --- a/tests/unit/workflows/nodejs_npm/test_workflow.py +++ b/tests/unit/workflows/nodejs_npm/test_workflow.py @@ -1,3 +1,5 @@ +import mock + from unittest import TestCase from aws_lambda_builders.actions import CopySourceAction @@ -8,6 +10,7 @@ NodejsNpmrcCopyAction, NodejsNpmrcCleanUpAction, ) +from aws_lambda_builders.workflows.nodejs_npm.utils import OSUtils class TestNodejsNpmWorkflow(TestCase): @@ -17,9 +20,14 @@ class TestNodejsNpmWorkflow(TestCase): this is just a quick wiring test to provide fast feedback if things are badly broken """ + def setUp(self): + self.osutils_mock = mock.Mock(spec=OSUtils()) + def test_workflow_sets_up_npm_actions(self): - workflow = NodejsNpmWorkflow("source", "artifacts", "scratch_dir", "manifest") + self.osutils_mock.file_exists.return_value = True + + workflow = NodejsNpmWorkflow("source", "artifacts", "scratch_dir", "manifest", osutils=self.osutils_mock) self.assertEqual(len(workflow.actions), 5) @@ -32,3 +40,12 @@ def test_workflow_sets_up_npm_actions(self): self.assertIsInstance(workflow.actions[3], NodejsNpmInstallAction) self.assertIsInstance(workflow.actions[4], NodejsNpmrcCleanUpAction) + + def test_workflow_only_copy_action(self): + self.osutils_mock.file_exists.return_value = False + + workflow = NodejsNpmWorkflow("source", "artifacts", "scratch_dir", "manifest", osutils=self.osutils_mock) + + self.assertEqual(len(workflow.actions), 1) + + self.assertIsInstance(workflow.actions[0], CopySourceAction) From 0bc7d65babec2622fa328e917163bf70a399a595 Mon Sep 17 00:00:00 2001 From: Mehmet Nuri Deveci <5735811+mndeveci@users.noreply.github.com> Date: Wed, 29 Sep 2021 11:53:27 -0700 Subject: [PATCH 16/19] feat: ARM support (#25) (#287) Co-authored-by: Mathieu Grandis <73313235+mgrandis@users.noreply.github.com> --- .gitignore | 1 + aws_lambda_builders/__main__.py | 2 + aws_lambda_builders/architecture.py | 5 + aws_lambda_builders/builder.py | 7 ++ aws_lambda_builders/exceptions.py | 24 ++++ aws_lambda_builders/utils.py | 16 +++ aws_lambda_builders/validator.py | 62 +++++++++- aws_lambda_builders/workflow.py | 87 +++++++------ .../workflows/dotnet_clipackage/actions.py | 25 +++- .../workflows/dotnet_clipackage/workflow.py | 7 +- .../workflows/go_dep/DESIGN.md | 15 +++ .../workflows/go_dep/actions.py | 8 +- .../workflows/go_dep/workflow.py | 10 +- .../workflows/go_modules/DESIGN.md | 8 +- .../workflows/go_modules/builder.py | 10 +- .../workflows/go_modules/validator.py | 38 +++--- .../workflows/go_modules/workflow.py | 4 +- .../workflows/java_gradle/gradle_validator.py | 27 ++++- .../workflows/java_gradle/workflow.py | 4 +- .../workflows/java_maven/maven_validator.py | 25 +++- .../workflows/java_maven/workflow.py | 3 +- .../workflows/nodejs_npm/workflow.py | 1 + .../workflows/python_pip/DESIGN.md | 10 +- .../workflows/python_pip/actions.py | 8 +- .../workflows/python_pip/packager.py | 95 +++++++++++---- .../workflows/python_pip/validator.py | 39 +++--- .../workflows/python_pip/workflow.py | 11 +- .../workflows/ruby_bundler/workflow.py | 1 + requirements/dev.txt | 2 + tests/functional/test_builder.py | 1 + tests/functional/test_cli.py | 3 +- tests/functional/test_utils.py | 7 +- .../workflows/python_pip/test_packager.py | 28 ++++- .../dotnet_clipackage/test_dotnet.py | 114 +++++++++++++++++- .../Function.cs | 0 .../WithDefaultsFile.csproj | 0 .../aws-lambda-tools-defaults.json | 0 .../testdata/WithDefaultsFile3.1/Function.cs | 27 +++++ .../WithDefaultsFile.csproj | 11 ++ .../aws-lambda-tools-defaults.json | 16 +++ .../workflows/go_modules/__init__.py | 0 .../workflows/go_modules/test_go.py | 49 ++++++++ .../integration/workflows/go_modules/utils.py | 20 +++ .../workflows/nodejs_npm/test_nodejs_npm.py | 2 +- .../workflows/python_pip/test_python_pip.py | 36 ++++++ tests/unit/test_builder.py | 2 + tests/unit/test_validator.py | 21 +++- tests/unit/test_workflow.py | 68 ++++++++++- .../workflows/custom_make/test_workflow.py | 17 +++ .../dotnet_clipackage/test_actions.py | 92 +++++++++++--- .../dotnet_clipackage/test_workflow.py | 21 ++++ tests/unit/workflows/go_dep/test_actions.py | 31 +++++ tests/unit/workflows/go_dep/test_workflow.py | 23 ++++ .../unit/workflows/go_modules/test_builder.py | 11 ++ .../workflows/go_modules/test_validator.py | 13 +- .../workflows/go_modules/test_workflow.py | 22 ++++ .../java_gradle/test_gradle_validator.py | 59 ++++++--- .../workflows/java_gradle/test_workflow.py | 31 +++-- .../java_maven/test_maven_validator.py | 49 +++++--- .../workflows/java_maven/test_workflow.py | 33 +++-- .../workflows/nodejs_npm/test_workflow.py | 26 +++- .../unit/workflows/python_pip/test_actions.py | 29 ++++- .../workflows/python_pip/test_packager.py | 13 ++ .../workflows/python_pip/test_validator.py | 25 ++-- .../workflows/python_pip/test_workflow.py | 40 ++++-- .../workflows/ruby_bundler/test_workflow.py | 21 ++++ 66 files changed, 1252 insertions(+), 264 deletions(-) create mode 100644 aws_lambda_builders/architecture.py rename tests/integration/workflows/dotnet_clipackage/testdata/{WithDefaultsFile => WithDefaultsFile2.1}/Function.cs (100%) rename tests/integration/workflows/dotnet_clipackage/testdata/{WithDefaultsFile => WithDefaultsFile2.1}/WithDefaultsFile.csproj (100%) rename tests/integration/workflows/dotnet_clipackage/testdata/{WithDefaultsFile => WithDefaultsFile2.1}/aws-lambda-tools-defaults.json (100%) create mode 100644 tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/Function.cs create mode 100644 tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/WithDefaultsFile.csproj create mode 100644 tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/aws-lambda-tools-defaults.json create mode 100644 tests/integration/workflows/go_modules/__init__.py create mode 100644 tests/integration/workflows/go_modules/utils.py diff --git a/.gitignore b/.gitignore index b031dced1..50664d49e 100644 --- a/.gitignore +++ b/.gitignore @@ -389,5 +389,6 @@ $RECYCLE.BIN/ /Dockerfile tests/integration/workflows/go_dep/data/src/*/vendor/* +tests/integration/workflows/go_dep/data/pkg/* # End of https://www.gitignore.io/api/osx,node,macos,linux,python,windows,pycharm,intellij,sublimetext,visualstudiocode diff --git a/aws_lambda_builders/__main__.py b/aws_lambda_builders/__main__.py index b4fc80cfa..4e44741d1 100644 --- a/aws_lambda_builders/__main__.py +++ b/aws_lambda_builders/__main__.py @@ -12,6 +12,7 @@ import logging import re +from aws_lambda_builders.architecture import X86_64 from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowNotFoundError, WorkflowUnknownError, WorkflowFailedError from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version @@ -124,6 +125,7 @@ def main(): # pylint: disable=too-many-statements optimizations=params["optimizations"], options=params["options"], mode=params.get("mode", None), + architecture=params.get("architecture", X86_64), ) # Return a success response diff --git a/aws_lambda_builders/architecture.py b/aws_lambda_builders/architecture.py new file mode 100644 index 000000000..703b67408 --- /dev/null +++ b/aws_lambda_builders/architecture.py @@ -0,0 +1,5 @@ +""" +Enum for determining type of architectures for Lambda Function. +""" +X86_64 = "x86_64" +ARM64 = "arm64" diff --git a/aws_lambda_builders/builder.py b/aws_lambda_builders/builder.py index 9a9a3d04d..f3793a63f 100644 --- a/aws_lambda_builders/builder.py +++ b/aws_lambda_builders/builder.py @@ -6,6 +6,7 @@ import os import logging +from aws_lambda_builders.architecture import X86_64, ARM64 from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY from aws_lambda_builders.workflow import Capability @@ -64,6 +65,7 @@ def build( options=None, executable_search_paths=None, mode=None, + architecture=X86_64, ): """ Actually build the code by running workflows @@ -105,6 +107,10 @@ def build( :type mode: str :param mode: Optional, Mode the build should produce + + :type architecture: str + :param architecture: + Type of architecture x86_64 and arm64 for Lambda Function """ if not os.path.exists(scratch_dir): @@ -120,6 +126,7 @@ def build( options=options, executable_search_paths=executable_search_paths, mode=mode, + architecture=architecture, ) return workflow.run() diff --git a/aws_lambda_builders/exceptions.py b/aws_lambda_builders/exceptions.py index 646b914dc..a6725f051 100644 --- a/aws_lambda_builders/exceptions.py +++ b/aws_lambda_builders/exceptions.py @@ -24,6 +24,30 @@ class MisMatchRuntimeError(LambdaBuilderError): ) +class RuntimeValidatorError(LambdaBuilderError): + """ + Raise when runtime is not supported or when runtime is not compatible with architecture + """ + + MESSAGE = "Runtime validation error for {runtime}" + + +class UnsupportedRuntimeError(RuntimeValidatorError): + """ + Raise when runtime is not supported + """ + + MESSAGE = "Runtime {runtime} is not suppported" + + +class UnsupportedArchitectureError(RuntimeValidatorError): + """ + Raise when runtime does not support architecture + """ + + MESSAGE = "Architecture {architecture} is not supported for runtime {runtime}" + + class WorkflowNotFoundError(LambdaBuilderError): """ Raised when a workflow matching the given capabilities was not found diff --git a/aws_lambda_builders/utils.py b/aws_lambda_builders/utils.py index 1ccd6dce5..9bb215637 100644 --- a/aws_lambda_builders/utils.py +++ b/aws_lambda_builders/utils.py @@ -7,6 +7,7 @@ import os import logging +from aws_lambda_builders.architecture import X86_64, ARM64 LOG = logging.getLogger(__name__) @@ -148,3 +149,18 @@ def _access_check(fn, mode): if _access_check(name, mode): paths.append(name) return paths + + +def get_goarch(architecture): + """ + Parameters + ---------- + architecture : str + name of the type of architecture + + Returns + ------- + str + returns a valid GO Architecture value + """ + return "arm64" if architecture == ARM64 else "amd64" diff --git a/aws_lambda_builders/validator.py b/aws_lambda_builders/validator.py index 4e014c103..ac0c99b6d 100644 --- a/aws_lambda_builders/validator.py +++ b/aws_lambda_builders/validator.py @@ -4,14 +4,74 @@ import logging +from aws_lambda_builders.architecture import ARM64, X86_64 +from aws_lambda_builders.exceptions import UnsupportedRuntimeError, UnsupportedArchitectureError + + LOG = logging.getLogger(__name__) +SUPPORTED_RUNTIMES = { + "nodejs10.x": [X86_64], + "nodejs12.x": [ARM64, X86_64], + "nodejs14.x": [ARM64, X86_64], + "python2.7": [X86_64], + "python3.6": [X86_64], + "python3.7": [X86_64], + "python3.8": [ARM64, X86_64], + "python3.9": [ARM64, X86_64], + "ruby2.5": [X86_64], + "ruby2.7": [ARM64, X86_64], + "java8": [ARM64, X86_64], + "java11": [ARM64, X86_64], + "go1.x": [ARM64, X86_64], + "dotnetcore2.1": [X86_64], + "dotnetcore3.1": [ARM64, X86_64], + "provided": [ARM64, X86_64], +} + class RuntimeValidator(object): - def __init__(self, runtime): + def __init__(self, runtime, architecture): + """ + + Parameters + ---------- + runtime : str + name of the AWS Lambda runtime that you are building for. This is sent to the builder for + informational purposes. + architecture : str + Architecture for which the build will be based on in AWS lambda + """ self.runtime = runtime self._runtime_path = None + self.architecture = architecture def validate(self, runtime_path): + """ + Parameters + ---------- + runtime_path : str + runtime to check eg: /usr/bin/runtime + + Returns + ------- + str + runtime to check eg: /usr/bin/runtime + + Raises + ------ + UnsupportedRuntimeError + Raised when runtime provided is not support. + + UnsupportedArchitectureError + Raised when runtime is not compatible with architecture + """ + runtime_architectures = SUPPORTED_RUNTIMES.get(self.runtime, None) + + if not runtime_architectures: + raise UnsupportedRuntimeError(runtime=self.runtime) + if self.architecture not in runtime_architectures: + raise UnsupportedArchitectureError(runtime=self.runtime, architecture=self.architecture) + self._runtime_path = runtime_path return runtime_path diff --git a/aws_lambda_builders/workflow.py b/aws_lambda_builders/workflow.py index 42263085b..66ad981f5 100644 --- a/aws_lambda_builders/workflow.py +++ b/aws_lambda_builders/workflow.py @@ -12,8 +12,15 @@ from aws_lambda_builders.path_resolver import PathResolver from aws_lambda_builders.validator import RuntimeValidator from aws_lambda_builders.registry import DEFAULT_REGISTRY -from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError, MisMatchRuntimeError +from aws_lambda_builders.exceptions import ( + WorkflowFailedError, + WorkflowUnknownError, + MisMatchRuntimeError, + RuntimeValidatorError, +) from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.architecture import X86_64 + LOG = logging.getLogger(__name__) @@ -32,16 +39,17 @@ class BuildMode(object): # TODO: Move sanitize out to its own class. -def sanitize(func): +def sanitize(func): # pylint: disable=too-many-statements """ sanitize the executable path of the runtime specified by validating it. :param func: Workflow's run method is sanitized """ @functools.wraps(func) - def wrapper(self, *args, **kwargs): + def wrapper(self, *args, **kwargs): # pylint: disable=too-many-statements valid_paths = {} invalid_paths = {} + validation_errors = [] # NOTE: we need to access binaries to get paths and resolvers, before validating. for binary, binary_checker in self.binaries.items(): invalid_paths[binary] = [] @@ -61,18 +69,30 @@ def wrapper(self, *args, **kwargs): except MisMatchRuntimeError as ex: LOG.debug("Invalid executable for %s at %s", binary, executable_path, exc_info=str(ex)) invalid_paths[binary].append(executable_path) + + except RuntimeValidatorError as ex: + LOG.debug("Runtime validation error for %s", binary, exc_info=str(ex)) + if str(ex) not in validation_errors: + validation_errors.append(str(ex)) + if valid_paths.get(binary, None): binary_checker.binary_path = valid_paths[binary] break + if validation_errors: + raise WorkflowFailedError( + workflow_name=self.NAME, action_name="Validation", reason="\n".join(validation_errors) + ) + if len(self.binaries) != len(valid_paths): validation_failed_binaries = set(self.binaries.keys()).difference(valid_paths.keys()) - messages = [] for validation_failed_binary in validation_failed_binaries: message = "Binary validation failed for {0}, searched for {0} in following locations : {1} which did not satisfy constraints for runtime: {2}. Do you have {0} for runtime: {2} on your PATH?".format( validation_failed_binary, invalid_paths[validation_failed_binary], self.runtime ) - messages.append(message) - raise WorkflowFailedError(workflow_name=self.NAME, action_name="Validation", reason="\n".join(messages)) + validation_errors.append(message) + raise WorkflowFailedError( + workflow_name=self.NAME, action_name="Validation", reason="\n".join(validation_errors) + ) func(self, *args, **kwargs) return wrapper @@ -140,48 +160,36 @@ def __init__( optimizations=None, options=None, mode=BuildMode.RELEASE, + architecture=X86_64, ): """ Initialize the builder with given arguments. These arguments together form the "public API" that each build action must support at the minimum. - :type source_dir: str - :param source_dir: + Parameters + ---------- + source_dir : str Path to a folder containing the source code - - :type artifacts_dir: str - :param artifacts_dir: + artifacts_dir : str Path to a folder where the built artifacts should be placed - - :type scratch_dir: str - :param scratch_dir: + scratch_dir : str Path to a directory that the workflow can use as scratch space. Workflows are expected to use this directory to write temporary files instead of ``/tmp`` or other OS-specific temp directories. - - :type manifest_path: str - :param manifest_path: + manifest_path : str Path to the dependency manifest - - :type runtime: str - :param runtime: - Optional, name of the AWS Lambda runtime that you are building for. This is sent to the builder for - informational purposes. - - :type optimizations: dict - :param optimizations: - Optional dictionary of optimization flags to pass to the build action. **Not supported**. - - :type options: dict - :param options: - Optional dictionary of options ot pass to build action. **Not supported**. - - :type executable_search_paths: list - :param executable_search_paths: - Optional, Additional list of paths to search for executables required by the workflow. - - :type mode: str - :param mode: - Optional, Mode the build should produce + runtime : str, optional + name of the AWS Lambda runtime that you are building for. This is sent to the builder for + informational purposes, by default None + executable_search_paths : list, optional + Additional list of paths to search for executables required by the workflow, by default None + optimizations : dict, optional + dictionary of optimization flags to pass to the build action. **Not supported**, by default None + options : dict, optional + dictionary of options ot pass to build action. **Not supported**., by default None + mode : str, optional + Mode the build should produce, by default BuildMode.RELEASE + architecture : str, optional + Architecture type either arm64 or x86_64 for which the build will be based on in AWS lambda, by default X86_64 """ self.source_dir = source_dir @@ -193,6 +201,7 @@ def __init__( self.options = options self.executable_search_paths = executable_search_paths self.mode = mode + self.architecture = architecture # Actions are registered by the subclasses as they seem fit self.actions = [] @@ -225,7 +234,7 @@ def get_validators(self): """ No-op validator that does not validate the runtime_path. """ - return [RuntimeValidator(runtime=self.runtime)] + return [RuntimeValidator(runtime=self.runtime, architecture=self.architecture)] @property def binaries(self): diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/actions.py b/aws_lambda_builders/workflows/dotnet_clipackage/actions.py index de7386d04..4a28770d6 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/actions.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/actions.py @@ -8,6 +8,7 @@ from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError from aws_lambda_builders.workflow import BuildMode +from aws_lambda_builders.architecture import ARM64 from .utils import OSUtils from .dotnetcli import DotnetCLIExecutionError @@ -66,13 +67,14 @@ class RunPackageAction(BaseAction): DESCRIPTION = "Execute the `dotnet lambda package` command." PURPOSE = Purpose.COMPILE_SOURCE - def __init__(self, source_dir, subprocess_dotnet, artifacts_dir, options, mode, os_utils=None): + def __init__(self, source_dir, subprocess_dotnet, artifacts_dir, options, mode, architecture=None, os_utils=None): super(RunPackageAction, self).__init__() self.source_dir = source_dir self.subprocess_dotnet = subprocess_dotnet self.artifacts_dir = artifacts_dir self.options = options self.mode = mode + self.architecture = architecture self.os_utils = os_utils if os_utils else OSUtils() def execute(self): @@ -82,7 +84,15 @@ def execute(self): zipfilename = os.path.basename(os.path.normpath(self.source_dir)) + ".zip" zipfullpath = os.path.join(self.artifacts_dir, zipfilename) - arguments = ["lambda", "package", "--output-package", zipfullpath] + arguments = [ + "lambda", + "package", + "--output-package", + zipfullpath, + # Specify the architecture with the --runtime MSBuild parameter + "--msbuild-parameters", + "--runtime " + self._get_runtime(), + ] if self.mode and self.mode.lower() == BuildMode.DEBUG: LOG.debug("Debug build requested: Setting configuration to Debug") @@ -102,3 +112,14 @@ def execute(self): except DotnetCLIExecutionError as ex: raise ActionFailedError(str(ex)) + + def _get_runtime(self): + """ + Returns the msbuild runtime for the action architecture + + Returns + ------- + str + linux-arm64 if ARM64, linux-x64 otherwise + """ + return "linux-arm64" if self.architecture == ARM64 else "linux-x64" diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py b/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py index 6756beffc..cd79dc8c1 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py @@ -30,7 +30,12 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim dotnetcli_install = GlobalToolInstallAction(subprocess_dotnet=subprocess_dotnetcli) dotnetcli_deployment = RunPackageAction( - source_dir, subprocess_dotnet=subprocess_dotnetcli, artifacts_dir=artifacts_dir, options=options, mode=mode + source_dir, + subprocess_dotnet=subprocess_dotnetcli, + artifacts_dir=artifacts_dir, + options=options, + mode=mode, + architecture=self.architecture, ) self.actions = [dotnetcli_install, dotnetcli_deployment] diff --git a/aws_lambda_builders/workflows/go_dep/DESIGN.md b/aws_lambda_builders/workflows/go_dep/DESIGN.md index e5cf63d00..f797465f9 100644 --- a/aws_lambda_builders/workflows/go_dep/DESIGN.md +++ b/aws_lambda_builders/workflows/go_dep/DESIGN.md @@ -4,10 +4,18 @@ Building Go projects using the dep tool (https://github.com/golang/dep) is rather simple, if you was to do this by hand, you would perform these commands: +For x86 architecture + - `dep ensure` - `GOOS=linux GOARCH=amd64 go build -o handler main.go` - `zip -r source.zip` +Or for ARM architecture + + - `dep ensure` + - `GOOS=linux GOARCH=arm64 go build -o handler main.go` + - `zip -r source.zip` + The scope of the Go dep builder is to create a macro for these commands to ensure that spelling and paths are correct. We don't have to care about versioning of the tooling of either Go or dep since Lambda doesn't have to care, and so it becomes user preference. @@ -15,9 +23,16 @@ user preference. ## Implementation The go-dep builder runs the above commands with some minor tweaks, the commands ran on behalf of the user are: +For x86 architecture: + 1. dep ensure 2. GOOS=linux GOARCH=amd64 go build -o $ARTIFACT_DIR/$HANDLER_NAME $SOURCE_DIR +For ARM architecture: + + 1. dep ensure + 2. GOOS=linux GOARCH=arm64 go build -o $ARTIFACT_DIR/$HANDLER_NAME $SOURCE_DIR + The main difference being we want to capture the compiled binary to package later, so the binary has the output path as the artifact dir set by the caller. diff --git a/aws_lambda_builders/workflows/go_dep/actions.py b/aws_lambda_builders/workflows/go_dep/actions.py index f424e7c0b..82c96f65d 100644 --- a/aws_lambda_builders/workflows/go_dep/actions.py +++ b/aws_lambda_builders/workflows/go_dep/actions.py @@ -6,6 +6,9 @@ import os from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError +from aws_lambda_builders.architecture import X86_64, ARM64 +from aws_lambda_builders.utils import get_goarch + from .subproc_exec import ExecutionError @@ -46,7 +49,7 @@ class GoBuildAction(BaseAction): DESCRIPTION = "Builds final binary" PURPOSE = Purpose.COMPILE_SOURCE - def __init__(self, base_dir, source_path, output_path, subprocess_go, env=None): + def __init__(self, base_dir, source_path, output_path, subprocess_go, architecture=X86_64, env=None): super(GoBuildAction, self).__init__() self.base_dir = base_dir @@ -54,11 +57,12 @@ def __init__(self, base_dir, source_path, output_path, subprocess_go, env=None): self.output_path = output_path self.subprocess_go = subprocess_go + self.goarch = get_goarch(architecture) self.env = env if not env is None else {} def execute(self): env = self.env - env.update({"GOOS": "linux", "GOARCH": "amd64"}) + env.update({"GOOS": "linux", "GOARCH": self.goarch}) try: self.subprocess_go.run(["build", "-o", self.output_path, self.source_path], cwd=self.source_path, env=env) diff --git a/aws_lambda_builders/workflows/go_dep/workflow.py b/aws_lambda_builders/workflows/go_dep/workflow.py index b357418fa..e9c309e31 100644 --- a/aws_lambda_builders/workflows/go_dep/workflow.py +++ b/aws_lambda_builders/workflows/go_dep/workflow.py @@ -7,7 +7,6 @@ from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.workflow import BaseWorkflow, Capability - from .actions import DepEnsureAction, GoBuildAction from .utils import OSUtils from .subproc_exec import SubprocessExec @@ -48,5 +47,12 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim self.actions = [ DepEnsureAction(base_dir, subprocess_dep), - GoBuildAction(base_dir, osutils.abspath(source_dir), output_path, subprocess_go, env=osutils.environ), + GoBuildAction( + base_dir, + osutils.abspath(source_dir), + output_path, + subprocess_go, + self.architecture, + env=osutils.environ, + ), ] diff --git a/aws_lambda_builders/workflows/go_modules/DESIGN.md b/aws_lambda_builders/workflows/go_modules/DESIGN.md index 22b0bf00c..8c13a3a18 100644 --- a/aws_lambda_builders/workflows/go_modules/DESIGN.md +++ b/aws_lambda_builders/workflows/go_modules/DESIGN.md @@ -31,7 +31,13 @@ def build(self, source_dir_path, artifacts_dir_path, executable_name): The general algorithm for preparing a Go package for use on AWS Lambda is very simple. It's as follows: -Pass in GOOS=linux and GOARCH=amd64 to the `go build` command to target the +Depending on the architecture pass in either: + + - `GOOS=linux and GOARCH=arm64` for ARM architecture or + + - `GOOS=linux and GOARCH=amd64` for an X86 architecture + +to the `go build` command to target the OS and architecture used on AWS Lambda. Let go tooling handle the cross-compilation, regardless of the build environment. Move the resulting static binary to the artifacts folder to be shipped as a single-file zip diff --git a/aws_lambda_builders/workflows/go_modules/builder.py b/aws_lambda_builders/workflows/go_modules/builder.py index ab3e36758..fb9247c46 100644 --- a/aws_lambda_builders/workflows/go_modules/builder.py +++ b/aws_lambda_builders/workflows/go_modules/builder.py @@ -4,6 +4,8 @@ import logging from aws_lambda_builders.workflow import BuildMode +from aws_lambda_builders.architecture import X86_64, ARM64 +from aws_lambda_builders.utils import get_goarch LOG = logging.getLogger(__name__) @@ -19,7 +21,7 @@ class GoModulesBuilder(object): LANGUAGE = "go" - def __init__(self, osutils, binaries, mode=BuildMode.RELEASE): + def __init__(self, osutils, binaries, mode=BuildMode.RELEASE, architecture=X86_64): """Initialize a GoModulesBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` @@ -28,10 +30,14 @@ def __init__(self, osutils, binaries, mode=BuildMode.RELEASE): :type binaries: dict :param binaries: A dict of language binaries + + :type architecture: str + :param architecture: name of the type of architecture """ self.osutils = osutils self.binaries = binaries self.mode = mode + self.goarch = get_goarch(architecture) def build(self, source_dir_path, output_path): """Builds a go project onto an output path. @@ -44,7 +50,7 @@ def build(self, source_dir_path, output_path): """ env = {} env.update(self.osutils.environ) - env.update({"GOOS": "linux", "GOARCH": "amd64"}) + env.update({"GOOS": "linux", "GOARCH": self.goarch}) runtime_path = self.binaries[self.LANGUAGE].binary_path cmd = [runtime_path, "build"] if self.mode and self.mode.lower() == BuildMode.DEBUG: diff --git a/aws_lambda_builders/workflows/go_modules/validator.py b/aws_lambda_builders/workflows/go_modules/validator.py index 6ac325b68..1e0949be0 100644 --- a/aws_lambda_builders/workflows/go_modules/validator.py +++ b/aws_lambda_builders/workflows/go_modules/validator.py @@ -7,28 +7,20 @@ import os import subprocess +from aws_lambda_builders.validator import RuntimeValidator from aws_lambda_builders.exceptions import MisMatchRuntimeError LOG = logging.getLogger(__name__) -class GoRuntimeValidator(object): +class GoRuntimeValidator(RuntimeValidator): LANGUAGE = "go" - SUPPORTED_RUNTIMES = {"go1.x"} GO_VERSION_REGEX = re.compile("go(\\d)\\.(x|\\d+)") - def __init__(self, runtime): - self.runtime = runtime + def __init__(self, runtime, architecture): + super(GoRuntimeValidator, self).__init__(runtime, architecture) self._valid_runtime_path = None - def has_runtime(self): - """ - Checks if the runtime is supported. - :param string runtime: Runtime to check - :return bool: True, if the runtime is supported. - """ - return self.runtime in self.SUPPORTED_RUNTIMES - @staticmethod def get_go_versions(version_string): parts = GoRuntimeValidator.GO_VERSION_REGEX.findall(version_string) @@ -41,12 +33,24 @@ def get_go_versions(version_string): def validate(self, runtime_path): """ Checks if the language supplied matches the required lambda runtime - :param string runtime_path: runtime to check eg: /usr/bin/go - :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime + + Parameters + ---------- + runtime_path : str + runtime to check eg: /usr/bin/go1.x + + Returns + ------- + str + runtime_path, runtime to check eg: /usr/bin/go1.x + + Raises + ------ + MisMatchRuntimeError + Raise runtime is not support or runtime does not support architecture. """ - if not self.has_runtime(): - LOG.warning("'%s' runtime is not " "a supported runtime", self.runtime) - return None + + runtime_path = super(GoRuntimeValidator, self).validate(runtime_path) expected_major_version = int(self.runtime.replace(self.LANGUAGE, "").split(".")[0]) min_expected_minor_version = 11 if expected_major_version == 1 else 0 diff --git a/aws_lambda_builders/workflows/go_modules/workflow.py b/aws_lambda_builders/workflows/go_modules/workflow.py index 0932e81f5..ceafdbd7a 100644 --- a/aws_lambda_builders/workflows/go_modules/workflow.py +++ b/aws_lambda_builders/workflows/go_modules/workflow.py @@ -31,8 +31,8 @@ def __init__( output_path = osutils.joinpath(artifacts_dir, handler) - builder = GoModulesBuilder(osutils, binaries=self.binaries, mode=mode) + builder = GoModulesBuilder(osutils, binaries=self.binaries, mode=mode, architecture=self.architecture) self.actions = [GoModulesBuildAction(source_dir, output_path, builder)] def get_validators(self): - return [GoRuntimeValidator(runtime=self.runtime)] + return [GoRuntimeValidator(runtime=self.runtime, architecture=self.architecture)] diff --git a/aws_lambda_builders/workflows/java_gradle/gradle_validator.py b/aws_lambda_builders/workflows/java_gradle/gradle_validator.py index 36dcb4817..71d1c5b20 100644 --- a/aws_lambda_builders/workflows/java_gradle/gradle_validator.py +++ b/aws_lambda_builders/workflows/java_gradle/gradle_validator.py @@ -5,12 +5,15 @@ import logging import re +from aws_lambda_builders.validator import RuntimeValidator + from .utils import OSUtils + LOG = logging.getLogger(__name__) -class GradleValidator(object): +class GradleValidator(RuntimeValidator): VERSION_STRING_WARNING = ( "%s failed to return a version string using the '-v' option. The workflow is unable to " "check that the version of the JVM used is compatible with AWS Lambda." @@ -22,17 +25,31 @@ class GradleValidator(object): "been configured to be compatible with Java %s using 'targetCompatibility' in Gradle." ) - def __init__(self, runtime, os_utils=None, log=None): + def __init__(self, runtime, architecture, os_utils=None, log=None): + super(GradleValidator, self).__init__(runtime, architecture) self.language = "java" self._valid_binary_path = None - self._runtime = runtime self.os_utils = OSUtils() if not os_utils else os_utils self.log = LOG if not log else log - def validate(self, gradle_path): + def validate(self, runtime_path): + """ + Parameters + ---------- + runtime_path : str + gradle path to check eg: /usr/bin/java8 + + Returns + ------- + str + runtime to check for the java binaries eg: /usr/bin/java8 + """ + + gradle_path = super(GradleValidator, self).validate(runtime_path) + jvm_mv = self._get_major_version(gradle_path) - language_version = self._runtime.replace("java", "") + language_version = self.runtime.replace("java", "") if jvm_mv: if int(jvm_mv) > int(language_version): diff --git a/aws_lambda_builders/workflows/java_gradle/workflow.py b/aws_lambda_builders/workflows/java_gradle/workflow.py index e28c06c94..f9f621db0 100644 --- a/aws_lambda_builders/workflows/java_gradle/workflow.py +++ b/aws_lambda_builders/workflows/java_gradle/workflow.py @@ -4,6 +4,7 @@ import hashlib import os from aws_lambda_builders.workflow import BaseWorkflow, Capability + from .actions import JavaGradleBuildAction, JavaGradleCopyArtifactsAction from .gradle import SubprocessGradle from .utils import OSUtils @@ -27,6 +28,7 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, **kwar self.os_utils = OSUtils() self.build_dir = None + subprocess_gradle = SubprocessGradle(gradle_binary=self.binaries["gradle"], os_utils=self.os_utils) self.actions = [ @@ -38,7 +40,7 @@ def get_resolvers(self): return [GradleResolver(executable_search_paths=self.executable_search_paths)] def get_validators(self): - return [GradleValidator(self.runtime, self.os_utils)] + return [GradleValidator(self.runtime, self.architecture, self.os_utils)] @property def build_output_dir(self): diff --git a/aws_lambda_builders/workflows/java_maven/maven_validator.py b/aws_lambda_builders/workflows/java_maven/maven_validator.py index ad69eb7ec..4fc0911bb 100644 --- a/aws_lambda_builders/workflows/java_maven/maven_validator.py +++ b/aws_lambda_builders/workflows/java_maven/maven_validator.py @@ -5,12 +5,14 @@ import logging import re +from aws_lambda_builders.validator import RuntimeValidator + from .utils import OSUtils LOG = logging.getLogger(__name__) -class MavenValidator(object): +class MavenValidator(RuntimeValidator): VERSION_STRING_WARNING = ( "%s failed to return a version string using the '-v' option. The workflow is unable to " "check that the version of the JVM used is compatible with AWS Lambda." @@ -22,17 +24,30 @@ class MavenValidator(object): "been configured to be compatible with Java %s using 'maven.compiler.target' in Maven." ) - def __init__(self, runtime, os_utils=None, log=None): + def __init__(self, runtime, architecture, os_utils=None, log=None): + super(MavenValidator, self).__init__(runtime, architecture) self.language = "java" self._valid_binary_path = None - self._runtime = runtime self.os_utils = OSUtils() if not os_utils else os_utils self.log = LOG if not log else log - def validate(self, maven_path): + def validate(self, runtime_path): + """ + Parameters + ---------- + runtime_path : str + maven_path to check eg: /usr/bin/java8 + + Returns + ------- + str + runtime to check for the java binaries eg: /usr/bin/java8 + """ + + maven_path = super(MavenValidator, self).validate(runtime_path) jvm_mv = self._get_major_version(maven_path) - language_version = self._runtime.replace("java", "") + language_version = self.runtime.replace("java", "") if jvm_mv: if int(jvm_mv) > int(language_version): diff --git a/aws_lambda_builders/workflows/java_maven/workflow.py b/aws_lambda_builders/workflows/java_maven/workflow.py index 6586ee4f0..e29c64c17 100644 --- a/aws_lambda_builders/workflows/java_maven/workflow.py +++ b/aws_lambda_builders/workflows/java_maven/workflow.py @@ -3,6 +3,7 @@ """ from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.actions import CopySourceAction + from .actions import JavaMavenBuildAction, JavaMavenCopyDependencyAction, JavaMavenCopyArtifactsAction from .maven import SubprocessMaven from .maven_resolver import MavenResolver @@ -40,4 +41,4 @@ def get_resolvers(self): return [MavenResolver(executable_search_paths=self.executable_search_paths)] def get_validators(self): - return [MavenValidator(self.runtime, self.os_utils)] + return [MavenValidator(self.runtime, self.architecture, self.os_utils)] diff --git a/aws_lambda_builders/workflows/nodejs_npm/workflow.py b/aws_lambda_builders/workflows/nodejs_npm/workflow.py index 9880a089c..2e892dccf 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/workflow.py +++ b/aws_lambda_builders/workflows/nodejs_npm/workflow.py @@ -6,6 +6,7 @@ from aws_lambda_builders.path_resolver import PathResolver from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.actions import CopySourceAction + from .actions import NodejsNpmPackAction, NodejsNpmInstallAction, NodejsNpmrcCopyAction, NodejsNpmrcCleanUpAction from .utils import OSUtils from .npm import SubprocessNpm diff --git a/aws_lambda_builders/workflows/python_pip/DESIGN.md b/aws_lambda_builders/workflows/python_pip/DESIGN.md index d619fa645..1d9da131a 100644 --- a/aws_lambda_builders/workflows/python_pip/DESIGN.md +++ b/aws_lambda_builders/workflows/python_pip/DESIGN.md @@ -89,10 +89,10 @@ Sort the downloaded packages into three categories: Pip will give us a wheel when it can, but some distributions do not ship with wheels at all in which case we will have an sdist for it. In some cases a platform specific wheel file may be availble so pip will have downloaded that, -if our platform does not match the platform lambda runs on -(linux_x86_64/manylinux) then the downloaded wheel file may not be compatible -with lambda. Pure python wheels still will be compatible because they have no -platform specific dependencies. +if our platform does not match the platform defined for the lambda function +(linux/manylinux x86_64 or aarch64) then the downloaded wheel file may not be +compatible with lambda. Pure python wheels still will be compatible because +they have no platform specific dependencies. #### Step 3: Try to download a compatible wheel for each incompatible package @@ -100,7 +100,7 @@ Next we need to go through the downloaded packages and pick out any dependencies that do not have a compatible wheel file downloaded. For these packages we need to explicitly try to download a compatible wheel file. A compatible wheel file means one that is explicitly for marked as supporting the -linux_x86_64 or manylinux1. +corresponding architecture for the function. #### Step 4: Try to compile wheel files ourselves diff --git a/aws_lambda_builders/workflows/python_pip/actions.py b/aws_lambda_builders/workflows/python_pip/actions.py index f4cd225bd..e1e5f9520 100644 --- a/aws_lambda_builders/workflows/python_pip/actions.py +++ b/aws_lambda_builders/workflows/python_pip/actions.py @@ -3,6 +3,7 @@ """ from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError +from aws_lambda_builders.architecture import X86_64 from aws_lambda_builders.workflows.python_pip.utils import OSUtils from .exceptions import MissingPipError from .packager import PythonPipDependencyBuilder, PackagerError, DependencyBuilder, SubprocessPip, PipRunner @@ -15,12 +16,13 @@ class PythonPipBuildAction(BaseAction): PURPOSE = Purpose.RESOLVE_DEPENDENCIES LANGUAGE = "python" - def __init__(self, artifacts_dir, scratch_dir, manifest_path, runtime, binaries): + def __init__(self, artifacts_dir, scratch_dir, manifest_path, runtime, binaries, architecture=X86_64): self.artifacts_dir = artifacts_dir self.manifest_path = manifest_path self.scratch_dir = scratch_dir self.runtime = runtime self.binaries = binaries + self.architecture = architecture def execute(self): os_utils = OSUtils() @@ -30,7 +32,9 @@ def execute(self): except MissingPipError as ex: raise ActionFailedError(str(ex)) pip_runner = PipRunner(python_exe=python_path, pip=pip) - dependency_builder = DependencyBuilder(osutils=os_utils, pip_runner=pip_runner, runtime=self.runtime) + dependency_builder = DependencyBuilder( + osutils=os_utils, pip_runner=pip_runner, runtime=self.runtime, architecture=self.architecture + ) package_builder = PythonPipDependencyBuilder( osutils=os_utils, runtime=self.runtime, dependency_builder=dependency_builder diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 0e884c8da..1f7864d47 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -8,7 +8,7 @@ import logging from email.parser import FeedParser - +from aws_lambda_builders.architecture import ARM64, X86_64 from .compat import pip_import_string from .compat import pip_no_compile_c_env_vars from .compat import pip_no_compile_c_shim @@ -95,7 +95,7 @@ def get_lambda_abi(runtime): class PythonPipDependencyBuilder(object): - def __init__(self, runtime, osutils=None, dependency_builder=None): + def __init__(self, runtime, osutils=None, dependency_builder=None, architecture=X86_64): """Initialize a PythonPipDependencyBuilder. :type runtime: str @@ -110,13 +110,17 @@ def __init__(self, runtime, osutils=None, dependency_builder=None): :type dependency_builder: :class:`DependencyBuilder` :param dependency_builder: This class will be used to build the dependencies of the project. + + :type architecture: str + :param description: Architecture used to build dependencies for. This can + be either arm64 or x86_64. The default value is x86_64 if it's not provided. """ self.osutils = osutils if osutils is None: self.osutils = OSUtils() if dependency_builder is None: - dependency_builder = DependencyBuilder(self.osutils, runtime) + dependency_builder = DependencyBuilder(self.osutils, runtime, architecture=architecture) self._dependency_builder = dependency_builder def build_dependencies(self, artifacts_dir_path, scratch_dir_path, requirements_path, ui=None, config=None): @@ -166,12 +170,32 @@ class DependencyBuilder(object): packager. """ - _ADDITIONAL_COMPATIBLE_PLATFORM = {"any", "linux_x86_64"} + _COMPATIBLE_PLATFORM_ARM64 = { + "any", + "manylinux2014_aarch64", + } + + _COMPATIBLE_PLATFORM_X86_64 = { + "any", + "linux_x86_64", + "manylinux1_x86_64", + "manylinux2010_x86_64", + "manylinux2014_x86_64", + } + + _COMPATIBLE_PLATFORMS = { + ARM64: _COMPATIBLE_PLATFORM_ARM64, + X86_64: _COMPATIBLE_PLATFORM_X86_64, + } + _MANYLINUX_LEGACY_MAP = { "manylinux1_x86_64": "manylinux_2_5_x86_64", "manylinux2010_x86_64": "manylinux_2_12_x86_64", "manylinux2014_x86_64": "manylinux_2_17_x86_64", } + + _COMPATIBLE_PACKAGE_ALLOWLIST = {"sqlalchemy"} + # Mapping of abi to glibc version in Lambda runtime. _RUNTIME_GLIBC = { "cp27mu": (2, 17), @@ -184,9 +208,8 @@ class DependencyBuilder(object): # not in _RUNTIME_GLIBC. # Unlikely to hit this case. _DEFAULT_GLIBC = (2, 17) - _COMPATIBLE_PACKAGE_ALLOWLIST = {"sqlalchemy"} - def __init__(self, osutils, runtime, pip_runner=None): + def __init__(self, osutils, runtime, pip_runner=None, architecture=X86_64): """Initialize a DependencyBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` @@ -199,12 +222,16 @@ def __init__(self, osutils, runtime, pip_runner=None): :type pip_runner: :class:`PipRunner` :param pip_runner: This class is responsible for executing our pip on our behalf. + + :type architecture: str + :param architecture: Architecture to build for. """ self._osutils = osutils if pip_runner is None: pip_runner = PipRunner(python_exe=None, pip=SubprocessPip(osutils)) self._pip = pip_runner self.runtime = runtime + self.architecture = architecture def build_site_packages(self, requirements_filepath, target_directory, scratch_directory): """Build site-packages directory for a set of requiremetns. @@ -262,9 +289,10 @@ def _download_dependencies(self, directory, requirements_filename): # ship with wheels at all in which case we will have an sdist for it. # In some cases a platform specific wheel file may be availble so pip # will have downloaded that, if our platform does not match the - # platform lambda runs on (linux_x86_64/manylinux) then the downloaded - # wheel file may not be compatible with lambda. Pure python wheels - # still will be compatible because they have no platform dependencies. + # platform that the function will run on (x86_64 or arm64) then the + # downloaded wheel file may not be compatible with Lambda. Pure python + # wheels still will be compatible because they have no platform + # dependencies. compatible_wheels = set() incompatible_wheels = set() sdists = set() @@ -343,7 +371,8 @@ def _download_binary_wheels(self, packages, directory): # Try to get binary wheels for each package that isn't compatible. LOG.debug("Downloading missing wheels: %s", packages) lambda_abi = get_lambda_abi(self.runtime) - self._pip.download_manylinux_wheels([pkg.identifier for pkg in packages], directory, lambda_abi) + platform = "manylinux2014_aarch64" if self.architecture == ARM64 else "manylinux2014_x86_64" + self._pip.download_manylinux_wheels([pkg.identifier for pkg in packages], directory, lambda_abi, platform) def _build_sdists(self, sdists, directory, compile_c=True): LOG.debug("Build missing wheels from sdists " "(C compiling %s): %s", compile_c, sdists) @@ -399,29 +428,43 @@ def _is_compatible_platform_tag(self, expected_abi, platform): In addition to checking the tag pattern, we also need to verify the glibc version """ - if platform in self._ADDITIONAL_COMPATIBLE_PLATFORM: + if platform in self._COMPATIBLE_PLATFORMS[self.architecture]: return True - elif platform.startswith("manylinux"): - perennial_tag = self._MANYLINUX_LEGACY_MAP.get(platform, platform) - m = re.match("manylinux_([0-9]+)_([0-9]+)_(.*)", perennial_tag) - if m is None: - return False - tag_major, tag_minor = [int(x) for x in m.groups()[:2]] - runtime_major, runtime_minor = self._RUNTIME_GLIBC.get(expected_abi, self._DEFAULT_GLIBC) - if (tag_major, tag_minor) <= (runtime_major, runtime_minor): - # glibc version is compatible with Lambda Runtime - return True - return False + + arch = "aarch64" if self.architecture == ARM64 else "x86_64" + + # Verify the tag pattern + # Try to get the matching value for legacy values or keep the current + perennial_tag = self._MANYLINUX_LEGACY_MAP.get(platform, platform) + + match = re.match("manylinux_([0-9]+)_([0-9]+)_" + arch, perennial_tag) + if match is None: + return False + + # Get the glibc major and minor versions and compare them with the expected ABI + # platform: manylinux_2_17_aarch64 -> 2 and 17 + # expected_abi: cp37m -> compat glibc -> 2 and 17 + # -> Compatible + tag_major, tag_minor = [int(x) for x in match.groups()[:2]] + runtime_major, runtime_minor = self._RUNTIME_GLIBC.get(expected_abi, self._DEFAULT_GLIBC) + + return (tag_major, tag_minor) <= (runtime_major, runtime_minor) def _iter_all_compatibility_tags(self, wheel): """ Generates all possible combination of tag sets as described in PEP 425 https://www.python.org/dev/peps/pep-0425/#compressed-tag-sets """ + # ex: wheel = numpy-1.20.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64 implementation_tag, abi_tag, platform_tag = wheel.split("-")[-3:] + # cp38, cp38, manylinux_2_17_aarch64.manylinux2014_aarch64 for implementation in implementation_tag.split("."): + # cp38 for abi in abi_tag.split("."): + # cp38 for platform in platform_tag.split("."): + # manylinux_2_17_aarch64 + # manylinux2014_aarch64 yield (implementation, abi, platform) def _apply_wheel_allowlist(self, compatible_wheels, incompatible_wheels): @@ -693,22 +736,22 @@ def download_all_dependencies(self, requirements_filename, directory): # complain at deployment time. self.build_wheel(wheel_package_path, directory) - def download_manylinux_wheels(self, packages, directory, lambda_abi): + def download_manylinux_wheels(self, packages, directory, lambda_abi, platform="manylinux2014_x86_64"): """Download wheel files for manylinux for all the given packages.""" # If any one of these dependencies fails pip will bail out. Since we # are only interested in all the ones we can download, we need to feed # each package to pip individually. The return code of pip doesn't # matter here since we will inspect the working directory to see which # wheels were downloaded. We are only interested in wheel files - # compatible with lambda, which means manylinux1_x86_64 platform and - # cpython implementation. The compatible abi depends on the python + # compatible with Lambda, which depends on the function architecture, + # and cpython implementation. The compatible abi depends on the python # version and is checked later. for package in packages: arguments = [ "--only-binary=:all:", "--no-deps", "--platform", - "manylinux2014_x86_64", + platform, "--implementation", "cp", "--abi", diff --git a/aws_lambda_builders/workflows/python_pip/validator.py b/aws_lambda_builders/workflows/python_pip/validator.py index d31a929bc..f3cd35808 100644 --- a/aws_lambda_builders/workflows/python_pip/validator.py +++ b/aws_lambda_builders/workflows/python_pip/validator.py @@ -6,37 +6,40 @@ import os import subprocess +from aws_lambda_builders.validator import RuntimeValidator from aws_lambda_builders.exceptions import MisMatchRuntimeError from .utils import OSUtils LOG = logging.getLogger(__name__) -class PythonRuntimeValidator(object): - SUPPORTED_RUNTIMES = {"python2.7", "python3.6", "python3.7", "python3.8", "python3.9"} - - def __init__(self, runtime): +class PythonRuntimeValidator(RuntimeValidator): + def __init__(self, runtime, architecture): + super(PythonRuntimeValidator, self).__init__(runtime, architecture) self.language = "python" - self.runtime = runtime self._valid_runtime_path = None - def has_runtime(self): - """ - Checks if the runtime is supported. - :param string runtime: Runtime to check - :return bool: True, if the runtime is supported. - """ - return self.runtime in self.SUPPORTED_RUNTIMES - def validate(self, runtime_path): """ Checks if the language supplied matches the required lambda runtime - :param string runtime_path: runtime to check eg: /usr/bin/python3.6 - :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime + + Parameters + ---------- + runtime_path : str + runtime to check eg: /usr/bin/go + + Returns + ------- + str + runtime_path, runtime to check eg: /usr/bin/python3.6 + + Raises + ------ + MisMatchRuntimeError + Raise runtime is not support or runtime does not support architecture. """ - if not self.has_runtime(): - LOG.warning("'%s' runtime is not " "a supported runtime", self.runtime) - return + + runtime_path = super(PythonRuntimeValidator, self).validate(runtime_path) cmd = self._validate_python_cmd(runtime_path) diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index c682df8cf..65bb46d43 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -76,7 +76,14 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim if osutils.file_exists(manifest_path): # If a requirements.txt exists, run pip builder before copy action. self.actions = [ - PythonPipBuildAction(artifacts_dir, scratch_dir, manifest_path, runtime, binaries=self.binaries), + PythonPipBuildAction( + artifacts_dir, + scratch_dir, + manifest_path, + runtime, + binaries=self.binaries, + architecture=self.architecture, + ), CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), ] else: @@ -86,4 +93,4 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim ] def get_validators(self): - return [PythonRuntimeValidator(runtime=self.runtime)] + return [PythonRuntimeValidator(runtime=self.runtime, architecture=self.architecture)] diff --git a/aws_lambda_builders/workflows/ruby_bundler/workflow.py b/aws_lambda_builders/workflows/ruby_bundler/workflow.py index f57f6cf60..e16d725d1 100644 --- a/aws_lambda_builders/workflows/ruby_bundler/workflow.py +++ b/aws_lambda_builders/workflows/ruby_bundler/workflow.py @@ -4,6 +4,7 @@ from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.actions import CopySourceAction + from .actions import RubyBundlerInstallAction, RubyBundlerVendorAction from .utils import OSUtils from .bundler import SubprocessBundler diff --git a/requirements/dev.txt b/requirements/dev.txt index c4ca79eef..750cd6f5c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -17,6 +17,8 @@ mock==4.0.2; python_version >= '3.6' parameterized==0.7.4 pathlib2==2.3.2; python_version<"3.4" futures==3.2.0; python_version<"3.2.3" +pyelftools~=0.27 # Used to verify the generated Go binary architecture in integration tests (utils.py) + # tempfile backport for < 3.6 backports.tempfile==1.0; python_version<"3.7" diff --git a/tests/functional/test_builder.py b/tests/functional/test_builder.py index 9561ef442..edc31a348 100644 --- a/tests/functional/test_builder.py +++ b/tests/functional/test_builder.py @@ -51,6 +51,7 @@ def test_run_hello_workflow_with_exec_paths(self): self.artifacts_dir, self.scratch_dir, "/ignored", + "python3.8", executable_search_paths=[str(pathlib.Path(sys.executable).parent)], ) diff --git a/tests/functional/test_cli.py b/tests/functional/test_cli.py index e937cf23a..772fde0e8 100644 --- a/tests/functional/test_cli.py +++ b/tests/functional/test_cli.py @@ -74,9 +74,10 @@ def test_run_hello_workflow_with_backcompat(self, flavor, protocol_version): "artifacts_dir": self.artifacts_dir, "scratch_dir": self.scratch_dir, "manifest_path": "/ignored", - "runtime": "ignored", + "runtime": "python3.8", "optimizations": {}, "options": {}, + "architecture": "x86_64", }, } diff --git a/tests/functional/test_utils.py b/tests/functional/test_utils.py index f19604dff..bb0cea0ae 100644 --- a/tests/functional/test_utils.py +++ b/tests/functional/test_utils.py @@ -4,7 +4,7 @@ from unittest import TestCase -from aws_lambda_builders.utils import copytree +from aws_lambda_builders.utils import copytree, get_goarch class TestCopyTree(TestCase): @@ -40,6 +40,11 @@ def test_must_respect_excludes_list(self): self.assertEqual(set(os.listdir(os.path.join(self.dest, "a"))), {"c"}) self.assertEqual(set(os.listdir(os.path.join(self.dest, "a"))), {"c"}) + def test_must_return_valid_go_architecture(self): + self.assertEqual(get_goarch("arm64"), "arm64") + self.assertEqual(get_goarch("x86_64"), "amd64") + self.assertEqual(get_goarch(""), "amd64") + def file(*args): path = os.path.join(*args) diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index 2b627fac0..643b08ad3 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -8,6 +8,7 @@ import pytest import mock +from aws_lambda_builders.architecture import ARM64 from aws_lambda_builders.workflows.python_pip.packager import PipRunner, UnsupportedPackageError from aws_lambda_builders.workflows.python_pip.packager import DependencyBuilder from aws_lambda_builders.workflows.python_pip.packager import Package @@ -200,10 +201,10 @@ def _write_requirements_txt(self, packages, directory): with open(filepath, "w") as f: f.write(contents) - def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner): + def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner, **kwargs): appdir = str(_create_app_structure(tmpdir)) self._write_requirements_txt(reqs, appdir) - builder = DependencyBuilder(OSUtils(), "python3.6", runner) + builder = DependencyBuilder(OSUtils(), "python3.6", runner, **kwargs) return appdir, builder def test_can_build_local_dir_as_whl(self, tmpdir, pip_runner, osutils): @@ -514,6 +515,29 @@ def test_can_get_py27_whls(self, tmpdir, osutils, pip_runner): for req in reqs: assert req in installed_packages + def test_can_get_arm64_whls(self, tmpdir, osutils, pip_runner): + reqs = ["foo", "bar", "baz"] + pip, runner = pip_runner + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner, architecture=ARM64) + requirements_file = os.path.join(appdir, "requirements.txt") + pip.packages_to_download( + expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], + packages=[ + "foo-1.0-cp36-none-any.whl", + "bar-1.2-cp36-none-manylinux2014_aarch64.whl", + "baz-1.5-cp36-cp36m-manylinux2014_aarch64.whl", + ], + ) + + site_packages = os.path.join(appdir, ".chalice.", "site-packages") + with osutils.tempdir() as scratch_dir: + builder.build_site_packages(requirements_file, site_packages, scratch_dir) + installed_packages = os.listdir(site_packages) + + pip.validate() + for req in reqs: + assert req in installed_packages + def test_does_fail_on_invalid_local_package(self, tmpdir, osutils, pip_runner): reqs = ["../foo"] pip, runner = pip_runner diff --git a/tests/integration/workflows/dotnet_clipackage/test_dotnet.py b/tests/integration/workflows/dotnet_clipackage/test_dotnet.py index 2565aae42..eb6e6cf46 100644 --- a/tests/integration/workflows/dotnet_clipackage/test_dotnet.py +++ b/tests/integration/workflows/dotnet_clipackage/test_dotnet.py @@ -1,30 +1,58 @@ import os import shutil import tempfile +import json +try: + import pathlib +except ImportError: + import pathlib2 as pathlib from unittest import TestCase from aws_lambda_builders.builder import LambdaBuilder +from aws_lambda_builders.architecture import ARM64, X86_64 -class TestDotnetDep(TestCase): +class TestDotnetBase(TestCase): + """ + Base class for dotnetcore tests + """ + TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata") def setUp(self): self.artifacts_dir = tempfile.mkdtemp() self.scratch_dir = tempfile.mkdtemp() - self.builder = LambdaBuilder(language="dotnet", dependency_manager="cli-package", application_framework=None) - - self.runtime = "dotnetcore2.1" + self.runtime = "dotnetcore2.1" # default to 2.1 def tearDown(self): shutil.rmtree(self.artifacts_dir) shutil.rmtree(self.scratch_dir) + def verify_architecture(self, deps_file_name, expected_architecture): + deps_file = pathlib.Path(self.artifacts_dir, deps_file_name) + + if not deps_file.exists(): + self.fail("Failed verifying architecture, {} file not found".format(deps_file_name)) + + with open(str(deps_file)) as f: + deps_json = json.loads(f.read()) + version = self.runtime[-3:] + target_name = ".NETCoreApp,Version=v{}/{}".format(version, expected_architecture) + target = deps_json.get("runtimeTarget").get("name") + + self.assertEqual(target, target_name) + + +class TestDotnet21(TestDotnetBase): + """ + Tests for dotnetcore 2.1 + """ + def test_with_defaults_file(self): - source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile") + source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile2.1") self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime) @@ -41,6 +69,7 @@ def test_with_defaults_file(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + self.verify_architecture("WithDefaultsFile.deps.json", "linux-x64") def test_require_parameters(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "RequireParameters") @@ -67,3 +96,78 @@ def test_require_parameters(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + self.verify_architecture("RequireParameters.deps.json", "linux-x64") + + +class TestDotnet31(TestDotnetBase): + """ + Tests for dotnetcore 3.1 + """ + + def setUp(self): + super(TestDotnet31, self).setUp() + self.runtime = "dotnetcore3.1" + + def test_with_defaults_file(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile3.1") + + self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime) + + expected_files = { + "Amazon.Lambda.Core.dll", + "Amazon.Lambda.Serialization.Json.dll", + "Newtonsoft.Json.dll", + "WithDefaultsFile.deps.json", + "WithDefaultsFile.dll", + "WithDefaultsFile.pdb", + "WithDefaultsFile.runtimeconfig.json", + } + + output_files = set(os.listdir(self.artifacts_dir)) + + self.assertEqual(expected_files, output_files) + self.verify_architecture("WithDefaultsFile.deps.json", "linux-x64") + + def test_with_defaults_file_x86(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile3.1") + + self.builder.build( + source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime, architecture=X86_64 + ) + + expected_files = { + "Amazon.Lambda.Core.dll", + "Amazon.Lambda.Serialization.Json.dll", + "Newtonsoft.Json.dll", + "WithDefaultsFile.deps.json", + "WithDefaultsFile.dll", + "WithDefaultsFile.pdb", + "WithDefaultsFile.runtimeconfig.json", + } + + output_files = set(os.listdir(self.artifacts_dir)) + + self.assertEqual(expected_files, output_files) + self.verify_architecture("WithDefaultsFile.deps.json", "linux-x64") + + def test_with_defaults_file_arm64(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile3.1") + + self.builder.build( + source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime, architecture=ARM64 + ) + + expected_files = { + "Amazon.Lambda.Core.dll", + "Amazon.Lambda.Serialization.Json.dll", + "Newtonsoft.Json.dll", + "WithDefaultsFile.deps.json", + "WithDefaultsFile.dll", + "WithDefaultsFile.pdb", + "WithDefaultsFile.runtimeconfig.json", + } + + output_files = set(os.listdir(self.artifacts_dir)) + + self.assertEqual(expected_files, output_files) + self.verify_architecture("WithDefaultsFile.deps.json", "linux-arm64") diff --git a/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile/Function.cs b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/Function.cs similarity index 100% rename from tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile/Function.cs rename to tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/Function.cs diff --git a/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile/WithDefaultsFile.csproj b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/WithDefaultsFile.csproj similarity index 100% rename from tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile/WithDefaultsFile.csproj rename to tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/WithDefaultsFile.csproj diff --git a/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile/aws-lambda-tools-defaults.json b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/aws-lambda-tools-defaults.json similarity index 100% rename from tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile/aws-lambda-tools-defaults.json rename to tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/aws-lambda-tools-defaults.json diff --git a/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/Function.cs b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/Function.cs new file mode 100644 index 000000000..23fc86994 --- /dev/null +++ b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/Function.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +using Amazon.Lambda.Core; + +// Assembly attribute to enable the Lambda function's JSON input to be converted into a .NET class. +[assembly: LambdaSerializer(typeof(Amazon.Lambda.Serialization.Json.JsonSerializer))] + +namespace WithDefaultsFile +{ + public class Function + { + + /// + /// A simple function that takes a string and does a ToUpper + /// + /// + /// + /// + public string FunctionHandler(string input, ILambdaContext context) + { + return input?.ToUpper(); + } + } +} diff --git a/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/WithDefaultsFile.csproj b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/WithDefaultsFile.csproj new file mode 100644 index 000000000..fc380827b --- /dev/null +++ b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/WithDefaultsFile.csproj @@ -0,0 +1,11 @@ + + + netcoreapp3.1 + true + Lambda + + + + + + \ No newline at end of file diff --git a/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/aws-lambda-tools-defaults.json b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/aws-lambda-tools-defaults.json new file mode 100644 index 000000000..56786ebd2 --- /dev/null +++ b/tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile3.1/aws-lambda-tools-defaults.json @@ -0,0 +1,16 @@ +{ + "Information": [ + "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", + "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", + "dotnet lambda help", + "All the command line options for the Lambda command can be specified in this file." + ], + "profile": "", + "region": "", + "configuration": "Release", + "framework": "netcoreapp3.1", + "function-runtime": "dotnetcore3.1", + "function-memory-size": 256, + "function-timeout": 30, + "function-handler": "WithDefaultsFile::WithDefaultsFile.Function::FunctionHandler" +} \ No newline at end of file diff --git a/tests/integration/workflows/go_modules/__init__.py b/tests/integration/workflows/go_modules/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/workflows/go_modules/test_go.py b/tests/integration/workflows/go_modules/test_go.py index 3fd79415e..17f0aad46 100644 --- a/tests/integration/workflows/go_modules/test_go.py +++ b/tests/integration/workflows/go_modules/test_go.py @@ -2,11 +2,18 @@ import shutil import tempfile +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + from unittest import TestCase from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowFailedError +from tests.integration.workflows.go_modules.utils import get_executable_arch + class TestGoWorkflow(TestCase): """ @@ -66,3 +73,45 @@ def test_fails_if_modules_cannot_resolve_dependencies(self): options={"artifact_executable_name": "failed"}, ) self.assertIn("GoModulesBuilder:Build - Builder Failed: ", str(ctx.exception)) + + def test_build_defaults_to_x86_architecture(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") + built_x86_architecture = self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "no-deps-main-amd64"}, + ) + pathname = Path(self.artifacts_dir, "no-deps-main-amd64") + self.assertEqual(get_executable_arch(pathname), "x64") + + def test_builds_x86_architecture(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") + built_x86_architecture = self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "no-deps-main-amd64"}, + architecture="x86_64", + ) + pathname = Path(self.artifacts_dir, "no-deps-main-amd64") + self.assertEqual(get_executable_arch(pathname), "x64") + + def test_builds_arm64_architecture(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") + built_arm_architecture = self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "no-deps-main-arm64"}, + architecture="arm64", + ) + + pathname = Path(self.artifacts_dir, "no-deps-main-arm64") + self.assertEqual(get_executable_arch(pathname), "AArch64") diff --git a/tests/integration/workflows/go_modules/utils.py b/tests/integration/workflows/go_modules/utils.py new file mode 100644 index 000000000..a5caae61e --- /dev/null +++ b/tests/integration/workflows/go_modules/utils.py @@ -0,0 +1,20 @@ +from elftools.elf.elffile import ELFFile + + +def get_executable_arch(path): + """ + Returns the architecture of an executable binary + + Parameters + ---------- + path : str + path to the Go binaries generated + + Returns + ------- + str + Architecture type of the generated binaries + """ + with open(str(path), "rb") as f: + e = ELFFile(f) + return e.get_machine_arch() diff --git a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py index f6bb38c4e..1077cf37f 100644 --- a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py +++ b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py @@ -27,7 +27,7 @@ def setUp(self): self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps") self.builder = LambdaBuilder(language="nodejs", dependency_manager="npm", application_framework=None) - self.runtime = "nodejs8.10" + self.runtime = "nodejs12.x" def tearDown(self): shutil.rmtree(self.artifacts_dir) diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 8ea5b842a..29825dec9 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -53,6 +53,21 @@ def tearDown(self): shutil.rmtree(self.artifacts_dir) shutil.rmtree(self.scratch_dir) + def check_architecture_in(self, library, architectures): + wheel_architectures = [] + with open(os.path.join(self.artifacts_dir, library, "WHEEL")) as wheel: + for line in wheel: + if line.startswith("Tag:"): + wheel_architecture = line.rstrip().split("-")[-1] + if wheel_architecture in architectures: + return # Success + wheel_architectures.append(wheel_architecture) + self.fail( + "Wheel architectures [{}] not found in [{}]".format( + ", ".join(wheel_architectures), ", ".join(architectures) + ) + ) + # Temporarily skipping this test in Windows # Fails and we are not sure why: pip version/multiple Python versions in path/os/pypa issue? # TODO: Revisit when we deprecate Python2 @@ -65,12 +80,33 @@ def test_must_build_python_project(self): if self.runtime == "python2.7": expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"}) elif self.runtime == "python3.6": + self.check_architecture_in("numpy-1.17.4.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"}) else: + self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) + + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + def test_must_build_python_project_with_arm_architecture(self): + if self.runtime != "python3.8": + self.skipTest("{} is not supported on ARM architecture".format(self.runtime)) + ### Check the wheels + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + self.manifest_path_valid, + runtime=self.runtime, + architecture="arm64", + ) + expected_files = self.test_data_files.union({"numpy", "numpy.libs", "numpy-1.20.3.dist-info"}) output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2014_aarch64"]) + # Temporarily skipping this test in Windows # Fails and we are not sure why: pip version/multiple Python versions in path/os/pypa issue? # TODO: Revisit when we deprecate Python2 diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 08148158c..0a32f430d 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -131,6 +131,7 @@ def test_with_mocks(self, scratch_dir_exists, get_workflow_mock, importlib_mock, "artifacts_dir", "scratch_dir", "manifest_path", + architecture="arm64", runtime="runtime", optimizations="optimizations", options="options", @@ -143,6 +144,7 @@ def test_with_mocks(self, scratch_dir_exists, get_workflow_mock, importlib_mock, "artifacts_dir", "scratch_dir", "manifest_path", + architecture="arm64", runtime="runtime", optimizations="optimizations", options="options", diff --git a/tests/unit/test_validator.py b/tests/unit/test_validator.py index e1cc8844a..710edd8ab 100644 --- a/tests/unit/test_validator.py +++ b/tests/unit/test_validator.py @@ -1,14 +1,29 @@ from unittest import TestCase from aws_lambda_builders.validator import RuntimeValidator +from aws_lambda_builders.exceptions import UnsupportedRuntimeError, UnsupportedArchitectureError class TestRuntimeValidator(TestCase): def setUp(self): - self.validator = RuntimeValidator(runtime="chitti2.0") + self.validator = RuntimeValidator(runtime="python3.8", architecture="arm64") def test_inits(self): - self.assertEqual(self.validator.runtime, "chitti2.0") + self.assertEqual(self.validator.runtime, "python3.8") + self.assertEqual(self.validator.architecture, "arm64") def test_validate_runtime(self): - self.validator.validate("/usr/bin/chitti") + self.validator.validate("/usr/bin/python3.8") + self.assertEqual(self.validator._runtime_path, "/usr/bin/python3.8") + + def test_validate_with_unsupported_runtime(self): + validator = RuntimeValidator(runtime="unknown_runtime", architecture="x86_64") + with self.assertRaises(UnsupportedRuntimeError): + validator.validate("/usr/bin/unknown_runtime") + + def test_validate_with_runtime_and_incompatible_architecture(self): + runtime_list = ["dotnetcore2.1", "nodejs10.x", "ruby2.5", "python3.6", "python3.7", "python2.7"] + for runtime in runtime_list: + validator = RuntimeValidator(runtime=runtime, architecture="arm64") + with self.assertRaises(UnsupportedArchitectureError): + validator.validate("/usr/bin/{}".format(runtime)) diff --git a/tests/unit/test_workflow.py b/tests/unit/test_workflow.py index 149a82fe7..41c0709c7 100644 --- a/tests/unit/test_workflow.py +++ b/tests/unit/test_workflow.py @@ -13,7 +13,13 @@ from aws_lambda_builders.validator import RuntimeValidator from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY -from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError, MisMatchRuntimeError +from aws_lambda_builders.exceptions import ( + WorkflowFailedError, + WorkflowUnknownError, + MisMatchRuntimeError, + UnsupportedRuntimeError, + UnsupportedArchitectureError, +) from aws_lambda_builders.actions import ActionFailedError @@ -112,6 +118,7 @@ def test_must_initialize_variables(self): self.assertEqual(self.work.executable_search_paths, [str(sys.executable)]) self.assertEqual(self.work.optimizations, {"a": "b"}) self.assertEqual(self.work.options, {"c": "d"}) + self.assertEqual(self.work.architecture, "x86_64") class TestBaseWorkflow_is_supported(TestCase): @@ -132,6 +139,7 @@ def setUp(self): executable_search_paths=[], optimizations={"a": "b"}, options={"c": "d"}, + architecture="arm64", ) def test_must_ignore_manifest_if_not_provided(self): @@ -150,6 +158,9 @@ def test_must_match_manifest_name_from_path(self): self.assertTrue(self.work.is_supported()) + def test_must_match_architecture_type(self): + self.assertEqual(self.work.architecture, "arm64") + def test_must_fail_if_manifest_not_in_list(self): self.work.SUPPORTED_MANIFESTS = ["someother_manifest"] @@ -269,7 +280,7 @@ def test_must_raise_if_action_crashed(self): self.assertIn("somevalueerror", str(ctx.exception)) def test_supply_executable_path(self): - # Run workflow with supplied executable path to search for executables. + # Run workflow with supplied executable path to search for executables action_mock = Mock() self.work = self.MyWorkflow( @@ -287,6 +298,59 @@ def test_supply_executable_path(self): self.work.run() + def test_must_raise_for_unknown_runtime(self): + action_mock = Mock() + validator_mock = Mock() + validator_mock.validate = Mock() + validator_mock.validate = MagicMock(side_effect=UnsupportedRuntimeError(runtime="runtime")) + + resolver_mock = Mock() + resolver_mock.exec_paths = ["/usr/bin/binary"] + binaries_mock = Mock() + binaries_mock.return_value = [] + + self.work.get_validators = lambda: validator_mock + self.work.get_resolvers = lambda: resolver_mock + self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] + self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")} + with self.assertRaises(WorkflowFailedError) as ex: + self.work.run() + + self.assertIn("Runtime runtime is not suppported", str(ex.exception)) + + def test_must_raise_for_incompatible_runtime_and_architecture(self): + self.work = self.MyWorkflow( + "source_dir", + "artifacts_dir", + "scratch_dir", + "manifest_path", + runtime="python2.7", + executable_search_paths=[str(pathlib.Path(os.getcwd()).parent)], + optimizations={"a": "b"}, + options={"c": "d"}, + ) + action_mock = Mock() + validator_mock = Mock() + validator_mock.validate = Mock() + validator_mock.validate = MagicMock( + side_effect=UnsupportedArchitectureError(runtime="python2.7", architecture="arm64") + ) + + resolver_mock = Mock() + resolver_mock.exec_paths = ["/usr/bin/binary"] + binaries_mock = Mock() + binaries_mock.return_value = [] + + self.work.architecture = "arm64" + self.work.get_validators = lambda: validator_mock + self.work.get_resolvers = lambda: resolver_mock + self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] + self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")} + with self.assertRaises(WorkflowFailedError) as ex: + self.work.run() + + self.assertIn("Architecture arm64 is not supported for runtime python2.7", str(ex.exception)) + class TestBaseWorkflow_repr(TestCase): class MyWorkflow(BaseWorkflow): diff --git a/tests/unit/workflows/custom_make/test_workflow.py b/tests/unit/workflows/custom_make/test_workflow.py index 856f39c76..8775c11b0 100644 --- a/tests/unit/workflows/custom_make/test_workflow.py +++ b/tests/unit/workflows/custom_make/test_workflow.py @@ -1,5 +1,6 @@ from unittest import TestCase +from aws_lambda_builders.architecture import X86_64, ARM64 from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.exceptions import WorkflowFailedError from aws_lambda_builders.workflows.custom_make.workflow import CustomMakeWorkflow @@ -29,3 +30,19 @@ def test_workflow_sets_up_make_actions_no_options(self): with self.assertRaises(WorkflowFailedError): CustomMakeWorkflow("source", "artifacts", "scratch_dir", "manifest") + + def test_must_validate_architecture(self): + workflow = CustomMakeWorkflow( + "source", "artifacts", "scratch_dir", "manifest", options={"build_logical_id": "hello"} + ) + workflow_with_arm = CustomMakeWorkflow( + "source", + "artifacts", + "scratch_dir", + "manifest", + options={"build_logical_id": "hello"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/dotnet_clipackage/test_actions.py b/tests/unit/workflows/dotnet_clipackage/test_actions.py index 1df2694a3..4b27d9b21 100644 --- a/tests/unit/workflows/dotnet_clipackage/test_actions.py +++ b/tests/unit/workflows/dotnet_clipackage/test_actions.py @@ -1,11 +1,12 @@ from unittest import TestCase -from concurrent.futures import ThreadPoolExecutor -from mock import patch import os import platform +from concurrent.futures import ThreadPoolExecutor +from mock import patch from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.architecture import ARM64, X86_64 from aws_lambda_builders.workflows.dotnet_clipackage.dotnetcli import DotnetCLIExecutionError from aws_lambda_builders.workflows.dotnet_clipackage.actions import GlobalToolInstallAction, RunPackageAction @@ -66,46 +67,89 @@ class TestRunPackageAction(TestCase): @patch("aws_lambda_builders.workflows.dotnet_clipackage.utils.OSUtils") def setUp(self, MockSubprocessDotnetCLI, MockOSUtils): self.subprocess_dotnet = MockSubprocessDotnetCLI.return_value - self.os_utils = MockOSUtils - self.source_dir = os.path.join("/source_dir") - self.artifacts_dir = os.path.join("/artifacts_dir") - self.scratch_dir = os.path.join("/scratch_dir") + self.os_utils = MockOSUtils.return_value + self.source_dir = "/source_dir" + self.artifacts_dir = "/artifacts_dir" + self.scratch_dir = "/scratch_dir" def tearDown(self): self.subprocess_dotnet.reset_mock() + self.os_utils.reset_mock() def test_build_package(self): mode = "Release" options = {} action = RunPackageAction( - self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, self.os_utils + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, os_utils=self.os_utils + ) + + action.execute() + + zip_path = os.path.join(self.artifacts_dir, "source_dir.zip") + + self.subprocess_dotnet.run.assert_called_once_with( + ["lambda", "package", "--output-package", zip_path, "--msbuild-parameters", "--runtime linux-x64"], + cwd="/source_dir", + ) + + def test_build_package_x86(self): + mode = "Release" + + options = {} + action = RunPackageAction( + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, X86_64, os_utils=self.os_utils + ) + + action.execute() + + zip_path = os.path.join(self.artifacts_dir, "source_dir.zip") + + self.subprocess_dotnet.run.assert_called_once_with( + ["lambda", "package", "--output-package", zip_path, "--msbuild-parameters", "--runtime linux-x64"], + cwd="/source_dir", + ) + + def test_build_package_arm64(self): + mode = "Release" + + options = {} + action = RunPackageAction( + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, ARM64, os_utils=self.os_utils ) action.execute() - zipFilePath = os.path.join("/", "artifacts_dir", "source_dir.zip") + zip_path = os.path.join(self.artifacts_dir, "source_dir.zip") self.subprocess_dotnet.run.assert_called_once_with( - ["lambda", "package", "--output-package", zipFilePath], cwd="/source_dir" + ["lambda", "package", "--output-package", zip_path, "--msbuild-parameters", "--runtime linux-arm64"], + cwd="/source_dir", ) def test_build_package_arguments(self): mode = "Release" options = {"--framework": "netcoreapp2.1"} action = RunPackageAction( - self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, self.os_utils + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, os_utils=self.os_utils ) action.execute() - if platform.system().lower() == "windows": - zipFilePath = "/artifacts_dir\\source_dir.zip" - else: - zipFilePath = "/artifacts_dir/source_dir.zip" + zip_path = self.artifacts_dir + ("\\" if platform.system().lower() == "windows" else "/") + "source_dir.zip" self.subprocess_dotnet.run.assert_called_once_with( - ["lambda", "package", "--output-package", zipFilePath, "--framework", "netcoreapp2.1"], cwd="/source_dir" + [ + "lambda", + "package", + "--output-package", + zip_path, + "--msbuild-parameters", + "--runtime linux-x64", + "--framework", + "netcoreapp2.1", + ], + cwd="/source_dir", ) def test_build_error(self): @@ -114,7 +158,7 @@ def test_build_error(self): self.subprocess_dotnet.run.side_effect = DotnetCLIExecutionError(message="Failed Package") options = {} action = RunPackageAction( - self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, self.os_utils + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, os_utils=self.os_utils ) self.assertRaises(ActionFailedError, action.execute) @@ -123,13 +167,23 @@ def test_debug_configuration_set(self): mode = "Debug" options = None action = RunPackageAction( - self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, self.os_utils + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, os_utils=self.os_utils ) - zipFilePath = os.path.join("/", "artifacts_dir", "source_dir.zip") + zip_path = os.path.join("/", "artifacts_dir", "source_dir.zip") action.execute() self.subprocess_dotnet.run.assert_called_once_with( - ["lambda", "package", "--output-package", zipFilePath, "--configuration", "Debug"], cwd="/source_dir" + [ + "lambda", + "package", + "--output-package", + zip_path, + "--msbuild-parameters", + "--runtime linux-x64", + "--configuration", + "Debug", + ], + cwd="/source_dir", ) diff --git a/tests/unit/workflows/dotnet_clipackage/test_workflow.py b/tests/unit/workflows/dotnet_clipackage/test_workflow.py index f520c40dd..c3cb9a9f3 100644 --- a/tests/unit/workflows/dotnet_clipackage/test_workflow.py +++ b/tests/unit/workflows/dotnet_clipackage/test_workflow.py @@ -1,5 +1,6 @@ from unittest import TestCase +from aws_lambda_builders.architecture import ARM64, X86_64 from aws_lambda_builders.workflows.dotnet_clipackage.workflow import DotnetCliPackageWorkflow from aws_lambda_builders.workflows.dotnet_clipackage.actions import GlobalToolInstallAction, RunPackageAction @@ -11,3 +12,23 @@ def test_actions(self): self.assertIsInstance(workflow.actions[0], GlobalToolInstallAction) self.assertIsInstance(workflow.actions[1], RunPackageAction) + + def test_must_validate_architecture(self): + workflow = DotnetCliPackageWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + ) + workflow_with_arm = DotnetCliPackageWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/go_dep/test_actions.py b/tests/unit/workflows/go_dep/test_actions.py index 33978182a..e0f73fd16 100644 --- a/tests/unit/workflows/go_dep/test_actions.py +++ b/tests/unit/workflows/go_dep/test_actions.py @@ -2,6 +2,7 @@ from mock import patch from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.architecture import X86_64, ARM64 from aws_lambda_builders.workflows.go_dep.actions import DepEnsureAction, GoBuildAction from aws_lambda_builders.workflows.go_dep.subproc_exec import ExecutionError @@ -67,3 +68,33 @@ def test_fails_go_build(self, SubProcMock): action.execute() self.assertEqual(raised.exception.args[0], "Exec Failed: boom!") + + @patch("aws_lambda_builders.workflows.go_dep.subproc_exec.SubprocessExec") + def test_runs_go_build_with_arm_architecture(self, SubProcMock): + """ + tests the happy path of running `dep ensure` + """ + + sub_proc_go = SubProcMock.return_value + action = GoBuildAction("base", "source", "output", sub_proc_go, ARM64, env={}) + + action.execute() + + sub_proc_go.run.assert_called_with( + ["build", "-o", "output", "source"], cwd="source", env={"GOOS": "linux", "GOARCH": "arm64"} + ) + + @patch("aws_lambda_builders.workflows.go_dep.subproc_exec.SubprocessExec") + def test_fails_go_build_with_arm_architecture(self, SubProcMock): + """ + tests failure, something being returned on stderr + """ + + sub_proc_go = SubProcMock.return_value + sub_proc_go.run.side_effect = ExecutionError(message="boom!") + action = GoBuildAction("base", "source", "output", sub_proc_go, "unknown_architecture", env={}) + + with self.assertRaises(ActionFailedError) as raised: + action.execute() + + self.assertEqual(raised.exception.args[0], "Exec Failed: boom!") diff --git a/tests/unit/workflows/go_dep/test_workflow.py b/tests/unit/workflows/go_dep/test_workflow.py index d4c14518a..76aa4eefc 100644 --- a/tests/unit/workflows/go_dep/test_workflow.py +++ b/tests/unit/workflows/go_dep/test_workflow.py @@ -1,5 +1,7 @@ from unittest import TestCase +from aws_lambda_builders.architecture import X86_64, ARM64 + from aws_lambda_builders.workflows.go_dep.workflow import GoDepWorkflow from aws_lambda_builders.workflows.go_dep.actions import DepEnsureAction, GoBuildAction @@ -14,6 +16,27 @@ def test_workflow_sets_up_workflow(self): workflow = GoDepWorkflow( "source", "artifacts", "scratch", "manifest", options={"artifact_executable_name": "foo"} ) + self.assertEqual(len(workflow.actions), 2) self.assertIsInstance(workflow.actions[0], DepEnsureAction) self.assertIsInstance(workflow.actions[1], GoBuildAction) + + def test_must_validate_architecture(self): + workflow = GoDepWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + ) + workflow_with_arm = GoDepWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/go_modules/test_builder.py b/tests/unit/workflows/go_modules/test_builder.py index 1c205af46..bc3d24bbb 100644 --- a/tests/unit/workflows/go_modules/test_builder.py +++ b/tests/unit/workflows/go_modules/test_builder.py @@ -60,3 +60,14 @@ def test_debug_configuration_set(self): stderr="PIPE", stdout="PIPE", ) + + def test_debug_configuration_set_with_arm_architecture(self): + self.under_test = GoModulesBuilder(self.osutils, self.binaries, "Debug", "arm64") + self.under_test.build("source_dir", "output_path") + self.osutils.popen.assert_called_with( + ["/path/to/go", "build", "-gcflags", "all=-N -l", "-o", "output_path", "source_dir"], + cwd="source_dir", + env={"GOOS": "linux", "GOARCH": "arm64"}, + stderr="PIPE", + stdout="PIPE", + ) diff --git a/tests/unit/workflows/go_modules/test_validator.py b/tests/unit/workflows/go_modules/test_validator.py index 221fea415..14acd8698 100644 --- a/tests/unit/workflows/go_modules/test_validator.py +++ b/tests/unit/workflows/go_modules/test_validator.py @@ -5,6 +5,7 @@ from aws_lambda_builders.exceptions import MisMatchRuntimeError from aws_lambda_builders.workflows.go_modules.validator import GoRuntimeValidator +from aws_lambda_builders.exceptions import UnsupportedRuntimeError class MockSubProcess(object): @@ -19,16 +20,12 @@ def communicate(self): class TestGoRuntimeValidator(TestCase): def setUp(self): - self.validator = GoRuntimeValidator(runtime="go1.x") - - @parameterized.expand(["go1.x"]) - def test_supported_runtimes(self, runtime): - validator = GoRuntimeValidator(runtime=runtime) - self.assertTrue(validator.has_runtime()) + self.validator = GoRuntimeValidator(runtime="go1.x", architecture="arm64") def test_runtime_validate_unsupported_language_fail_open(self): - validator = GoRuntimeValidator(runtime="go2.x") - validator.validate(runtime_path="/usr/bin/go2") + validator = GoRuntimeValidator(runtime="go2.x", architecture="arm64") + with self.assertRaises(UnsupportedRuntimeError): + validator.validate(runtime_path="/usr/bin/go2") @parameterized.expand( [ diff --git a/tests/unit/workflows/go_modules/test_workflow.py b/tests/unit/workflows/go_modules/test_workflow.py index ddab1449d..86e42e306 100644 --- a/tests/unit/workflows/go_modules/test_workflow.py +++ b/tests/unit/workflows/go_modules/test_workflow.py @@ -2,6 +2,7 @@ from aws_lambda_builders.workflows.go_modules.workflow import GoModulesWorkflow from aws_lambda_builders.workflows.go_modules.actions import GoModulesBuildAction +from aws_lambda_builders.architecture import X86_64, ARM64 class TestGoModulesWorkflow(TestCase): @@ -19,5 +20,26 @@ def test_workflow_sets_up_builder_actions(self): runtime="go1.x", options={"artifact_executable_name": "main"}, ) + self.assertEqual(len(workflow.actions), 1) self.assertIsInstance(workflow.actions[0], GoModulesBuildAction) + + def test_must_validate_architecture(self): + workflow = GoModulesWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + ) + workflow_with_arm = GoModulesWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/java_gradle/test_gradle_validator.py b/tests/unit/workflows/java_gradle/test_gradle_validator.py index 19a7768b6..1b8eb6117 100644 --- a/tests/unit/workflows/java_gradle/test_gradle_validator.py +++ b/tests/unit/workflows/java_gradle/test_gradle_validator.py @@ -3,6 +3,7 @@ from mock import patch, Mock from parameterized import parameterized from aws_lambda_builders.workflows.java_gradle.gradle_validator import GradleValidator +from aws_lambda_builders.exceptions import UnsupportedRuntimeError, UnsupportedArchitectureError class FakePopen(object): @@ -24,51 +25,69 @@ class TestGradleBinaryValidator(TestCase): def setUp(self, MockOSUtils): self.mock_os_utils = MockOSUtils.return_value self.mock_log = Mock() - self.gradle_path = "/path/to/gradle" + self.runtime_path = "/path/to/gradle" self.runtime = "java8" + self.architecture = "x86_64" @parameterized.expand(["1.7.0", "1.8.9", "11.0.0", "12 (Fluff)", "12"]) def test_accepts_any_jvm_mv(self, version): version_string = ("JVM: %s" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils) - self.assertTrue(validator.validate(gradle_path=self.gradle_path)) - self.assertEqual(validator.validated_binary_path, self.gradle_path) + validator = GradleValidator(runtime=self.runtime, architecture=self.architecture, os_utils=self.mock_os_utils) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) def test_emits_warning_when_jvm_mv_greater_than_8(self): version_string = "JVM: 9.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - self.assertTrue(validator.validate(gradle_path=self.gradle_path)) - self.assertEqual(validator.validated_binary_path, self.gradle_path) - self.mock_log.warning.assert_called_with(GradleValidator.MAJOR_VERSION_WARNING, self.gradle_path, "9", "8", "8") + validator = GradleValidator( + runtime=self.runtime, architecture=self.architecture, os_utils=self.mock_os_utils, log=self.mock_log + ) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) + self.mock_log.warning.assert_called_with( + GradleValidator.MAJOR_VERSION_WARNING, self.runtime_path, "9", "8", "8" + ) @parameterized.expand(["1.6.0", "1.7.0", "1.8.9"]) def test_does_not_emit_warning_when_jvm_mv_8_or_less(self, version): version_string = ("JVM: %s" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - self.assertTrue(validator.validate(gradle_path=self.gradle_path)) - self.assertEqual(validator.validated_binary_path, self.gradle_path) + validator = GradleValidator( + runtime=self.runtime, architecture=self.architecture, os_utils=self.mock_os_utils, log=self.mock_log + ) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) self.mock_log.warning.assert_not_called() def test_emits_warning_when_gradle_excutable_fails(self): version_string = "JVM: 9.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string, returncode=1)] - validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - validator.validate(gradle_path=self.gradle_path) - self.mock_log.warning.assert_called_with(GradleValidator.VERSION_STRING_WARNING, self.gradle_path) + validator = GradleValidator( + runtime=self.runtime, architecture=self.architecture, os_utils=self.mock_os_utils, log=self.mock_log + ) + validator.validate(runtime_path=self.runtime_path) + self.mock_log.warning.assert_called_with(GradleValidator.VERSION_STRING_WARNING, self.runtime_path) def test_emits_warning_when_version_string_not_found(self): version_string = "The Java Version: 9.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string, returncode=0)] - validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - validator.validate(gradle_path=self.gradle_path) - self.mock_log.warning.assert_called_with(GradleValidator.VERSION_STRING_WARNING, self.gradle_path) + validator = GradleValidator( + runtime=self.runtime, architecture=self.architecture, os_utils=self.mock_os_utils, log=self.mock_log + ) + validator.validate(runtime_path=self.runtime_path) + self.mock_log.warning.assert_called_with(GradleValidator.VERSION_STRING_WARNING, self.runtime_path) def test_no_warning_when_jvm_mv_11_and_java11_runtime(self): version_string = "JVM: 11.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = GradleValidator(runtime="java11", os_utils=self.mock_os_utils, log=self.mock_log) - self.assertTrue(validator.validate(gradle_path=self.gradle_path)) - self.assertEqual(validator.validated_binary_path, self.gradle_path) + validator = GradleValidator( + runtime="java11", architecture=self.architecture, os_utils=self.mock_os_utils, log=self.mock_log + ) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) + + def test_runtime_validate_unsupported_language_fail_open(self): + validator = GradleValidator(runtime="java2.0", architecture="arm64") + with self.assertRaises(UnsupportedRuntimeError): + validator.validate(runtime_path="/usr/bin/java2.0") diff --git a/tests/unit/workflows/java_gradle/test_workflow.py b/tests/unit/workflows/java_gradle/test_workflow.py index 889f01ce7..ed6bd13bb 100644 --- a/tests/unit/workflows/java_gradle/test_workflow.py +++ b/tests/unit/workflows/java_gradle/test_workflow.py @@ -6,6 +6,7 @@ from aws_lambda_builders.workflows.java_gradle.actions import JavaGradleBuildAction, JavaGradleCopyArtifactsAction from aws_lambda_builders.workflows.java_gradle.gradle_resolver import GradleResolver from aws_lambda_builders.workflows.java_gradle.gradle_validator import GradleValidator +from aws_lambda_builders.architecture import ARM64 class TestJavaGradleWorkflow(TestCase): @@ -16,35 +17,45 @@ class TestJavaGradleWorkflow(TestCase): def test_workflow_sets_up_gradle_actions(self): workflow = JavaGradleWorkflow("source", "artifacts", "scratch_dir", "manifest") - self.assertEqual(len(workflow.actions), 2) - self.assertIsInstance(workflow.actions[0], JavaGradleBuildAction) - self.assertIsInstance(workflow.actions[1], JavaGradleCopyArtifactsAction) def test_workflow_sets_up_resolvers(self): workflow = JavaGradleWorkflow("source", "artifacts", "scratch_dir", "manifest") - resolvers = workflow.get_resolvers() self.assertEqual(len(resolvers), 1) - self.assertIsInstance(resolvers[0], GradleResolver) def test_workflow_sets_up_validators(self): workflow = JavaGradleWorkflow("source", "artifacts", "scratch_dir", "manifest") - validators = workflow.get_validators() self.assertEqual(len(validators), 1) - self.assertIsInstance(validators[0], GradleValidator) def test_computes_correct_build_dir(self): workflow = JavaGradleWorkflow("source", "artifacts", "scratch_dir", "manifest") - sha1 = hashlib.sha1() sha1.update(os.path.abspath(workflow.source_dir).encode("utf8")) - expected_build_dir = os.path.join(workflow.scratch_dir, sha1.hexdigest()) - self.assertEqual(expected_build_dir, workflow.build_output_dir) + + def test_must_validate_architecture(self): + workflow = JavaGradleWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + ) + workflow_with_arm = JavaGradleWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/java_maven/test_maven_validator.py b/tests/unit/workflows/java_maven/test_maven_validator.py index 819e81735..e7466e2f3 100644 --- a/tests/unit/workflows/java_maven/test_maven_validator.py +++ b/tests/unit/workflows/java_maven/test_maven_validator.py @@ -3,6 +3,7 @@ from mock import patch, Mock from parameterized import parameterized from aws_lambda_builders.workflows.java_maven.maven_validator import MavenValidator +from aws_lambda_builders.exceptions import UnsupportedRuntimeError, UnsupportedArchitectureError class FakePopen(object): @@ -24,52 +25,60 @@ class TestMavenBinaryValidator(TestCase): def setUp(self, MockOSUtils): self.mock_os_utils = MockOSUtils.return_value self.mock_log = Mock() - self.maven_path = "/path/to/maven" + self.runtime_path = "/path/to/maven" self.runtime = "java8" + self.architecture = "x86_64" @parameterized.expand(["1.7.0", "1.8.9", "11.0.0"]) def test_accepts_any_jvm_mv(self, version): version_string = ("Java version: %s, vendor: Oracle Corporation" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils) - self.assertTrue(validator.validate(maven_path=self.maven_path)) - self.assertEqual(validator.validated_binary_path, self.maven_path) + validator = MavenValidator(self.runtime, self.architecture, os_utils=self.mock_os_utils) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) @parameterized.expand(["12"]) def test_accepts_major_version_only_jvm_mv(self, version): version_string = ("Java version: %s, vendor: Oracle Corporation" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils) - self.assertTrue(validator.validate(maven_path=self.maven_path)) - self.assertEqual(validator.validated_binary_path, self.maven_path) + validator = MavenValidator(self.runtime, self.architecture, os_utils=self.mock_os_utils) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) def test_emits_warning_when_jvm_mv_greater_than_8(self): version_string = "Java version: 10.0.1, vendor: Oracle Corporation".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - self.assertTrue(validator.validate(maven_path=self.maven_path)) - self.assertEqual(validator.validated_binary_path, self.maven_path) - self.mock_log.warning.assert_called_with(MavenValidator.MAJOR_VERSION_WARNING, self.maven_path, "10", "8", "8") + validator = MavenValidator(self.runtime, self.architecture, os_utils=self.mock_os_utils, log=self.mock_log) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) + self.mock_log.warning.assert_called_with( + MavenValidator.MAJOR_VERSION_WARNING, self.runtime_path, "10", "8", "8" + ) @parameterized.expand(["1.6.0", "1.7.0", "1.8.9"]) def test_does_not_emit_warning_when_jvm_mv_8_or_less(self, version): version_string = ("Java version: %s, vendor: Oracle Corporation" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] - validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - self.assertTrue(validator.validate(maven_path=self.maven_path)) - self.assertEqual(validator.validated_binary_path, self.maven_path) + validator = MavenValidator(self.runtime, self.architecture, os_utils=self.mock_os_utils, log=self.mock_log) + self.assertTrue(validator.validate(runtime_path=self.runtime_path)) + self.assertEqual(validator.validated_binary_path, self.runtime_path) self.mock_log.warning.assert_not_called() def test_emits_warning_when_maven_excutable_fails(self): version_string = "Java version: %s, vendor: Oracle Corporation".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string, returncode=1)] - validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - validator.validate(maven_path=self.maven_path) - self.mock_log.warning.assert_called_with(MavenValidator.VERSION_STRING_WARNING, self.maven_path) + validator = MavenValidator(self.runtime, self.architecture, os_utils=self.mock_os_utils, log=self.mock_log) + validator.validate(runtime_path=self.runtime_path) + self.mock_log.warning.assert_called_with(MavenValidator.VERSION_STRING_WARNING, self.runtime_path) def test_emits_warning_when_version_string_not_found(self): version_string = "Blah: 9.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string, returncode=0)] - validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) - validator.validate(maven_path=self.maven_path) - self.mock_log.warning.assert_called_with(MavenValidator.VERSION_STRING_WARNING, self.maven_path) + validator = MavenValidator(self.runtime, self.architecture, os_utils=self.mock_os_utils, log=self.mock_log) + validator.validate(runtime_path=self.runtime_path) + self.mock_log.warning.assert_called_with(MavenValidator.VERSION_STRING_WARNING, self.runtime_path) + + def test_runtime_validate_unsupported_language_fail_open(self): + validator = MavenValidator(runtime="java2.0", architecture="arm64") + with self.assertRaises(UnsupportedRuntimeError): + validator.validate(runtime_path="/usr/bin/java2.0") diff --git a/tests/unit/workflows/java_maven/test_workflow.py b/tests/unit/workflows/java_maven/test_workflow.py index 7c830b605..670d04950 100644 --- a/tests/unit/workflows/java_maven/test_workflow.py +++ b/tests/unit/workflows/java_maven/test_workflow.py @@ -9,6 +9,7 @@ from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.workflows.java_maven.maven_resolver import MavenResolver from aws_lambda_builders.workflows.java_maven.maven_validator import MavenValidator +from aws_lambda_builders.architecture import ARM64 class TestJavaMavenWorkflow(TestCase): @@ -19,38 +20,46 @@ class TestJavaMavenWorkflow(TestCase): def test_workflow_sets_up_maven_actions(self): workflow = JavaMavenWorkflow("source", "artifacts", "scratch_dir", "manifest") - self.assertEqual(len(workflow.actions), 4) - self.assertIsInstance(workflow.actions[0], CopySourceAction) - self.assertIsInstance(workflow.actions[1], JavaMavenBuildAction) - self.assertIsInstance(workflow.actions[2], JavaMavenCopyDependencyAction) - self.assertIsInstance(workflow.actions[3], JavaMavenCopyArtifactsAction) def test_workflow_sets_up_resolvers(self): workflow = JavaMavenWorkflow("source", "artifacts", "scratch_dir", "manifest") - resolvers = workflow.get_resolvers() self.assertEqual(len(resolvers), 1) - self.assertIsInstance(resolvers[0], MavenResolver) def test_workflow_sets_up_validators(self): workflow = JavaMavenWorkflow("source", "artifacts", "scratch_dir", "manifest") - validators = workflow.get_validators() self.assertEqual(len(validators), 1) - self.assertIsInstance(validators[0], MavenValidator) def test_workflow_excluded_files(self): workflow = JavaMavenWorkflow("source", "artifacts", "scratch_dir", "manifest") - self.assertIsInstance(workflow.actions[0], CopySourceAction) - self.assertEqual(".aws-sam", workflow.actions[0].excludes[0]) - self.assertEqual(".git", workflow.actions[0].excludes[1]) + + def test_must_validate_architecture(self): + workflow = JavaMavenWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + ) + workflow_with_arm = JavaMavenWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/nodejs_npm/test_workflow.py b/tests/unit/workflows/nodejs_npm/test_workflow.py index f63565203..91fac0c48 100644 --- a/tests/unit/workflows/nodejs_npm/test_workflow.py +++ b/tests/unit/workflows/nodejs_npm/test_workflow.py @@ -3,6 +3,7 @@ from unittest import TestCase from aws_lambda_builders.actions import CopySourceAction +from aws_lambda_builders.architecture import ARM64 from aws_lambda_builders.workflows.nodejs_npm.workflow import NodejsNpmWorkflow from aws_lambda_builders.workflows.nodejs_npm.actions import ( NodejsNpmPackAction, @@ -30,15 +31,10 @@ def test_workflow_sets_up_npm_actions(self): workflow = NodejsNpmWorkflow("source", "artifacts", "scratch_dir", "manifest", osutils=self.osutils_mock) self.assertEqual(len(workflow.actions), 5) - self.assertIsInstance(workflow.actions[0], NodejsNpmPackAction) - self.assertIsInstance(workflow.actions[1], NodejsNpmrcCopyAction) - self.assertIsInstance(workflow.actions[2], CopySourceAction) - self.assertIsInstance(workflow.actions[3], NodejsNpmInstallAction) - self.assertIsInstance(workflow.actions[4], NodejsNpmrcCleanUpAction) def test_workflow_only_copy_action(self): @@ -49,3 +45,23 @@ def test_workflow_only_copy_action(self): self.assertEqual(len(workflow.actions), 1) self.assertIsInstance(workflow.actions[0], CopySourceAction) + + def test_must_validate_architecture(self): + workflow = NodejsNpmWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + ) + workflow_with_arm = NodejsNpmWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") diff --git a/tests/unit/workflows/python_pip/test_actions.py b/tests/unit/workflows/python_pip/test_actions.py index 24c1f3d2e..76f157419 100644 --- a/tests/unit/workflows/python_pip/test_actions.py +++ b/tests/unit/workflows/python_pip/test_actions.py @@ -1,9 +1,10 @@ import sys from unittest import TestCase -from mock import patch, Mock +from mock import patch, Mock, ANY from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.architecture import ARM64, X86_64 from aws_lambda_builders.binary_path import BinaryPath from aws_lambda_builders.workflows.python_pip.actions import PythonPipBuildAction @@ -13,7 +14,8 @@ class TestPythonPipBuildAction(TestCase): @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder") - def test_action_must_call_builder(self, PythonPipDependencyBuilderMock): + @patch("aws_lambda_builders.workflows.python_pip.actions.DependencyBuilder") + def test_action_must_call_builder(self, DependencyBuilderMock, PythonPipDependencyBuilderMock): builder_instance = PythonPipDependencyBuilderMock.return_value action = PythonPipBuildAction( @@ -25,6 +27,29 @@ def test_action_must_call_builder(self, PythonPipDependencyBuilderMock): ) action.execute() + DependencyBuilderMock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=X86_64) + + builder_instance.build_dependencies.assert_called_with( + artifacts_dir_path="artifacts", scratch_dir_path="scratch_dir", requirements_path="manifest" + ) + + @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder") + @patch("aws_lambda_builders.workflows.python_pip.actions.DependencyBuilder") + def test_action_must_call_builder_with_architecture(self, DependencyBuilderMock, PythonPipDependencyBuilderMock): + builder_instance = PythonPipDependencyBuilderMock.return_value + + action = PythonPipBuildAction( + "artifacts", + "scratch_dir", + "manifest", + "runtime", + {"python": BinaryPath(resolver=Mock(), validator=Mock(), binary="python", binary_path=sys.executable)}, + ARM64, + ) + action.execute() + + DependencyBuilderMock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=ARM64) + builder_instance.build_dependencies.assert_called_with( artifacts_dir_path="artifacts", scratch_dir_path="scratch_dir", requirements_path="manifest" ) diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 6fa61a7de..ddc3e4c46 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -5,6 +5,7 @@ import mock import pytest +from aws_lambda_builders.architecture import ARM64, X86_64 from aws_lambda_builders.workflows.python_pip.utils import OSUtils from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_env_vars from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_shim @@ -117,6 +118,18 @@ def test_can_call_dependency_builder(self, osutils): "path/to/requirements.txt", "artifacts/path/", "scratch_dir/path/" ) + @mock.patch("aws_lambda_builders.workflows.python_pip.packager.DependencyBuilder") + def test_can_create_new_dependency_builder(self, DependencyBuilderMock, osutils): + osutils_mock = mock.Mock(spec=osutils) + builder = PythonPipDependencyBuilder(osutils=osutils_mock, runtime="runtime") + DependencyBuilderMock.assert_called_with(osutils_mock, "runtime", architecture=X86_64) + + @mock.patch("aws_lambda_builders.workflows.python_pip.packager.DependencyBuilder") + def test_can_call_dependency_builder_with_architecture(self, DependencyBuilderMock, osutils): + osutils_mock = mock.Mock(spec=osutils) + builder = PythonPipDependencyBuilder(osutils=osutils_mock, runtime="runtime", architecture=ARM64) + DependencyBuilderMock.assert_called_with(osutils_mock, "runtime", architecture=ARM64) + class TestPackage(object): def test_can_create_package_with_custom_osutils(self, osutils): diff --git a/tests/unit/workflows/python_pip/test_validator.py b/tests/unit/workflows/python_pip/test_validator.py index d1db8a17a..95caf6773 100644 --- a/tests/unit/workflows/python_pip/test_validator.py +++ b/tests/unit/workflows/python_pip/test_validator.py @@ -5,6 +5,7 @@ from aws_lambda_builders.exceptions import MisMatchRuntimeError from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator +from aws_lambda_builders.exceptions import UnsupportedRuntimeError, UnsupportedArchitectureError class MockSubProcess(object): @@ -17,16 +18,12 @@ def communicate(self): class TestPythonRuntimeValidator(TestCase): def setUp(self): - self.validator = PythonRuntimeValidator(runtime="python3.7") - - @parameterized.expand(["python2.7", "python3.6", "python3.7", "python3.8", "python3.9"]) - def test_supported_runtimes(self, runtime): - validator = PythonRuntimeValidator(runtime=runtime) - self.assertTrue(validator.has_runtime()) + self.validator = PythonRuntimeValidator(runtime="python3.7", architecture="x86_64") def test_runtime_validate_unsupported_language_fail_open(self): - validator = PythonRuntimeValidator(runtime="python2.6") - validator.validate(runtime_path="/usr/bin/python2.6") + validator = PythonRuntimeValidator(runtime="python2.6", architecture="arm64") + with self.assertRaises(UnsupportedRuntimeError): + validator.validate(runtime_path="/usr/bin/python2.6") def test_runtime_validate_supported_version_runtime(self): with mock.patch("subprocess.Popen") as mock_subprocess: @@ -46,3 +43,15 @@ def test_python_command(self): version_strings = ["sys.version_info.major == 3", "sys.version_info.minor == 7"] for version_string in version_strings: self.assertTrue(all([part for part in cmd if version_string in part])) + + @parameterized.expand( + [ + ("python2.7", "arm64"), + ("python3.6", "arm64"), + ("python3.7", "arm64"), + ] + ) + def test_runtime_validate_with_incompatible_architecture(self, runtime, architecture): + validator = PythonRuntimeValidator(runtime=runtime, architecture=architecture) + with self.assertRaises(UnsupportedArchitectureError): + validator.validate(runtime_path="/usr/bin/python") diff --git a/tests/unit/workflows/python_pip/test_workflow.py b/tests/unit/workflows/python_pip/test_workflow.py index 01e960543..20446422d 100644 --- a/tests/unit/workflows/python_pip/test_workflow.py +++ b/tests/unit/workflows/python_pip/test_workflow.py @@ -1,5 +1,4 @@ -import mock - +from mock import patch, ANY, Mock from unittest import TestCase from aws_lambda_builders.actions import CopySourceAction @@ -11,27 +10,46 @@ class TestPythonPipWorkflow(TestCase): def setUp(self): self.osutils = OSUtils() - - def test_workflow_sets_up_actions(self): - osutils_mock = mock.Mock(spec=self.osutils) - osutils_mock.file_exists.return_value = True + self.osutils_mock = Mock(spec=self.osutils) + self.osutils_mock.file_exists.return_value = True self.workflow = PythonPipWorkflow( - "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=osutils_mock + "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=self.osutils_mock ) + + def test_workflow_sets_up_actions(self): self.assertEqual(len(self.workflow.actions), 2) self.assertIsInstance(self.workflow.actions[0], PythonPipBuildAction) self.assertIsInstance(self.workflow.actions[1], CopySourceAction) def test_workflow_sets_up_actions_without_requirements(self): - osutils_mock = mock.Mock(spec=self.osutils) - osutils_mock.file_exists.return_value = False + self.osutils_mock.file_exists.return_value = False self.workflow = PythonPipWorkflow( - "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=osutils_mock + "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=self.osutils_mock ) self.assertEqual(len(self.workflow.actions), 1) self.assertIsInstance(self.workflow.actions[0], CopySourceAction) def test_workflow_validator(self): - self.workflow = PythonPipWorkflow("source", "artifacts", "scratch_dir", "manifest", runtime="python3.7") for validator in self.workflow.get_validators(): self.assertTrue(isinstance(validator, PythonRuntimeValidator)) + + @patch("aws_lambda_builders.workflows.python_pip.workflow.PythonPipBuildAction") + def test_must_build_with_architecture(self, PythonPipBuildActionMock): + self.workflow = PythonPipWorkflow( + "source", + "artifacts", + "scratch_dir", + "manifest", + runtime="python3.7", + architecture="ARM64", + osutils=self.osutils_mock, + ) + PythonPipBuildActionMock.assert_called_with( + "artifacts", + "scratch_dir", + "manifest", + "python3.7", + binaries=ANY, + architecture="ARM64", + ) + self.assertEqual(2, len(self.workflow.actions)) diff --git a/tests/unit/workflows/ruby_bundler/test_workflow.py b/tests/unit/workflows/ruby_bundler/test_workflow.py index aa7ccafc3..804e0712f 100644 --- a/tests/unit/workflows/ruby_bundler/test_workflow.py +++ b/tests/unit/workflows/ruby_bundler/test_workflow.py @@ -1,6 +1,7 @@ from unittest import TestCase from aws_lambda_builders.actions import CopySourceAction +from aws_lambda_builders.architecture import X86_64, ARM64 from aws_lambda_builders.workflows.ruby_bundler.workflow import RubyBundlerWorkflow from aws_lambda_builders.workflows.ruby_bundler.actions import RubyBundlerInstallAction, RubyBundlerVendorAction @@ -17,3 +18,23 @@ def test_workflow_sets_up_bundler_actions(self): self.assertIsInstance(workflow.actions[0], CopySourceAction) self.assertIsInstance(workflow.actions[1], RubyBundlerInstallAction) self.assertIsInstance(workflow.actions[2], RubyBundlerVendorAction) + + def test_must_validate_architecture(self): + workflow = RubyBundlerWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + ) + workflow_with_arm = RubyBundlerWorkflow( + "source", + "artifacts", + "scratch", + "manifest", + options={"artifact_executable_name": "foo"}, + architecture=ARM64, + ) + + self.assertEqual(workflow.architecture, "x86_64") + self.assertEqual(workflow_with_arm.architecture, "arm64") From 713de19f24b2044314665a854aaeb78314bc335d Mon Sep 17 00:00:00 2001 From: Mehmet Nuri Deveci <5735811+mndeveci@users.noreply.github.com> Date: Wed, 29 Sep 2021 11:59:31 -0700 Subject: [PATCH 17/19] chore: bump version to 1.8.0 (#288) --- aws_lambda_builders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index cb6f432a3..8cda8731c 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = "1.7.0" +__version__ = "1.8.0" RPC_PROTOCOL_VERSION = "0.3" From 4303bc9af159ced954b4470701c8f1f4aaecf07c Mon Sep 17 00:00:00 2001 From: Mathieu Grandis <73313235+mgrandis@users.noreply.github.com> Date: Thu, 30 Sep 2021 01:24:08 +0200 Subject: [PATCH 18/19] fix: CustomMake validator (#293) * fix: Custom Make runtime validation * chore: Add doc * chore: black format --- aws_lambda_builders/exceptions.py | 2 +- .../workflows/custom_make/validator.py | 19 +++++++++++++++++++ .../workflows/custom_make/workflow.py | 4 ++++ tests/unit/test_workflow.py | 2 +- 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 aws_lambda_builders/workflows/custom_make/validator.py diff --git a/aws_lambda_builders/exceptions.py b/aws_lambda_builders/exceptions.py index a6725f051..2ed3f3f8e 100644 --- a/aws_lambda_builders/exceptions.py +++ b/aws_lambda_builders/exceptions.py @@ -37,7 +37,7 @@ class UnsupportedRuntimeError(RuntimeValidatorError): Raise when runtime is not supported """ - MESSAGE = "Runtime {runtime} is not suppported" + MESSAGE = "Runtime {runtime} is not supported" class UnsupportedArchitectureError(RuntimeValidatorError): diff --git a/aws_lambda_builders/workflows/custom_make/validator.py b/aws_lambda_builders/workflows/custom_make/validator.py new file mode 100644 index 000000000..ed1216dc8 --- /dev/null +++ b/aws_lambda_builders/workflows/custom_make/validator.py @@ -0,0 +1,19 @@ +""" +Custom Make Runtime Validation +""" + +from aws_lambda_builders.validator import RuntimeValidator + + +class CustomMakeRuntimeValidator(RuntimeValidator): + """ + Default runtime validator for CustomMake workflow + """ + + def __init__(self, runtime, architecture): + super(CustomMakeRuntimeValidator, self).__init__(runtime, architecture) + self._valid_runtime_path = None + + def validate(self, runtime_path): + self._runtime_path = runtime_path + return runtime_path diff --git a/aws_lambda_builders/workflows/custom_make/workflow.py b/aws_lambda_builders/workflows/custom_make/workflow.py index e1f66e0ba..04c680237 100644 --- a/aws_lambda_builders/workflows/custom_make/workflow.py +++ b/aws_lambda_builders/workflows/custom_make/workflow.py @@ -1,6 +1,7 @@ """ ProvidedMakeWorkflow """ +from aws_lambda_builders.workflows.custom_make.validator import CustomMakeRuntimeValidator from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.path_resolver import PathResolver @@ -56,3 +57,6 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim def get_resolvers(self): return [PathResolver(runtime="provided", binary="make", executable_search_paths=self.executable_search_paths)] + + def get_validators(self): + return [CustomMakeRuntimeValidator(runtime=self.runtime, architecture=self.architecture)] diff --git a/tests/unit/test_workflow.py b/tests/unit/test_workflow.py index 41c0709c7..47013c0ed 100644 --- a/tests/unit/test_workflow.py +++ b/tests/unit/test_workflow.py @@ -316,7 +316,7 @@ def test_must_raise_for_unknown_runtime(self): with self.assertRaises(WorkflowFailedError) as ex: self.work.run() - self.assertIn("Runtime runtime is not suppported", str(ex.exception)) + self.assertIn("Runtime runtime is not supported", str(ex.exception)) def test_must_raise_for_incompatible_runtime_and_architecture(self): self.work = self.MyWorkflow( From 01031a52ea314217e1c373f6ab2edc0a265ce552 Mon Sep 17 00:00:00 2001 From: Mathieu Grandis <73313235+mgrandis@users.noreply.github.com> Date: Thu, 30 Sep 2021 01:29:29 +0200 Subject: [PATCH 19/19] chore: bump version to 1.8.1 (#295) * chore: bump version to 1.8.1 --- aws_lambda_builders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index 8cda8731c..b8725094b 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = "1.8.0" +__version__ = "1.8.1" RPC_PROTOCOL_VERSION = "0.3"