diff --git a/aws_lambda_builders/__main__.py b/aws_lambda_builders/__main__.py index 142f4eb1e..25e778393 100644 --- a/aws_lambda_builders/__main__.py +++ b/aws_lambda_builders/__main__.py @@ -94,6 +94,7 @@ def main(): # pylint: disable=too-many-statements params["scratch_dir"], params["manifest_path"], runtime=params["runtime"], + runtime_path=params["runtime_path"], optimizations=params["optimizations"], options=params["options"]) diff --git a/aws_lambda_builders/actions.py b/aws_lambda_builders/actions.py index d37d361c1..abf034ab9 100644 --- a/aws_lambda_builders/actions.py +++ b/aws_lambda_builders/actions.py @@ -23,6 +23,9 @@ class Purpose(object): Enum like object to describe the purpose of each action. """ + # Action is identifying language version against provided runtime + RESOLVE_LANGUAGE_VERSION = "RESOLVE_LANGUAGE_VERSION" + # Action is identifying dependencies, downloading, compiling and resolving them RESOLVE_DEPENDENCIES = "RESOLVE_DEPENDENCIES" diff --git a/aws_lambda_builders/builder.py b/aws_lambda_builders/builder.py index 57eda2f9a..31a61845b 100644 --- a/aws_lambda_builders/builder.py +++ b/aws_lambda_builders/builder.py @@ -6,7 +6,6 @@ import logging from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY -from aws_lambda_builders.validate import RuntimeValidator from aws_lambda_builders.workflow import Capability LOG = logging.getLogger(__name__) @@ -56,7 +55,7 @@ def __init__(self, language, dependency_manager, application_framework, supporte LOG.debug("Found workflow '%s' to support capabilities '%s'", self.selected_workflow_cls.NAME, self.capability) def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path, - runtime=None, optimizations=None, options=None): + runtime=None, runtime_path=None, optimizations=None, options=None): """ Actually build the code by running workflows @@ -90,30 +89,18 @@ def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path, :param options: Optional dictionary of options ot pass to build action. **Not supported**. """ - if runtime: - self._validate_runtime(runtime) - - + # import ipdb + # ipdb.set_trace() workflow = self.selected_workflow_cls(source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=runtime, + runtime_path=runtime_path, optimizations=optimizations, options=options) return workflow.run() - def _validate_runtime(self, runtime): - """ - validate runtime and local runtime version to make sure they match - - :type runtime: str - :param runtime: - String matching a lambda runtime eg: python3.6 - """ - RuntimeValidator.validate_runtime(required_language=self.capability.language, - required_runtime=runtime) - def _clear_workflows(self): DEFAULT_REGISTRY.clear() diff --git a/aws_lambda_builders/path_resolver.py b/aws_lambda_builders/path_resolver.py new file mode 100644 index 000000000..30f57de6c --- /dev/null +++ b/aws_lambda_builders/path_resolver.py @@ -0,0 +1,21 @@ +from aws_lambda_builders.utils import which + + +class PathResolver(object): + + def __init__(self, language, runtime): + self.language = language + self.runtime = runtime + self.executables = [self.runtime, self.language] + + def _which(self): + for executable in self.executables: + path = which(executable) + if path: + return path + raise ValueError("Path resolution for runtime: {} of language: " + "{} was not successful".format(self.runtime, self.language)) + + @property + def path(self): + return self._which() diff --git a/aws_lambda_builders/utils.py b/aws_lambda_builders/utils.py index e7ebc3941..428b0d069 100644 --- a/aws_lambda_builders/utils.py +++ b/aws_lambda_builders/utils.py @@ -3,6 +3,7 @@ """ import shutil +import sys import os import logging @@ -10,6 +11,69 @@ LOG = logging.getLogger(__name__) +def which(cmd, mode=os.F_OK | os.X_OK, path=None): + """Given a command, mode, and a PATH string, return the path which + conforms to the given mode on the PATH, or None if there is no such + file. + + `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result + of os.environ.get("PATH"), or can be overridden with a custom search + path. + + """ + + # Check that a given file can be accessed with the correct mode. + # Additionally check that `file` is not a directory, as on Windows + # directories pass the os.access check. + def _access_check(fn, mode): + return (os.path.exists(fn) and os.access(fn, mode) + and not os.path.isdir(fn)) + + # If we're given a path with a directory part, look it up directly rather + # than referring to PATH directories. This includes checking relative to the + # current directory, e.g. ./script + if os.path.dirname(cmd): + if _access_check(cmd, mode): + return cmd + return None + + if path is None: + path = os.environ.get("PATH", os.defpath) + if not path: + return None + path = path.split(os.pathsep) + + if sys.platform == "win32": + # The current directory takes precedence on Windows. + if not os.curdir in path: + path.insert(0, os.curdir) + + # PATHEXT is necessary to check on Windows. + pathext = os.environ.get("PATHEXT", "").split(os.pathsep) + # See if the given file matches any of the expected path extensions. + # This will allow us to short circuit when given "python.exe". + # If it does match, only test that one, otherwise we have to try + # others. + if any(cmd.lower().endswith(ext.lower()) for ext in pathext): + files = [cmd] + else: + files = [cmd + ext for ext in pathext] + else: + # On other platforms you don't have things like PATHEXT to tell you + # what file suffixes are executable, so just pass on cmd as-is. + files = [cmd] + + seen = set() + for dir in path: + normdir = os.path.normcase(dir) + if not normdir in seen: + seen.add(normdir) + for thefile in files: + name = os.path.join(dir, thefile) + if _access_check(name, mode): + return name + return None + def copytree(source, destination, ignore=None): """ Similar to shutil.copytree except that it removes the limitation that the destination directory should diff --git a/aws_lambda_builders/validate.py b/aws_lambda_builders/validate.py deleted file mode 100644 index 7f4cbeb9b..000000000 --- a/aws_lambda_builders/validate.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -Supported Runtimes and their validations. -""" - -import logging -import os -import subprocess - -from aws_lambda_builders.exceptions import MisMatchRuntimeError - -LOG = logging.getLogger(__name__) - - -def validate_python_cmd(required_language, required_runtime_version): - major, minor = required_runtime_version.replace(required_language, "").split('.') - cmd = [ - "python", - "-c", - "import sys; " - "assert sys.version_info.major == {major} " - "and sys.version_info.minor == {minor}".format( - major=major, - minor=minor)] - return cmd - - -_RUNTIME_VERSION_RESOLVER = { - "python": validate_python_cmd -} - - -class RuntimeValidator(object): - SUPPORTED_RUNTIMES = [ - "python2.7", - "python3.6" - ] - - @classmethod - def has_runtime(cls, runtime): - """ - Checks if the runtime is supported. - :param string runtime: Runtime to check - :return bool: True, if the runtime is supported. - """ - return runtime in cls.SUPPORTED_RUNTIMES - - @classmethod - def validate_runtime(cls, required_language, required_runtime): - """ - Checks if the language supplied matches the required lambda runtime - :param string required_language: language to check eg: python - :param string required_runtime: runtime to check eg: python3.6 - :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime - """ - if required_language in _RUNTIME_VERSION_RESOLVER: - if not RuntimeValidator.has_runtime(required_runtime): - LOG.warning("'%s' runtime is not " - "a supported runtime", required_runtime) - return - cmd = _RUNTIME_VERSION_RESOLVER[required_language](required_language, required_runtime) - - p = subprocess.Popen(cmd, - cwd=os.getcwd(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - p.communicate() - if p.returncode != 0: - raise MisMatchRuntimeError(language=required_language, - required_runtime=required_runtime) - else: - LOG.warning("'%s' runtime has not " - "been validated!", required_language) diff --git a/aws_lambda_builders/validator.py b/aws_lambda_builders/validator.py new file mode 100644 index 000000000..ae319d023 --- /dev/null +++ b/aws_lambda_builders/validator.py @@ -0,0 +1,29 @@ +""" +Base Class for Runtime Validators +""" + +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class RuntimeValidator(object): + SUPPORTED_RUNTIMES = [] + + @abc.abstractmethod + def has_runtime(self): + """ + Checks if the runtime is supported. + :param string runtime: Runtime to check + :return bool: True, if the runtime is supported. + """ + raise NotImplementedError + + @abc.abstractmethod + def validate_runtime(self, runtime_path): + """ + Checks if the language supplied matches the required lambda runtime + :param string runtime_path: runtime language path to validate + :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime + """ + raise NotImplementedError diff --git a/aws_lambda_builders/workflow.py b/aws_lambda_builders/workflow.py index 140887de8..4f8943e43 100644 --- a/aws_lambda_builders/workflow.py +++ b/aws_lambda_builders/workflow.py @@ -2,15 +2,16 @@ Implementation of a base workflow """ -import os import logging - +import os from collections import namedtuple + import six -from aws_lambda_builders.registry import DEFAULT_REGISTRY -from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError, MisMatchRuntimeError +from aws_lambda_builders.registry import DEFAULT_REGISTRY +from aws_lambda_builders.validator import RuntimeValidator LOG = logging.getLogger(__name__) @@ -78,6 +79,7 @@ def __init__(self, artifacts_dir, scratch_dir, manifest_path, + runtime_path, runtime=None, optimizations=None, options=None): @@ -123,10 +125,24 @@ def __init__(self, self.runtime = runtime self.optimizations = optimizations self.options = options + self.runtime_path = runtime_path # Actions are registered by the subclasses as they seem fit self.actions = [] + def get_validator(self, runtime): + + class EmptyRuntimeValidator(RuntimeValidator): + def __init__(self): + pass + + def has_runtime(self): + pass + + def validate_runtime(self, runtime_path): + pass + return EmptyRuntimeValidator() + def is_supported(self): """ Is the given manifest supported? If the workflow exposes no manifests names, then we it is assumed that @@ -138,6 +154,10 @@ def is_supported(self): return True + def resolve_runtime(self): + if self.runtime: + self.get_validator(self.runtime).validate_runtime(self.runtime_path) + def run(self): """ Actually perform the build by executing registered actions. @@ -150,6 +170,15 @@ def run(self): LOG.debug("Running workflow '%s'", self.NAME) + try: + self.resolve_runtime() + except MisMatchRuntimeError as ex: + raise WorkflowFailedError(workflow_name=self.NAME, + action_name=None, + reason=str(ex)) + + LOG.info("%s path : '%s'", self.CAPABILITY.language, self.runtime_path) + if not self.actions: raise WorkflowFailedError(workflow_name=self.NAME, action_name=None, diff --git a/aws_lambda_builders/workflows/python_pip/actions.py b/aws_lambda_builders/workflows/python_pip/actions.py index cd8a6590e..ac1bf16da 100644 --- a/aws_lambda_builders/workflows/python_pip/actions.py +++ b/aws_lambda_builders/workflows/python_pip/actions.py @@ -3,7 +3,8 @@ """ from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError -from .packager import PythonPipDependencyBuilder, PackagerError +from aws_lambda_builders.workflows.python_pip.utils import OSUtils +from .packager import PythonPipDependencyBuilder, PackagerError, DependencyBuilder, SubprocessPip, PipRunner class PythonPipBuildAction(BaseAction): @@ -12,15 +13,22 @@ class PythonPipBuildAction(BaseAction): DESCRIPTION = "Installing dependencies from PIP" PURPOSE = Purpose.RESOLVE_DEPENDENCIES - def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime): + def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime, runtime_path): self.artifacts_dir = artifacts_dir self.manifest_path = manifest_path self.scratch_dir = scratch_dir self.runtime = runtime - self.package_builder = PythonPipDependencyBuilder() + self.language_exec_path = runtime_path + self.pip = SubprocessPip(osutils=OSUtils(), python_exe=runtime_path) + self.pip_runner = PipRunner(python_exe=runtime_path, pip=self.pip) + self.package_builder = PythonPipDependencyBuilder( + dependency_builder=DependencyBuilder(osutils=OSUtils(), + pip_runner=self.pip_runner)) def execute(self): try: + # import ipdb + # ipdb.set_trace() self.package_builder.build_dependencies( self.artifacts_dir, self.manifest_path, diff --git a/aws_lambda_builders/workflows/python_pip/compat.py b/aws_lambda_builders/workflows/python_pip/compat.py index 1a08b645b..0c3df584d 100644 --- a/aws_lambda_builders/workflows/python_pip/compat.py +++ b/aws_lambda_builders/workflows/python_pip/compat.py @@ -1,14 +1,21 @@ import os -import six +from aws_lambda_builders.workflows.python_pip.utils import OSUtils -def pip_import_string(): - import pip - pip_major_version = pip.__version__.split('.')[0] + +def pip_import_string(python_exe=None): + os_utils = OSUtils() + cmd = [ + python_exe, + "-c", + "import pip; assert pip.__version__.split('.')[0] == 9" + ] + p = os_utils.popen(cmd,stdout=os_utils.pipe, stderr=os_utils.pipe) + p.communicate() # Pip moved its internals to an _internal module in version 10. # In order to be compatible with version 9 which has it at at the # top level we need to figure out the correct import path here. - if pip_major_version == '9': + if p.returncode == 0: return 'from pip import main' else: return 'from pip._internal import main' @@ -103,7 +110,3 @@ def raise_compile_error(*args, **kwargs): } -if six.PY3: - lambda_abi = 'cp36m' -else: - lambda_abi = 'cp27mu' diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index d9c78cd74..c124dcb01 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -8,11 +8,10 @@ from email.parser import FeedParser -from .compat import lambda_abi from .compat import pip_import_string from .compat import pip_no_compile_c_env_vars from .compat import pip_no_compile_c_shim -from .utils import OSUtils +from aws_lambda_builders.workflows.python_pip.utils import OSUtils # TODO update the wording here @@ -440,10 +439,11 @@ class SDistMetadataFetcher(object): "exec(compile(code, __file__, 'exec'))" ) - def __init__(self, osutils=None): + def __init__(self, osutils=None, python_exe=None): if osutils is None: osutils = OSUtils() self._osutils = osutils + self.python_exe = python_exe def _parse_pkg_info_file(self, filepath): # The PKG-INFO generated by the egg-info command is in an email feed @@ -493,25 +493,26 @@ def get_package_name_and_version(self, sdist_path): class SubprocessPip(object): """Wrapper around calling pip through a subprocess.""" - def __init__(self, osutils=None, import_string=None): + def __init__(self, osutils=None, import_string=None, python_exe=None): if osutils is None: osutils = OSUtils() self._osutils = osutils if import_string is None: - import_string = pip_import_string() + import_string = pip_import_string(python_exe=python_exe) self._import_string = import_string + self.python_exe = python_exe def main(self, args, env_vars=None, shim=None): if env_vars is None: env_vars = self._osutils.environ() if shim is None: shim = '' - python_exe = sys.executable run_pip = ( 'import sys; %s; sys.exit(main(%s))' ) % (self._import_string, args) exec_string = '%s%s' % (shim, run_pip) - invoke_pip = [python_exe, '-c', exec_string] + invoke_pip = [self.python_exe, '-c', exec_string] + print(invoke_pip) p = self._osutils.popen(invoke_pip, stdout=self._osutils.pipe, stderr=self._osutils.pipe, @@ -528,9 +529,10 @@ class PipRunner(object): " Link is a directory," " ignoring download_dir") - def __init__(self, pip, osutils=None): + def __init__(self, python_exe, pip, osutils=None): if osutils is None: osutils = OSUtils() + self.python_exe = python_exe self._wrapped_pip = pip self._osutils = osutils @@ -541,6 +543,16 @@ def _execute(self, command, args, env_vars=None, shim=None): shim=shim) return rc, out, err + def _determine_abi(self): + cmd = [ + self.python_exe, + "-c", + "import sys; assert sys.version_info.major == 3" + ] + p = self._osutils.popen(cmd, stdout=self._osutils.pipe, stderr=self._osutils.pipe) + p.communicate() + return 'cp36m' if p.returncode == 0 else 'cp27mu' + def build_wheel(self, wheel, directory, compile_c=True): """Build an sdist into a wheel file.""" arguments = ['--no-deps', '--wheel-dir', directory, wheel] @@ -602,5 +614,5 @@ def download_manylinux_wheels(self, packages, directory): for package in packages: arguments = ['--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', directory, package] + '--abi', self._determine_abi(), '--dest', directory, package] self._execute('download', arguments) diff --git a/aws_lambda_builders/workflows/python_pip/utils.py b/aws_lambda_builders/workflows/python_pip/utils.py index 2cdd1941c..16ec6718e 100644 --- a/aws_lambda_builders/workflows/python_pip/utils.py +++ b/aws_lambda_builders/workflows/python_pip/utils.py @@ -3,6 +3,7 @@ """ import io +import sys import os import zipfile import contextlib diff --git a/aws_lambda_builders/workflows/python_pip/validator.py b/aws_lambda_builders/workflows/python_pip/validator.py new file mode 100644 index 000000000..9c040210f --- /dev/null +++ b/aws_lambda_builders/workflows/python_pip/validator.py @@ -0,0 +1,64 @@ +""" +Supported Runtimes and their validations. +""" + +import logging +import os +import subprocess + +from aws_lambda_builders.exceptions import MisMatchRuntimeError +from aws_lambda_builders.validator import RuntimeValidator + +LOG = logging.getLogger(__name__) + + +class PythonRuntimeValidator(RuntimeValidator): + SUPPORTED_RUNTIMES = [ + "python2.7", + "python3.6" + ] + + def __init__(self, language_runtime): + self.language = "python" + self.language_runtime = language_runtime + + def has_runtime(self): + """ + Checks if the runtime is supported. + :param string runtime: Runtime to check + :return bool: True, if the runtime is supported. + """ + return self.language_runtime in self.SUPPORTED_RUNTIMES + + def validate_runtime(self, runtime_path): + """ + Checks if the language supplied matches the required lambda runtime + :param string required_language: language to check eg: python + :param string required_runtime: runtime to check eg: python3.6 + :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime + """ + if not self.has_runtime(): + LOG.warning("'%s' runtime is not " + "a supported runtime", self.language_runtime) + return + cmd = self._validate_python_cmd(runtime_path) + + p = subprocess.Popen(cmd, + cwd=os.getcwd(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.communicate() + if p.returncode != 0: + raise MisMatchRuntimeError(language=self.language, + required_runtime=self.language_runtime) + + def _validate_python_cmd(self, runtime_path): + major, minor = self.language_runtime.replace(self.language, "").split('.') + cmd = [ + runtime_path, + "-c", + "import sys; " + "assert sys.version_info.major == {major} " + "and sys.version_info.minor == {minor}".format( + major=major, + minor=minor)] + return cmd diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index fb2582b0d..526737f49 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -1,12 +1,17 @@ """ Python PIP Workflow """ +import logging +from aws_lambda_builders.path_resolver import PathResolver from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.actions import CopySourceAction +from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator from .actions import PythonPipBuildAction +LOG = logging.getLogger(__name__) + class PythonPipWorkflow(BaseWorkflow): @@ -54,17 +59,25 @@ def __init__(self, artifacts_dir, scratch_dir, manifest_path, - runtime=None, **kwargs): + runtime_path=None, + runtime=None, + **kwargs): + + self.runtime_path = PathResolver(language=self.CAPABILITY.language, runtime=runtime).path super(PythonPipWorkflow, self).__init__(source_dir, artifacts_dir, scratch_dir, manifest_path, + runtime_path=self.runtime_path, runtime=runtime, **kwargs) self.actions = [ PythonPipBuildAction(artifacts_dir, scratch_dir, - manifest_path, runtime), + manifest_path, runtime, runtime_path=self.runtime_path), CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), ] + + def get_validator(self, runtime): + return PythonRuntimeValidator(language_runtime=runtime) \ No newline at end of file diff --git a/tests/functional/test_cli.py b/tests/functional/test_cli.py index f72180e78..6103bcd4a 100644 --- a/tests/functional/test_cli.py +++ b/tests/functional/test_cli.py @@ -61,6 +61,7 @@ def test_run_hello_workflow(self, flavor): "scratch_dir": "/ignored", "manifest_path": "/ignored", "runtime": "ignored", + "runtime_path": "ignored_path", "optimizations": {}, "options": {}, } diff --git a/tests/functional/test_utils.py b/tests/functional/test_utils.py index 69dcd3390..2ba3d1368 100644 --- a/tests/functional/test_utils.py +++ b/tests/functional/test_utils.py @@ -6,6 +6,7 @@ from aws_lambda_builders.utils import copytree + class TestCopyTree(TestCase): def setUp(self): diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index f51c1b4b9..57c8d0174 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -1,4 +1,5 @@ import os +import sys import zipfile import tarfile import io @@ -16,7 +17,6 @@ from aws_lambda_builders.workflows.python_pip.packager import SDistMetadataFetcher from aws_lambda_builders.workflows.python_pip.packager import \ InvalidSourceDistributionNameError -from aws_lambda_builders.workflows.python_pip.compat import lambda_abi from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_env_vars from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_shim from aws_lambda_builders.workflows.python_pip.utils import OSUtils @@ -644,7 +644,7 @@ def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner): expected_args=[ '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', mock.ANY, + '--abi', 'cp36m', '--dest', mock.ANY, 'bar==1.2' ], packages=[ @@ -677,7 +677,7 @@ def test_whitelist_sqlalchemy(self, tmpdir, osutils, pip_runner): expected_args=[ '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', mock.ANY, + '--abi', 'cp36m', '--dest', mock.ANY, 'sqlalchemy==1.1.18' ], packages=[ @@ -839,7 +839,7 @@ def test_build_into_existing_dir_with_preinstalled_packages( expected_args=[ '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', mock.ANY, + '--abi', 'cp36m', '--dest', mock.ANY, 'foo==1.2' ], packages=[ diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 965f7bcce..09c47361f 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -107,12 +107,11 @@ def test_with_mocks(self, get_workflow_mock, importlib_mock): get_workflow_mock.return_value = workflow_cls - with patch.object(LambdaBuilder, "_validate_runtime"): - builder = LambdaBuilder(self.lang, self.lang_framework, self.app_framework, supported_workflows=[]) + builder = LambdaBuilder(self.lang, self.lang_framework, self.app_framework, supported_workflows=[]) - builder.build("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", - runtime="runtime", optimizations="optimizations", options="options") + builder.build("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", + runtime="runtime", runtime_path="runtime_path", optimizations="optimizations", options="options") - workflow_cls.assert_called_with("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", - runtime="runtime", optimizations="optimizations", options="options") - workflow_instance.run.assert_called_once() + workflow_cls.assert_called_with("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", + runtime="runtime", runtime_path="runtime_path", optimizations="optimizations", options="options") + workflow_instance.run.assert_called_once() diff --git a/tests/unit/test_path_resolver.py b/tests/unit/test_path_resolver.py new file mode 100644 index 000000000..6428e5e1b --- /dev/null +++ b/tests/unit/test_path_resolver.py @@ -0,0 +1,29 @@ +import os +from unittest import TestCase + +import mock + +from aws_lambda_builders.path_resolver import PathResolver + + +class TestPathResolver(TestCase): + + def setUp(self): + self.path_resolver = PathResolver(language="chitti",runtime="chitti2.0") + + def test_inits(self): + self.assertEquals(self.path_resolver.language, "chitti") + self.assertEquals(self.path_resolver.runtime, "chitti2.0") + self.assertEquals(self.path_resolver.executables, + [self.path_resolver.runtime, self.path_resolver.language]) + + def test_which_fails(self): + with self.assertRaises(ValueError): + self.path_resolver.path + + def test_which_success_immediate(self): + with mock.patch.object(self.path_resolver,'_which') as which_mock: + which_mock.return_value = os.getcwd() + self.assertEquals(self.path_resolver.path, os.getcwd()) + + diff --git a/tests/unit/test_runtime.py b/tests/unit/test_runtime.py deleted file mode 100644 index 25799806d..000000000 --- a/tests/unit/test_runtime.py +++ /dev/null @@ -1,49 +0,0 @@ -from unittest import TestCase - -import mock - -from aws_lambda_builders.exceptions import MisMatchRuntimeError -from aws_lambda_builders.validate import validate_python_cmd -from aws_lambda_builders.validate import RuntimeValidator - - -class MockSubProcess(object): - - def __init__(self, returncode): - self.returncode = returncode - - def communicate(self): - pass - - -class TestRuntime(TestCase): - - def test_supported_runtimes(self): - self.assertTrue(RuntimeValidator.has_runtime("python2.7")) - self.assertTrue(RuntimeValidator.has_runtime("python3.6")) - self.assertFalse(RuntimeValidator.has_runtime("test_language")) - - def test_runtime_validate_unsupported_language_fail_open(self): - RuntimeValidator.validate_runtime("test_language", "test_language2.7") - - def test_runtime_validate_unsupported_runtime_version_fail_open(self): - RuntimeValidator.validate_runtime("python", "python2.8") - - def test_runtime_validate_supported_version_runtime(self): - with mock.patch('subprocess.Popen') as mock_subprocess: - mock_subprocess.return_value = MockSubProcess(0) - RuntimeValidator.validate_runtime("python", "python3.6") - self.assertTrue(mock_subprocess.call_count, 1) - - def test_runtime_validate_mismatch_version_runtime(self): - with mock.patch('subprocess.Popen') as mock_subprocess: - mock_subprocess.return_value = MockSubProcess(1) - with self.assertRaises(MisMatchRuntimeError): - RuntimeValidator.validate_runtime("python", "python2.7") - self.assertTrue(mock_subprocess.call_count, 1) - - def test_python_command(self): - cmd = validate_python_cmd("python", "python2.7") - version_strings = ["sys.version_info.major == 2", "sys.version_info.minor == 7"] - for version_string in version_strings: - self.assertTrue(any([part for part in cmd if version_string in part])) diff --git a/tests/unit/test_workflow.py b/tests/unit/test_workflow.py index b137f8f13..7d85a02be 100644 --- a/tests/unit/test_workflow.py +++ b/tests/unit/test_workflow.py @@ -87,6 +87,7 @@ class MyWorkflow(BaseWorkflow): def test_must_initialize_variables(self): self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", runtime="runtime", + runtime_path="runtime_path", optimizations={"a": "b"}, options={"c": "d"}) @@ -95,6 +96,7 @@ def test_must_initialize_variables(self): self.assertEquals(self.work.scratch_dir, "scratch_dir") self.assertEquals(self.work.manifest_path, "manifest_path") self.assertEquals(self.work.runtime, "runtime") + self.assertEquals(self.work.runtime_path, "runtime_path") self.assertEquals(self.work.optimizations, {"a": "b"}) self.assertEquals(self.work.options, {"c": "d"}) @@ -111,6 +113,7 @@ class MyWorkflow(BaseWorkflow): def setUp(self): self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", runtime="runtime", + runtime_path="runtime_path", optimizations={"a": "b"}, options={"c": "d"}) @@ -148,6 +151,7 @@ class MyWorkflow(BaseWorkflow): def setUp(self): self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", runtime="runtime", + runtime_path="runtime_path", optimizations={"a": "b"}, options={"c": "d"}) @@ -216,6 +220,7 @@ def setUp(self): self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", runtime="runtime", + runtime_path="runtime_path", optimizations={"a": "b"}, options={"c": "d"}) diff --git a/tests/unit/workflows/python_pip/test_actions.py b/tests/unit/workflows/python_pip/test_actions.py index 208681a52..9975ba109 100644 --- a/tests/unit/workflows/python_pip/test_actions.py +++ b/tests/unit/workflows/python_pip/test_actions.py @@ -1,3 +1,4 @@ +import sys from unittest import TestCase from mock import patch @@ -15,7 +16,7 @@ def test_action_must_call_builder(self, PythonPipDependencyBuilderMock): builder_instance = PythonPipDependencyBuilderMock.return_value action = PythonPipBuildAction("artifacts", "scratch_dir", - "manifest", "runtime") + "manifest", "runtime", sys.executable) action.execute() builder_instance.build_dependencies.assert_called_with("artifacts", @@ -29,7 +30,7 @@ def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock): builder_instance.build_dependencies.side_effect = PackagerError() action = PythonPipBuildAction("artifacts", "scratch_dir", - "manifest", "runtime") + "manifest", "runtime", sys.executable) with self.assertRaises(ActionFailedError): action.execute() diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 01f81a735..1161c07db 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -47,7 +47,7 @@ def calls(self): def pip_factory(): def create_pip_runner(osutils=None): pip = FakePip() - pip_runner = PipRunner(pip, osutils=osutils) + pip_runner = PipRunner(sys.executable, pip, osutils=osutils) return pip, pip_runner return create_pip_runner diff --git a/tests/unit/workflows/python_pip/test_validator.py b/tests/unit/workflows/python_pip/test_validator.py new file mode 100644 index 000000000..e81797e56 --- /dev/null +++ b/tests/unit/workflows/python_pip/test_validator.py @@ -0,0 +1,49 @@ +import sys + +from unittest import TestCase +import mock + +from aws_lambda_builders.exceptions import MisMatchRuntimeError +from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator + + +class MockSubProcess(object): + + def __init__(self, returncode): + self.returncode = returncode + + def communicate(self): + pass + + +class TestPythonRuntimeValidator(TestCase): + + def setUp(self): + self.runtime_validator = PythonRuntimeValidator("python3.6") + + def test_python_validator_inits(self): + self.assertEquals(self.runtime_validator.SUPPORTED_RUNTIMES, + ["python2.7","python3.6"]) + self.assertEquals(self.runtime_validator.language, "python") + self.assertEquals(self.runtime_validator.language_runtime, "python3.6") + + def test_python_valid_runtimes(self): + self.assertTrue(self.runtime_validator.has_runtime()) + + def test_python_invalid_runtime(self): + runtime_validator = PythonRuntimeValidator("python2.8") + self.assertFalse(runtime_validator.has_runtime()) + + def test_python_valid_runtime_path(self): + with mock.patch('subprocess.Popen') as mock_subprocess: + mock_subprocess.return_value = MockSubProcess(0) + runtime_validator = PythonRuntimeValidator("python{}.{}".format(sys.version_info.major, sys.version_info.minor)) + runtime_validator.validate_runtime(sys.executable) + + def test_python_invalid_runtime_path(self): + with mock.patch('subprocess.Popen') as mock_subprocess: + mock_subprocess.return_value = MockSubProcess(1) + runtime_validator = PythonRuntimeValidator("python{}.{}".format(sys.version_info.major, sys.version_info.minor)) + with self.assertRaises(MisMatchRuntimeError): + runtime_validator.validate_runtime(sys.executable) +