fix: Parameterize processor and py_version for test runs (#208)

* fix: Parameterize processor and py_version for test runs

* move py_version and processor fixture to utils.py

* Fix flake8 error

* Update buildspec.yml

Author: icywang86rui

buildspec-release.yml

@@ -16,7 +16,7 @@ phases:
   build:
     commands:
       # prepare the release (update versions, changelog etc.)
-      - git-release --prepare
+      - git-release --min-version 2.0.0 --prepare
 
       # run linter
       - tox -e flake8
@@ -71,10 +71,7 @@ phases:
             "dest": ["1.13.1-gpu-py3", "1.13-gpu-py3", "1.13.1-gpu-py3-'${CODEBUILD_BUILD_ID#*:}'"]
           }],
           "test": [
-            "IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker -n auto --region {region} --account-id 520713654638 --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.13.1 --processor cpu",
-            "IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker -n auto --region {region} --account-id 520713654638 --docker-base-name sagemaker-tensorflow-scriptmode --py-version 2 --framework-version 1.13.1 --processor cpu",
-            "IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker -n auto --region {region} --account-id 520713654638 --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.13.1 --processor gpu",
-            "IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker -n auto --region {region} --account-id 520713654638 --docker-base-name sagemaker-tensorflow-scriptmode --py-version 2 --framework-version 1.13.1 --processor gpu"
+            "IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker -n 32 --region {region} --account-id 520713654638 --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.13.1 --processor cpu,gpu --py-version 2,3",
           ]
         }]' > deployments.json
 

buildspec.yml

@@ -5,10 +5,8 @@ env:
     FRAMEWORK_VERSION: '1.13.1'
     CPU_FRAMEWORK_BINARY: 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/cpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl'
     CPU_PY_VERSION: '3'
-    CPU_INSTANCE_TYPE: 'ml.c4.xlarge'
     GPU_FRAMEWORK_BINARY: 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/gpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl'
     GPU_PY_VERSION: '3'
-    GPU_INSTANCE_TYPE: 'ml.p2.xlarge'
     ECR_REPO: 'sagemaker-test'
     GITHUB_REPO: 'sagemaker-tensorflow-container'
     SETUP_FILE: 'setup_cmds.sh'
@@ -74,8 +72,7 @@ phases:
       - docker push $PREPROD_IMAGE:$GPU_TAG
 
       # launch remote gpu instance
-      - prefix='ml.'
-      - instance_type=${GPU_INSTANCE_TYPE#"$prefix"}
+      - instance_type='p2.xlarge'
       - create-key-pair
       - launch-ec2-instance --instance-type $instance_type --ami-name dlami-ubuntu
 
@@ -97,20 +94,13 @@ phases:
           echo "skipping gpu integration tests"
         fi
 
-      # run cpu sagemaker tests
+      # run sagemaker tests
       - |
         if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
-          pytest test/integration/sagemaker -n auto --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG --py-version $CPU_PY_VERSION --instance-type $CPU_INSTANCE_TYPE
+          pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG --py-version $CPU_PY_VERSION --processor cpu
+          pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG --py-version $GPU_PY_VERSION --processor gpu
         else
-          echo "skipping cpu sagemaker tests"
-        fi
-
-      # run gpu sagemaker tests
-      - |
-        if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
-          pytest test/integration/sagemaker -n auto --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG --py-version $GPU_PY_VERSION --instance-type $GPU_INSTANCE_TYPE
-        else
-          echo "skipping gpu sagemaker tests"
+          echo "skipping sagemaker tests"
         fi
 
     finally:

test/conftest.py renamed to test/integration/conftest.py

@@ -1,4 +1,4 @@
-#  Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#  Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License").
 #  You may not use this file except in compliance with the License.
@@ -33,42 +33,37 @@
 
 
 def pytest_addoption(parser):
-    parser.addoption('--docker-base-name', default='preprod-tensorflow')
+    parser.addoption('--docker-base-name', default='sagemaker-tensorflow-scriptmode')
     parser.addoption('--tag', default=None)
     parser.addoption('--region', default='us-west-2')
     parser.addoption('--framework-version', default=TensorFlow.LATEST_VERSION)
-    parser.addoption('--processor', default='cpu', choices=['gpu', 'cpu'])
-    parser.addoption('--py-version', default='3', choices=['2', '3'])
+    parser.addoption('--processor', default='cpu', choices=['cpu', 'gpu', 'cpu,gpu'])
+    parser.addoption('--py-version', default='3', choices=['2', '3', '2,3'])
     parser.addoption('--account-id', default='142577830533')
     parser.addoption('--instance-type', default=None)
 
 
-@pytest.fixture(scope='session')
-def docker_base_name(request):
-    return request.config.getoption('--docker-base-name')
+def pytest_configure(config):
+    os.environ['TEST_PY_VERSIONS'] = config.getoption('--py-version')
+    os.environ['TEST_PROCESSORS'] = config.getoption('--processor')
 
 
 @pytest.fixture(scope='session')
-def processor(request):
-    return request.config.getoption('--processor')
+def docker_base_name(request):
+    return request.config.getoption('--docker-base-name')
 
 
 @pytest.fixture(scope='session')
 def region(request):
     return request.config.getoption('--region')
 
 
-@pytest.fixture(scope='session')
-def py_version(request):
-    return request.config.getoption('--py-version')
-
-
 @pytest.fixture(scope='session')
 def framework_version(request):
     return request.config.getoption('--framework-version')
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture
 def tag(request, framework_version, processor, py_version):
     provided_tag = request.config.getoption('--tag')
     default_tag = '{}-{}-py{}'.format(framework_version, processor, py_version)
@@ -90,7 +85,7 @@ def account_id(request):
     return request.config.getoption('--account-id')
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture
 def instance_type(request, processor):
     provided_instance_type = request.config.getoption('--instance-type')
     default_instance_type = 'ml.c4.xlarge' if processor == 'cpu' else 'ml.p2.xlarge'
@@ -112,12 +107,12 @@ def skip_gpu_instance_restricted_regions(region, instance_type):
         pytest.skip('Skipping GPU test in region {}'.format(region))
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture
 def docker_image(docker_base_name, tag):
     return '{}:{}'.format(docker_base_name, tag)
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture
 def ecr_image(account_id, docker_base_name, tag, region):
     return '{}.dkr.ecr.{}.amazonaws.com/{}:{}'.format(
         account_id, region, docker_base_name, tag)

test/integration/local/test_horovod.py

@@ -19,6 +19,8 @@
 import pytest
 from sagemaker.tensorflow import TensorFlow
 
+from test.integration.sagemaker.utils import processor, py_version  # noqa: F401
+
 RESOURCE_PATH = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
 
 

test/integration/local/test_keras.py

@@ -20,6 +20,7 @@
 from sagemaker.tensorflow import serving, TensorFlow
 
 from test.integration import RESOURCE_PATH
+from test.integration.sagemaker.utils import processor, py_version  # noqa: F401
 
 
 logging.basicConfig(level=logging.DEBUG)

test/integration/local/test_training.py

@@ -18,12 +18,14 @@
 import pytest
 from sagemaker.tensorflow import TensorFlow
 
+from test.integration.sagemaker.utils import processor, py_version  # noqa: F401
+
 RESOURCE_PATH = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
 TF_CHECKPOINT_FILES = ['graph.pbtxt', 'model.ckpt-0.index', 'model.ckpt-0.meta']
 
 
-@pytest.fixture
-def py_full_version(py_version):
+@pytest.fixture  # noqa: F811
+def py_full_version(py_version):  # noqa: F811
     if py_version == '2':
         return '2.7'
     else:

test/integration/sagemaker/test_horovod.py

@@ -17,7 +17,7 @@
 import sagemaker
 from sagemaker.tensorflow import TensorFlow
 
-from utils import unique_name_from_base
+from utils import processor, py_version, unique_name_from_base  # noqa: F401
 
 RESOURCE_PATH = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
 

test/integration/sagemaker/test_mnist.py

@@ -19,7 +19,7 @@
 from six.moves.urllib.parse import urlparse
 
 from sagemaker_tensorflow_container.training import SAGEMAKER_PARAMETER_SERVER_ENABLED
-from utils import unique_name_from_base
+from utils import processor, py_version, unique_name_from_base  # noqa: F401
 
 
 def test_mnist(sagemaker_session, ecr_image, instance_type, framework_version):

test/integration/sagemaker/test_tuning_model_dir.py

@@ -17,7 +17,7 @@
 from sagemaker.tensorflow import TensorFlow
 from sagemaker.tuner import HyperparameterTuner, IntegerParameter
 
-from utils import unique_name_from_base
+from utils import processor, py_version, unique_name_from_base  # noqa: F401
 
 
 def test_model_dir_with_training_job_name(sagemaker_session, ecr_image, instance_type, framework_version):

test/integration/sagemaker/utils.py

@@ -12,13 +12,26 @@
 # language governing permissions and limitations under the License.
 from __future__ import absolute_import
 
+import os
 import random
 import time
 
+import pytest
+
 
 def unique_name_from_base(base, max_length=63):
     unique = '%04x' % random.randrange(16**4)  # 4-digit hex
     ts = str(int(time.time()))
     available_length = max_length - 2 - len(ts) - len(unique)
     trimmed = base[:available_length]
     return '{}-{}-{}'.format(trimmed, ts, unique)
+
+
+@pytest.fixture(params=os.environ['TEST_PY_VERSIONS'].split(','))
+def py_version(request):
+    return request.param
+
+
+@pytest.fixture(params=os.environ['TEST_PROCESSORS'].split(','))
+def processor(request):
+    return request.param