ML Pipeline CI/CD #53
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ML Pipeline CI/CD | |
| on: | |
| push: | |
| branches: [main, develop] | |
| pull_request: | |
| branches: [main] | |
| schedule: | |
| - cron: '0 2 * * *' # Daily at 2 AM | |
| env: | |
| AWS_REGION: us-east-1 | |
| PYTHON_VERSION: '3.9' | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install -r requirements-dev.txt | |
| - name: Run tests | |
| run: | | |
| pytest tests/ -v --cov=src --cov-report=xml | |
| - name: Upload coverage | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| file: ./coverage.xml | |
| build: | |
| needs: test | |
| runs-on: ubuntu-latest | |
| if: github.ref == 'refs/heads/main' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| - name: Build and push Docker images | |
| env: | |
| ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} | |
| IMAGE_TAG: ${{ github.sha }} | |
| run: | | |
| # Build training image | |
| docker build -t $ECR_REGISTRY/ml-training:$IMAGE_TAG -f docker/training/Dockerfile . | |
| docker push $ECR_REGISTRY/ml-training:$IMAGE_TAG | |
| # Build inference image | |
| docker build -t $ECR_REGISTRY/ml-inference:$IMAGE_TAG -f docker/inference/Dockerfile . | |
| docker push $ECR_REGISTRY/ml-inference:$IMAGE_TAG | |
| deploy: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| if: github.ref == 'refs/heads/main' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Deploy SageMaker Pipeline | |
| run: | | |
| pip install sagemaker boto3 | |
| python pipelines/sagemaker/pipeline.py |