Merge branch 'main' into leave_input_staging_directory

Author: mr-c

.coveragerc

@@ -9,6 +9,7 @@ exclude_lines =
     pragma: no cover
     raise NotImplementedError
     if __name__ == .__main__.:
+    if TYPE_CHECKING:
 ignore_errors = True
 omit =
     tests/*

.flake8

@@ -0,0 +1,8 @@
+[flake8]
+ignore = E203,W503
+max-line-length = 100
+select = B,C,E,F,W,T4
+exclude = cwltool/schemas
+extend-ignore = E501,B905
+# when Python 3.10 is the minimum version, re-enable check B905 for zip + strict
+extend-select = B9

.git-blame-ignore-revs

@@ -0,0 +1,8 @@
+# isort
+46e0485a889453dc178a878b5b5ebbfc7e4eb5f1
+
+# upgrade to black 20.8b1
+6fd6fdb381fe3f347627fd517a8f2dba7b0a7029
+
+# upgrade to black 23.1, longer lines (100)
+7fe8c0739b0515d00daabc7db87bc5fad926d345

.github/workflows/ci-tests.yml

@@ -1,4 +1,4 @@
-name: Continuous integration tests
+name: CI Tests
 
 on:
   push:
@@ -12,20 +12,35 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  singularity_version: 3.8.3
+  TOX_SKIP_MISSING_INTERPRETERS: False
+  # Rich (pip)
+  FORCE_COLOR: 1
+  # Tox
+  PY_COLORS: 1
+  # Mypy (see https://github.com/python/mypy/issues/7771)
+  TERM: xterm-color
+  MYPY_FORCE_COLOR: 1
+  MYPY_FORCE_TERMINAL_WIDTH: 200
+  # Pytest
+  PYTEST_ADDOPTS: --color=yes
 
 jobs:
 
   tox:
-    name: CI tests via Tox
-
-    runs-on: ubuntu-20.04
-
+    name: Tox
+    runs-on: ubuntu-20.04  # 22.04 doesn't support Python 3.6
     strategy:
       matrix:
         py-ver-major: [3]
-        py-ver-minor: [6, 7, 8, 9, 10]
+        py-ver-minor: [6, 7, 8, 9, 10, 11]
         step: [lint, unit, bandit, mypy]
+        exclude:
+          - py-ver-major: 3
+            py-ver-minor: 6
+            step: mypy
+          - py-ver-major: 3
+            py-ver-minor: 6
+            step: lint
 
     env:
       py-semver: ${{ format('{0}.{1}', matrix.py-ver-major, matrix.py-ver-minor) }}
@@ -38,13 +53,13 @@ jobs:
 
       - name: Set up Singularity
         if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }}
-        uses: eWaterCycle/setup-singularity@v7
-        with:
-          singularity-version: ${{ env.singularity_version }}
+        run: |
+          wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb
+          sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb
 
       - name: Give the test runner user a name to make provenance happy.
         if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }}
-        run: sudo usermod -c 'CI Runner' $(whoami)
+        run: sudo usermod -c 'CI Runner' "$(whoami)"
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -58,7 +73,7 @@ jobs:
       - name: Upgrade setuptools and install tox
         run: |
           pip install -U pip setuptools wheel
-          pip install tox tox-gh-actions
+          pip install "tox<4" "tox-gh-actions<3"
 
       - name: MyPy cache
         if: ${{ matrix.step == 'mypy' }}
@@ -75,19 +90,19 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   tox-style:
-    name: CI linters via Tox
-
+    name: Linters
     runs-on: ubuntu-20.04
 
     strategy:
       matrix:
         step: [lintreadme, shellcheck, pydocstyle]
 
     env:
-      py-semver: "3.10"
-      TOXENV: ${{ format('py310-{0}', matrix.step) }}
+      py-semver: "3.11"
+      TOXENV: ${{ format('py311-{0}', matrix.step) }}
 
     steps:
       - uses: actions/checkout@v3
@@ -103,7 +118,7 @@ jobs:
       - name: Upgrade setuptools and install tox
         run: |
           pip install -U pip setuptools wheel
-          pip install tox tox-gh-actions
+          pip install "tox<4" "tox-gh-actions<3"
 
       - if: ${{ matrix.step == 'pydocstyle' && github.event_name == 'pull_request'}}
         name: Create local branch for diff-quality for PRs
@@ -112,61 +127,119 @@ jobs:
       - name: Test with tox
         run: tox
 
-  conformance_tests:
-    name: CWL spec conformance tests
+  clean_working_dir:
+    name: No leftovers
+    runs-on: ubuntu-22.04
+    env:
+      py-semver: "3.11"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
-    runs-on: ubuntu-20.04
+      - name: Set up Singularity
+        run: |
+          wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb
+          sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb
+
+      - name: Give the test runner user a name to make provenance happy.
+        run: sudo usermod -c 'CI Runner' "$(whoami)"
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.py-semver }}
+          cache: pip
+
+      - name: install with test dependencies
+        run: |
+          pip install -U pip setuptools wheel
+          pip install --no-build-isolation -rtest-requirements.txt .[deps]
+
+      - name: make working directory read-only
+        run: |
+          mkdir .pytest_cache
+          chmod a-w .
+
+      - name: run tests
+        run: make test
+
+
+  conformance_tests:
+    name: CWL conformance
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
         cwl-version: [v1.0, v1.1, v1.2]
         container: [docker, singularity, podman]
+        extras: [""]
+        include:
+          - cwl-version: v1.2
+            container: docker
+            extras: "--fast-parser"
 
     steps:
       - uses: actions/checkout@v3
 
       - name: Set up Singularity
-        uses: eWaterCycle/setup-singularity@v7
+        if: ${{ matrix.container == 'singularity' }}
+        run: |
+          wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb
+          sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb
+
+      - name: Singularity cache
+        if: ${{ matrix.container == 'singularity' }}
+        uses: actions/cache@v3
         with:
-          singularity-version: ${{ env.singularity_version }}
+          path: sifcache
+          key: singularity
+
+      - name: Set up Podman
+        if: ${{ matrix.container == 'podman' }}
+        run: sudo rm -f /usr/bin/docker ; sudo apt-get install -y podman
 
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: 3.11
           cache: pip
 
-      - name: Run CWL conformance tests ${{ matrix.cwl-version }}
+      - name: "Test CWL ${{ matrix.cwl-version }} conformance"
         env:
-          version: ${{ matrix.cwl-version }}
-          container: ${{ matrix.container }}
-          spec_branch: main
+          VERSION: ${{ matrix.cwl-version }}
+          CONTAINER: ${{ matrix.container }}
+          GIT_TARGET: main
+          CWLTOOL_OPTIONS: ${{ matrix.extras }}
         run: ./conformance-test.sh
-
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
   release_test:
     name: cwltool release test
-
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
       - uses: actions/checkout@v3
 
       - name: Set up Singularity
-        uses: eWaterCycle/setup-singularity@v7
-        with:
-          singularity-version: ${{ env.singularity_version }}
+        run: |
+          wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb
+          sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb
 
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"  # quoted, otherwise that turns into the number 3.1
+          python-version: 3.11
           cache: pip
           cache-dependency-path: |
             requirements.txt
             test-requirements.txt
 
       - name: Give the test runner user a name to make provenance happy.
-        run: sudo usermod -c 'CI Runner' $(whoami)
+        run: sudo usermod -c 'CI Runner' "$(whoami)"
 
       - name: Install packages
         run: |
@@ -177,3 +250,50 @@ jobs:
         env:
           RELEASE_SKIP: head
         run: ./release-test.sh
+
+  build_test_container:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: build & test cwltool_module container
+        run: ./build-cwltool-docker.sh
+
+  macos:
+    name: Test on macos-latest
+    runs-on: macos-latest
+    env:
+      TOXENV: py311-unit
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          cache: pip
+          cache-dependency-path: |
+            requirements.txt
+            tox.ini
+      - name: Upgrade setuptools and install tox
+        run: |
+          pip install -U pip setuptools wheel
+          pip install "tox<4" "tox-gh-actions<3"
+      # # docker for mac install is not currently stable
+      # - name: 'SETUP MacOS: load Homebrew cache'
+      #   uses: actions/cache@v3
+      #   if: runner.os == 'macOS'
+      #   with:
+      #      path: |
+      #        ~/Library/Caches/Homebrew/downloads/*--Docker.dmg
+      #      key: brew-actions-setup-docker-1.0.11
+      #      restore-keys: brew-actions-setup-docker-
+      # - name: setup docker on macos (default stable version)
+      #   uses: docker-practice/actions-setup-docker@master
+      - name: Test with tox
+        run: tox
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/quay-publish.yml

@@ -12,29 +12,40 @@ jobs:
       - name: Get image tags
         id: image_tags
         run: |
-          echo -n ::set-output name=IMAGE_TAGS::${GITHUB_REF#refs/*/}
+          echo -n ::set-output "name=IMAGE_TAGS::${GITHUB_REF#refs/*/}"
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-multi-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-multi-buildx
       - name: Login to Quay.io
         uses: docker/login-action@v2
         with:
           registry: ${{ secrets.REGISTRY_SERVER }}
           username: ${{ secrets.REGISTRY_USERNAME }}
           password: ${{ secrets.REGISTRY_PASSWORD }}
       - name: Build and publish cwltool_module image to Quay
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: cwltool.Dockerfile
-          tags: quay.io/commonwl/cwltool_module:${{ steps.image_tags.outputs.IMAGE_TAGS }}
+          tags: quay.io/commonwl/cwltool_module:${{ steps.image_tags.outputs.IMAGE_TAGS }},quay.io/commonwl/cwltool_module:latest
           target: module
+          platforms: linux/amd64,linux/arm64
           push: true
           cache-from: type=gha
           cache-to: type=gha,mode=max
       - name: Build and publish cwltool image to Quay
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: cwltool.Dockerfile
-          tags: quay.io/commonwl/cwltool:${{ steps.image_tags.outputs.IMAGE_TAGS }}
+          tags: quay.io/commonwl/cwltool:${{ steps.image_tags.outputs.IMAGE_TAGS }},quay.io/commonwl/cwltool:latest
+          platforms: linux/amd64,linux/arm64
           push: true
           cache-from: type=gha
           cache-to: type=gha,mode=max

.gitignore

@@ -28,10 +28,8 @@ venv3/
 # pycharm
 .idea/
 
-# typshed repo
-typeshed/2and3/schema_salad
-typeshed/2and3/ruamel/yaml
-
+# local stubs
+mypy-stubs/ruamel/yaml
 
 #mypy
 .mypy_cache/
@@ -54,3 +52,9 @@ time.txt
 value
 
 .python-version
+
+# Folder created when using make
+cwltool_deps
+docs/_build/
+docs/autoapi/
+