add shellcheck + fixes

Author: mr-c

Makefile

@@ -171,6 +171,10 @@ mypy: $(filter-out setup.py gittagger.py,${PYSOURCES})
 mypyc: ${PYSOURCES}
 	MYPYPATH=typeshed/2and3/:typeshed/3 CWLTOOL_USE_MYPYC=1 pip install --verbose -e . && pytest --ignore cwltool/schemas --basetemp ./tmp
 
+shellcheck: FORCE
+	shellcheck build-cwl-docker.sh cwl-docker.sh release-test.sh travis.bash \
+		cwltool-in-docker.sh
+
 release-test: FORCE
 	git diff-index --quiet HEAD -- || ( echo You have uncommited changes, please commit them and try again; false )
 	./release-test.sh

build-cwl-docker.sh

@@ -4,8 +4,7 @@ docker build --file=cwltool.Dockerfile --tag=commonworkflowlanguage/cwltool_modu
 docker build --file=cwltool.Dockerfile --tag=commonworkflowlanguage/cwltool .
 
 version=$(git describe --tags)
-echo $version | grep -vq '\-' >& /dev/null
-if [ $? -eq 0 ];then
-    docker tag commonworkflowlanguage/cwltool_module commonworkflowlanguage/cwltool_module:$version
-    docker tag commonworkflowlanguage/cwltool commonworkflowlanguage/cwltool:$version
+if echo "$version" | grep -vq '\-' >& /dev/null ; then
+    docker tag commonworkflowlanguage/cwltool_module commonworkflowlanguage/cwltool_module:"$version"
+    docker tag commonworkflowlanguage/cwltool commonworkflowlanguage/cwltool:"$version"
 fi

cwltool-in-docker.sh

@@ -1,11 +1,13 @@
-#!/usr/bin/env sh
-if [ \( ! -S /var/run/docker.sock \) -a \( -z "$DOCKER_HOST" \) ]; then
+#!/bin/sh
+if [ -S /var/run/docker.sock ] && [ -z "$DOCKER_HOST" ]; then
   >&2 echo 'ERROR: cwltool cannot work inside a container without access to docker'
   >&2 echo 'Launch the container with the option -v /var/run/docker.sock:/var/run/docker.sock'
+  # shellcheck disable=SC2016
   >&2 echo 'or launch the container with the option to set $DOCKER_HOST'
   exit 1
 elif [ "$PWD" = '/error' ]; then
   >&2 echo 'ERROR: cwltool cannot work without access to the current path'
+  # shellcheck disable=SC2016
   >&2 echo 'Launch the container with the options -v "$PWD":"$PWD" -w="$PWD"'
   exit 1
 else

release-test.sh

@@ -14,7 +14,7 @@ run_tests() {
 	local mod_loc
 	mod_loc=$(pip show ${package} | 
 		grep ^Location | awk '{print $2}')/${module}
-	${test_prefix}bin/py.test "--ignore=${mod_loc}/schemas/" \
+	"${test_prefix}"bin/py.test "--ignore=${mod_loc}/schemas/" \
 		--pyargs -x ${module} -n auto --dist=loadfile
 }
 pipver=7.0.2 # minimum required version of pip
@@ -81,9 +81,9 @@ source bin/activate
 rm lib/python-wheels/setuptools* \
 	&& pip install --force-reinstall -U pip==${pipver} \
         && pip install setuptools==${setuptoolsver} wheel
-package_tar=${package}*tar.gz
+package_tar=$(find . -name "${package}*tar.gz")
 pip install "-r${DIR}/test-requirements.txt"
-pip install ${package_tar}  # [deps]
+pip install "${package_tar}"  # [deps]
 mkdir out
 tar --extract --directory=out -z -f ${package}*.tar.gz
 cd out/${package}*

tox.ini

@@ -5,6 +5,7 @@ envlist =
   py{36,37,38,39}-bandit,
   py{36,37,38}-mypy,
   py38-lint-readme,
+  py38-shellcheck,
   py38-pydocstyle
 
 skipsdist = True
@@ -24,14 +25,15 @@ description =
   py{36,37,38,39}-bandit: Search for common security issues
   py{36,37,38}-mypy: Check for type safety
   py38-pydocstyle: docstring style checker
+  py38-shellcheck: syntax check for shell scripts
 
 passenv =
   CI
   TRAVIS
   TRAVIS_*
   PROOT_NO_SECCOMP
 deps =
-  -rrequirements.txt
+  py{36,37,38,39}-{unit,lint,bandit,mypy}: -rrequirements.txt
   py{36,37,38,39}-unit: codecov
   py{36,37,38,39}-unit: pytest-xdist
   py{36,37,38,39}-unit: pytest-cov
@@ -58,11 +60,13 @@ commands =
   py{36,37,38,39}-lint: flake8 cwltool setup.py
   py{36,37,38}-mypy: make mypy
   py{36,37,38}-mypy: make mypyc
+  py38-shellcheck: make shellcheck
 
 whitelist_externals =
   py{36,37,38,39}-lint: flake8
   py{36,37,38,39}-lint: black
-  py{36,37,38}-mypy: make
+  py{36,37,38}-{mypy,shellcheck}: make
+  py38-shellcheck: shellcheck
 
 [testenv:py38-pydocstyle]
 whitelist_externals = make

travis.bash

@@ -1,41 +1,40 @@
 #!/bin/bash
 venv() {
         if ! test -d "$1" ; then
-                virtualenv -p python"${PYTHON_VERSION}" "$1"
+                virtualenv -p python3 "$1"
         fi
 	# shellcheck source=/dev/null
         source "$1"/bin/activate
 }
-
+version=${version:-v1.0}
 if [[ "$version" = "v1.0" ]] ; then
    wget https://github.com/common-workflow-language/common-workflow-language/archive/main.tar.gz
    tar xzf main.tar.gz && rm main.tar.gz
 else
-    repo=$(echo $version | sed 's/\(v[0-9]*\.\)\([0-9]*\).*/\1\2/')
-    wget https://github.com/common-workflow-language/cwl-${repo}/archive/main.tar.gz
+    # shellcheck disable=SC2001
+    repo=$(echo "$version" | sed 's/\(v[0-9]*\.\)\([0-9]*\).*/\1\2/')  
+    wget https://github.com/common-workflow-language/cwl-"${repo}"/archive/main.tar.gz
     tar xzf main.tar.gz && rm main.tar.gz
 fi
 
 docker pull node:slim
 
-for PYTHON_VERSION in 3
-do
+# shellcheck disable=SC2043
 for CONTAINER in docker
 # for CONTAINER in docker singularity
 # singularity having issues on ci.commonwl.org; tests pass with https://gist.github.com/mr-c/0ec90d717617d074017c0cb38b72d1a4
 do
-	venv cwltool-venv${PYTHON_VERSION}
-	# use pip2.7 and pip3 in separate loop runs
-	pip${PYTHON_VERSION} install -U setuptools wheel pip
-	pip${PYTHON_VERSION} uninstall -y cwltool
-	pip${PYTHON_VERSION} install -e .
-	pip${PYTHON_VERSION} install "cwltest>=1.0.20180518074130" codecov
+	venv cwltool-venv3
+	pip3 install -U setuptools wheel pip
+	pip3 uninstall -y cwltool
+	pip3 install -e .
+	pip3 install "cwltest>=1.0.20180518074130" codecov
 	if [[ "$version" = "v1.0" ]]
 	then
 		DRAFT="DRAFT=v1.0"
 		pushd common-workflow-language-main || exit 1
 	else
-		pushd cwl-${repo}-main || exit 1
+		pushd cwl-"${repo}"-main || exit 1
 	fi
 	rm -f .coverage* coverage.xml
 	source=$(realpath ../cwltool)
@@ -55,7 +54,7 @@ ignore_errors = True
 omit =
     tests/*
 EOF
-	CWLTOOL_WITH_COV=${PWD}/cwltool_with_cov${PYTHON_VERSION}
+	CWLTOOL_WITH_COV=${PWD}/cwltool_with_cov3
 	cat > "${CWLTOOL_WITH_COV}" <<EOF
 #!/bin/bash
 coverage run --parallel-mode --rcfile=${COVERAGE_RC} \
@@ -76,17 +75,18 @@ EOF
 	then
 		EXTRA="EXTRA=${EXTRA}"
 	fi
-	if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$version" = "v1.0" ]] && [[ "$CONTAINER" = "docker" ]] && [ $PYTHON_VERSION -eq 3 ]
+	if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$version" = "v1.0" ]] && [[ "$CONTAINER" = "docker" ]]
 	then
 		rm -Rf conformance
+                # shellcheck disable=SC2154
 		git clone http://"${jenkins_cwl_conformance}"@github.com/common-workflow-language/conformance.git
 
 		git -C conformance config user.email "[email protected]"
 		git -C conformance config user.name "CWL Jenkins build bot"
 		CONFORMANCE_MSG=$(cat << EOM
 Conformance test of cwltool ${tool_ver} for CWL ${version}
 Commit: ${GIT_COMMIT}
-Python version: ${PYTHON_VERSION}
+Python version: 3
 Container: ${CONTAINER}
 EOM
 )
@@ -98,28 +98,27 @@ EOM
 		BADGE=" --badgedir=${badgedir}"
 	fi
 	# shellcheck disable=SC2086
-	LC_ALL=C.UTF-8 ./run_test.sh --junit-xml=result${PYTHON_VERSION}.xml \
+	LC_ALL=C.UTF-8 ./run_test.sh --junit-xml=result3.xml \
 		RUNNER=${CWLTOOL_WITH_COV} "-j$(nproc)" ${BADGE} \
 		${DRAFT} "${EXTRA}" \
-		"--classname=py${PYTHON_VERSION}_${CONTAINER}"
+		"--classname=py3_${CONTAINER}"
 	# LC_ALL=C is to work around junit-xml ASCII only bug
 	CODE=$((CODE+$?)) # capture return code of ./run_test.sh
-	coverage combine "--rcfile=${COVERAGE_RC}" $(find . -name '.coverage.*')
+	coverage combine "--rcfile=${COVERAGE_RC}" "$(find . -name '.coverage.*')"
 	coverage xml "--rcfile=${COVERAGE_RC}"
 	codecov --file coverage.xml
 
 	if [ -d conformance ]
 	then
-		rm -rf conformance/cwltool/cwl_${version}/cwltool_latest
-		cp -r conformance/cwltool/cwl_${version}/cwltool_${tool_ver} conformance/cwltool/cwl_${version}/cwltool_latest
+		rm -rf conformance/cwltool/cwl_"${version}"/cwltool_latest
+		cp -r conformance/cwltool/cwl_"${version}"/cwltool_"${tool_ver}" conformance/cwltool/cwl_"${version}"/cwltool_latest
 		git -C conformance add --all
 		git -C conformance diff-index --quiet HEAD || git -C conformance commit -m "${CONFORMANCE_MSG}"
-		git -C conformance push http://${jenkins_cwl_conformance}:[email protected]/common-workflow-language/conformance.git
+		git -C conformance push http://"${jenkins_cwl_conformance}":[email protected]/common-workflow-language/conformance.git
 	fi
 
 	deactivate
-	popd
-done
+	popd || exit
 done
 # build new docker container
 if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$version" = "v1.0" ]]