diff --git a/cwltool/job.py b/cwltool/job.py index 28bed29d7..1e23d8d3e 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -79,7 +79,7 @@ def relink_initialworkdir( container_outdir: str, inplace_update: bool = False, ) -> None: - for _, vol in pathmapper.items(): + for _, vol in pathmapper.items_exclude_children(): if not vol.staged: continue diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index c89fac5f4..6e39c7540 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -4,6 +4,7 @@ import stat import urllib import uuid +from pathlib import Path from typing import Dict, Iterator, List, Optional, Tuple, cast from schema_salad.exceptions import ValidationException @@ -191,6 +192,16 @@ def files(self) -> List[str]: def items(self) -> List[Tuple[str, MapperEnt]]: return list(self._pathmap.items()) + def items_exclude_children(self) -> List[Tuple[str, MapperEnt]]: + newitems = {} + keys = [key for key, entry in self.items()] + for key, entry in self.items(): + parents = Path(key).parents + if any([Path(key_) in parents for key_ in keys]): + continue + newitems[key] = entry + return list(newitems.items()) + def reversemap( self, target: str, diff --git a/cwltool/process.py b/cwltool/process.py index c55f5c101..a21b1f291 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -263,8 +263,9 @@ def stage_files( fix_conflicts: bool = False, ) -> None: """Link or copy files to their targets. Create them as needed.""" + items = pathmapper.items() if not symlink else pathmapper.items_exclude_children() targets = {} # type: Dict[str, MapperEnt] - for key, entry in pathmapper.items(): + for key, entry in items: if "File" not in entry.type: continue if entry.target not in targets: @@ -287,7 +288,7 @@ def stage_files( % (targets[entry.target].resolved, entry.resolved, entry.target) ) - for key, entry in pathmapper.items(): + for key, entry in items: if not entry.staged: continue if not os.path.exists(os.path.dirname(entry.target)): diff --git a/tests/test_iwdr.py b/tests/test_iwdr.py index 48712da23..163492c89 100644 --- a/tests/test_iwdr.py +++ b/tests/test_iwdr.py @@ -25,6 +25,25 @@ def test_empty_file_creation() -> None: assert err_code == 0 +def test_passthrough_successive(tmp_path: Path) -> None: + """An empty file can be successively passed through a subdir of InitialWorkingDirectory.""" + err_code, _, _ = get_main_output( + [ + "--outdir", + str(tmp_path), + get_data("tests/wf/iwdr-passthrough-successive.cwl"), + ] + ) + assert err_code == 0 + children = sorted( + tmp_path.glob("*") + ) # This input directory should be left pristine. + assert len(children) == 1 + subdir = tmp_path / children[0] + assert len(sorted(subdir.glob("*"))) == 1 + assert (subdir / "file").exists() + + @needs_docker def test_directory_literal_with_real_inputs_inside(tmp_path: Path) -> None: """Cope with unmoveable files in the output directory created by Docker+IWDR.""" diff --git a/tests/wf/iwdr-passthrough-successive.cwl b/tests/wf/iwdr-passthrough-successive.cwl new file mode 100644 index 000000000..ef7d79983 --- /dev/null +++ b/tests/wf/iwdr-passthrough-successive.cwl @@ -0,0 +1,71 @@ +#!/usr/bin/env cwl-runner + +cwlVersion: v1.0 +class: Workflow + +inputs: [] + +steps: + # Create a test directory structure; could be done outside CWL and passed in as input. + # This input directory should be left pristine. + mkdirs: + run: + class: CommandLineTool + baseCommand: [bash, '-c', 'mkdir dir dir/subdir && touch dir/subdir/file', '-'] + inputs: [] + outputs: + mkdirs_out: + type: Directory + outputBinding: + glob: dir + in: [] + out: [mkdirs_out] + + # Given an input directory, emit a subdirectory as output. + passthrough1: + run: + class: CommandLineTool + requirements: + - class: InitialWorkDirRequirement + listing: + - entry: $(inputs.passthrough1_in) + writable: false + baseCommand: ["true"] + inputs: + passthrough1_in: + type: Directory + outputs: + passthrough1_subdir: + type: Directory + outputBinding: + glob: $(inputs.passthrough1_in.basename)/subdir + in: + passthrough1_in: mkdirs/mkdirs_out + out: [passthrough1_subdir] + + # Given a (sub-)directory, emit it unchanged. + passthrough2: + run: + class: CommandLineTool + requirements: + - class: InitialWorkDirRequirement + listing: + - entry: $(inputs.passthrough2_in) + writable: false + baseCommand: ["true"] + inputs: + passthrough2_in: + type: Directory + outputs: + passthrough2_subdir: + type: Directory + outputBinding: + glob: $(inputs.passthrough2_in.basename) + in: + passthrough2_in: passthrough1/passthrough1_subdir + out: [passthrough2_subdir] + +outputs: + out: + type: Directory + outputSource: passthrough2/passthrough2_subdir \ No newline at end of file