Rework the modeling of install actions and report failures

art/_install.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+
+import os
+import shutil
+import stat
+import click
+
+from . import _cache
+from . import _paths
+
+class InstallUnmatchedError(click.ClickException):
+    """An exception raised when an artifact was not found"""
+
+    error = '''Source path(s) did not match any files/non-empty directories in the archive:
+  archive: {archive}
+  project: {project}
+  job: {job}
+  ref: {ref}
+    {unmatched}'''
+
+    def __init__(self, filename, entry, unmatched):
+        unmatched_desc = ('{} => {}'.format(src, dst) for src, dst in unmatched.items())
+
+        message = self.error.format(
+            archive=_cache.cache_path(filename),
+            unmatched='\n    '.join(unmatched_desc),
+            **entry)
+
+        super().__init__(message)
+
+class InstallAction():
+    """Represents a user request to install a file from an artifact archive"""
+
+    S_IRWXUGO = 0o0777
+
+    def __init__(self, source, destination):
+        self.src = source
+        self.dest = destination
+
+        if source == '.':
+            # "copy all" filter
+            self._match = lambda f: True
+            self.translate = lambda f: os.path.join(self.dest, f)
+        elif source.endswith('/'):
+            # 1:1 directory filter
+            self._match = lambda f: f.startswith(self.src)
+            self.translate = lambda f: os.path.join(self.dest, f[len(self.src):])
+        else:
+            # 1:1 file filter
+            self._match = lambda f: f == self.src
+            self.translate = lambda f: self.dest
+
+    def match(self, filepath):
+        """Compare an archive file using the source file pattern
+        Arguments:
+            filepath    The archive filepath to evaluate
+
+        Returns: True if the pattern matched the filepath, otherwise False
+        """
+        return self._match(filepath)
+
+    def install(self, archive, member):
+        """Perform the install action on a zip archive member
+
+        Parameters:
+        archive     Archive from which to extract the file
+        member      ZipInfo identifying the file to install
+        """
+        access = None
+        filemode_str = ""
+
+        # Translate the archive path to the install destination
+        target = self.translate(member.filename)
+
+        # if create_system is Unix (3), external_attr contains filesystem permissions
+        if member.create_system == 3:
+            filemode = member.external_attr >> 16
+
+            # Keep only the normal permissions bits;
+            # ignore special bits like setuid, setgid, sticky
+            access = filemode & InstallAction.S_IRWXUGO
+            filemode_str = '   ' + stat.filemode(stat.S_IFMT(filemode) | access)
+
+        click.echo('* install: %s => %s%s' % (member.filename, target, filemode_str))
+
+        if member.filename.endswith('/'):
+            _paths.mkdirs(target)
+        else:
+            if os.sep in target:
+                _paths.mkdirs(os.path.dirname(target))
+            with archive.open(member) as fmember:
+                with open(target, 'wb') as ftarget:
+                    shutil.copyfileobj(fmember, ftarget)
+
+        if access is not None:
+            os.chmod(target, access)
+
+    def __str__(self):
+        return '{} => {}'.format(self.src, self.dest)

art/command_line.py

@@ -3,20 +3,17 @@
 from __future__ import absolute_import
 
 import os
-import shutil
-import stat
 import sys
 import zipfile
 import click
 from gitlab import Gitlab
 from . import _cache
 from . import _config
+from . import _install
 from . import _paths
 from . import _yaml
 from . import __version__ as version
 
-S_IRWXUGO = 0o0777
-
 def get_gitlab():
     config = _config.load()
     if config['token_type'] == 'private':
@@ -57,37 +54,6 @@ def zip_name(project, job_id):
     return os.path.join(project, '{}.zip'.format(job_id))
 
 
-def install_member(archive, member, target):
-    """Install a zip archive member
-
-    Parameters:
-    archive     Archive from which to extract the file
-    member      ZipInfo identifying the file to extract
-    target      Path to which the file is extracted
-    """
-    access = None
-    filemode_str = ""
-
-    # if create_system is Unix (3), external_attr contains filesystem permissions
-    if member.create_system == 3:
-        filemode = member.external_attr >> 16
-
-        # Keep only the normal permissions bits;
-        # ignore special bits like setuid, setgid, sticky
-        access = filemode & S_IRWXUGO
-        filemode_str = '   ' + stat.filemode(stat.S_IFMT(filemode) | access)
-
-    click.echo('* install: %s => %s%s' % (member.filename, target, filemode_str))
-    if os.sep in target:
-        _paths.mkdirs(os.path.dirname(target))
-    with archive.open(member) as fmember:
-        with open(target, 'wb') as ftarget:
-            shutil.copyfileobj(fmember, ftarget)
-
-    if access is not None:
-        os.chmod(target, access)
-
-
 @click.group()
 @click.version_option(version, prog_name='art')
 @click.option('--cache', '-c', help='Download cache directory.')
@@ -158,37 +124,18 @@ def download():
 
 
 @main.command()
-def install():
+@click.option('--keep-empty-dirs', '-k', default=False, is_flag=True, help='Do not prune empty directories.')
+def install(keep_empty_dirs):
     """Install artifacts to current directory."""
 
     artifacts_lock = _yaml.load(_paths.artifacts_lock_file)
 
     for entry in artifacts_lock:
-        # convert the "install" dictionary to list of (match, translate)
-        installs = []
-        for source, destination in entry['install'].items():
-            # Nb. Defaults parameters on lambda are required due to derpy
-            #     Python closure semantics (scope capture).
-            if source == '.':
-                # "copy all" filter
-                installs.append((
-                    lambda f, s=source, d=destination: True,
-                    lambda f, s=source, d=destination: os.path.join(d, f)
-                ))
-            elif source.endswith('/'):
-                # 1:1 directory filter
-                installs.append((
-                    lambda f, s=source, d=destination: f.startswith(s),
-                    lambda f, s=source, d=destination: os.path.join(d, f[len(s):])
-                ))
-            else:
-                # 1:1 file filter
-                installs.append((
-                    lambda f, s=source, d=destination: f == s,
-                    lambda f, s=source, d=destination: d
-                ))
-        # make sure there are no bugs in the lambdas above
-        del source, destination # pylint: disable=undefined-loop-variable
+        # dictionary of src:dest pairs representing artifacts to install
+        install_requests = entry['install']
+
+        # create an InstallAction (file match and translate) for each request
+        actions = [_install.InstallAction(src, dest) for src, dest in install_requests.items()]
 
         # open the artifacts.zip archive
         filename = zip_name(entry['project'], entry['job_id'])
@@ -197,10 +144,22 @@ def install():
 
         # iterate over the zip archive
         for member in archive.infolist():
-            if member.filename.endswith('/'):
-                # skip directories, they will be created as-is
+            # Skip directory members
+            # - Parent directories are created when installing files
+            # - The keep_empty_dirs option preserves the original archive tree
+            if not keep_empty_dirs and member.filename.endswith('/'):
                 continue
-            for match, translate in installs:
-                if match(member.filename):
-                    target = translate(member.filename)
-                    install_member(archive, member, target)
+
+            # perform installs that match this member
+            for action in actions:
+                if not action.match(member.filename):
+                    continue
+
+                action.install(archive, member)
+
+                # remove requests that are successfully installed
+                install_requests.pop(action.src, None)
+
+        # Report an error if any requested artifacts were not installed
+        if install_requests:
+            raise _install.InstallUnmatchedError(filename, entry, install_requests)