Add support for CI_JOB_TOKEN

README.md

@@ -7,7 +7,7 @@
 1. Create a Gitlab private token and save it in `art` configuration:
 
     ```shell
-    art configure https://gitlab.example.com/ 'as1!df2@gh3#jk4$'
+    art configure https://gitlab.example.com/ --token-type private 'as1!df2@gh3#jk4$'
     ```
 
 2. Create `artifacts.yml` with definitions of needed artifacts:
@@ -56,7 +56,7 @@ Add the following commands to your `.gitlab-ci.yml`:
 ```yaml
 before_script:
   - sudo pip install https://github.com/kosma/art
-  - art configure <url> <token>
+  - art configure <url> --token-type {private,job} <token>
   - art download
   - art install
 cache:
@@ -72,9 +72,6 @@ automatically set to `.art-cache` so it can be preserved across jobs.
 
 ## Bugs and limitations
 
-* Gitlab's `$CI_BUILD_TOKEN` infrastructure doesn't support accessing artifacts,
-  so a private token must be used. This is very unfortunate and kludgey.
-  This might be fixed in future Gitlab releases (if I bug them hard enough).
 * Multiple Gitlab instances are not supported (and would be non-trivial to support).
 * Error handling is very rudimentary: any non-trivial exceptions simply propagate
   until Python dumps a stack trace.

art/_config.py

@@ -2,20 +2,66 @@
 
 from __future__ import absolute_import
 
-import re
 import os
+
+import click
+
 from . import _paths
 from . import _yaml
 
 
-def save(gitlab_url, private_token):
+class ConfigException(click.ClickException):
+    """An exception caused by invalid configuration settings."""
+    def __init__(self, config_key, message):
+        msg = 'config.{}: {}'.format(config_key, message)
+        super().__init__(msg)
+
+def save(gitlab_url, token_type, token):
     config = {
         'gitlab_url': gitlab_url,
-        'private_token': private_token,
+        'token_type': token_type,
+        'token': token,
     }
     _paths.mkdirs(os.path.dirname(_paths.config_file))
     _yaml.save(_paths.config_file, config)
 
-
 def load():
-    return _yaml.load(_paths.config_file)
+    config = _yaml.load(_paths.config_file)
+
+    if not config:
+        raise click.ClickException('No configuration found. Run "art configure" first.')
+
+    # convert old config to current representation
+    migrate(config)
+
+    validate(config)
+
+    return config
+
+def migrate(config):
+    """Perform conversions to maintain backwards-compatibility"""
+
+    # migrate legacy private_token value if it can be
+    # done without overwriting an existing value
+    if 'private_token' in config and 'token' not in config:
+        config['token'] = config['private_token']
+        del config['private_token']
+
+    # default to private tokens if not specified
+    if 'token_type' not in config:
+        config['token_type'] = 'private'
+
+def validate(config):
+    """Ensure the configuration meets expectations"""
+
+    required_fields = ('token', 'token_type', 'gitlab_url')
+    for field in required_fields:
+        if field not in config:
+            raise ConfigException(field, 'Required config element is missing. Run "art configure".')
+
+    # warn the user if they have both private_token and token configured
+    # the private_token element is a legacy field and should be removed
+    if 'private_token' in config and 'token' in config:
+        click.secho('Warning: ', nl=False, fg='yellow')
+        click.echo('Config includes both "token" and "private_token" elements. ', nl=False)
+        click.echo('Only the "token" value will be used.')

art/_paths.py

@@ -1,6 +1,5 @@
 import errno
 import os
-import sys
 
 import appdirs
 

art/command_line.py

@@ -19,8 +19,26 @@
 
 def get_gitlab():
     config = _config.load()
-    return Gitlab(config['gitlab_url'], private_token=config['private_token'])
+    if config['token_type'] == 'private':
+        return Gitlab(config['gitlab_url'], private_token=config['token'])
+    if config['token_type'] == 'job':
+        return Gitlab(config['gitlab_url'], job_token=config['token'])
 
+    raise _config.ConfigException('token_type', 'Unknown token type: {}'.format(config['token_type']))
+
+def is_using_job_token(gitlab):
+    """Determine if the GitLab client will use a job token to authenticate.
+
+    Job tokens cannot access the full GitLab API. See the documentation here:
+    https://docs.gitlab.com/ee/ci/jobs/ci_job_token.html
+
+    The client only uses a job token when other tokens are unavailable.
+    """
+    # private and oauth tokens will be used, if available
+    if gitlab.private_token is not None or gitlab.oauth_token is not None:
+        return False
+
+    return gitlab.job_token is not None
 
 def get_ref_last_successful_job(project, ref, job_name):
     pipelines = project.pipelines.list(as_list=False, ref=ref, order_by='id', sort='desc')
@@ -31,7 +49,7 @@ def get_ref_last_successful_job(project, ref, job_name):
                 # Turn ProjectPipelineJob into ProjectJob
                 return project.jobs.get(job.id, lazy=True)
 
-    raise Exception("Could not find latest successful '{}' job for {} ref {}".format(
+    raise click.ClickException("Could not find latest successful '{}' job for {} ref {}".format(
         job_name, project.path_with_namespace, ref))
 
 
@@ -86,7 +104,8 @@ def main(cache):
 
 @main.command()
 @click.argument('gitlab_url')
-@click.argument('private_token')
+@click.option('--token-type', type=click.Choice(['private', 'job']), default='private')
+@click.argument('token')
 def configure(**kwargs):
     """Configure Gitlab URL and access token."""
 
@@ -98,8 +117,13 @@ def update():
     """Update latest tag/branch job IDs."""
 
     gitlab = get_gitlab()
-    artifacts = _yaml.load(_paths.artifacts_file)
 
+    # With current GitLab (16.3, as of this writing)
+    # You cannot access the projects and jobs API endpoints using a job token
+    if is_using_job_token(gitlab):
+        raise _config.ConfigException('token_type', 'A job token cannot be used to update artifacts')
+
+    artifacts = _yaml.load(_paths.artifacts_file)
     for entry in artifacts:
         proj = gitlab.projects.get(entry['project'])
         entry['job_id'] = get_ref_last_successful_job(proj, entry['ref'], entry['job']).id
@@ -122,7 +146,9 @@ def download():
             _cache.get(filename)
         except KeyError:
             click.echo('* %s: %s => downloading...' % (entry['project'], entry['job_id']))
-            proj = gitlab.projects.get(entry['project'])
+            # Use shallow objects for proj and job to allow compatibility with
+            # job tokens where only the artifacts endpoint is accessible.
+            proj = gitlab.projects.get(entry['project'], lazy=True)
             job = proj.jobs.get(entry['job_id'], lazy=True)
             with _cache.save_file(filename) as f:
                 job.artifacts(streamed=True, action=f.write)

setup.py

@@ -23,7 +23,7 @@ def read_project_file(path):
         'PyYAML',
         'appdirs',
         'click',
-        'python-gitlab',
+        'python-gitlab>=1.12.0',
     ],
     entry_points={
         'console_scripts': [