Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cirro/cirro_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class CirroApi:
"""
Client for interacting directly with the Cirro API
"""
def __init__(self, auth_info: AuthInfo = None, base_url: str = None):
def __init__(self, auth_info: AuthInfo = None, base_url: str = None, user_agent: str = 'Cirro SDK'):
"""
Instantiates the Cirro API object

Expand Down Expand Up @@ -40,7 +40,7 @@ def __init__(self, auth_info: AuthInfo = None, base_url: str = None):
self._api_client = CirroApiClient(
base_url=self._configuration.rest_endpoint,
auth_method=auth_info.get_auth_method(),
client_name='Cirro SDK',
client_name=user_agent,
package_name='cirro'
)

Expand Down
80 changes: 31 additions & 49 deletions cirro/cli/controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
from cirro.models.process import PipelineDefinition, ConfigAppStatus, CONFIG_APP_URL
from cirro.services.service_helpers import list_all_datasets

NO_PROJECTS = "No projects available"
# Log to STDOUT
log_formatter = logging.Formatter(
'%(asctime)s %(levelname)-8s [Cirro CLI] %(message)s'
Expand All @@ -37,13 +36,8 @@

def run_list_datasets(input_params: ListArguments, interactive=False):
"""List the datasets available in a particular project."""
_check_configure()
cirro = CirroApi()
logger.info(f"Collecting data from {cirro.configuration.base_url}")
projects = cirro.projects.list()

if len(projects) == 0:
raise InputError(NO_PROJECTS)
cirro = _init_cirro_client()
projects = _get_projects(cirro)

if interactive:
# Prompt the user for the project
Expand All @@ -63,17 +57,10 @@ def run_list_datasets(input_params: ListArguments, interactive=False):


def run_ingest(input_params: UploadArguments, interactive=False):
_check_configure()
cirro = CirroApi()
logger.info(f"Collecting data from {cirro.configuration.base_url}")
cirro = _init_cirro_client()
projects = _get_projects(cirro)
processes = cirro.processes.list(process_type=Executor.INGEST)

logger.info("Listing available projects")
projects = cirro.projects.list()

if len(projects) == 0:
raise InputError(NO_PROJECTS)

if interactive:
input_params, files = gather_upload_arguments(input_params, projects, processes)
directory = input_params['data_directory']
Expand Down Expand Up @@ -121,15 +108,8 @@ def run_ingest(input_params: UploadArguments, interactive=False):


def run_validate_folder(input_params: ValidateArguments, interactive=False):
_check_configure()
cirro = CirroApi()
logger.info(f"Collecting data from {cirro.configuration.base_url}")

logger.info("Listing available projects")
projects = cirro.projects.list()

if len(projects) == 0:
raise InputError(NO_PROJECTS)
cirro = _init_cirro_client()
projects = _get_projects(cirro)

if interactive:
input_params = gather_validate_arguments(input_params, projects)
Expand All @@ -154,34 +134,27 @@ def run_validate_folder(input_params: ValidateArguments, interactive=False):

logger.info("Validating files")

validation_results = cirro.datasets.validate_folder(
results = cirro.datasets.validate_folder(
project_id=project_id,
dataset_id=dataset_id,
local_folder=input_params['data_directory']
)

for file_list, label, log_level in [
(validation_results.files_matching, "✅ Matched Files (identical in Cirro and locally)", logging.INFO),
(validation_results.files_not_matching, "⚠️ Checksum Mismatches (same file name, different content)", logging.WARNING),
(validation_results.files_missing, "⚠️ Missing Locally (present in system but not found locally)", logging.WARNING),
(validation_results.local_only_files, "⚠️ Unexpected Local Files (present locally but not in system)", logging.WARNING),
(validation_results.validate_errors, "⚠️ Validation Failed (checksums may not be available)", logging.WARNING)
(results.files_matching, "✅ Matched Files (identical in Cirro and locally)", logging.INFO),
(results.files_not_matching, "⚠️ Checksum Mismatches (same file name, different content)", logging.WARNING),
(results.files_missing, "⚠️ Missing Locally (present in system but not found locally)", logging.WARNING),
(results.local_only_files, "⚠️ Unexpected Local Files (present locally but not in system)", logging.WARNING),
(results.validate_errors, "⚠️ Validation Failed (checksums may not be available)", logging.WARNING)
]:
logger.log(level=log_level, msg=f"{label}: {len(file_list):,}")
for file in file_list:
logger.log(level=log_level, msg=f" - {file}")


def run_download(input_params: DownloadArguments, interactive=False):
_check_configure()
cirro = CirroApi()
logger.info(f"Collecting data from {cirro.configuration.base_url}")

logger.info("Listing available projects")
projects = cirro.projects.list()

if len(projects) == 0:
raise InputError(NO_PROJECTS)
cirro = _init_cirro_client()
projects = _get_projects(cirro)

files_to_download = None
if interactive:
Expand Down Expand Up @@ -229,15 +202,9 @@ def run_download(input_params: DownloadArguments, interactive=False):


def run_upload_reference(input_params: UploadReferenceArguments, interactive=False):
_check_configure()
cirro = CirroApi()
logger.info(f"Collecting data from {cirro.configuration.base_url}")

cirro = _init_cirro_client()
projects = _get_projects(cirro)
reference_types = cirro.references.get_types()
projects = cirro.projects.list()

if len(projects) == 0:
raise InputError(NO_PROJECTS)

if interactive:
input_params, files = gather_reference_upload_arguments(input_params, projects, reference_types)
Expand Down Expand Up @@ -308,6 +275,21 @@ def run_create_pipeline_config(input_params: CreatePipelineConfigArguments, inte
f"{CONFIG_APP_URL}")


def _init_cirro_client():
_check_configure()
cirro = CirroApi(user_agent="Cirro CLI")
logger.info(f"Collecting data from {cirro.configuration.base_url}")
return cirro


def _get_projects(cirro: CirroApi):
logger.info("Listing available projects")
projects = cirro.projects.list()
if len(projects) == 0:
raise InputError("No projects available")
return projects


def _check_configure():
"""
Prompts the user to do initial configuration if needed
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "cirro"
version = "1.8.0"
version = "1.9.0"
description = "CLI tool and SDK for interacting with the Cirro platform"
authors = ["Cirro Bio <[email protected]>"]
license = "MIT"
Expand Down
Loading