diff --git a/src/acquisition/covidcast/database.py b/src/acquisition/covidcast/database.py index 70660b60c..24597c8e1 100644 --- a/src/acquisition/covidcast/database.py +++ b/src/acquisition/covidcast/database.py @@ -326,6 +326,7 @@ def delete_batch(self, cc_deletions): ''' drop_tmp_table_sql = f'DROP TABLE {tmp_table_name}' + total = None try: self._cursor.execute(create_tmp_table_sql) self._cursor.execute(amend_tmp_table_sql) @@ -337,12 +338,18 @@ def delete_batch(self, cc_deletions): raise Exception(f"Bad deletions argument: need a filename or a list of tuples; got a {type(cc_deletions)}") self._cursor.execute(add_id_sql) self._cursor.execute(delete_sql) + total = self._cursor.rowcount self._cursor.execute(fix_latest_issue_sql) self._connection.commit() + + if total == -1: + # the SQL connector does not support returning number of rows affected (see PEP 249) + total = None except Exception as e: raise e finally: self._cursor.execute(drop_tmp_table_sql) + return total def compute_covidcast_meta(self, table_name='covidcast', use_index=True): """Compute and return metadata on all non-WIP COVIDcast signals.""" diff --git a/src/acquisition/covidcast/delete_batch.py b/src/acquisition/covidcast/delete_batch.py index 33bc6751d..fe40897fd 100644 --- a/src/acquisition/covidcast/delete_batch.py +++ b/src/acquisition/covidcast/delete_batch.py @@ -2,6 +2,7 @@ # standard library import argparse +import glob import os import time @@ -22,19 +23,21 @@ def get_argument_parser(): help="filename for log output (defaults to stdout)") return parser -def handle_file(deletion_file, database): +def handle_file(deletion_file, database, logger): logger.info("Deleting from csv file", filename=deletion_file) rows = [] with open(deletion_file) as f: for line in f: - rows.append(line.strip().split(",")) + fields = line.strip().split(",") + if len(fields) < 9: continue + rows.append(fields + ["day"]) rows = rows[1:] try: n = database.delete_batch(rows) logger.info("Deleted database rows", row_count=n) return n except Exception as e: - logger.exception('Exception while deleting rows:', e) + logger.exception('Exception while deleting rows', exception=e) database.rollback() return 0 @@ -49,7 +52,11 @@ def main(args): try: for deletion_file in sorted(glob.glob(os.path.join(args.deletion_dir, '*.csv'))): - all_n += handle_file(deletion_file) + n = handle_file(deletion_file, database, logger) + if n is not None: + all_n += n + else: + all_n = "rowcount unsupported" finally: database.disconnect(True)