✨ [open-api-framework#188] add csv option to data dump script

.github/workflows/bin-check.yml

@@ -39,6 +39,7 @@ jobs:
           python src/manage.py migrate
           src/manage.py loaddata demodata
           SCRIPTPATH=bin DUMP_FILE=dump.sql bin/dump_data.sh --combined
+          SCRIPTPATH=bin ZIP_FILE=dump.zip bin/dump_data.sh --csv
         env:
           DB_PASSWORD: ""
           DB_USER: postgres
@@ -56,3 +57,12 @@ jobs:
         run: |
           createdb -h localhost -U postgres test
           psql -v ON_ERROR_STOP=1 -h localhost -U postgres -d test -f dump.sql
+
+      - name: validate csv dump
+        run: |
+          unzip dump.zip
+          test -f core_object.csv || exit 1
+          ! test -f auth_group.csv || exit 1
+          grep "id,uuid,object_type_id,created_on,modified_on" core_object.csv
+
+

bin/dump_data.sh

@@ -10,7 +10,9 @@
 # or --combined which appends the data dump to the schema dump.
 # The schema dump could not use -t to filter tables because this excludes extensions like postgis in the dump.
 # pg_dump also does not add related tables automatically, so `dump_data.sh` does not add related data from accounts to the dump.
-
+#
+# with --csv a csv dump can be created for all tables in the given components. The csv files will be generated in the temporary directory csv_dumps
+# and combined into a single ZIP archive csv_dumps.
 
 set -e
 
@@ -27,32 +29,38 @@ SCRIPTPATH=$(dirname "$SCRIPT")
 
 ${SCRIPTPATH}/wait_for_db.sh
 
-DUMP_FILE=${DUMP_FILE:-"dump_$(date +'%Y-%m-%d_%H-%M-%S').sql"}
+DEFAULT_FILE_NAME="dump_$(date +'%Y-%m-%d_%H-%M-%S')"
+DUMP_FILE=${DUMP_FILE:-"$DEFAULT_FILE_NAME.sql"}
+ZIP_FILE=${ZIP_FILE:-"$DEFAULT_FILE_NAME.zip"}
+CSV_OUTPUT_DIR="csv_dumps"
 
+CSV=false
 SCHEMA=true
 DATA=true
 COMBINED=false
 
 for arg in "$@"; do
-  case "$arg" in
+    case "$arg" in
+    --csv) CSV=true ;;
     --schema-only) DATA=false ;;
-    --data-only)   SCHEMA=false ;;
-    --combined)    COMBINED=true ;;
+    --data-only) SCHEMA=false ;;
+    --combined) COMBINED=true ;;
     --*)
-      echo "Unknown flag: $arg"
-      exit 1
-      ;;
+        echo "Unknown flag: $arg"
+        exit 1
+        ;;
     *)
-      APPS+=("$arg") ;;
-  esac
+        APPS+=("$arg")
+        ;;
+    esac
 done
 
 # export given apps or export DEFAULT_APPS
 if [ "${#APPS[@]}" -eq 0 ]; then
-  APPS=("${DEFAULT_APPS[@]}")
+    APPS=("${DEFAULT_APPS[@]}")
 fi
 
->&2 echo "exporting: ${APPS[*]}"
+echo >&2 "exporting: ${APPS[*]}"
 
 # create -t flags for each app
 INCLUDES=()
@@ -61,32 +69,59 @@ for app in "${APPS[@]}"; do
 done
 
 dump_schema() {
-  echo "Dumping schema to $1..."
-  pg_dump --schema-only -f "$1"
+    echo "Dumping schema to $1..."
+    pg_dump --schema-only -f "$1"
 }
 
 dump_data() {
-  echo "Dumping data to $1..."
-  pg_dump "${INCLUDES[@]}" --disable-triggers --data-only > "$1"
+    echo "Dumping data to $1..."
+    pg_dump "${INCLUDES[@]}" --disable-triggers --data-only >"$1"
 }
 
 append_data() {
-  echo "Appending data to $1..."
-  pg_dump "${INCLUDES[@]}" --disable-triggers --data-only \
-    | sed '/^SET\|^SELECT pg_catalog.set_config/d' >> "$1"
+    echo "Appending data to $1..."
+    pg_dump "${INCLUDES[@]}" --disable-triggers --data-only |
+        sed '/^SET\|^SELECT pg_catalog.set_config/d' >>"$1"
 }
 
+dump_csv() {
+    mkdir -p $CSV_OUTPUT_DIR
+    echo "Dumping data to csv..."
+
+    WHERE_CLAUSE=""
+    for app in "${APPS[@]}"; do
+        if [ -n "$WHERE_CLAUSE" ]; then
+            WHERE_CLAUSE+=" OR "
+        fi
+        WHERE_CLAUSE+="tablename LIKE '${app}_%'"
+    done
+
+    TABLES=$(psql -Atc "SELECT tablename FROM pg_tables WHERE schemaname='public' AND ($WHERE_CLAUSE);")
+
+    for table in $TABLES; do
+        echo "dumping $table..."
+        psql -c "\copy $table TO '$CSV_OUTPUT_DIR/$table.csv' WITH CSV HEADER"
+    done
+
+    zip -j "$ZIP_FILE" "$CSV_OUTPUT_DIR"/*.csv
+    rm -rf "$CSV_OUTPUT_DIR"
+}
+
+if $CSV; then
+    dump_csv
+    exit 0
+fi
 
 if $COMBINED; then
-  dump_schema "$DUMP_FILE"
-  append_data "$DUMP_FILE"
-  exit 0
+    dump_schema "$DUMP_FILE"
+    append_data "$DUMP_FILE"
+    exit 0
 fi
 
 if $SCHEMA; then
-  dump_schema "schema__$DUMP_FILE"
+    dump_schema "schema__$DUMP_FILE"
 fi
 
 if $DATA; then
-  dump_data "data__$DUMP_FILE"
+    dump_data "data__$DUMP_FILE"
 fi

docs/manual/scripts.rst

@@ -7,7 +7,7 @@ Scripts
 Dump data
 ---------
 
-Met het script ``dump_data.sh`` kan de data van alle componenten (core) worden geëxporteerd naar een sql bestand.
+Met het script ``dump_data.sh`` kan de data van alle componenten (core) worden geëxporteerd naar een sql of csv bestand(en).
 
 Dit script is niet bedoeld voor een data migratie naar een andere Objects Api of Objecttypes Api instantie.
 
@@ -23,6 +23,8 @@ Om alleen specifieke data te exporteren kunnen de gewenste component namen worde
 
     om een postgres 17 db te exporteren is de package postgres-client-17 vereist.
 
+Met de flag ``--csv`` worden alle tabellen in de meegegeven componenten geëxporteerd naar csv bestanden. Deze bestanden worden tijdelijk in ``csv_dumps`` geplaatst en gecombineerd in een ZIP bestand.
+
 Environment variabelen
 ----------------------
 
@@ -32,6 +34,7 @@ Environment variabelen
 * DB_NAME (objects/objecttypes)
 * DB_PASSWORD ("")
 * DUMP_FILE ("dump_$(date +'%Y-%m-%d_%H-%M-%S').sql")
+* CSV_FILE ("dump_$(date +'%Y-%m-%d_%H-%M-%S').zip")
 
 .. code-block:: shell