Replace filetracker with s3dedup in dev docker compose

Dockerfile

@@ -98,6 +98,8 @@ FROM base AS development
 COPY --from=development-sandboxes /sio2/sandboxes /sio2/sandboxes
 RUN chmod +x /sio2/oioioi/download_sandboxes.sh
 
+# For production: Upload sandboxes to built-in filetracker during build
+# For dev: These will be re-uploaded to s3dedup at runtime via oioioi_init.sh
 RUN ./manage.py supervisor > /dev/null --daemonize --nolaunch=uwsgi && \
     /sio2/oioioi/wait-for-it.sh -t 60 "127.0.0.1:9999" && \
     ./manage.py upload_sandboxes_to_filetracker -d /sio2/sandboxes && \

UPGRADING.rst

@@ -1121,3 +1121,49 @@ List of changes since the *CONFIG_VERSION* numbering was introduced:
         +# )
 
      * Remove all sentry and raven reminiscent from settings.py in LOGGING SECTION.
+
+Migrating from Filetracker to s3dedup (currently only for docker-compose-dev.yml deployments)
+----------------------------------------------------------------
+
+This guide covers migrating from the built-in Filetracker to s3dedup in Docker Compose
+development deployments. This is currently only needed deployments using ``docker-compose-dev.yml``.
+If you don't have any data you want to keep, you don't have to do anything.
+
+#. Stop the s3dedup container::
+
+    docker-compose -f docker-compose-dev.yml stop s3dedup
+
+#. In ``docker-compose-dev.yml``, modify the web container environment variables:
+
+   * Remove ``FILETRACKER_SERVER_ENABLED=False`` (or set to ``True``)
+   * Set ``FILETRACKER_LISTEN_ADDR=0.0.0.0``
+   * Set ``FILETRACKER_LISTEN_PORT=9999``
+
+#. Restart the web container::
+
+    docker-compose -f docker-compose-dev.yml down web
+    docker-compose -f docker-compose-dev.yml up -d web
+
+#. Run the migration::
+
+    docker-compose -f docker-compose-dev.yml run --rm s3dedup migrate --env \
+      --filetracker-url http://web:9999 \
+      --max-concurrency 10
+
+   The migration will display progress and statistics about files migrated and
+   deduplication savings.
+
+#. Once migration completes, restore the environment variables in ``docker-compose-dev.yml``:
+
+   * Remove or comment out ``FILETRACKER_LISTEN_ADDR`` and ``FILETRACKER_LISTEN_PORT``
+   * Set ``FILETRACKER_SERVER_ENABLED=False``
+
+#. Restart the web container and start s3dedup::
+
+    docker-compose -f docker-compose-dev.yml down web
+    docker-compose -f docker-compose-dev.yml up -d web s3dedup
+
+#. Verify the migration by checking that files are accessible through s3dedup.
+
+For more details about the migration process, see the s3dedup documentation at
+``https://github.com/sio2project/s3dedup``.

docker-compose-dev.yml

@@ -1,5 +1,5 @@
 x-common-envs: &common-envs
- FILETRACKER_URL: 'http://web:9999'
+ FILETRACKER_URL: 'http://s3dedup:9999/ft'
  DATABASE_HOST: 'db'
  DATABASE_PORT: '5432'
 
@@ -30,8 +30,7 @@ services:
       RABBITMQ_PORT: '5672'
       RABBITMQ_USER: 'oioioi'
       RABBITMQ_PASSWORD: 'oioioi'
-      FILETRACKER_LISTEN_ADDR: '0.0.0.0'
-      FILETRACKER_LISTEN_PORT: '9999'
+      FILETRACKER_SERVER_ENABLED: 'False'
       <<: *common-envs
     ports:
     # web server
@@ -49,6 +48,7 @@ services:
     depends_on:
       - db
       - broker
+      - s3dedup
   worker:
     image: sio2project/oioioi-dev
     command: ["/sio2/oioioi/worker_init.sh"]
@@ -68,8 +68,55 @@ services:
       RABBITMQ_DEFAULT_USER: oioioi
       RABBITMQ_DEFAULT_PASS: oioioi
     stop_grace_period: 1m
+  minio:
+    image: minio/minio:RELEASE.2024-10-02T17-50-41Z
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    volumes:
+      - minio-data-dev:/data
+    command: server /data --console-address ":9001"
+    ports:
+      - "9000:9000"  # MinIO API
+      - "9001:9001"  # MinIO Console
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+      start_period: 5s
+    stop_grace_period: 1m
+  s3dedup:
+    image: ghcr.io/sio2project/s3dedup:latest-dev
+    ports:
+      - "9999:9999"
+    environment:
+      LOG_LEVEL: debug
+      BUCKET_NAME: oioioi-filetracker
+      LISTEN_ADDRESS: 0.0.0.0
+      LISTEN_PORT: 9999
+      KVSTORAGE_TYPE: sqlite
+      SQLITE_PATH: /app/data/kv.db
+      SQLITE_MAX_CONNECTIONS: 10
+      S3_ENDPOINT: http://minio:9000
+      S3_ACCESS_KEY: minioadmin
+      S3_SECRET_KEY: minioadmin
+      S3_FORCE_PATH_STYLE: "true"
+      CLEANER_ENABLED: "true"
+      CLEANER_INTERVAL: 3600
+      CLEANER_BATCH_SIZE: 1000
+      CLEANER_MAX_DELETES: 10000
+    volumes:
+      - s3dedup-data-dev:/app/data
+    depends_on:
+      minio:
+        condition: service_healthy
+    restart: on-failure:5
+    stop_grace_period: 1m
 volumes:
   postgress-data-dev:
   filetracker-data-dev:
   node_modules-dev:
   dist_webpack-dev:
+  minio-data-dev:
+  s3dedup-data-dev:

oioioi/default_settings.py

@@ -359,7 +359,7 @@
     },
 }
 
-FILETRACKER_SERVER_ENABLED = True
+FILETRACKER_SERVER_ENABLED = os.getenv('FILETRACKER_SERVER_ENABLED', 'True').lower() not in ('false', '0', 'no', 'off')
 FILETRACKER_LISTEN_ADDR = os.getenv('FILETRACKER_LISTEN_ADDR', '127.0.0.1')
 FILETRACKER_LISTEN_PORT = os.getenv('FILETRACKER_LISTEN_PORT', 9999)
 

oioioi_init.sh

@@ -7,8 +7,22 @@ set -x
 if [ "$1" == "--dev" ]; then
     ./manage.py migrate 2>&1 | tee /sio2/deployment/logs/migrate.log
     ./manage.py loaddata ../oioioi/extra/dbdata/default_admin.json
+
+    # Upload sandboxes to filetracker (s3dedup) on first run
+    SANDBOX_MARKER="/sio2/deployment/media/.sandboxes_uploaded"
+    if [ ! -f "$SANDBOX_MARKER" ]; then
+        echo "Uploading sandboxes to filetracker (this may take ~30 seconds)..."
+        # Extract filetracker host from FILETRACKER_URL (format: http://host:port/path)
+        FT_HOST=$(echo $FILETRACKER_URL | sed 's|http://||' | sed 's|/.*||')
+        /sio2/oioioi/wait-for-it.sh -t 120 "$FT_HOST" && \
+            ./manage.py upload_sandboxes_to_filetracker -d /sio2/sandboxes && \
+            touch "$SANDBOX_MARKER" && \
+            echo "Sandboxes uploaded successfully"
+    else
+        echo "Sandboxes already uploaded, skipping..."
+    fi
 fi
 
 echo "Init Finished"
 
-exec ./manage.py supervisor --logfile=/sio2/deployment/logs/supervisor.log
+exec ./manage.py supervisor --logfile=/sio2/deployment/logs/supervisor.log