diff --git a/Dockerfile b/Dockerfile index e0a5d9f5c..d154e5408 100644 --- a/Dockerfile +++ b/Dockerfile @@ -98,6 +98,8 @@ FROM base AS development COPY --from=development-sandboxes /sio2/sandboxes /sio2/sandboxes RUN chmod +x /sio2/oioioi/download_sandboxes.sh +# For production: Upload sandboxes to built-in filetracker during build +# For dev: These will be re-uploaded to s3dedup at runtime via oioioi_init.sh RUN ./manage.py supervisor > /dev/null --daemonize --nolaunch=uwsgi && \ /sio2/oioioi/wait-for-it.sh -t 60 "127.0.0.1:9999" && \ ./manage.py upload_sandboxes_to_filetracker -d /sio2/sandboxes && \ diff --git a/UPGRADING.rst b/UPGRADING.rst index 7bb33f39d..735bd2cf1 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -1121,3 +1121,49 @@ List of changes since the *CONFIG_VERSION* numbering was introduced: +# ) * Remove all sentry and raven reminiscent from settings.py in LOGGING SECTION. + +Migrating from Filetracker to s3dedup (currently only for docker-compose-dev.yml deployments) +---------------------------------------------------------------- + +This guide covers migrating from the built-in Filetracker to s3dedup in Docker Compose +development deployments. This is currently only needed deployments using ``docker-compose-dev.yml``. +If you don't have any data you want to keep, you don't have to do anything. + +#. Stop the s3dedup container:: + + docker-compose -f docker-compose-dev.yml stop s3dedup + +#. In ``docker-compose-dev.yml``, modify the web container environment variables: + + * Remove ``FILETRACKER_SERVER_ENABLED=False`` (or set to ``True``) + * Set ``FILETRACKER_LISTEN_ADDR=0.0.0.0`` + * Set ``FILETRACKER_LISTEN_PORT=9999`` + +#. Restart the web container:: + + docker-compose -f docker-compose-dev.yml down web + docker-compose -f docker-compose-dev.yml up -d web + +#. Run the migration:: + + docker-compose -f docker-compose-dev.yml run --rm s3dedup migrate --env \ + --filetracker-url http://web:9999 \ + --max-concurrency 10 + + The migration will display progress and statistics about files migrated and + deduplication savings. + +#. Once migration completes, restore the environment variables in ``docker-compose-dev.yml``: + + * Remove or comment out ``FILETRACKER_LISTEN_ADDR`` and ``FILETRACKER_LISTEN_PORT`` + * Set ``FILETRACKER_SERVER_ENABLED=False`` + +#. Restart the web container and start s3dedup:: + + docker-compose -f docker-compose-dev.yml down web + docker-compose -f docker-compose-dev.yml up -d web s3dedup + +#. Verify the migration by checking that files are accessible through s3dedup. + +For more details about the migration process, see the s3dedup documentation at +``https://github.com/sio2project/s3dedup``. diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 6a4a1c5c7..505d3f878 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,5 +1,5 @@ x-common-envs: &common-envs - FILETRACKER_URL: 'http://web:9999' + FILETRACKER_URL: 'http://s3dedup:9999/ft' DATABASE_HOST: 'db' DATABASE_PORT: '5432' @@ -30,8 +30,7 @@ services: RABBITMQ_PORT: '5672' RABBITMQ_USER: 'oioioi' RABBITMQ_PASSWORD: 'oioioi' - FILETRACKER_LISTEN_ADDR: '0.0.0.0' - FILETRACKER_LISTEN_PORT: '9999' + FILETRACKER_SERVER_ENABLED: 'False' <<: *common-envs ports: # web server @@ -49,6 +48,7 @@ services: depends_on: - db - broker + - s3dedup worker: image: sio2project/oioioi-dev command: ["/sio2/oioioi/worker_init.sh"] @@ -68,8 +68,55 @@ services: RABBITMQ_DEFAULT_USER: oioioi RABBITMQ_DEFAULT_PASS: oioioi stop_grace_period: 1m + minio: + image: minio/minio:RELEASE.2024-10-02T17-50-41Z + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + volumes: + - minio-data-dev:/data + command: server /data --console-address ":9001" + ports: + - "9000:9000" # MinIO API + - "9001:9001" # MinIO Console + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + stop_grace_period: 1m + s3dedup: + image: ghcr.io/sio2project/s3dedup:latest-dev + ports: + - "9999:9999" + environment: + LOG_LEVEL: debug + BUCKET_NAME: oioioi-filetracker + LISTEN_ADDRESS: 0.0.0.0 + LISTEN_PORT: 9999 + KVSTORAGE_TYPE: sqlite + SQLITE_PATH: /app/data/kv.db + SQLITE_MAX_CONNECTIONS: 10 + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: minioadmin + S3_SECRET_KEY: minioadmin + S3_FORCE_PATH_STYLE: "true" + CLEANER_ENABLED: "true" + CLEANER_INTERVAL: 3600 + CLEANER_BATCH_SIZE: 1000 + CLEANER_MAX_DELETES: 10000 + volumes: + - s3dedup-data-dev:/app/data + depends_on: + minio: + condition: service_healthy + restart: on-failure:5 + stop_grace_period: 1m volumes: postgress-data-dev: filetracker-data-dev: node_modules-dev: dist_webpack-dev: + minio-data-dev: + s3dedup-data-dev: diff --git a/oioioi/default_settings.py b/oioioi/default_settings.py index 3cb2be490..108a0dcf4 100755 --- a/oioioi/default_settings.py +++ b/oioioi/default_settings.py @@ -359,7 +359,7 @@ }, } -FILETRACKER_SERVER_ENABLED = True +FILETRACKER_SERVER_ENABLED = os.getenv('FILETRACKER_SERVER_ENABLED', 'True').lower() not in ('false', '0', 'no', 'off') FILETRACKER_LISTEN_ADDR = os.getenv('FILETRACKER_LISTEN_ADDR', '127.0.0.1') FILETRACKER_LISTEN_PORT = os.getenv('FILETRACKER_LISTEN_PORT', 9999) diff --git a/oioioi_init.sh b/oioioi_init.sh index 5954d0481..cb82b0b5a 100755 --- a/oioioi_init.sh +++ b/oioioi_init.sh @@ -7,8 +7,22 @@ set -x if [ "$1" == "--dev" ]; then ./manage.py migrate 2>&1 | tee /sio2/deployment/logs/migrate.log ./manage.py loaddata ../oioioi/extra/dbdata/default_admin.json + + # Upload sandboxes to filetracker (s3dedup) on first run + SANDBOX_MARKER="/sio2/deployment/media/.sandboxes_uploaded" + if [ ! -f "$SANDBOX_MARKER" ]; then + echo "Uploading sandboxes to filetracker (this may take ~30 seconds)..." + # Extract filetracker host from FILETRACKER_URL (format: http://host:port/path) + FT_HOST=$(echo $FILETRACKER_URL | sed 's|http://||' | sed 's|/.*||') + /sio2/oioioi/wait-for-it.sh -t 120 "$FT_HOST" && \ + ./manage.py upload_sandboxes_to_filetracker -d /sio2/sandboxes && \ + touch "$SANDBOX_MARKER" && \ + echo "Sandboxes uploaded successfully" + else + echo "Sandboxes already uploaded, skipping..." + fi fi echo "Init Finished" -exec ./manage.py supervisor --logfile=/sio2/deployment/logs/supervisor.log \ No newline at end of file +exec ./manage.py supervisor --logfile=/sio2/deployment/logs/supervisor.log