[WIP] LM Workload

.github/workflows/CI.yml

@@ -19,8 +19,8 @@ jobs:
         pip install .[pytorch_cpu]
         pip install .[full]
         pip install -e .
-        python tests/reference_algorithm_tests.py --workload=fastmri     --framework=pytorch     --global_batch_size=8     --submission_path=reference_algorithms/target_setting_algorithms/pytorch_momentum.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/fastmri/tuning_search_space.json
-        python tests/reference_algorithm_tests.py --workload=fastmri     --framework=jax     --global_batch_size=8     --submission_path=reference_algorithms/target_setting_algorithms/jax_momentum.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/fastmri/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=fastmri     --framework=pytorch     --global_batch_size=8     --submission_path=algorithms/target_setting_algorithms/pytorch_momentum.py     --tuning_search_space=algorithms/target_setting_algorithms/fastmri/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=fastmri     --framework=jax     --global_batch_size=8     --submission_path=algorithms/target_setting_algorithms/jax_momentum.py     --tuning_search_space=algorithms/target_setting_algorithms/fastmri/tuning_search_space.json
   wmt_jax:
     runs-on: ubuntu-latest
     steps:
@@ -37,7 +37,7 @@ jobs:
         pip install .[pytorch_cpu]
         pip install .[full]
         pip install -e .
-        python tests/reference_algorithm_tests.py --workload=wmt     --framework=jax     --global_batch_size=2     --submission_path=reference_algorithms/target_setting_algorithms/jax_nadamw.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/wmt/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=wmt     --framework=jax     --global_batch_size=2     --submission_path=algorithms/target_setting_algorithms/jax_nadamw.py     --tuning_search_space=algorithms/target_setting_algorithms/wmt/tuning_search_space.json
   wmt_pytorch:
     runs-on: ubuntu-latest
     steps:
@@ -54,7 +54,7 @@ jobs:
         pip install .[pytorch_cpu]
         pip install .[full]
         pip install -e .
-        python tests/reference_algorithm_tests.py --workload=wmt     --framework=pytorch     --global_batch_size=2     --submission_path=reference_algorithms/target_setting_algorithms/pytorch_nadamw.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/wmt/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=wmt     --framework=pytorch     --global_batch_size=2     --submission_path=algorithms/target_setting_algorithms/pytorch_nadamw.py     --tuning_search_space=algorithms/target_setting_algorithms/wmt/tuning_search_space.json
   imagenet_jax:
     runs-on: ubuntu-latest
     steps:
@@ -71,8 +71,8 @@ jobs:
         pip install .[pytorch_cpu]
         pip install .[full]
         pip install -e .
-        python tests/reference_algorithm_tests.py --workload=imagenet_vit     --framework=jax     --global_batch_size=2     --submission_path=reference_algorithms/target_setting_algorithms/jax_adamw.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json
-        python tests/reference_algorithm_tests.py --workload=imagenet_resnet     --framework=jax     --global_batch_size=2     --submission_path=reference_algorithms/target_setting_algorithms/jax_momentum.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=imagenet_vit     --framework=jax     --global_batch_size=2     --submission_path=algorithms/target_setting_algorithms/jax_adamw.py     --tuning_search_space=algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=imagenet_resnet     --framework=jax     --global_batch_size=2     --submission_path=algorithms/target_setting_algorithms/jax_momentum.py     --tuning_search_space=algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json
   imagenet_pytorch:
     runs-on: ubuntu-latest
     steps:
@@ -89,8 +89,8 @@ jobs:
         pip install .[pytorch_cpu]
         pip install .[full]
         pip install -e .
-        python tests/reference_algorithm_tests.py --workload=imagenet_resnet     --framework=pytorch     --global_batch_size=2     --submission_path=reference_algorithms/target_setting_algorithms/pytorch_momentum.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json
-        python tests/reference_algorithm_tests.py --workload=imagenet_vit     --framework=pytorch     --global_batch_size=2     --submission_path=reference_algorithms/target_setting_algorithms/pytorch_adamw.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=imagenet_resnet     --framework=pytorch     --global_batch_size=2     --submission_path=algorithms/target_setting_algorithms/pytorch_momentum.py     --tuning_search_space=algorithms/target_setting_algorithms/imagenet_resnet/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=imagenet_vit     --framework=pytorch     --global_batch_size=2     --submission_path=algorithms/target_setting_algorithms/pytorch_adamw.py     --tuning_search_space=algorithms/target_setting_algorithms/imagenet_vit/tuning_search_space.json
   # uncomment when https://github.com/mlcommons/algorithmic-efficiency/issues/339 is resolved.
   criteo_jax:
     runs-on: ubuntu-latest
@@ -108,7 +108,7 @@ jobs:
         pip install .[pytorch_cpu]
         pip install .[full]
         pip install -e .
-        python tests/reference_algorithm_tests.py --workload=criteo1tb_test     --framework=jax     --global_batch_size=1     --submission_path=reference_algorithms/target_setting_algorithms/jax_adamw.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=criteo1tb_test     --framework=jax     --global_batch_size=1     --submission_path=algorithms/target_setting_algorithms/jax_adamw.py     --tuning_search_space=algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json
   criteo_pytorch:
     runs-on: ubuntu-latest
     steps:
@@ -125,7 +125,7 @@ jobs:
         pip install .[pytorch_cpu]
         pip install .[full]
         pip install -e .
-        python tests/reference_algorithm_tests.py --workload=criteo1tb_test     --framework=pytorch     --global_batch_size=1     --submission_path=reference_algorithms/target_setting_algorithms/pytorch_adamw.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=criteo1tb_test     --framework=pytorch     --global_batch_size=1     --submission_path=algorithms/target_setting_algorithms/pytorch_adamw.py     --tuning_search_space=algorithms/target_setting_algorithms/criteo1tb/tuning_search_space.json
   speech_jax:
     runs-on: ubuntu-latest
     steps:
@@ -142,8 +142,8 @@ jobs:
         pip install .[pytorch_cpu]
         pip install .[full]
         pip install -e .
-        python tests/reference_algorithm_tests.py --workload=librispeech_conformer     --framework=jax     --global_batch_size=2     --submission_path=reference_algorithms/target_setting_algorithms/jax_adamw.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json
-        python tests/reference_algorithm_tests.py --workload=librispeech_deepspeech     --framework=jax     --global_batch_size=2     --submission_path=reference_algorithms/target_setting_algorithms/jax_adamw.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=librispeech_conformer     --framework=jax     --global_batch_size=2     --submission_path=algorithms/target_setting_algorithms/jax_adamw.py     --tuning_search_space=algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=librispeech_deepspeech     --framework=jax     --global_batch_size=2     --submission_path=algorithms/target_setting_algorithms/jax_adamw.py     --tuning_search_space=algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json
   speech_pytorch:
     runs-on: ubuntu-latest
     steps:
@@ -160,8 +160,8 @@ jobs:
         pip install .[pytorch_cpu]
         pip install .[full]
         pip install -e .
-        python tests/reference_algorithm_tests.py --workload=librispeech_deepspeech     --framework=pytorch     --global_batch_size=2     --submission_path=reference_algorithms/target_setting_algorithms/pytorch_adamw.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json
-        python tests/reference_algorithm_tests.py --workload=librispeech_conformer     --framework=pytorch     --global_batch_size=2     --submission_path=reference_algorithms/target_setting_algorithms/pytorch_adamw.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=librispeech_deepspeech     --framework=pytorch     --global_batch_size=2     --submission_path=algorithms/target_setting_algorithms/pytorch_adamw.py     --tuning_search_space=algorithms/target_setting_algorithms/librispeech_deepspeech/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=librispeech_conformer     --framework=pytorch     --global_batch_size=2     --submission_path=algorithms/target_setting_algorithms/pytorch_adamw.py     --tuning_search_space=algorithms/target_setting_algorithms/librispeech_conformer/tuning_search_space.json
   ogbg:
     runs-on: ubuntu-latest
     steps:
@@ -178,9 +178,9 @@ jobs:
         pip install .[pytorch_cpu]
         pip install .[full]
         pip install -e .
-        python tests/reference_algorithm_tests.py --workload=ogbg     --framework=pytorch     --global_batch_size=8     --submission_path=reference_algorithms/target_setting_algorithms/pytorch_nesterov.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/ogbg/tuning_search_space.json
-        python tests/reference_algorithm_tests.py --workload=ogbg     --framework=jax     --global_batch_size=8     --submission_path=reference_algorithms/target_setting_algorithms/jax_nesterov.py     --tuning_search_space=reference_algorithms/target_setting_algorithms/ogbg/tuning_search_space.json
-  pytest:
+        python tests/reference_algorithm_tests.py --workload=ogbg     --framework=pytorch     --global_batch_size=8     --submission_path=algorithms/target_setting_algorithms/pytorch_nesterov.py     --tuning_search_space=algorithms/target_setting_algorithms/ogbg/tuning_search_space.json
+        python tests/reference_algorithm_tests.py --workload=ogbg     --framework=jax     --global_batch_size=8     --submission_path=algorithms/target_setting_algorithms/jax_nesterov.py     --tuning_search_space=algorithms/target_setting_algorithms/ogbg/tuning_search_space.json
+  pytest-params:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3

.github/workflows/linting.yml

@@ -3,54 +3,35 @@ name: Linting
 on: [push, pull_request]
 
 jobs:
-  pylint:
+  ruff-linting:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python 3.11.10
       uses: actions/setup-python@v2
       with:
         python-version: 3.11.10
-    - name: Install pylint
+    - name: Install ruff
       run: |
         python -m pip install --upgrade pip
-        pip install pylint==2.16.1
-    - name: Run pylint
+        pip install ruff==0.12.0
+    - name: Run ruff linter
       run: |
-        pylint algoperf
-        pylint reference_algorithms
-        pylint prize_qualification_baselines
-        pylint submission_runner.py
-        pylint tests
+        ruff check
 
-  isort:
+  ruff-formatter:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python 3.11.10
       uses: actions/setup-python@v2
       with:
         python-version: 3.11.10
-    - name: Install isort
+    - name: Install ruff
       run: |
         python -m pip install --upgrade pip
-        pip install isort==5.12.0
-    - name: Run isort
+        pip install ruff==0.12.0
+    - name: Run ruff formatter
       run: |
-        isort . --check --diff
+        ruff format --check
 
-  yapf:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.11.10
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.11.10
-    - name: Install yapf
-      run: |
-        python -m pip install --upgrade pip
-        pip install yapf==0.32 toml
-    - name: Run yapf
-      run: |
-        yapf . --diff --recursive