[Profiling/GitHub workflow] Added profiling infrastructure & improvements to GitHub Action workflows

.github/workflows/a100_profiler.yml

@@ -0,0 +1,40 @@
+name: A100 Performance Profiler
+
+on:
+  schedule:
+    - cron:  '00 00 * * *'
+
+jobs:
+  sycl_A100_Profiling:
+    name: SYCL A100 Profiling
+    env:
+      SYCL_NAME_PREFIX: sycl_AMD-Epyc-7313_a100_gcc-11.3_cuda-12.0.1
+      ENABLE_CI_PROFILER: 1
+
+      MADGRAPH4GPU_DB_URL: ${{ secrets.MADGRAPH4GPU_DB_URL }}
+      MADGRAPH4GPU_DB_SECRET: ${{ secrets.MADGRAPH4GPU_DB_SECRET }}
+    runs-on: [self-hosted, linux, a100]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Runs SYCL performanceProfiler.py script
+      run: cd tools/profiling/;
+           python3 performanceProfiler.py -l 'SYCL' -b 'master'
+    - name: Uploads SYCL JSON files to DB
+      run: cd tools/profiling/; python3 sendData.py --absLayer SYCL --profiler 1 --branch master
+
+  cuda_a100_Profiling:
+    name: CUDA A100 Profiling
+    env:
+      CUDA_NAME_PREFIX: cudacpp_AMD-Epyc-7313_a100_gcc-11.2.1_cuda-12.0.1
+      ENABLE_CI_PROFILER: 1
+
+      MADGRAPH4GPU_DB_URL: ${{ secrets.MADGRAPH4GPU_DB_URL }}
+      MADGRAPH4GPU_DB_SECRET: ${{ secrets.MADGRAPH4GPU_DB_SECRET }}
+    runs-on: [self-hosted, linux, a100]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Runs CUDA performanceProfiler.py script
+      run: cd tools/profiling/;
+           python3 performanceProfiler.py -l 'CUDA' -b 'master'
+    - name: Uploads CUDA JSON files to DB
+      run: cd tools/profiling/; python3 sendData.py --absLayer CUDA --profiler 1 --branch master

.github/workflows/c-cpp.yml

@@ -11,17 +11,26 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        folder: [ epochX/cudacpp/ee_mumu.sa/SubProcesses/P1_Sigma_sm_epem_mupmum , epoch1/cuda/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum , epoch2/cuda/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum ]
+        folder: [ epochX/cudacpp/ee_mumu.sa/SubProcesses/P1_Sigma_sm_epem_mupmum,
+                  epochX/cudacpp/gg_tt.sa/SubProcesses/P1_Sigma_sm_gg_ttx,
+                  epochX/cudacpp/gg_ttg.sa/SubProcesses/P1_Sigma_sm_gg_ttxg,
+                  epochX/cudacpp/gg_ttgg.sa/SubProcesses/P1_Sigma_sm_gg_ttxgg,
+                  epochX/cudacpp/gg_ttggg.sa/SubProcesses/P1_Sigma_sm_gg_ttxggg ]
       fail-fast: false
     steps:
+
     - uses: actions/checkout@v2
-    - name: make epoch1
+    - name: make epochX
       run: make -C ${{ matrix.folder }} debug
   CPU:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        folder: [ epochX/cudacpp/ee_mumu.sa/SubProcesses/P1_Sigma_sm_epem_mupmum , epochX/cudacpp/gg_ttgg.sa/SubProcesses/P1_Sigma_sm_gg_ttxgg ]
+        folder: [ epochX/cudacpp/ee_mumu.sa/SubProcesses/P1_Sigma_sm_epem_mupmum,
+                  epochX/cudacpp/gg_tt.sa/SubProcesses/P1_Sigma_sm_gg_ttx,
+                  epochX/cudacpp/gg_ttg.sa/SubProcesses/P1_Sigma_sm_gg_ttxg,
+                  epochX/cudacpp/gg_ttgg.sa/SubProcesses/P1_Sigma_sm_gg_ttxgg,
+                  epochX/cudacpp/gg_ttggg.sa/SubProcesses/P1_Sigma_sm_gg_ttxggg ]
         precision: [ d , f , m ]
       fail-fast: false
     steps:
@@ -38,7 +47,11 @@ jobs:
       FC: gfortran-11
     strategy:
       matrix:
-        folder: [ epochX/cudacpp/ee_mumu.sa/SubProcesses/P1_Sigma_sm_epem_mupmum, epochX/cudacpp/gg_ttgg.sa/SubProcesses/P1_Sigma_sm_gg_ttxgg ]
+        folder: [ epochX/cudacpp/ee_mumu.sa/SubProcesses/P1_Sigma_sm_epem_mupmum,
+                  epochX/cudacpp/gg_tt.sa/SubProcesses/P1_Sigma_sm_gg_ttx,
+                  epochX/cudacpp/gg_ttg.sa/SubProcesses/P1_Sigma_sm_gg_ttxg,
+                  epochX/cudacpp/gg_ttgg.sa/SubProcesses/P1_Sigma_sm_gg_ttxgg,
+                  epochX/cudacpp/gg_ttggg.sa/SubProcesses/P1_Sigma_sm_gg_ttxggg ]
         precision: [ d , f , m ]
       fail-fast: false
     steps:
@@ -50,20 +63,22 @@ jobs:
     - name: make check
       run: make AVX=none OMPFLAGS= FPTYPE=${{ matrix.precision }} -C ${{ matrix.folder }} check
   GPU:
-    runs-on: self-hosted
+    runs-on: [self-hosted, linux, a100]
     env:
       CUDA_HOME: /usr/local/cuda/
-      FC: gfortran
       REQUIRE_CUDA: 1
+      FC: gfortran
     strategy:
       matrix:
-        folder: [ epochX/cudacpp/ee_mumu.sa/SubProcesses/P1_Sigma_sm_epem_mupmum , epochX/cudacpp/gg_ttgg.sa/SubProcesses/P1_Sigma_sm_gg_ttxgg ]
+        folder: [ epochX/cudacpp/ee_mumu.sa/SubProcesses/P1_Sigma_sm_epem_mupmum,
+                  epochX/cudacpp/gg_tt.sa/SubProcesses/P1_Sigma_sm_gg_ttx,
+                  epochX/cudacpp/gg_ttg.sa/SubProcesses/P1_Sigma_sm_gg_ttxg,
+                  epochX/cudacpp/gg_ttgg.sa/SubProcesses/P1_Sigma_sm_gg_ttxgg,
+                  epochX/cudacpp/gg_ttggg.sa/SubProcesses/P1_Sigma_sm_gg_ttxggg ]
         precision: [ d , f , m ]
       fail-fast: false
     steps:
     - uses: actions/checkout@v2
-    - name: path
-      run: echo "PATH=$PATH"
     - name: make info
       run: make FPTYPE=${{ matrix.precision }} -C ${{ matrix.folder }} info
     - name: make

.github/workflows/mi250x_profiler.yml

@@ -0,0 +1,80 @@
+name: MI250X Performance Profiler
+
+on:
+  push:
+    branches: [ gpu_abstraction ]
+
+jobs:
+  Container_Setup_and_Execution:
+    runs-on: [self-hosted, linux, a100]
+    name: Container Setup and Execution
+    steps:
+    - name: Generate runner token
+      id: generate_token
+      run: |
+        TOKEN=$(curl -XPOST -fsSL \
+                    -H "Authorization: token ${{ secrets.PAT }}" \
+                    -H "Accept: application/vnd.github.v3+json" \
+                    "https://api.github.com/repos/${{ github.repository }}/actions/runners/registration-token" \
+              | grep -o '"token": *"[^"]*"' | cut -d '"' -f 4)
+        echo "token=$TOKEN" >> $GITHUB_OUTPUT
+    - name: SSH and run Docker container
+      env:
+        SSH_PRIVATE_KEY:       ${{ secrets.SSH_KEY }}
+        MI250X_PROFILING_HOST: ${{ secrets.MI250X_PROFILING_HOST }}
+        MI250X_PROFILING_USER: ${{ secrets.MI250X_PROFILING_USER }}
+        HPC_ACCOUNT:           ${{ secrets.HPC_ACCOUNT }}
+        HPC_PROJECT:           ${{ secrets.HPC_PROJECT }}
+        SINGULARITY_CACHEDIR:  /scratch/$HPC_ACCOUNT/$MI250X_PROFILING_USER/
+        SINGULARITY_TMPDIR:    /scratch/$HPC_ACCOUNT/$MI250X_PROFILING_USER/
+      continue-on-error: true
+      run: |
+        echo "$SSH_PRIVATE_KEY" > id_rsa
+        chmod 600 id_rsa
+        ssh -o StrictHostKeyChecking=no -i id_rsa $MI250X_PROFILING_USER@$MI250X_PROFILING_HOST "\
+          cd /scratch/$HPC_ACCOUNT/$MI250X_PROFILING_USER/ && \
+          singularity pull --force oras://ghcr.io/${{ github.repository_owner }}/github_runner_mi250x:latest && \
+          srun --account=$HPC_ACCOUNT -p $HPC_PROJECT --gpus=1 --time=03:00:00 singularity run --rocm \
+            --env GITHUB_TOKEN=${{ steps.generate_token.outputs.token }} \
+            --env REPO_URL=https://github.com/${{ github.repository }} \
+            --env RUNNER_NAME=github_runner_mi250x \
+            --env GITHUB_RUNNER_TAGS='Linux,x64,mi250x' \
+            --env RUNNER_URL=https://github.com/actions/runner/releases/download/v2.303.0/actions-runner-linux-x64-2.303.0.tar.gz \
+            github_runner_mi250x_latest.sif"
+
+  HIP_MI250X_Profiling:
+    runs-on: [self-hosted, linux, mi250x]
+    name: HIP MI250X Profiling
+    env:
+      HIP_NAME_PREFIX: hip_AMD-Epyc-7A53_MI250X_gcc-11.2.1_rocm-5.2.3
+      ENABLE_CI_PROFILER: 1
+    steps:
+    - uses: actions/checkout@v2
+    - name: Runs HIP performanceProfiler.py script
+      run: cd tools/profiling/;
+           python3 performanceProfiler.py -l 'HIP' -b 'master'
+
+    - name: Uploads workplace_mg4gpu directory as an artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: profiling-results
+        path: tools/profiling/workplace_mg4gpu
+
+  Upload_JSON_files:
+    needs: HIP_MI250X_Profiling
+    runs-on: [self-hosted, linux]
+    name: Upload JSON files to DB
+    env:
+      HIP_NAME_PREFIX: hip_AMD-Epyc-7A53_MI250X_gcc-11.2.1_rocm-5.2.3
+      ENABLE_CI_PROFILER: 1
+      MADGRAPH4GPU_DB_URL: ${{ secrets.MADGRAPH4GPU_DB_URL }}
+      MADGRAPH4GPU_DB_SECRET: ${{ secrets.MADGRAPH4GPU_DB_SECRET }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Download artifact containing profiling data
+      uses: actions/download-artifact@v3
+      with:
+        name: profiling-results
+        path: tools/profiling
+    - name: Uploads HIP JSON files to DB
+      run: cd tools/profiling; python3 sendData.py --absLayer HIP --profiler 1 --branch master

.github/workflows/sycl.yml

@@ -0,0 +1,49 @@
+name: SYCL CI
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - 'epochX/sycl/**'
+  pull_request:
+    branches: [ master ]
+    paths:
+      - 'epochX/sycl/**'
+
+jobs:
+  GPU:
+    runs-on: [self-hosted, linux, a100]
+    env:
+      FC: gfortran
+      REQUIRE_CUDA: 1
+      SYCLFLAGS: -fsycl -fsycl-targets=nvptx64-nvidia-cuda -Xsycl-target-backend --cuda-gpu-arch=sm_80 -Xclang -fdenormal-fp-math=ieee
+      ENABLE_CI_PROFILER: 1
+    strategy:
+      matrix:
+        folder: [ epochX/sycl/ee_mumu.sa/SubProcesses/P1_Sigma_sm_epem_mupmum,
+                  epochX/sycl/gg_tt.sa/SubProcesses/P1_Sigma_sm_gg_ttx,
+                  epochX/sycl/gg_ttg.sa/SubProcesses/P1_Sigma_sm_gg_ttxg,
+                  epochX/sycl/gg_ttgg.sa/SubProcesses/P1_Sigma_sm_gg_ttxgg,
+                  epochX/sycl/gg_ttggg.sa/SubProcesses/P1_Sigma_sm_gg_ttxggg ]
+        precision: [ d , f ]
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v2
+    - name: make info
+      run: source /cvmfs/sft.cern.ch/lcg/releases/gcc/11.3.0-ad0f5/x86_64-centos8/setup.sh;
+           source /cvmfs/projects.cern.ch/intelsw/oneAPI/linux/x86_64/2023/setvars.sh --include-intel-llvm;
+           CXX=/cvmfs/projects.cern.ch/intelsw/oneAPI/linux/x86_64/2023/compiler/2023.0.0/linux/bin-llvm/clang++;
+           LD_LIBRARY_PATH=${{ github.workspace }}/${{ matrix.folder }}/../../lib:$LD_LIBRARY_PATH;
+           make FPTYPE=${{ matrix.precision }} -C ${{ matrix.folder }} info
+    - name: make
+      run: source /cvmfs/sft.cern.ch/lcg/releases/gcc/11.3.0-ad0f5/x86_64-centos8/setup.sh;
+           source /cvmfs/projects.cern.ch/intelsw/oneAPI/linux/x86_64/2023/setvars.sh --include-intel-llvm;
+           CXX=/cvmfs/projects.cern.ch/intelsw/oneAPI/linux/x86_64/2023/compiler/2023.0.0/linux/bin-llvm/clang++;
+           LD_LIBRARY_PATH=${{ github.workspace }}/${{ matrix.folder }}/../../lib:$LD_LIBRARY_PATH;
+           make FPTYPE=${{ matrix.precision }} -C ${{ matrix.folder }}
+    - name: make check
+      run: source /cvmfs/sft.cern.ch/lcg/releases/gcc/11.3.0-ad0f5/x86_64-centos8/setup.sh;
+           source /cvmfs/projects.cern.ch/intelsw/oneAPI/linux/x86_64/2023/setvars.sh --include-intel-llvm;
+           CXX=/cvmfs/projects.cern.ch/intelsw/oneAPI/linux/x86_64/2023/compiler/2023.0.0/linux/bin-llvm/clang++;
+           LD_LIBRARY_PATH=${{ github.workspace }}/${{ matrix.folder }}/../../lib:$LD_LIBRARY_PATH;
+           make FPTYPE=${{ matrix.precision }} -C ${{ matrix.folder }} check

.github/workflows/v100s_profiler.yml

@@ -0,0 +1,41 @@
+name: V100s Performance Profiler
+
+on:
+  schedule:
+    - cron:  '00 00 * * *'
+
+jobs:
+
+  sycl_v100s_Profiling:
+    name: SYCL V100S Profiling
+    env:
+      SYCL_NAME_PREFIX: sycl_Xeon-Silver-4216_v100s_gcc-11.3_cuda-12.0.1
+      ENABLE_CI_PROFILER: 1
+
+      MADGRAPH4GPU_DB_URL: ${{ secrets.MADGRAPH4GPU_DB_URL }}
+      MADGRAPH4GPU_DB_SECRET: ${{ secrets.MADGRAPH4GPU_DB_SECRET }}
+    runs-on: [self-hosted, linux, v100s]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Runs SYCL performanceProfiler.py script
+      run: cd tools/profiling/;
+           python3 performanceProfiler.py -l 'SYCL' -b 'master'
+    - name: Uploads SYCL JSON files to DB
+      run: cd tools/profiling/; python3 sendData.py --absLayer SYCL --profiler 1 --branch master
+
+  cuda_v100s_Profiling:
+    name: CUDA V100S Profiling
+    env:
+      CUDA_NAME_PREFIX: cudacpp_Xeon-Silver-4216_v100s_gcc-11.3_cuda-12.0.1
+      ENABLE_CI_PROFILER: 1
+
+      MADGRAPH4GPU_DB_URL: ${{ secrets.MADGRAPH4GPU_DB_URL }}
+      MADGRAPH4GPU_DB_SECRET: ${{ secrets.MADGRAPH4GPU_DB_SECRET }}
+    runs-on: [self-hosted, linux, v100s]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Runs CUDA performanceProfiler.py script
+      run: cd tools/profiling/;
+           python3 performanceProfiler.py -l 'CUDA' -b 'master'
+    - name: Uploads CUDA JSON files to DB
+      run: cd tools/profiling/; python3 sendData.py --absLayer CUDA --profiler 1 --branch master