Merge pull request #249 from NVIDIA/ksimpson/fix-gh-actions

Fix the build portion of the gh-actions

Author: ksimpson-work

.github/actions/build/action.yml.j2 renamed to .github/actions/build/action.yml

@@ -22,33 +22,14 @@ inputs:
   upload-enabled:
     required: true
     type: boolean
+  python-version:
+    required: true
+    type: string
 
 runs:
   using: composite
   steps:
 
-<% for package_id, package_info in packages.items() %>
-    - name: Download <<package_info.repo>> (artifacts)
-      uses: ./.github/actions/download-artifacts
-      with:
-        artifact-repo: "<<package_info.repo>>"
-        artifact-name: "<<package_info.artifact_name | replace_placeholder('repo', package_info.repo) | replace_placeholder('git_tag', package_info.git_tag) >>"
-        target-device: "${{ inputs.target-device }}"
-        git_sha: "<<package_info.git_tag>>"
-        host-platform: ${{ inputs.host-platform }}
-        dest-dir: ${{ env.ARTIFACTS_DIR }}
-        dependencies-workflow: <<package_info.artifact_workflow>>
-<% endfor %>
-
-<% if packages %>
-
-    - name: Display structure of downloaded artifacts
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        pwd
-        ls -lahR ${{ env.ARTIFACTS_DIR }}
-<% endif %>
-
     - if: ${{ inputs.use-container }}
       name: Build (in container)
       shell: bash --noprofile --norc -xeuo pipefail {0}
@@ -60,14 +41,16 @@ runs:
           -e AWS_ACCESS_KEY_ID \
           -e AWS_SECRET_ACCESS_KEY \
           -e GITHUB_TOKEN \
-          -e ARTIFACTS_DIR="$ARTIFACTS_DIR" \
+          -e BINDINGS_ARTIFACTS_DIR="$BINDINGS_ARTIFACTS_DIR" \
+          -e CORE_ARTIFACTS_DIR="$CORE_ARTIFACTS_DIR" \
           -e UPLOAD_ENABLED="$UPLOAD_ENABLED" \
           -e USE_CUDA="$USE_CUDA" \
           -e REPO_DIR="$REPO_DIR" \
           -e LEGATE_CORE_BUILD_MODE="$LEGATE_CORE_BUILD_MODE" \
           -e PYTHON_VERSION="$PYTHON_VERSION" \
           -v "${{ env.REPO_DIR }}:${{ env.REPO_DIR }}" \
-          -v "${{ env.ARTIFACTS_DIR }}:${{ env.ARTIFACTS_DIR }}" \
+          -v "${{ env.BINDINGS_ARTIFACTS_DIR }}:${{ env.BINDINGS_ARTIFACTS_DIR }}" \
+          -v "${{ env.CORE_ARTIFACTS_DIR }}:${{ env.CORE_ARTIFACTS_DIR }}" \
           --rm "${{ inputs.docker-image }}" \
           /bin/bash -c "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint ${{ env.REPO_DIR }}/continuous_integration/scripts/build ${{ inputs.build-type}} ${{ inputs.target-device }}"
 
@@ -77,14 +60,26 @@ runs:
       run: |
         "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/build" "${{ inputs.build-type}}" "${{ inputs.target-device }}"
 
-    - name: Display structure of the artifacts folder (post build)
+    - name: Display structure of the bindings artifacts folder (post build)
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+        sudo chown -R $(whoami) ${{ env.BINDINGS_ARTIFACTS_DIR }}
+        ls -lahR ${{ env.BINDINGS_ARTIFACTS_DIR }}
+
+    - name: Upload bindings build artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ env.BINDINGS_ARTIFACT_NAME }}
+        path: ${{ env.BINDINGS_ARTIFACTS_DIR }}
+
+    - name: Display structure of the core artifacts folder (post build)
       shell: bash --noprofile --norc -xeuo pipefail {0}
       run: |
-        sudo chown -R $(whoami) ${{ env.ARTIFACTS_DIR }}
-        ls -lahR ${{ env.ARTIFACTS_DIR }}
+        sudo chown -R $(whoami) ${{ env.CORE_ARTIFACTS_DIR }}
+        ls -lahR ${{ env.CORE_ARTIFACTS_DIR }}
 
-    - name: Upload build artifacts
+    - name: Upload core build artifacts
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ env.ARTIFACT_NAME }}
-        path: ${{ env.ARTIFACTS_DIR }}
+        name: ${{ env.CORE_ARTIFACT_NAME }}
+        path: ${{ env.CORE_ARTIFACTS_DIR }}

.github/actions/download-artifacts/action.yml

@@ -20,7 +20,7 @@ inputs:
     required: true
     type: boolean
   python-version:
-    required: false
+    required: true
     type: string
 
 runs:
@@ -57,12 +57,16 @@ runs:
           PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}"
         fi
 
-        echo "ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ inputs.client-repo }}-python${{ inputs.python-version }}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV
-        echo "ARTIFACTS_DIR=$(realpath "$(pwd)/dist")" >> $GITHUB_ENV
+        PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.')
+
+        echo "BINDINGS_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_bindings-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV
+        echo "BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV
+        echo "CORE_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV
+        echo "CORE_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_core/dist")" >> $GITHUB_ENV
         echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV
         echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled  == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV
         echo "LEGATE_CORE_BUILD_MODE=${BUILD_MODE}" >> $GITHUB_ENV
         echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
         echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV
         echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV
-        echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV
+        echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV

.github/actions/setup/action.yml

@@ -24,6 +24,9 @@ jobs:
           - release
         upload-enabled:
           - false
+        python-version:
+          #TODO cover the whole python and cuda matrix
+          - 3.12
     uses:
       ./.github/workflows/gh-build-and-test.yml
     with:
@@ -32,4 +35,5 @@ jobs:
       build-mode: ${{ matrix.build-mode }}
       build-type: ci
       upload-enabled: ${{ matrix.upload-enabled }}
+      python-version: ${{ matrix.python-version }}
     secrets: inherit

.github/workflows/ci-gh.yml

@@ -16,6 +16,9 @@ on:
       upload-enabled:
         type: boolean
         required: true
+      python-version:
+        type: string
+        required: true
 jobs:
   build:
     if: ${{ github.repository_owner == 'nvidia' }}
@@ -31,4 +34,5 @@ jobs:
       dependencies-file: ""
       build-mode: ${{ inputs.build-mode }}
       upload-enabled: ${{ inputs.upload-enabled }}
+      python-version: ${{ inputs.python-version }}
     secrets: inherit

.github/workflows/gh-build-and-test.yml

@@ -33,7 +33,7 @@ on:
         required: true
         type: boolean
       python-version:
-        required: false
+        required: true
         type: string
 
 jobs:
@@ -63,27 +63,6 @@ jobs:
           upload-enabled: ${{ inputs.upload-enabled }}
           python-version: ${{ inputs.python-version }}
 
-      - name: Render templates
-        shell: bash --noprofile --norc -xeuo pipefail {0}
-        run: |
-          pip -q install jinja2
-
-          DEPENDENCIES_FILE=""
-
-          if [ -z "${{ inputs.dependencies-file }}" ]; then
-            DEPENDENCIES_FILE="${REPO_DIR}/continuous_integration/no_dependencies.json"
-          else
-            DEPENDENCIES_FILE="${REPO_DIR}/${{ inputs.dependencies-file }}"
-          fi
-
-          ${REPO_DIR}/continuous_integration/scripts/render-template.py .github/actions/build/action.yml.j2 "${DEPENDENCIES_FILE}" .github/actions/build/action.yml
-
-      - name: Dump templates
-        shell: bash --noprofile --norc -xeuo pipefail {0}
-        run: |
-          echo ${REPO_DIR}/.github/actions/build/action.yml
-          cat ${REPO_DIR}/.github/actions/build/action.yml
-
       - name: Call build action
         uses: ./.github/actions/build
         with:

.github/workflows/gh-build.yml

@@ -3,11 +3,13 @@
 build_ci() {
     set -xeou pipefail
 
-    cd "${REPO_DIR}"
-
     export CUDA_HOME="${CONDA_PREFIX}/targets/x86_64-linux"
     export PARALLEL_LEVEL=$(nproc --ignore 1)
 
+    cd "${REPO_DIR}/cuda_bindings"
+    python setup.py bdist_wheel
+    
+    cd "${REPO_DIR}/cuda_core"
     python setup.py bdist_wheel
 }
 

continuous_integration/no_dependencies.json

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Function to generate environment.yml
+generate_environment_yml() {
+    local python_version=$1
+    local cuda_version=$2
+    local output_path=$3
+
+    cat <<EOF > "${output_path}/environment.yml"
+name: cuda_python
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - python=${python_version}
+  - cython
+  - pytest
+  - numpy
+  - setuptools
+  - wheel
+  - pip
+  - cuda-version=${cuda_version}
+  - cuda-cudart-static
+  - cuda-driver-dev
+  - cuda-cudart-dev
+  - cuda-profiler-api
+  - cuda-nvrtc-dev
+  - cuda-nvcc
+  - pip:
+    - pytest-benchmark
+    - pyclibrary
+    - versioneer==0.29
+    - tomli; python_version < "3.11"
+    - pywin32; sys_platform == 'win32'
+EOF
+}

continuous_integration/scripts/build

@@ -3,24 +3,16 @@
 set -x
 
 make_ci_env() {
-    mamba env create -n "${CONDA_ENV}" -f "${REPO_DIR}/continuous_integration/environment.yml"
-}
-
-make_test_env() {
-    . conda-utils
-
-    mamba env create -n "${CONDA_ENV}" -f "${REPO_DIR}/continuous_integration/environment.yml"
-
-    activate_conda_env
-
-    pip install "${ARTIFACTS_DIR}"/*.whl
-
+    #TODO wire cuda version as a top level matrix argument
+    generate_environment_yml "${PYTHON_VERSION}" 12.6 .
+    mamba env create -n "${CONDA_ENV}" -f ./environment.yml
 }
 
 make_conda_env() {
     set -xeuo pipefail
 
     . setup-utils;
+    . generate-environment
     set_base_defs;
 
     case "$1" in