Enable testing of Python limited API builds

Author: vyasr

.github/workflows/build-all-rapids-repos.yml

@@ -43,7 +43,19 @@ jobs:
       build_command: |
         sccache -z;
         sccache --show-adv-stats;
-        clone-all -j$(nproc) -v -q --clone-upstream --single-branch --shallow-submodules;
+        # Need Python 3.11 for the buffer protocol in the limited API
+        export PYTHON_VERSION="3.11"
+        clone-all -j$(nproc) -v -q --clone-upstream --single-branch --shallow-submodules --no-update-env;
+        rapids-generate-scripts
+        . rapids-make-${PYTHON_PACKAGE_MANAGER}-env
+        # The latest version of this branch includes both my patches (required
+        # for RAPIDS Cython usage to work) and upstream patches required for
+        # Cython itself to work under the limited API.
+        export NO_CYTHON_COMPILE=true
+        python -m pip install git+https://github.com/cython/cython.git@limited-api-preview
+        # Need to install my branch of scikit-build-core for the way I'm specifying CMAKE_CXX_FLAGS via CMAKE_ARGS to work
+        python -m pip install git+https://github.com/vyasr/scikit-build-core.git@fix/cmake_args_quoted_spaces
+
         build-all                        \
           -v                             \
           -j$(nproc --ignore=1)          \

features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml

@@ -24,7 +24,7 @@ repos:
     - name: rmm
       sub_dir: python/rmm
       depends: [rmm]
-      args: {install: *rapids_build_backend_args}
+      args: {install: *rapids_build_backend_args, cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'"}
 
 - name: ucxx
   path: ucxx
@@ -82,11 +82,11 @@ repos:
     - name: pylibcudf
       sub_dir: python/pylibcudf
       depends: [cudf]
-      args: {install: *rapids_build_backend_args}
+      args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
     - name: cudf
       sub_dir: python/cudf
       depends: [cudf]
-      args: {install: *rapids_build_backend_args}
+      args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
     - name: dask_cudf
       sub_dir: python/dask_cudf
       args: {install: *rapids_build_backend_args}
@@ -115,11 +115,11 @@ repos:
     - name: pylibraft
       sub_dir: python/pylibraft
       depends: [raft]
-      args: {cmake: -DFIND_RAFT_CPP=ON, install: *rapids_build_backend_args}
+      args: {cmake: -DFIND_RAFT_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
     - name: raft-dask
       sub_dir: python/raft-dask
       depends: [ucxx, raft]
-      args: {cmake: -DFIND_RAFT_CPP=ON, install: *rapids_build_backend_args}
+      args: {cmake: -DFIND_RAFT_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
 
 - name: cuvs
   path: cuvs
@@ -135,7 +135,7 @@ repos:
     - name: cuvs
       sub_dir: python/cuvs
       depends: [cuvs]
-      args: {cmake: -DFIND_CUVS_CPP=ON, install: *rapids_build_backend_args}
+      args: {cmake: -DFIND_CUVS_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
 
 - name: cumlprims_mg
   path: cumlprims_mg
@@ -158,7 +158,7 @@ repos:
     - name: cuml
       sub_dir: python/cuml
       depends: [cuml]
-      args: {cmake: -DFIND_CUML_CPP=ON, install: *rapids_build_backend_args}
+      args: {cmake: -DFIND_CUML_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
 
 - name: cugraph-ops
   path: cugraph-ops
@@ -220,11 +220,11 @@ repos:
     - name: pylibcugraph
       sub_dir: python/pylibcugraph
       depends: [cugraph]
-      args: {cmake: -DFIND_CUGRAPH_CPP=ON, install: *rapids_build_backend_args}
+      args: {cmake: -DFIND_CUGRAPH_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
     - name: cugraph
       sub_dir: python/cugraph
       depends: [cugraph]
-      args: {cmake: -DFIND_CUGRAPH_CPP=ON, install: *rapids_build_backend_args}
+      args: {cmake: -DFIND_CUGRAPH_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}
     - name: nx-cugraph
       sub_dir: python/nx-cugraph
       args: {install: *rapids_build_backend_args}
@@ -263,11 +263,12 @@ repos:
       # it doesn't have a build dir of its own. That means we can't list it as
       # a cpp repo after cuspatial (above), and we have to manually point CMake
       # to the cuspatial build dir here.
+      # Limited API compilation of cuproje currently fails due to https://github.com/cython/cython/issues/6405
       args:
         cmake: |
           -DFIND_CUPROJ_CPP=ON
           -Dcuproj_ROOT="${HOME}/cuspatial/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest"
     - name: cuspatial
       sub_dir: python/cuspatial
       depends: [cuspatial]
-      args: {install: *rapids_build_backend_args}
+      args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args}