Re-enable libtiledb grafting with auditwheel

.github/workflows/build-wheels.yml

@@ -49,17 +49,24 @@ jobs:
       - name: Build wheels
         uses: pypa/[email protected]
         env:
+          CIBW_ENVIRONMENT:
+            CI_WHEEL_BUILD=1
           CIBW_BUILD_VERBOSITY: 3
-          CIBW_ENVIRONMENT_PASS_LINUX: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_TILEDB S3_BUCKET TILEDB_TOKEN TILEDB_NAMESPACE
+          CIBW_ENVIRONMENT_PASS_LINUX: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_TILEDB S3_BUCKET TILEDB_TOKEN TILEDB_NAMESPACE RUNNER_TEMP
           CIBW_ENVIRONMENT_MACOS: >
             CC=clang
             CXX=clang++
+            RUNNER_TEMP=${{ runner.temp }}
           MACOSX_DEPLOYMENT_TARGET: "11.0"
           CIBW_ARCHS: all
           CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python }}-${{ matrix.buildplat[1] }}
           CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
           CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: "bash -c 'if [ -n \"$RUNNER_TEMP\" ] && [ -d \"$RUNNER_TEMP/tiledb-external\" ]; then export LD_LIBRARY_PATH=\"$RUNNER_TEMP/tiledb-external:$LD_LIBRARY_PATH\"; fi; auditwheel repair -w {dest_dir} {wheel}'"
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: "bash -c 'if [ -n \"$RUNNER_TEMP\" ] && [ -d \"$RUNNER_TEMP/tiledb-external\" ]; then export DYLD_LIBRARY_PATH=\"$RUNNER_TEMP/tiledb-external:$DYLD_LIBRARY_PATH\"; fi; delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}'"
+          CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel"
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path \"%RUNNER_TEMP%\\tiledb-external\" -w {dest_dir} {wheel}"
           # __init__.py interferes with the tests and is included as local file instead of
           # used from wheels. To be honest, tests should not be in the source folder at all.
           CIBW_BEFORE_TEST: rm {project}/tiledb/__init__.py
@@ -74,6 +81,9 @@ jobs:
   build_sdist:
     name: Build source distribution
     runs-on: ubuntu-latest
+    env:
+      RUNNER_TEMP: ""
+      CI_WHEEL_BUILD: 0
     outputs:
       sdist_name: ${{ steps.get_sdist_name.outputs.sdist_name }}
     steps:
@@ -93,50 +103,53 @@ jobs:
           path: dist/*.tar.gz
 
   test_sdist:
-    name: Test source distribution package
-    needs: [build_sdist]
-    strategy:
-      matrix:
-        os:
-          - macos-13
-          - macos-14
-          - windows-2022
-          - ubuntu-22.04
-          - ubuntu-24.04-arm
-        python: ["3.9", "3.10", "3.11", "3.12", "3.13"]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Download sdist artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: sdist
-          path: dist
-
-      - name: Install sdist artifact
-        run: pip install --verbose dist/${{ needs.build_sdist.outputs.sdist_name }}
-
-      - uses: actions/checkout@v4
-
-      - name: Install test dependencies
-        run: pip install pytest pytest-rerunfailures hypothesis psutil pyarrow
-
-      - name: Run tests
-        shell: bash
-        run: |
-          PROJECT_CWD=$PWD
-          rm tiledb/__init__.py
-          cd ..
-          pytest -vv --showlocals $PROJECT_CWD
-
-      - name: "Re-run tests without pandas"
-        run: |
-          pip uninstall -y pandas
-          pytest -vv --showlocals $PROJECT_CWD
+      name: Test source distribution package
+      needs: [build_sdist]
+      strategy:
+        matrix:
+          os:
+            - macos-13
+            - macos-14
+            - windows-2022
+            - ubuntu-22.04
+            - ubuntu-24.04-arm
+          python: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+      runs-on: ${{ matrix.os }}
+      env:
+        CI_WHEEL_BUILD: 0
+        RUNNER_TEMP: ""
+      steps:
+        - name: Set up Python ${{ matrix.python }}
+          uses: actions/setup-python@v5
+          with:
+            python-version: ${{ matrix.python }}
+
+        - name: Download sdist artifact
+          uses: actions/download-artifact@v4
+          with:
+            name: sdist
+            path: dist
+
+        - name: Install sdist artifact
+          run: pip install --verbose dist/${{ needs.build_sdist.outputs.sdist_name }}
+
+        - uses: actions/checkout@v4
+
+        - name: Install test dependencies
+          run: pip install pytest pytest-rerunfailures hypothesis psutil pyarrow
+
+        - name: Run tests
+          shell: bash
+          run: |
+            PROJECT_CWD=$PWD
+            rm tiledb/__init__.py
+            cd ..
+            pytest -vv --showlocals $PROJECT_CWD
+
+        - name: "Re-run tests without pandas"
+          run: |
+            pip uninstall -y pandas
+            pytest -vv --showlocals $PROJECT_CWD
 
   upload_pypi:
     needs: [build_wheels, test_sdist]
@@ -164,4 +177,4 @@ jobs:
 
       - name: Upload to pypi
         if: startsWith(github.ref, 'refs/tags/')
-        uses: pypa/gh-action-pypi-publish@release/v1
+        uses: pypa/gh-action-pypi-publish@release/v1

tiledb/CMakeLists.txt

@@ -1,5 +1,10 @@
 # Pybind11
 
+# Enable @rpath support on macOS for proper library isolation
+if(APPLE)
+    set(CMAKE_MACOSX_RPATH ON)
+endif()
+
 pybind11_add_module(
     main
     main.cc
@@ -38,22 +43,64 @@ endif()
 
 install(TARGETS main DESTINATION tiledb)
 
+# Handle TileDB shared library installation and RPATH setup
 if(TILEDB_DOWNLOADED)
-    message(STATUS "Adding \"libtiledb\" into install group")
-
-    install(IMPORTED_RUNTIME_ARTIFACTS TileDB::tiledb_shared DESTINATION tiledb)
-
-    if (APPLE)
-        set_target_properties(main PROPERTIES INSTALL_RPATH "@loader_path")
-    elseif(UNIX)
-        set_target_properties(main PROPERTIES INSTALL_RPATH "\$ORIGIN")
+    message(STATUS "CI_WHEEL_BUILD is set to: '$ENV{CI_WHEEL_BUILD}'")
+    set(CI_BUILD_VAR "$ENV{CI_WHEEL_BUILD}")
+    if(CI_BUILD_VAR OR CIBUILD_VAR STREQUAL "1")
+        # For CI builds - install to external directory
+        set(TILEDB_EXTERNAL_DIR "$ENV{RUNNER_TEMP}/tiledb-external")
+        message(STATUS "CI Build - installing TileDB to: ${TILEDB_EXTERNAL_DIR}")
+        install(IMPORTED_RUNTIME_ARTIFACTS TileDB::tiledb_shared 
+                DESTINATION "${TILEDB_EXTERNAL_DIR}")
+
+        # Set RPATH for CI builds
+        if(APPLE)
+            set_target_properties(main PROPERTIES 
+                INSTALL_RPATH "@rpath"
+                BUILD_WITH_INSTALL_RPATH TRUE
+            )
+        else()
+            set_target_properties(main PROPERTIES 
+                INSTALL_RPATH "${TILEDB_EXTERNAL_DIR}"
+                BUILD_WITH_INSTALL_RPATH TRUE
+            )
+        endif()
+    else()
+        # For local builds - install to tiledb directory
+        message(STATUS "Local Build - installing TileDB shared library to tiledb directory")
+        install(IMPORTED_RUNTIME_ARTIFACTS TileDB::tiledb_shared DESTINATION tiledb)
+
+        # Set RPATH for local builds
+        if(APPLE)
+            set_target_properties(main PROPERTIES 
+                INSTALL_RPATH "@loader_path"
+                BUILD_WITH_INSTALL_RPATH TRUE
+            )
+        elseif(UNIX)
+            set_target_properties(main PROPERTIES 
+                INSTALL_RPATH "\$ORIGIN"
+                BUILD_WITH_INSTALL_RPATH TRUE
+            )
+        endif()
     endif()
 else()
-    # If using external TileDB core library force it to be linked at runtime using RPATH
+    # For external TileDB installations - no installation needed, just set RPATH
     get_property(TILEDB_LOCATION TARGET TileDB::tiledb_shared PROPERTY LOCATION)
     get_filename_component(TILEDB_LOCATION ${TILEDB_LOCATION} DIRECTORY)
-    message(STATUS "Setting RPATH for targets \"main\" and \"libtiledb\" to ${TILEDB_LOCATION}")
-    set_target_properties(main PROPERTIES INSTALL_RPATH ${TILEDB_LOCATION})
+    message(STATUS "External TileDB - setting RPATH to: ${TILEDB_LOCATION}")
+
+    if(APPLE)
+        set_target_properties(main PROPERTIES 
+            INSTALL_RPATH "@rpath"
+            BUILD_WITH_INSTALL_RPATH TRUE
+        )
+    else()
+        set_target_properties(main PROPERTIES 
+            INSTALL_RPATH ${TILEDB_LOCATION}
+            BUILD_WITH_INSTALL_RPATH TRUE
+        )
+    endif()
 endif()
 
 add_subdirectory(libtiledb)

tiledb/libtiledb/CMakeLists.txt

@@ -1,3 +1,8 @@
+# Enable @rpath support on macOS for proper library isolation
+if(APPLE)
+    set(CMAKE_MACOSX_RPATH ON)
+endif()
+
 pybind11_add_module(
     libtiledb
     array.cc
@@ -55,15 +60,41 @@ endif()
 install(TARGETS libtiledb DESTINATION tiledb)
 
 if(TILEDB_DOWNLOADED)
-    if (APPLE)
-    set_target_properties(libtiledb PROPERTIES INSTALL_RPATH "@loader_path")
-    elseif(UNIX)
-    set_target_properties(libtiledb PROPERTIES INSTALL_RPATH "\$ORIGIN")
+    set(CI_BUILD_VAR "$ENV{CI_WHEEL_BUILD}")
+    if(CI_BUILD_VAR OR CIBUILD_VAR STREQUAL "1")
+        # CI builds - shared library is in external directory
+        set(TILEDB_EXTERNAL_DIR "$ENV{RUNNER_TEMP}/tiledb-external")
+        if(APPLE)
+            set_target_properties(libtiledb PROPERTIES 
+                INSTALL_RPATH "@rpath"
+                BUILD_WITH_INSTALL_RPATH TRUE
+            )
+        else()
+            set_target_properties(libtiledb PROPERTIES 
+                INSTALL_RPATH "${TILEDB_EXTERNAL_DIR}"
+                BUILD_WITH_INSTALL_RPATH TRUE
+            )
+        endif()
+    else()
+        # Local builds - shared library is in same directory
+        if(APPLE)
+            set_target_properties(libtiledb PROPERTIES 
+                INSTALL_RPATH "@loader_path"
+                BUILD_WITH_INSTALL_RPATH TRUE
+            )
+        elseif(UNIX)
+            set_target_properties(libtiledb PROPERTIES 
+                INSTALL_RPATH "\$ORIGIN"
+                BUILD_WITH_INSTALL_RPATH TRUE
+            )
+        endif()
     endif()
 else()
-    # If using external TileDB core library force it to be linked at runtime using RPATH
     get_property(TILEDB_LOCATION TARGET TileDB::tiledb_shared PROPERTY LOCATION)
     get_filename_component(TILEDB_LOCATION ${TILEDB_LOCATION} DIRECTORY)
     message(STATUS "Setting RPATH for target \"libtiledb\" to ${TILEDB_LOCATION}")
-    set_target_properties(libtiledb PROPERTIES INSTALL_RPATH ${TILEDB_LOCATION})
+    set_target_properties(libtiledb PROPERTIES 
+        INSTALL_RPATH ${TILEDB_LOCATION}
+        BUILD_WITH_INSTALL_RPATH TRUE
+    )
 endif()