diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index bed1ba503f..73d2b7fce5 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -49,17 +49,24 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.23.3 env: + CIBW_ENVIRONMENT: + CI_WHEEL_BUILD=1 CIBW_BUILD_VERBOSITY: 3 - CIBW_ENVIRONMENT_PASS_LINUX: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_TILEDB S3_BUCKET TILEDB_TOKEN TILEDB_NAMESPACE + CIBW_ENVIRONMENT_PASS_LINUX: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_TILEDB S3_BUCKET TILEDB_TOKEN TILEDB_NAMESPACE RUNNER_TEMP CIBW_ENVIRONMENT_MACOS: > CC=clang CXX=clang++ + RUNNER_TEMP=${{ runner.temp }} MACOSX_DEPLOYMENT_TARGET: "11.0" CIBW_ARCHS: all CIBW_PRERELEASE_PYTHONS: True CIBW_BUILD: ${{ matrix.python }}-${{ matrix.buildplat[1] }} CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28" CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28" + CIBW_REPAIR_WHEEL_COMMAND_LINUX: "bash -c 'if [ -n \"$RUNNER_TEMP\" ] && [ -d \"$RUNNER_TEMP/tiledb-external\" ]; then export LD_LIBRARY_PATH=\"$RUNNER_TEMP/tiledb-external:$LD_LIBRARY_PATH\"; fi; auditwheel repair -w {dest_dir} {wheel}'" + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "bash -c 'if [ -n \"$RUNNER_TEMP\" ] && [ -d \"$RUNNER_TEMP/tiledb-external\" ]; then export DYLD_LIBRARY_PATH=\"$RUNNER_TEMP/tiledb-external:$DYLD_LIBRARY_PATH\"; fi; delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}'" + CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path \"%RUNNER_TEMP%\\tiledb-external\" -w {dest_dir} {wheel}" # __init__.py interferes with the tests and is included as local file instead of # used from wheels. To be honest, tests should not be in the source folder at all. CIBW_BEFORE_TEST: rm {project}/tiledb/__init__.py @@ -74,6 +81,9 @@ jobs: build_sdist: name: Build source distribution runs-on: ubuntu-latest + env: + RUNNER_TEMP: "" + CI_WHEEL_BUILD: 0 outputs: sdist_name: ${{ steps.get_sdist_name.outputs.sdist_name }} steps: @@ -93,50 +103,53 @@ jobs: path: dist/*.tar.gz test_sdist: - name: Test source distribution package - needs: [build_sdist] - strategy: - matrix: - os: - - macos-13 - - macos-14 - - windows-2022 - - ubuntu-22.04 - - ubuntu-24.04-arm - python: ["3.9", "3.10", "3.11", "3.12", "3.13"] - runs-on: ${{ matrix.os }} - steps: - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - - - name: Download sdist artifact - uses: actions/download-artifact@v4 - with: - name: sdist - path: dist - - - name: Install sdist artifact - run: pip install --verbose dist/${{ needs.build_sdist.outputs.sdist_name }} - - - uses: actions/checkout@v4 - - - name: Install test dependencies - run: pip install pytest pytest-rerunfailures hypothesis psutil pyarrow - - - name: Run tests - shell: bash - run: | - PROJECT_CWD=$PWD - rm tiledb/__init__.py - cd .. - pytest -vv --showlocals $PROJECT_CWD - - - name: "Re-run tests without pandas" - run: | - pip uninstall -y pandas - pytest -vv --showlocals $PROJECT_CWD + name: Test source distribution package + needs: [build_sdist] + strategy: + matrix: + os: + - macos-13 + - macos-14 + - windows-2022 + - ubuntu-22.04 + - ubuntu-24.04-arm + python: ["3.9", "3.10", "3.11", "3.12", "3.13"] + runs-on: ${{ matrix.os }} + env: + CI_WHEEL_BUILD: 0 + RUNNER_TEMP: "" + steps: + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Download sdist artifact + uses: actions/download-artifact@v4 + with: + name: sdist + path: dist + + - name: Install sdist artifact + run: pip install --verbose dist/${{ needs.build_sdist.outputs.sdist_name }} + + - uses: actions/checkout@v4 + + - name: Install test dependencies + run: pip install pytest pytest-rerunfailures hypothesis psutil pyarrow + + - name: Run tests + shell: bash + run: | + PROJECT_CWD=$PWD + rm tiledb/__init__.py + cd .. + pytest -vv --showlocals $PROJECT_CWD + + - name: "Re-run tests without pandas" + run: | + pip uninstall -y pandas + pytest -vv --showlocals $PROJECT_CWD upload_pypi: needs: [build_wheels, test_sdist] @@ -164,4 +177,4 @@ jobs: - name: Upload to pypi if: startsWith(github.ref, 'refs/tags/') - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 0ec03bbee0..a73a7e649b 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -1,5 +1,10 @@ # Pybind11 +# Enable @rpath support on macOS for proper library isolation +if(APPLE) + set(CMAKE_MACOSX_RPATH ON) +endif() + pybind11_add_module( main main.cc @@ -38,22 +43,64 @@ endif() install(TARGETS main DESTINATION tiledb) +# Handle TileDB shared library installation and RPATH setup if(TILEDB_DOWNLOADED) - message(STATUS "Adding \"libtiledb\" into install group") - - install(IMPORTED_RUNTIME_ARTIFACTS TileDB::tiledb_shared DESTINATION tiledb) - - if (APPLE) - set_target_properties(main PROPERTIES INSTALL_RPATH "@loader_path") - elseif(UNIX) - set_target_properties(main PROPERTIES INSTALL_RPATH "\$ORIGIN") + message(STATUS "CI_WHEEL_BUILD is set to: '$ENV{CI_WHEEL_BUILD}'") + set(CI_BUILD_VAR "$ENV{CI_WHEEL_BUILD}") + if(CI_BUILD_VAR OR CIBUILD_VAR STREQUAL "1") + # For CI builds - install to external directory + set(TILEDB_EXTERNAL_DIR "$ENV{RUNNER_TEMP}/tiledb-external") + message(STATUS "CI Build - installing TileDB to: ${TILEDB_EXTERNAL_DIR}") + install(IMPORTED_RUNTIME_ARTIFACTS TileDB::tiledb_shared + DESTINATION "${TILEDB_EXTERNAL_DIR}") + + # Set RPATH for CI builds + if(APPLE) + set_target_properties(main PROPERTIES + INSTALL_RPATH "@rpath" + BUILD_WITH_INSTALL_RPATH TRUE + ) + else() + set_target_properties(main PROPERTIES + INSTALL_RPATH "${TILEDB_EXTERNAL_DIR}" + BUILD_WITH_INSTALL_RPATH TRUE + ) + endif() + else() + # For local builds - install to tiledb directory + message(STATUS "Local Build - installing TileDB shared library to tiledb directory") + install(IMPORTED_RUNTIME_ARTIFACTS TileDB::tiledb_shared DESTINATION tiledb) + + # Set RPATH for local builds + if(APPLE) + set_target_properties(main PROPERTIES + INSTALL_RPATH "@loader_path" + BUILD_WITH_INSTALL_RPATH TRUE + ) + elseif(UNIX) + set_target_properties(main PROPERTIES + INSTALL_RPATH "\$ORIGIN" + BUILD_WITH_INSTALL_RPATH TRUE + ) + endif() endif() else() - # If using external TileDB core library force it to be linked at runtime using RPATH + # For external TileDB installations - no installation needed, just set RPATH get_property(TILEDB_LOCATION TARGET TileDB::tiledb_shared PROPERTY LOCATION) get_filename_component(TILEDB_LOCATION ${TILEDB_LOCATION} DIRECTORY) - message(STATUS "Setting RPATH for targets \"main\" and \"libtiledb\" to ${TILEDB_LOCATION}") - set_target_properties(main PROPERTIES INSTALL_RPATH ${TILEDB_LOCATION}) + message(STATUS "External TileDB - setting RPATH to: ${TILEDB_LOCATION}") + + if(APPLE) + set_target_properties(main PROPERTIES + INSTALL_RPATH "@rpath" + BUILD_WITH_INSTALL_RPATH TRUE + ) + else() + set_target_properties(main PROPERTIES + INSTALL_RPATH ${TILEDB_LOCATION} + BUILD_WITH_INSTALL_RPATH TRUE + ) + endif() endif() add_subdirectory(libtiledb) \ No newline at end of file diff --git a/tiledb/libtiledb/CMakeLists.txt b/tiledb/libtiledb/CMakeLists.txt index 59a63cda95..d5f30f894b 100644 --- a/tiledb/libtiledb/CMakeLists.txt +++ b/tiledb/libtiledb/CMakeLists.txt @@ -1,3 +1,8 @@ +# Enable @rpath support on macOS for proper library isolation +if(APPLE) + set(CMAKE_MACOSX_RPATH ON) +endif() + pybind11_add_module( libtiledb array.cc @@ -55,15 +60,41 @@ endif() install(TARGETS libtiledb DESTINATION tiledb) if(TILEDB_DOWNLOADED) - if (APPLE) - set_target_properties(libtiledb PROPERTIES INSTALL_RPATH "@loader_path") - elseif(UNIX) - set_target_properties(libtiledb PROPERTIES INSTALL_RPATH "\$ORIGIN") + set(CI_BUILD_VAR "$ENV{CI_WHEEL_BUILD}") + if(CI_BUILD_VAR OR CIBUILD_VAR STREQUAL "1") + # CI builds - shared library is in external directory + set(TILEDB_EXTERNAL_DIR "$ENV{RUNNER_TEMP}/tiledb-external") + if(APPLE) + set_target_properties(libtiledb PROPERTIES + INSTALL_RPATH "@rpath" + BUILD_WITH_INSTALL_RPATH TRUE + ) + else() + set_target_properties(libtiledb PROPERTIES + INSTALL_RPATH "${TILEDB_EXTERNAL_DIR}" + BUILD_WITH_INSTALL_RPATH TRUE + ) + endif() + else() + # Local builds - shared library is in same directory + if(APPLE) + set_target_properties(libtiledb PROPERTIES + INSTALL_RPATH "@loader_path" + BUILD_WITH_INSTALL_RPATH TRUE + ) + elseif(UNIX) + set_target_properties(libtiledb PROPERTIES + INSTALL_RPATH "\$ORIGIN" + BUILD_WITH_INSTALL_RPATH TRUE + ) + endif() endif() else() - # If using external TileDB core library force it to be linked at runtime using RPATH get_property(TILEDB_LOCATION TARGET TileDB::tiledb_shared PROPERTY LOCATION) get_filename_component(TILEDB_LOCATION ${TILEDB_LOCATION} DIRECTORY) message(STATUS "Setting RPATH for target \"libtiledb\" to ${TILEDB_LOCATION}") - set_target_properties(libtiledb PROPERTIES INSTALL_RPATH ${TILEDB_LOCATION}) + set_target_properties(libtiledb PROPERTIES + INSTALL_RPATH ${TILEDB_LOCATION} + BUILD_WITH_INSTALL_RPATH TRUE + ) endif()