Merge pull request #300 from lonvia/rewrite-build-system

Modernize build system

Author: lonvia

.github/actions/install-dependencies/action.yml

@@ -7,62 +7,51 @@ on:
 
 jobs:
   build_wheels:
-    name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }}
+    name: Build wheels on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-2019, macos-13, macos-14]
-        arch: [native]
+        os: [ubuntu-latest, ubuntu-24.04-arm, macos-13, macos-latest]
+        vcpkg: [none]
         include:
-          - os: ubuntu-20.04
-            arch: aarch64
+         - os: windows-latest
+           vcpkg: 'C:/vcpkg/installed/x64-windows'
+         - os: windows-11-arm
+           vcpkg: 'C:/vcpkg/installed/arm64-windows'
 
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/checkout@v4
-        with:
-          repository: pybind/pybind11
-          ref: v2.11.1
-          path: contrib/pybind11
-
-      - uses: actions/checkout@v4
-        with:
-          repository: mapbox/protozero
-          ref: v1.7.1
-          path: contrib/protozero
-
-      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          repository: osmcode/libosmium
-          ref: v2.20.0
-          path: contrib/libosmium
+          python-version: '3.13'
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        if: ${{ matrix.arch == 'aarch64' }}
+      - name: Install requirements
+        run: python -m pip install cibuildwheel build
 
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.8'
+      - name: Get dependencies via sdist
+        run: python -m build --sdist
 
       - name: Build wheels
-        uses: pypa/[email protected]
+        run: |
+          python -m cibuildwheel --output-dir wheelhouse
+        shell: bash
         env:
-           CIBW_ARCHS: ${{ matrix.arch }}
-           CIBW_SKIP: "pp* *musllinux* cp37-macosx_* {cp37,cp38}-*linux_aarch64"
-           CIBW_TEST_REQUIRES: pytest pytest-httpserver shapely
-           CIBW_TEST_REQUIRES_LINUX: urllib3<2.0 pytest pytest-httpserver shapely
+           CIBW_ENABLE: cpython-freethreading
+           CIBW_ARCHS: "native"
+           CIBW_SKIP: "*musllinux*"
+           CIBW_TEST_REQUIRES: pytest pytest-httpserver
            CIBW_TEST_COMMAND: pytest {project}/test
            CIBW_BUILD_FRONTEND: build
            CIBW_BEFORE_BUILD_LINUX: yum install -y expat-devel boost-devel zlib-devel bzip2-devel lz4-devel
            CIBW_BEFORE_BUILD_MACOS: brew install boost
-           CIBW_BEFORE_BUILD_WINDOWS: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows boost-variant:x64-windows boost-iterator:x64-windows lz4:x86-windows
+           CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=11.0 SKBUILD_CMAKE_ARGS=-DWITH_LZ4=OFF
+           CIBW_BEFORE_BUILD_WINDOWS: vcpkg install bzip2 expat zlib boost-variant boost-iterator lz4
            CIBW_ENVIRONMENT_WINDOWS: 'CMAKE_TOOLCHAIN_FILE="C:/vcpkg/scripts/buildsystems/vcpkg.cmake"'
-           CIBW_ENVIRONMENT_MACOS: CMAKE_WITH_LZ4=OFF MACOSX_DEPLOYMENT_TARGET=11.0
+           CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: 'pipx run delvewheel repair --add-path ${{ matrix.vcpkg }}/bin/ --add-path ${{ matrix.vcpkg }}/debug/bin -w {dest_dir} {wheel}'
 
       - uses: actions/upload-artifact@v4
         with:
-          name: pyosmium-wheels-${{ matrix.os }}-${{ matrix.arch }}
+          name: pyosmium-wheels-${{ matrix.os }}
           path: ./wheelhouse/*.whl

.github/workflows/build_wheels.yml

@@ -268,6 +268,7 @@ jobs:
         runs-on: ${{ matrix.platform }}
 
         strategy:
+            fail-fast: false
             matrix:
                 compiler: [gcc-old, clang-old, gcc, clang, macos-intel, macos-arm]
                 include:
@@ -317,47 +318,60 @@ jobs:
         steps:
             - uses: actions/checkout@v4
 
-            - name: Install prerequisites
+            - name: Clean up build environment
               run: |
                   # Workaround for github/brew problem. Python is already install
                   # on the Github action runner and then homebrew comes along...
                   # See also: https://github.com/Homebrew/homebrew-core/issues/173191
                   rm -f /usr/local/bin/2to3* /usr/local/bin/idle3* /usr/local/bin/pydoc3* /usr/local/bin/python3*
               if: ${{ matrix.flavour == 'macos' }}
 
-            - uses: ./.github/actions/install-dependencies
-              with:
-                  version: ${{ matrix.deps }}
-
-            - uses: actions/setup-python@v5
-              with:
-                  python-version: "${{ matrix.python }}"
-                  allow-prereleases: true
-
             - name: Install packages
               run: |
                 sudo apt-get update -y -qq
-                sudo apt-get install -y -qq libboost-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev liblz4-dev
+                sudo apt-get install -y -qq libboost-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev liblz4-dev virtualenv
               if: ${{ matrix.flavour == 'linux' }}
 
             - name: Install packages
-              run: brew install boost geos
+              run: brew install boost geos virtualenv
               shell: bash
               if: ${{ matrix.flavour == 'macos' }}
 
-            - name: Install prerequisites
+            - name: Setup virtualenv
               run: |
-                  python -m pip install --upgrade pip
-                  pip install -U pytest pytest-httpserver shapely setuptools requests
+                virtualenv venv
+                ./venv/bin/pip install build
+
+            - name: Install package dependencies (develop)
+              run: |
+                  git clone --quiet --depth 1 https://github.com/osmcode/libosmium.git contrib/libosmium
+                  git clone --quiet --depth 1 https://github.com/mapbox/protozero.git contrib/protozero
+                  git clone --quiet https://github.com/pybind/pybind11.git contrib/pybind11
               shell: bash
+              if: ${{ matrix.deps == 'develop' }}
+
+            - name: Install package dependencies (release)
+              run: |
+                  ./venv/bin/python -m build --sdist
+              shell: bash
+              if: ${{ matrix.deps == 'release' }}
+
+            - uses: actions/setup-python@v5
+              with:
+                  python-version: "${{ matrix.python }}"
+                  allow-prereleases: true
 
             - name: Build package
-              run: python setup.py build
+              run: |
+                ./venv/bin/python -m build -w
               shell: bash
 
             - name: Run tests
               run: |
-                pytest test
+                WHEEL=`ls dist/osmium*.whl`
+                echo "Installing $WHEEL"
+                ./venv/bin/pip install dist/osmium*whl ${WHEEL}[tests]
+                ./venv/bin/pytest test
               shell: bash
 
 
@@ -476,11 +490,16 @@ jobs:
               env:
                   CMAKE_TOOLCHAIN_FILE: C:/vcpkg/scripts/buildsystems/vcpkg.cmake
 
+            - name: Repair wheels
+              run: |
+                pip install delvewheel
+                delvewheel repair --add-path C:/vcpkg/installed/x64-windows/bin/ --add-path C:/vcpkg/installed/x64-windows/debug/bin dist/*.whl
+
             - name: 'Upload Artifact'
               uses: actions/upload-artifact@v4
               with:
                   name: pyosmium-win64-dist
-                  path: dist
+                  path: wheelhouse
 
     build-windows-free-threaded:
         runs-on: windows-2022
@@ -502,7 +521,8 @@ jobs:
               shell: bash
 
             - name: Install packages
-              run: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows boost-variant:x64-windows boost-iterator:x64-windows lz4:x86-windows
+              run: |
+                vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows boost-variant:x64-windows boost-iterator:x64-windows lz4:x86-windows
               shell: bash
 
             - name: Set up Python 3.13t
@@ -532,11 +552,16 @@ jobs:
               env:
                   CMAKE_TOOLCHAIN_FILE: C:/vcpkg/scripts/buildsystems/vcpkg.cmake
 
+            - name: Repair wheels
+              run: |
+                pip install delvewheel
+                delvewheel repair --add-path C:/vcpkg/installed/x64-windows/bin/ --add-path C:/vcpkg/installed/x64-windows/debug/bin dist/*.whl
+
             - name: 'Upload Artifact'
               uses: actions/upload-artifact@v4
               with:
                   name: pyosmium-win64-dist-t
-                  path: dist
+                  path: wheelhouse
 
 
     test-windows:
@@ -599,7 +624,7 @@ jobs:
 
     test-windows-free-threaded:
         runs-on: windows-2022
-        needs: build-windows
+        needs: build-windows-free-threaded
 
         strategy:
             fail-fast: false
@@ -646,7 +671,8 @@ jobs:
               if: matrix.python-version == '3.14t'
 
             - name: Run tests
-              run: ./osmium-test/Scripts/pytest test --parallel-threads 5 --iterations 5
+              run: |
+                ./osmium-test/Scripts/pytest test --parallel-threads 5 --iterations 5
               shell: bash
 
             - name: Check tool availability