adapt CI work flow to new build system

Author: lonvia

.github/actions/install-dependencies/action.yml

@@ -268,6 +268,7 @@ jobs:
         runs-on: ${{ matrix.platform }}
 
         strategy:
+            fail-fast: false
             matrix:
                 compiler: [gcc-old, clang-old, gcc, clang, macos-intel, macos-arm]
                 include:
@@ -317,47 +318,60 @@ jobs:
         steps:
             - uses: actions/checkout@v4
 
-            - name: Install prerequisites
+            - name: Clean up build environment
               run: |
                   # Workaround for github/brew problem. Python is already install
                   # on the Github action runner and then homebrew comes along...
                   # See also: https://github.com/Homebrew/homebrew-core/issues/173191
                   rm -f /usr/local/bin/2to3* /usr/local/bin/idle3* /usr/local/bin/pydoc3* /usr/local/bin/python3*
               if: ${{ matrix.flavour == 'macos' }}
 
-            - uses: ./.github/actions/install-dependencies
-              with:
-                  version: ${{ matrix.deps }}
-
-            - uses: actions/setup-python@v5
-              with:
-                  python-version: "${{ matrix.python }}"
-                  allow-prereleases: true
-
             - name: Install packages
               run: |
                 sudo apt-get update -y -qq
-                sudo apt-get install -y -qq libboost-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev liblz4-dev
+                sudo apt-get install -y -qq libboost-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev liblz4-dev virtualenv
               if: ${{ matrix.flavour == 'linux' }}
 
             - name: Install packages
-              run: brew install boost geos
+              run: brew install boost geos virtualenv
               shell: bash
               if: ${{ matrix.flavour == 'macos' }}
 
-            - name: Install prerequisites
+            - name: Setup virtualenv
               run: |
-                  python -m pip install --upgrade pip
-                  pip install -U pytest pytest-httpserver shapely setuptools requests
+                virtualenv venv
+                ./venv/bin/pip install build
+
+            - name: Install package dependencies (develop)
+              run: |
+                  git clone --quiet --depth 1 https://github.com/osmcode/libosmium.git contrib/libosmium
+                  git clone --quiet --depth 1 https://github.com/mapbox/protozero.git contrib/protozero
+                  git clone --quiet https://github.com/pybind/pybind11.git contrib/pybind11
               shell: bash
+              if: ${{ matrix.deps == 'develop' }}
+
+            - name: Install package dependencies (release)
+              run: |
+                  ./venv/bin/python -m build --sdist
+              shell: bash
+              if: ${{ matrix.deps == 'release' }}
+
+            - uses: actions/setup-python@v5
+              with:
+                  python-version: "${{ matrix.python }}"
+                  allow-prereleases: true
 
             - name: Build package
-              run: python setup.py build
+              run: |
+                ./venv/bin/python -m build -w
               shell: bash
 
             - name: Run tests
               run: |
-                pytest test
+                WHEEL=`ls dist/osmium*.whl`
+                echo "Installing $WHEEL"
+                ./venv/bin/pip install dist/osmium*whl ${WHEEL}[tests]
+                ./venv/bin/pytest test
               shell: bash
 
 
@@ -476,11 +490,16 @@ jobs:
               env:
                   CMAKE_TOOLCHAIN_FILE: C:/vcpkg/scripts/buildsystems/vcpkg.cmake
 
+            - name: Repair wheels
+              run: |
+                pip install delvewheel
+                delvewheel repair --add-path C:/vcpkg/installed/x64-windows/bin/ --add-path C:/vcpkg/installed/x64-windows/debug/bin dist/*.whl
+
             - name: 'Upload Artifact'
               uses: actions/upload-artifact@v4
               with:
                   name: pyosmium-win64-dist
-                  path: dist
+                  path: wheelhouse
 
     build-windows-free-threaded:
         runs-on: windows-2022
@@ -502,7 +521,8 @@ jobs:
               shell: bash
 
             - name: Install packages
-              run: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows boost-variant:x64-windows boost-iterator:x64-windows lz4:x86-windows
+              run: |
+                vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows boost-variant:x64-windows boost-iterator:x64-windows lz4:x86-windows
               shell: bash
 
             - name: Set up Python 3.13t
@@ -532,11 +552,16 @@ jobs:
               env:
                   CMAKE_TOOLCHAIN_FILE: C:/vcpkg/scripts/buildsystems/vcpkg.cmake
 
+            - name: Repair wheels
+              run: |
+                pip install delvewheel
+                delvewheel repair --add-path C:/vcpkg/installed/x64-windows/bin/ --add-path C:/vcpkg/installed/x64-windows/debug/bin dist/*.whl
+
             - name: 'Upload Artifact'
               uses: actions/upload-artifact@v4
               with:
                   name: pyosmium-win64-dist-t
-                  path: dist
+                  path: wheelhouse
 
 
     test-windows:
@@ -599,7 +624,7 @@ jobs:
 
     test-windows-free-threaded:
         runs-on: windows-2022
-        needs: build-windows
+        needs: build-windows-free-threaded
 
         strategy:
             fail-fast: false
@@ -646,7 +671,8 @@ jobs:
               if: matrix.python-version == '3.14t'
 
             - name: Run tests
-              run: ./osmium-test/Scripts/pytest test --parallel-threads 5 --iterations 5
+              run: |
+                ./osmium-test/Scripts/pytest test --parallel-threads 5 --iterations 5
               shell: bash
 
             - name: Check tool availability

.github/workflows/ci.yml

@@ -67,13 +67,19 @@ foreach(PYMOD geom index io area)
     pybind11_add_module(${PYMOD} lib/${PYMOD}.cc)
     target_link_libraries(${PYMOD} PRIVATE ${OSMIUM_LIBRARIES})
     install(TARGETS ${PYMOD} DESTINATION osmium)
+    if(APPLE)
+        set_target_properties(${PYMOD} PROPERTIES CXX_VISIBILITY_PRESET "default")
+    endif()
 endforeach()
 
 # Modules where additional Python code is in src (C++-part will be private).
 foreach(PYMOD osm replication)
     pybind11_add_module(_${PYMOD} lib/${PYMOD}.cc)
     target_link_libraries(_${PYMOD} PRIVATE ${OSMIUM_LIBRARIES})
     install(TARGETS _${PYMOD} DESTINATION osmium/${PYMOD})
+    if(APPLE)
+        set_target_properties(_${PYMOD} PROPERTIES CXX_VISIBILITY_PRESET "default")
+    endif()
 endforeach()
 
 # Modules with multiple source files.
@@ -101,4 +107,11 @@ target_link_libraries(filter PRIVATE ${OSMIUM_LIBRARIES})
 configure_file(cmake/version.py.tmpl version.py)
 install(FILES ${PROJECT_BINARY_DIR}/version.py DESTINATION osmium)
 
+# workaround for https://github.com/pybind/pybind11/issues/1272
+if(APPLE)
+    set_target_properties(_osmium PROPERTIES CXX_VISIBILITY_PRESET "default")
+    set_target_properties(filter PROPERTIES CXX_VISIBILITY_PRESET "default")
+endif()
+
+
 endif()

CMakeLists.txt

@@ -56,6 +56,10 @@ docs = [
     'argparse-manpage '
     ]
 
+[project.scripts]
+pyosmium-get-changes = "osmium.tools.pyosmium_get_changes:main"
+pyosmium-up-to-date = "osmium.tools.pyosmium_up_to_date:main"
+
 [tool.scikit-build]
 minimum-version = "build-system.requires"
 cmake.version = ">=3.15"
@@ -65,8 +69,8 @@ cmake = true
 exclude = ['**']
 include = ['/src/**/*.py',
            '/src/**/*.pyi',
+           '/src/osmium/py.typed',
            '/test/**/*.py',
-           '/tools/*',
            '/lib/**/*.cc',
            '/lib/**/*.h',
            '/cmake/*',

pyproject.toml

@@ -1,4 +1,3 @@
-#!/usr/bin/python
 """
 Fetch diffs from an OSM planet server.
 
@@ -24,6 +23,9 @@
 However, it can read cookies from a Netscape-style cookie jar file, send these
 cookies to the server and will save received cookies to the jar file.
 """
+import sys
+import logging
+from textwrap import dedent as msgfmt
 
 from argparse import ArgumentParser, RawDescriptionHelpFormatter, ArgumentTypeError
 import datetime as dt
@@ -33,16 +35,11 @@
 from osmium.replication import newest_change_from_file
 from osmium.replication.utils import get_replication_header
 from osmium.version import pyosmium_release
-from osmium import SimpleHandler, SimpleWriter
-
-
-import re
-import sys
-import logging
-from textwrap import dedent as msgfmt
+from osmium import SimpleWriter
 
 log = logging.getLogger()
 
+
 class ReplicationStart(object):
     """ Represents the point where changeset download should begin.
     """
@@ -78,7 +75,8 @@ def from_date(datestr):
             date = dt.datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ")
             date = date.replace(tzinfo=dt.timezone.utc)
         except ValueError:
-            raise ArgumentTypeError("Date needs to be in ISO8601 format (e.g. 2015-12-24T08:08:08Z).")
+            raise ArgumentTypeError(
+                "Date needs to be in ISO8601 format (e.g. 2015-12-24T08:08:08Z).")
 
         return ReplicationStart(date=date)
 
@@ -106,6 +104,7 @@ def from_osm_file(fname, ignore_headers):
 
         return ReplicationStart(seq_id=seq, date=ts, src=url)
 
+
 def write_end_sequence(fname, seqid):
     """Either writes out the sequence file or prints the sequence id to stdout.
     """
@@ -115,6 +114,7 @@ def write_end_sequence(fname, seqid):
         with open(fname, 'w') as fd:
             fd.write(str(seqid))
 
+
 def get_arg_parser(from_main=False):
     parser = ArgumentParser(prog='pyosmium-get-changes',
                             description=__doc__,
@@ -146,10 +146,10 @@ def get_arg_parser(from_main=False):
     group.add_argument('-O', '--start-osm-data', dest='start_file', metavar='OSMFILE',
                        help='start at the date of the newest OSM object in the file')
     parser.add_argument('-f', '--sequence-file', dest='seq_file',
-                       help='Sequence file. If the file exists, then updates '
-                            'will start after the id given in the file. At the '
-                            'end of the process, the last sequence ID contained '
-                            'in the diff is written.')
+                        help='Sequence file. If the file exists, then updates '
+                             'will start after the id given in the file. At the '
+                             'end of the process, the last sequence ID contained '
+                             'in the diff is written.')
     parser.add_argument('--ignore-osmosis-headers', dest='ignore_headers',
                         action='store_true',
                         help='When determining the start from an OSM file, '
@@ -164,7 +164,7 @@ def get_arg_parser(from_main=False):
     return parser
 
 
-def main(args):
+def pyosmium_get_changes(args):
     logging.basicConfig(stream=sys.stderr,
                         format='%(asctime)s %(levelname)s: %(message)s',
                         datefmt='%Y-%m-%d %H:%M:%S')
@@ -194,17 +194,16 @@ def main(args):
 
     if options.server_url is not None and options.start.source is not None:
         if options.server_url != options.start.source:
-            log.error(msgfmt("""
-              You asked to use server URL:
-                %s
-              but the referenced OSM file points to replication server:
-                %s
-              If you really mean to overwrite the URL, use --ignore-osmosis-headers."""
-              % (options.server_url, options.start.source)))
+            log.error(msgfmt(f"""
+                You asked to use server URL:
+                  {options.server_url}
+                but the referenced OSM file points to replication server:
+                  {options.start.source}
+                If you really mean to overwrite the URL, use --ignore-osmosis-headers."""))
             return 2
     url = options.server_url \
-            or options.start.source \
-            or 'https://planet.osm.org/replication/minute/'
+        or options.start.source \
+        or 'https://planet.osm.org/replication/minute/'
     logging.info("Using replication server at %s" % url)
 
     with rserv.ReplicationServer(url, diff_type=options.server_diff_type) as svr:
@@ -248,5 +247,9 @@ def main(args):
     return 0
 
 
-if __name__ == '__main__':
-    exit(main(sys.argv[1:]))
+def main():
+    logging.basicConfig(stream=sys.stderr,
+                        format='%(asctime)s %(levelname)s: %(message)s',
+                        datefmt='%Y-%m-%d %H:%M:%S')
+
+    return pyosmium_get_changes(sys.argv[1:])