diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 52621caf9..43fe9cd6b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,9 +79,13 @@ jobs: - os: macos-15 python_version: '3.13' test_select: ios + # Exercise iOS on a non-default simulator. + test_runtime: 'args: --simulator "iPhone 16e,OS=18.5"' - os: macos-15-intel python_version: '3.13' test_select: android + # Exercise Android on a non-default simulator + test_runtime: 'args: --managed minVersion' - os: macos-15 python_version: '3.13' test_select: android @@ -154,6 +158,7 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_BUILD_FRONTEND: ${{ matrix.test_select && 'build' || 'build[uv]' }} CIBW_PLATFORM: ${{ matrix.test_select }} + CIBW_TEST_RUNTIME: ${{ matrix.test_runtime }} - name: Run a sample build (GitHub Action, only) uses: ./ @@ -179,6 +184,7 @@ jobs: uses: ./ env: CIBW_PLATFORM: ${{ matrix.test_select }} + CIBW_TEST_RUNTIME: ${{ matrix.test_runtime }} with: package-dir: sample_proj output-dir: wheelhouse_config_file @@ -202,6 +208,8 @@ jobs: path: wheelhouse/*.whl - name: Test cibuildwheel + env: + CIBW_TEST_RUNTIME: ${{ matrix.test_runtime }} run: | uv run --no-sync bin/run_tests.py --test-select=${{ matrix.test_select || 'native' }} ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }} diff --git a/README.md b/README.md index 5c4eb6219..f3847b88e 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,8 @@ Usage | | Linux | macOS | Windows | Linux ARM | macOS ARM | Windows ARM | Android | iOS | |-----------------|-------|-------|---------|-----------|-----------|-------------|---------|-----| -| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅⁴ | ✅³⁵ | -| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅⁴ | ✅³⁵ | +| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅⁴ | ✅³ | +| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅⁴ | ✅³ | | Travis CI | ✅ | | ✅ | ✅ | | | ✅⁴ | | | CircleCI | ✅ | ✅ | | ✅ | ✅ | | ✅⁴ | ✅³ | | Gitlab CI | ✅ | ✅ | ✅ | ✅¹ | ✅ | | ✅⁴ | ✅³ | @@ -70,7 +70,6 @@ Usage ² [Uses cross-compilation](https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform.
³ Requires a macOS runner; runs tests on the simulator for the runner's architecture.
⁴ Building for Android requires the runner to be Linux x86_64, macOS ARM64 or macOS x86_64. Testing has [additional requirements](https://cibuildwheel.pypa.io/en/stable/platforms/#android).
-⁵ The `macos-15` and `macos-latest` images are [incompatible with cibuildwheel at this time](https://cibuildwheel.pypa.io/en/stable/platforms/#ios-system-requirements) when building iOS wheels.
@@ -160,12 +159,13 @@ The following diagram summarises the steps that cibuildwheel takes on each platf | | [`test-groups`](https://cibuildwheel.pypa.io/en/stable/options/#test-groups) | Specify test dependencies from your project's `dependency-groups` | | | [`test-skip`](https://cibuildwheel.pypa.io/en/stable/options/#test-skip) | Skip running tests on some builds | | | [`test-environment`](https://cibuildwheel.pypa.io/en/stable/options/#test-environment) | Set environment variables for the test environment | +| | [`test-runtime`](https://cibuildwheel.pypa.io/en/stable/options/#test-runtime) | Controls how the tests will be executed. | | **Debugging** | [`debug-keep-container`](https://cibuildwheel.pypa.io/en/stable/options/#debug-keep-container) | Keep the container after running for debugging. | | | [`debug-traceback`](https://cibuildwheel.pypa.io/en/stable/options/#debug-traceback) | Print full traceback when errors occur. | | | [`build-verbosity`](https://cibuildwheel.pypa.io/en/stable/options/#build-verbosity) | Increase/decrease the output of the build | - + These options can be specified in a pyproject.toml file, or as environment variables, see [configuration docs](https://cibuildwheel.pypa.io/en/latest/configuration/). diff --git a/bin/generate_schema.py b/bin/generate_schema.py index fe5f09187..611d19e5a 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -224,6 +224,24 @@ test-environment: description: Set environment variables for the test environment type: string_table + test-runtime: + description: Additional configuration for the test runner + oneOf: + - type: string + pattern: '^$' + - type: object + additionalProperties: false + - type: string + pattern: 'args:' + - type: object + additionalProperties: false + required: [args] + properties: + args: + type: array + items: + type: string + """ schema = yaml.safe_load(starter) @@ -304,6 +322,7 @@ test-sources: {"$ref": "#/$defs/inherit"} test-requires: {"$ref": "#/$defs/inherit"} test-environment: {"$ref": "#/$defs/inherit"} + test-runtime: {"$ref": "#/$defs/inherit"} """ ) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 22dc48d1a..af946a883 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -24,7 +24,7 @@ from .selector import BuildSelector, EnableGroup, TestSelector, selector_matches from .typing import PLATFORMS, PlatformName from .util import resources -from .util.helpers import format_safe, strtobool, unwrap +from .util.helpers import format_safe, parse_key_value_string, strtobool, unwrap from .util.packaging import DependencyConstraints MANYLINUX_ARCHS: Final[tuple[str, ...]] = ( @@ -92,6 +92,20 @@ class GlobalOptions: allow_empty: bool +@dataclasses.dataclass(frozen=True) +class TestRuntimeConfig: + args: Sequence[str] = () + + @classmethod + def from_config_string(cls, config_string: str) -> Self: + config_dict = parse_key_value_string(config_string, [], ["args"]) + args = config_dict.get("args") or [] + return cls(args=args) + + def options_summary(self) -> str | dict[str, str]: + return {"args": repr(self.args)} + + @dataclasses.dataclass(frozen=True, kw_only=True) class BuildOptions: globals: GlobalOptions @@ -110,6 +124,7 @@ class BuildOptions: test_extras: str test_groups: list[str] test_environment: ParsedEnvironment + test_runtime: TestRuntimeConfig build_verbosity: int build_frontend: BuildFrontendConfig config_settings: str @@ -761,6 +776,20 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: msg = f"Malformed environment option {test_environment_config!r}" raise errors.ConfigurationError(msg) from e + test_runtime_str = self.reader.get( + "test-runtime", + env_plat=False, + option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False), + ) + if not test_runtime_str: + test_runtime = TestRuntimeConfig() + else: + try: + test_runtime = TestRuntimeConfig.from_config_string(test_runtime_str) + except ValueError as e: + msg = f"Failed to parse test runtime config. {e}" + raise errors.ConfigurationError(msg) from e + test_requires = self.reader.get( "test-requires", option_format=ListFormat(sep=" ") ).split() @@ -868,6 +897,7 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: test_command=test_command, test_sources=test_sources, test_environment=test_environment, + test_runtime=test_runtime, test_requires=[*test_requires, *test_requirements_from_groups], test_extras=test_extras, test_groups=test_groups, diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index dca2f5117..13438250e 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -638,17 +638,27 @@ def test_wheel(state: BuildState, wheel: Path) -> None: ) raise errors.FatalError(msg) + # By default, run on a testbed managed emulator running the newest supported + # Android version. However, if the user specifies a --managed or --connected + # test execution argument, that argument takes precedence. + test_runtime_args = state.options.test_runtime.args + + if any(arg.startswith(("--managed", "--connected")) for arg in test_runtime_args): + emulator_args = [] + else: + emulator_args = ["--managed", "maxVersion"] + # Run the test app. call( state.python_dir / "android.py", "test", - "--managed", - "maxVersion", "--site-packages", site_packages_dir, "--cwd", cwd_dir, + *emulator_args, *(["-v"] if state.options.build_verbosity > 0 else []), + *test_runtime_args, "--", *test_args, env=state.build_env, diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index eba04c35d..cf9ba4eba 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -2,6 +2,7 @@ import dataclasses import os +import platform import shlex import shutil import subprocess @@ -653,11 +654,35 @@ def build(options: Options, tmp_path: Path) -> None: ) raise errors.FatalError(msg) + test_runtime_args = build_options.test_runtime.args + + # 2025-10: The GitHub Actions macos-15 runner has a known issue where + # the default simulator won't start due to a disk performance issue; + # see https://github.com/actions/runner-images/issues/12777 for details. + # In the meantime, if it looks like we're running on a GitHub Actions + # macos-15 runner, use a simulator that is known to work, unless the + # user explicitly specifies a simulator. + os_version, _, arch = platform.mac_ver() + if ( + "GITHUB_ACTIONS" in os.environ + and os_version.startswith("15.") + and arch == "arm64" + and not any( + arg.startswith("--simulator") for arg in test_runtime_args + ) + ): + test_runtime_args = [ + "--simulator", + "iPhone 16e,OS=18.5", + *test_runtime_args, + ] + call( "python", testbed_path, "run", *(["--verbose"] if build_options.build_verbosity > 0 else []), + *test_runtime_args, "--", *final_command, env=test_env, diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index bae1f9eb5..2c52aada4 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -569,6 +569,39 @@ ], "title": "CIBW_TEST_ENVIRONMENT" }, + "test-runtime": { + "description": "Additional configuration for the test runner", + "oneOf": [ + { + "type": "string", + "pattern": "^$" + }, + { + "type": "object", + "additionalProperties": false + }, + { + "type": "string", + "pattern": "args:" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "args" + ], + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + ], + "title": "CIBW_TEST_RUNTIME" + }, "overrides": { "type": "array", "description": "An overrides array", @@ -638,6 +671,9 @@ }, "test-environment": { "$ref": "#/$defs/inherit" + }, + "test-runtime": { + "$ref": "#/$defs/inherit" } } }, @@ -748,6 +784,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } } @@ -876,6 +915,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -936,6 +978,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -1009,6 +1054,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -1069,6 +1117,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -1129,6 +1180,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -1189,6 +1243,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } } diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index d5c176f27..78895bf9a 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -25,6 +25,7 @@ test-requires = [] test-extras = [] test-groups = [] test-environment = {} +test-runtime = {} container-engine = "docker" diff --git a/docs/options.md b/docs/options.md index 92caf5afe..88a377d30 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1672,6 +1672,41 @@ Platform-specific environment variables are also available:
CIBW_TEST_ENVIRONMENT: PYTHONSAFEPATH=1 ``` +### `test-runtime` {: #test-runtime toml env-var } + +> Controls how the tests will be executed. + +On desktop environments, the tests are executed on the same machine/container as the wheel was built. However on Android and iOS, the tests are run inside a virtual machine – a simulator or emulator – representing the target. + +For these embedded platforms, a testbed project is used to run the tests. The `test-runtime` setting can define an `args` key that defines additional arguments that will be used when starting the testbed project. + +Platform-specific environment variables are also available:
+`CIBW_TEST_RUNTIME_ANDROID` |`CIBW_TEST_RUNTIME_IOS` + +#### Examples + +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel.ios] + # Run the tests on an iPhone 16e simulator running iOS 18.5. + test-runtime = { args = ["--simulator='iPhone 16e,OS=18.5'"] } + + [tool.cibuildwheel.android] + # Run the Android tests on the minimum supported Android version. + test-runtime = { args = ["--managed", "minVersion"] } + ``` + +!!! tab examples "Environment variables" + + ```yaml + # Run the tests on an iPhone 16e simulator running iOS 18.5. + CIBW_TEST_RUNTIME_IOS: "args: --simulator='iPhone 16e,OS=18.5'" + + # Run the Android tests on the minimum supported Android version. + CIBW_TEST_RUNTIME_ANDROID: "args: --managed minVersion" + ``` + ## Debugging diff --git a/docs/platforms.md b/docs/platforms.md index 1cb931ebc..dee822490 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -233,6 +233,8 @@ machine – for example, if you're building on an ARM64 machine, then you can te ARM64 wheel. Wheels of other architectures can still be built, but testing will automatically be skipped. +Any arguments specified using [`test-runtime`](options.md#test-runtime) will be passed as arguments to the Python script that starts the [testbed project](https://github.com/python/cpython/blob/main/Android/README.md#testing). cibuildwheel will automatically start the testbed project with `--site-packages` and `--cwd` arguments matching your test environment, as well as enabling verbose output with `-v` if [`build-verbosity`](options.md#build-verbosity) is enabled. The most common additional arguments to use will be `--managed minVersion` or `--managed maxVersion`, specifying the use of a managed Android emulator with the minimum or maximum supported Android version; or `--connected `, specifying the use of an existing booted Android emulator or device. By default, the testbed project will run with `--managed maxVersion`. + Running an emulator requires the build machine to either be bare-metal or support nested virtualization. CI platforms known to meet this requirement are: @@ -320,4 +322,6 @@ If tests have been configured, the test suite will be executed on the simulator The iOS test environment can't support running shell scripts, so the [`test-command`](options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. -The test process uses the same testbed used by CPython itself to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. +The test process uses the [same testbed used by CPython itself](https://github.com/python/cpython/tree/main/Apple/iOS#testing-python-on-ios) to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. + +Any arguments specified using [`test-runtime`](options.md#test-runtime) will be passed as arguments to the Python script that starts the testbed project. The testbed project will be started with `-v` enabling verbose output if [`build-verbosity`](options.md#build-verbosity) is enabled; the most common additional argument to use will be `--simulator`, which allows the specification of a specific device or iOS version for the test simulator. By default, the testbed project will attempt to find an "SE class" simulator (i.e., an iPhone SE, iPhone 16e, or similar), running the newest iOS version available. diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 52f2f1adf..b9847ed5c 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -2,6 +2,7 @@ import platform as platform_module import textwrap import unittest.mock +from collections.abc import Sequence from pathlib import Path from typing import Literal @@ -626,6 +627,39 @@ def test_get_build_frontend_extra_flags_warning( mock_warning.assert_called_once() +@pytest.mark.parametrize( + ("definition", "expected_args"), + [ + ("", ()), + ('test-runtime = ""', ()), + ("test-runtime = {}", ()), + ('test-runtime = {args = ""}', []), + ('test-runtime = "args: --simulator foo"', ["--simulator", "foo"]), + ('test-runtime = {args = ["--simulator", "foo"]}', ["--simulator", "foo"]), + ], +) +def test_test_runtime_handling( + tmp_path: Path, definition: str, expected_args: Sequence[str] | None +) -> None: + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + textwrap.dedent( + f"""\ + [tool.cibuildwheel] + {definition} + """ + ) + ) + + options = Options(platform="ios", command_line_arguments=args, env={}) + + local = options.build_options("cp313-ios_13_0_arm64_iphoneos") + assert local.test_runtime.args == expected_args + + @pytest.mark.parametrize( ("definition", "expected"), [