From 0382c2593d840817d47cfdb94c04f2380584d869 Mon Sep 17 00:00:00 2001 From: Kyle-Neale Date: Fri, 17 Oct 2025 22:21:27 -0400 Subject: [PATCH 01/10] New upgrade python to ddev and rename meta script --- ddev/src/ddev/cli/meta/scripts/__init__.py | 6 +- .../ddev/cli/meta/scripts/update_python.py | 397 ++++++++++++++++++ ...upgrade_python.py => upgrade_py_config.py} | 10 +- ddev/src/ddev/repo/constants.py | 1 + .../cli/meta/scripts/test_upgrade_python.py | 4 +- 5 files changed, 409 insertions(+), 9 deletions(-) create mode 100644 ddev/src/ddev/cli/meta/scripts/update_python.py rename ddev/src/ddev/cli/meta/scripts/{upgrade_python.py => upgrade_py_config.py} (95%) diff --git a/ddev/src/ddev/cli/meta/scripts/__init__.py b/ddev/src/ddev/cli/meta/scripts/__init__.py index d70d059bee64c..07bec84c4cf9f 100644 --- a/ddev/src/ddev/cli/meta/scripts/__init__.py +++ b/ddev/src/ddev/cli/meta/scripts/__init__.py @@ -10,7 +10,8 @@ from ddev.cli.meta.scripts.monitor import monitor from ddev.cli.meta.scripts.saved_views import sv from ddev.cli.meta.scripts.serve_openmetrics_payload import serve_openmetrics_payload -from ddev.cli.meta.scripts.upgrade_python import upgrade_python +from ddev.cli.meta.scripts.update_python import update_python_version +from ddev.cli.meta.scripts.upgrade_py_config import update_python_config @click.group(short_help='Miscellaneous scripts that may be useful') @@ -25,6 +26,7 @@ def scripts(): scripts.add_command(metrics2md) scripts.add_command(remove_labels) scripts.add_command(serve_openmetrics_payload) -scripts.add_command(upgrade_python) +scripts.add_command(update_python_config) +scripts.add_command(update_python_version) scripts.add_command(sv) scripts.add_command(monitor) diff --git a/ddev/src/ddev/cli/meta/scripts/update_python.py b/ddev/src/ddev/cli/meta/scripts/update_python.py new file mode 100644 index 0000000000000..014ac212c494f --- /dev/null +++ b/ddev/src/ddev/cli/meta/scripts/update_python.py @@ -0,0 +1,397 @@ +# (C) Datadog, Inc. 2023-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +import asyncio +import re +from typing import TYPE_CHECKING + +import click +import httpx +import orjson +import requests +from packaging.version import Version + +if TYPE_CHECKING: + from ddev.cli.application import Application + from ddev.validation.tracker import ValidationTracker + +@click.command('update-python-version', short_help='Upgrade the Python version used in the repository.') +@click.pass_obj +def update_python_version(app: Application): + """Upgrade the Python version used in the repository. + + Automatically detects the latest Python version, fetches official SHA256 hashes, + and updates version references across: + - ddev/src/ddev/repo/constants.py + - .builders/images/*/Dockerfile (Linux and Windows) + - .github/workflows/resolve-build-deps.yaml (macOS) + + \b + `$ ddev meta scripts update-python-version` + """ + from ddev.repo.constants import PYTHON_VERSION_FULL as current_version + from ddev.repo.constants import PYTHON_VERSION as major_minor + + tracker = app.create_validation_tracker('Python version updates') + + # Check for new version + latest_version = get_latest_python_version(app, major_minor) + if latest_version is None: + app.display_error(f"Could not find latest Python version for {major_minor}") + app.abort() + + # Validate version string format for security (prevent injection) + if not validate_version_string(latest_version): + app.display_error(f"Invalid version format detected: {latest_version}") + app.abort() + + if Version(latest_version) <= Version(current_version): + app.display_info(f"Already at latest Python version: {current_version}") + return + + app.display_info(f"Updating Python from {current_version} to {latest_version}") + + # Fetch hashes + try: + new_version_hashes = get_python_sha256_hashes(app, latest_version) + except Exception as e: + tracker.error(('SHA256 hashes',), message=f"Failed to fetch: {e}") + tracker.display() + app.abort() + + # Validate hashes + if not validate_sha256(new_version_hashes.get('linux_source_sha256', '')): + tracker.error(('SHA256 validation',), message="Invalid Linux SHA256 hash format") + if not validate_sha256(new_version_hashes.get('windows_amd64_sha256', '')): + tracker.error(('SHA256 validation',), message="Invalid Windows SHA256 hash format") + + if tracker.errors: + tracker.display() + app.abort() + + # Perform updates + update_python_version_full_constant(app, latest_version, tracker) + update_dockerfiles_python_version(app, latest_version, new_version_hashes, tracker) + update_macos_python_version(app, latest_version, tracker) + + # Display results + tracker.display() + + if tracker.errors: + app.display_warning("Some updates failed. Please review the errors above.") + app.abort() + + app.display_success(f"Python version updated from {current_version} to {latest_version}") + +def validate_version_string(version: str) -> bool: + """ + Validate that version string is safe and matches expected format. + + Args: + version: Version string to validate + + Returns: + True if valid, False otherwise + """ + return bool(re.match(r'^\d+\.\d+\.\d+$', version)) + + +def validate_sha256(hash_str: str) -> bool: + """Validate SHA256 hash format (64 hex characters).""" + return bool(re.match(r'^[0-9a-f]{64}$', hash_str)) + + +def update_dockerfiles_python_version( + app: Application, new_version: str, hashes: dict[str, str], tracker: ValidationTracker +): + dockerfiles = [ + app.repo.path / '.builders' / 'images' / 'linux-aarch64' / 'Dockerfile', + app.repo.path / '.builders' / 'images' / 'linux-x86_64' / 'Dockerfile', + app.repo.path / '.builders' / 'images' / 'windows-x86_64' / 'Dockerfile', + ] + + try: + linux_sha = hashes['linux_source_sha256'] + windows_sha = hashes['windows_amd64_sha256'] + except KeyError as error: + tracker.error(('Dockerfiles',), message=f'Missing SHA256 hash entry: {error}') + return + + # Linux: ENV PYTHON3_VERSION=3.13.7 (no quotes, matches version at end of line) + # Windows: ENV PYTHON_VERSION="3.13.7" (with quotes) + linux_version_pattern = re.compile(r'(ENV PYTHON3_VERSION=)(\d+\.\d+\.\d+)$', re.MULTILINE) + windows_version_pattern = re.compile(r'(ENV PYTHON_VERSION=")(\d+\.\d+\.\d+)(")', re.MULTILINE) + + # SHA256 patterns must match the Python-specific ones: + # Linux: SHA256 that comes after VERSION="${PYTHON3_VERSION}" + # Windows: -Hash in the same RUN block with python-$Env:PYTHON_VERSION-amd64.exe + linux_sha_pattern = re.compile( + r'VERSION="\$\{PYTHON3_VERSION\}"[^\n]*\n[^\n]*SHA256="([0-9a-f]+)"', + re.MULTILINE + ) + windows_sha_pattern = re.compile( + r'python-\$Env:PYTHON_VERSION-amd64\.exe[^\n]*\n[^\n]*-Hash\s+\'([0-9a-f]+)\'', + re.MULTILINE + ) + + for dockerfile in dockerfiles: + if not dockerfile.exists(): + tracker.error((dockerfile.name,), message=f'File not found: {dockerfile}') + continue + + try: + content = dockerfile.read_text() + except Exception as e: + tracker.error((dockerfile.name,), message=f'Failed to read: {e}') + continue + + is_windows = 'windows-x86_64' in dockerfile.parts + version_pattern = windows_version_pattern if is_windows else linux_version_pattern + sha_pattern = windows_sha_pattern if is_windows else linux_sha_pattern + target_sha = windows_sha if is_windows else linux_sha + + def replace_version(match: re.Match[str]) -> str: + if is_windows: + prefix, _old_version, suffix = match.groups() + return f'{prefix}{new_version}{suffix}' + else: + prefix, _old_version = match.groups() + return f'{prefix}{new_version}' + + def replace_sha(match: re.Match[str]) -> str: + # The entire match contains the old hash, replace just the hash part + old_match = match.group(0) + old_hash = match.group(1) + return old_match.replace(old_hash, target_sha) + + # Helper to apply pattern substitution with error tracking + def apply_substitution(pattern: re.Pattern, replace_func, error_msg: str) -> str | None: + nonlocal content + content, count = pattern.subn(replace_func, content, count=1) + if count == 0: + tracker.error((dockerfile.name,), message=error_msg) + return None + return content + + # Apply version update + if apply_substitution(version_pattern, replace_version, 'Could not find Python version pattern') is None: + continue + + # Apply SHA256 update + if apply_substitution(sha_pattern, replace_sha, 'Could not find SHA256 pattern') is None: + continue + + try: + dockerfile.write_text(content) + tracker.success() + except Exception as e: + tracker.error((dockerfile.name,), message=f'Failed to write: {e}') + + +def update_macos_python_version(app: Application, new_version: str, tracker: ValidationTracker): + macos_python_file = app.repo.path / '.github' / 'workflows' / 'resolve-build-deps.yaml' + + if not macos_python_file.exists(): + tracker.error(('macOS workflow',), message=f'File not found: {macos_python_file}') + return + + try: + content = macos_python_file.read_text() + except Exception as e: + tracker.error(('macOS workflow',), message=f'Failed to read: {e}') + return + + target_line = next((line for line in content.splitlines() if 'PYTHON3_DOWNLOAD_URL' in line), None) + + if target_line is None: + tracker.error(('macOS workflow',), message='Could not find PYTHON3_DOWNLOAD_URL') + return + + new_url = f'https://www.python.org/ftp/python/{new_version}/python-{new_version}-macos11.pkg' + indent = target_line[: target_line.index('PYTHON3_DOWNLOAD_URL')] + new_line = f'{indent}PYTHON3_DOWNLOAD_URL: "{new_url}"' + + if target_line == new_line: + return + + updated_content = content.replace(target_line, new_line, 1) + + try: + macos_python_file.write_text(updated_content) + tracker.success() + except Exception as e: + tracker.error(('macOS workflow',), message=f'Failed to write: {e}') + + +def update_python_version_full_constant(app: Application, new_version: str, tracker: ValidationTracker): + constants_file = app.repo.path / 'ddev' / 'src' / 'ddev' / 'repo' / 'constants.py' + + if not constants_file.exists(): + tracker.error(('constants.py',), message=f'File not found: {constants_file}') + return + + try: + content = constants_file.read_text() + except Exception as e: + tracker.error(('constants.py',), message=f'Failed to read: {e}') + return + + prefix = 'PYTHON_VERSION_FULL = ' + target_line = next((line for line in content.splitlines() if line.startswith(prefix)), None) + + if target_line is None: + tracker.error(('constants.py',), message='Could not find PYTHON_VERSION_FULL constant') + return + + new_line = f"{prefix}'{new_version}'" + if target_line == new_line: + return + + updated_content = content.replace(target_line, new_line, 1) + + try: + constants_file.write_text(updated_content) + tracker.success() + except Exception as e: + tracker.error(('constants.py',), message=f'Failed to write: {e}') + + +def get_latest_python_version(app: Application, major_minor: str) -> str | None: + """ + Get the latest Python version from python.org FTP directory. + + Args: + major_minor: Python version in format "3.13" + + Returns: + Latest version string (e.g., "3.13.1") or None if not found + """ + url = "https://www.python.org/ftp/python/" + + try: + # Explicitly verify SSL/TLS certificate + response = requests.get(url, timeout=30, verify=True) + response.raise_for_status() + except requests.RequestException as e: + app.display_error(f"Error fetching Python versions: {e}") + return None + + # Parse directory listing for version folders + # Looking for patterns like: 3.13.0/ + pattern = rf'' + versions = [] + + for line in response.text.split("\n"): + match = re.search(pattern, line) + if match: + version_str = match.group(1) + try: + versions.append(Version(version_str)) + except Exception: + # Skip invalid versions + continue + + if not versions: + return None + + # Sort and return the latest version + versions.sort() + return str(versions[-1]) + + +def get_python_sha256_hashes(app: Application, version: str) -> dict[str, str]: + """ + Fetch SHA256 hashes for Python release artifacts using SBOM files. + + Python.org provides SBOM (Software Bill of Materials) files in SPDX JSON format + for each release artifact. These files contain SHA256 checksums. + + Args: + version: Python version string (e.g., "3.13.7") + + Returns: + Dictionary with SHA256 hashes: + { + 'linux_source_sha256': '<64-char hex hash>', + 'windows_amd64_sha256': '<64-char hex hash>' + } + + Raises: + ValueError: If version format is invalid + RuntimeError: If SBOM files cannot be fetched or parsed + """ + # Validate version format before using in URL construction (security) + if not validate_version_string(version): + raise ValueError(f"Invalid version format: {version}") + + # Steps: + # 1. Construct SBOM URLs for the files we need: + # - Linux source tarball: + # https://www.python.org/ftp/python/{version}/Python-{version}.tgz.spdx.json + # - Windows AMD64 installer: + # https://www.python.org/ftp/python/{version}/python-{version}-amd64.exe.spdx.json + SBOM_URLS = [ + f"https://www.python.org/ftp/python/{version}/Python-{version}.tgz.spdx.json", + f"https://www.python.org/ftp/python/{version}/python-{version}-amd64.exe.spdx.json" + ] + # 2. Download and parse each SBOM JSON file + async def get_sbom_data(client, url): + try: + # Explicitly verify SSL/TLS certificates + response = await client.get(url, timeout=30.0) + response.raise_for_status() + data = orjson.loads(response.text) + + # Validate SBOM structure + if not isinstance(data, dict): + raise ValueError(f"Invalid SBOM format: expected dict, got {type(data)}") + if 'packages' not in data: + raise ValueError("Invalid SBOM format: missing 'packages' field") + + return data.get('packages', []) + except Exception as e: + raise RuntimeError(f'Error processing URL {url}: {e}') from e + + async def fetch_sbom_data(urls): + # Create client with explicit SSL verification + async with httpx.AsyncClient(verify=True) as client: + return await asyncio.gather(*(get_sbom_data(client, url) for url in urls)) + + sbom_packages = asyncio.run(fetch_sbom_data(SBOM_URLS)) + + # Find the CPython package in the SBOM packages + linux_cpython_package = next((package for package in sbom_packages[0] if package.get('name') == "CPython"), None) + windows_cpython_package = next((package for package in sbom_packages[1] if package.get('name') == "CPython"), None) + + if linux_cpython_package is None: + raise ValueError("Could not find CPython package in Linux SBOM") + if windows_cpython_package is None: + raise ValueError("Could not find CPython package in Windows SBOM") + + # Find the SHA256 checksum in the CPython package checksums + linux_checksums = linux_cpython_package.get('checksums', []) + windows_checksums = windows_cpython_package.get('checksums', []) + + linux_checksum = next((checksum for checksum in linux_checksums if checksum.get('algorithm') == "SHA256"), None) + windows_checksum = next((checksum for checksum in windows_checksums if checksum.get('algorithm') == "SHA256"), None) + + if linux_checksum is None: + raise ValueError("Could not find SHA256 checksum in Linux SBOM") + if windows_checksum is None: + raise ValueError("Could not find SHA256 checksum in Windows SBOM") + + # Extract hash values and validate format + linux_hash = linux_checksum.get('checksumValue', '') + windows_hash = windows_checksum.get('checksumValue', '') + + if not validate_sha256(linux_hash): + raise ValueError(f"Invalid Linux SHA256 hash format from SBOM: {linux_hash}") + if not validate_sha256(windows_hash): + raise ValueError(f"Invalid Windows SHA256 hash format from SBOM: {windows_hash}") + + return { + 'linux_source_sha256': linux_hash, + 'windows_amd64_sha256': windows_hash + } diff --git a/ddev/src/ddev/cli/meta/scripts/upgrade_python.py b/ddev/src/ddev/cli/meta/scripts/upgrade_py_config.py similarity index 95% rename from ddev/src/ddev/cli/meta/scripts/upgrade_python.py rename to ddev/src/ddev/cli/meta/scripts/upgrade_py_config.py index ddfd4e984fb17..226a4aca210f8 100644 --- a/ddev/src/ddev/cli/meta/scripts/upgrade_python.py +++ b/ddev/src/ddev/cli/meta/scripts/upgrade_py_config.py @@ -15,19 +15,19 @@ from ddev.src.ddev.validation.tracker import ValidationTracker -@click.command('upgrade-python', short_help='Upgrade the Python version throughout the repository') +@click.command('update-python-config', short_help='Update Python version references in repository config files') @click.argument('version') @click.pass_obj -def upgrade_python(app: Application, version: str): - """Upgrade the Python version of all test environments. +def update_python_config(app: Application, version: str): + """Update Python version references in config files across all integrations. \b - `$ ddev meta scripts upgrade-python 3.11` + `$ ddev meta scripts update-python-config 3.11` """ from ddev.repo.constants import PYTHON_VERSION as old_version - tracker = app.create_validation_tracker('Python upgrades') + tracker = app.create_validation_tracker('Python config updates') for target in integrations(app): update_hatch_file(app, target.path, version, old_version, tracker) diff --git a/ddev/src/ddev/repo/constants.py b/ddev/src/ddev/repo/constants.py index 31a907f78fb65..3b7d281f80606 100644 --- a/ddev/src/ddev/repo/constants.py +++ b/ddev/src/ddev/repo/constants.py @@ -12,3 +12,4 @@ # This is automatically maintained PYTHON_VERSION = '3.13' +PYTHON_VERSION_FULL = '3.13.7' \ No newline at end of file diff --git a/ddev/tests/cli/meta/scripts/test_upgrade_python.py b/ddev/tests/cli/meta/scripts/test_upgrade_python.py index 0f72910563c37..28ac7aac0e485 100644 --- a/ddev/tests/cli/meta/scripts/test_upgrade_python.py +++ b/ddev/tests/cli/meta/scripts/test_upgrade_python.py @@ -4,14 +4,14 @@ from .conftest import NEW_PYTHON_VERSION, OLD_PYTHON_VERSION -def test_upgrade_python(fake_repo, ddev): +def test_update_py_config(fake_repo, ddev): constant_file = fake_repo.path / 'ddev' / 'src' / 'ddev' / 'repo' / 'constants.py' contents = constant_file.read_text() assert f'PYTHON_VERSION = {OLD_PYTHON_VERSION!r}' in contents assert f'PYTHON_VERSION = {NEW_PYTHON_VERSION!r}' not in contents - result = ddev('meta', 'scripts', 'upgrade-python', NEW_PYTHON_VERSION) + result = ddev('meta', 'scripts', 'update-python-config', NEW_PYTHON_VERSION) assert result.exit_code == 0, result.output assert result.output.endswith('Python upgrades\n\nPassed: 9\n') From 1207a0c3cfde4ac8199e50f40ca39de10c7d5bf6 Mon Sep 17 00:00:00 2001 From: Kyle-Neale Date: Fri, 17 Oct 2025 22:47:38 -0400 Subject: [PATCH 02/10] Lint --- .../ddev/cli/meta/scripts/update_python.py | 128 +++++++++--------- ddev/src/ddev/repo/constants.py | 2 +- 2 files changed, 65 insertions(+), 65 deletions(-) diff --git a/ddev/src/ddev/cli/meta/scripts/update_python.py b/ddev/src/ddev/cli/meta/scripts/update_python.py index 014ac212c494f..a46347e97dc49 100644 --- a/ddev/src/ddev/cli/meta/scripts/update_python.py +++ b/ddev/src/ddev/cli/meta/scripts/update_python.py @@ -17,6 +17,7 @@ from ddev.cli.application import Application from ddev.validation.tracker import ValidationTracker + @click.command('update-python-version', short_help='Upgrade the Python version used in the repository.') @click.pass_obj def update_python_version(app: Application): @@ -31,28 +32,30 @@ def update_python_version(app: Application): \b `$ ddev meta scripts update-python-version` """ - from ddev.repo.constants import PYTHON_VERSION_FULL as current_version from ddev.repo.constants import PYTHON_VERSION as major_minor + from ddev.repo.constants import PYTHON_VERSION_FULL as current_version tracker = app.create_validation_tracker('Python version updates') - + # Check for new version - latest_version = get_latest_python_version(app, major_minor) - if latest_version is None: + latest_version: str | None = get_latest_python_version(app, major_minor) + if not latest_version: app.display_error(f"Could not find latest Python version for {major_minor}") app.abort() - + return # Unreachable but helps type checker + # Validate version string format for security (prevent injection) if not validate_version_string(latest_version): app.display_error(f"Invalid version format detected: {latest_version}") app.abort() - + return # Unreachable but helps type checker + if Version(latest_version) <= Version(current_version): app.display_info(f"Already at latest Python version: {current_version}") return - + app.display_info(f"Updating Python from {current_version} to {latest_version}") - + # Fetch hashes try: new_version_hashes = get_python_sha256_hashes(app, latest_version) @@ -60,38 +63,41 @@ def update_python_version(app: Application): tracker.error(('SHA256 hashes',), message=f"Failed to fetch: {e}") tracker.display() app.abort() - + return # Unreachable but helps type checker + # Validate hashes if not validate_sha256(new_version_hashes.get('linux_source_sha256', '')): tracker.error(('SHA256 validation',), message="Invalid Linux SHA256 hash format") if not validate_sha256(new_version_hashes.get('windows_amd64_sha256', '')): tracker.error(('SHA256 validation',), message="Invalid Windows SHA256 hash format") - + if tracker.errors: tracker.display() app.abort() - + return # Unreachable but helps type checker + # Perform updates update_python_version_full_constant(app, latest_version, tracker) update_dockerfiles_python_version(app, latest_version, new_version_hashes, tracker) update_macos_python_version(app, latest_version, tracker) - + # Display results tracker.display() - + if tracker.errors: app.display_warning("Some updates failed. Please review the errors above.") app.abort() - + app.display_success(f"Python version updated from {current_version} to {latest_version}") + def validate_version_string(version: str) -> bool: """ Validate that version string is safe and matches expected format. - + Args: version: Version string to validate - + Returns: True if valid, False otherwise """ @@ -111,7 +117,7 @@ def update_dockerfiles_python_version( app.repo.path / '.builders' / 'images' / 'linux-x86_64' / 'Dockerfile', app.repo.path / '.builders' / 'images' / 'windows-x86_64' / 'Dockerfile', ] - + try: linux_sha = hashes['linux_source_sha256'] windows_sha = hashes['windows_amd64_sha256'] @@ -127,20 +133,16 @@ def update_dockerfiles_python_version( # SHA256 patterns must match the Python-specific ones: # Linux: SHA256 that comes after VERSION="${PYTHON3_VERSION}" # Windows: -Hash in the same RUN block with python-$Env:PYTHON_VERSION-amd64.exe - linux_sha_pattern = re.compile( - r'VERSION="\$\{PYTHON3_VERSION\}"[^\n]*\n[^\n]*SHA256="([0-9a-f]+)"', - re.MULTILINE - ) + linux_sha_pattern = re.compile(r'VERSION="\$\{PYTHON3_VERSION\}"[^\n]*\n[^\n]*SHA256="([0-9a-f]+)"', re.MULTILINE) windows_sha_pattern = re.compile( - r'python-\$Env:PYTHON_VERSION-amd64\.exe[^\n]*\n[^\n]*-Hash\s+\'([0-9a-f]+)\'', - re.MULTILINE + r'python-\$Env:PYTHON_VERSION-amd64\.exe[^\n]*\n[^\n]*-Hash\s+\'([0-9a-f]+)\'', re.MULTILINE ) for dockerfile in dockerfiles: if not dockerfile.exists(): tracker.error((dockerfile.name,), message=f'File not found: {dockerfile}') continue - + try: content = dockerfile.read_text() except Exception as e: @@ -152,26 +154,26 @@ def update_dockerfiles_python_version( sha_pattern = windows_sha_pattern if is_windows else linux_sha_pattern target_sha = windows_sha if is_windows else linux_sha - def replace_version(match: re.Match[str]) -> str: - if is_windows: + def replace_version(match: re.Match[str], _is_windows=is_windows) -> str: + if _is_windows: prefix, _old_version, suffix = match.groups() return f'{prefix}{new_version}{suffix}' else: prefix, _old_version = match.groups() return f'{prefix}{new_version}' - def replace_sha(match: re.Match[str]) -> str: + def replace_sha(match: re.Match[str], _target_sha=target_sha) -> str: # The entire match contains the old hash, replace just the hash part old_match = match.group(0) old_hash = match.group(1) - return old_match.replace(old_hash, target_sha) + return old_match.replace(old_hash, _target_sha) # Helper to apply pattern substitution with error tracking - def apply_substitution(pattern: re.Pattern, replace_func, error_msg: str) -> str | None: + def apply_substitution(pattern: re.Pattern, replace_func, error_msg: str, _dockerfile=dockerfile) -> str | None: nonlocal content content, count = pattern.subn(replace_func, content, count=1) if count == 0: - tracker.error((dockerfile.name,), message=error_msg) + tracker.error((_dockerfile.name,), message=error_msg) return None return content @@ -192,17 +194,17 @@ def apply_substitution(pattern: re.Pattern, replace_func, error_msg: str) -> str def update_macos_python_version(app: Application, new_version: str, tracker: ValidationTracker): macos_python_file = app.repo.path / '.github' / 'workflows' / 'resolve-build-deps.yaml' - + if not macos_python_file.exists(): tracker.error(('macOS workflow',), message=f'File not found: {macos_python_file}') return - + try: content = macos_python_file.read_text() except Exception as e: tracker.error(('macOS workflow',), message=f'Failed to read: {e}') return - + target_line = next((line for line in content.splitlines() if 'PYTHON3_DOWNLOAD_URL' in line), None) if target_line is None: @@ -217,7 +219,7 @@ def update_macos_python_version(app: Application, new_version: str, tracker: Val return updated_content = content.replace(target_line, new_line, 1) - + try: macos_python_file.write_text(updated_content) tracker.success() @@ -227,11 +229,11 @@ def update_macos_python_version(app: Application, new_version: str, tracker: Val def update_python_version_full_constant(app: Application, new_version: str, tracker: ValidationTracker): constants_file = app.repo.path / 'ddev' / 'src' / 'ddev' / 'repo' / 'constants.py' - + if not constants_file.exists(): tracker.error(('constants.py',), message=f'File not found: {constants_file}') return - + try: content = constants_file.read_text() except Exception as e: @@ -250,7 +252,7 @@ def update_python_version_full_constant(app: Application, new_version: str, trac return updated_content = content.replace(target_line, new_line, 1) - + try: constants_file.write_text(updated_content) tracker.success() @@ -261,15 +263,15 @@ def update_python_version_full_constant(app: Application, new_version: str, trac def get_latest_python_version(app: Application, major_minor: str) -> str | None: """ Get the latest Python version from python.org FTP directory. - + Args: major_minor: Python version in format "3.13" - + Returns: Latest version string (e.g., "3.13.1") or None if not found """ url = "https://www.python.org/ftp/python/" - + try: # Explicitly verify SSL/TLS certificate response = requests.get(url, timeout=30, verify=True) @@ -277,12 +279,12 @@ def get_latest_python_version(app: Application, major_minor: str) -> str | None: except requests.RequestException as e: app.display_error(f"Error fetching Python versions: {e}") return None - + # Parse directory listing for version folders # Looking for patterns like: 3.13.0/ pattern = rf'' versions = [] - + for line in response.text.split("\n"): match = re.search(pattern, line) if match: @@ -292,10 +294,10 @@ def get_latest_python_version(app: Application, major_minor: str) -> str | None: except Exception: # Skip invalid versions continue - + if not versions: return None - + # Sort and return the latest version versions.sort() return str(versions[-1]) @@ -304,20 +306,20 @@ def get_latest_python_version(app: Application, major_minor: str) -> str | None: def get_python_sha256_hashes(app: Application, version: str) -> dict[str, str]: """ Fetch SHA256 hashes for Python release artifacts using SBOM files. - + Python.org provides SBOM (Software Bill of Materials) files in SPDX JSON format for each release artifact. These files contain SHA256 checksums. - + Args: version: Python version string (e.g., "3.13.7") - + Returns: Dictionary with SHA256 hashes: { 'linux_source_sha256': '<64-char hex hash>', 'windows_amd64_sha256': '<64-char hex hash>' } - + Raises: ValueError: If version format is invalid RuntimeError: If SBOM files cannot be fetched or parsed @@ -325,7 +327,7 @@ def get_python_sha256_hashes(app: Application, version: str) -> dict[str, str]: # Validate version format before using in URL construction (security) if not validate_version_string(version): raise ValueError(f"Invalid version format: {version}") - + # Steps: # 1. Construct SBOM URLs for the files we need: # - Linux source tarball: @@ -334,8 +336,9 @@ def get_python_sha256_hashes(app: Application, version: str) -> dict[str, str]: # https://www.python.org/ftp/python/{version}/python-{version}-amd64.exe.spdx.json SBOM_URLS = [ f"https://www.python.org/ftp/python/{version}/Python-{version}.tgz.spdx.json", - f"https://www.python.org/ftp/python/{version}/python-{version}-amd64.exe.spdx.json" + f"https://www.python.org/ftp/python/{version}/python-{version}-amd64.exe.spdx.json", ] + # 2. Download and parse each SBOM JSON file async def get_sbom_data(client, url): try: @@ -343,13 +346,13 @@ async def get_sbom_data(client, url): response = await client.get(url, timeout=30.0) response.raise_for_status() data = orjson.loads(response.text) - + # Validate SBOM structure if not isinstance(data, dict): raise ValueError(f"Invalid SBOM format: expected dict, got {type(data)}") if 'packages' not in data: raise ValueError("Invalid SBOM format: missing 'packages' field") - + return data.get('packages', []) except Exception as e: raise RuntimeError(f'Error processing URL {url}: {e}') from e @@ -358,40 +361,37 @@ async def fetch_sbom_data(urls): # Create client with explicit SSL verification async with httpx.AsyncClient(verify=True) as client: return await asyncio.gather(*(get_sbom_data(client, url) for url in urls)) - + sbom_packages = asyncio.run(fetch_sbom_data(SBOM_URLS)) # Find the CPython package in the SBOM packages linux_cpython_package = next((package for package in sbom_packages[0] if package.get('name') == "CPython"), None) windows_cpython_package = next((package for package in sbom_packages[1] if package.get('name') == "CPython"), None) - + if linux_cpython_package is None: raise ValueError("Could not find CPython package in Linux SBOM") if windows_cpython_package is None: raise ValueError("Could not find CPython package in Windows SBOM") - + # Find the SHA256 checksum in the CPython package checksums linux_checksums = linux_cpython_package.get('checksums', []) windows_checksums = windows_cpython_package.get('checksums', []) - + linux_checksum = next((checksum for checksum in linux_checksums if checksum.get('algorithm') == "SHA256"), None) windows_checksum = next((checksum for checksum in windows_checksums if checksum.get('algorithm') == "SHA256"), None) - + if linux_checksum is None: raise ValueError("Could not find SHA256 checksum in Linux SBOM") if windows_checksum is None: raise ValueError("Could not find SHA256 checksum in Windows SBOM") - + # Extract hash values and validate format linux_hash = linux_checksum.get('checksumValue', '') windows_hash = windows_checksum.get('checksumValue', '') - + if not validate_sha256(linux_hash): raise ValueError(f"Invalid Linux SHA256 hash format from SBOM: {linux_hash}") if not validate_sha256(windows_hash): raise ValueError(f"Invalid Windows SHA256 hash format from SBOM: {windows_hash}") - - return { - 'linux_source_sha256': linux_hash, - 'windows_amd64_sha256': windows_hash - } + + return {'linux_source_sha256': linux_hash, 'windows_amd64_sha256': windows_hash} diff --git a/ddev/src/ddev/repo/constants.py b/ddev/src/ddev/repo/constants.py index 3b7d281f80606..24fd14fc77705 100644 --- a/ddev/src/ddev/repo/constants.py +++ b/ddev/src/ddev/repo/constants.py @@ -12,4 +12,4 @@ # This is automatically maintained PYTHON_VERSION = '3.13' -PYTHON_VERSION_FULL = '3.13.7' \ No newline at end of file +PYTHON_VERSION_FULL = '3.13.7' From b673ce126a057d38a88efa7ae62d7cc5a8c4ea37 Mon Sep 17 00:00:00 2001 From: Kyle-Neale Date: Fri, 17 Oct 2025 22:53:35 -0400 Subject: [PATCH 03/10] Add changelog --- ddev/changelog.d/21695.added | 1 + 1 file changed, 1 insertion(+) create mode 100644 ddev/changelog.d/21695.added diff --git a/ddev/changelog.d/21695.added b/ddev/changelog.d/21695.added new file mode 100644 index 0000000000000..7fd26385e5825 --- /dev/null +++ b/ddev/changelog.d/21695.added @@ -0,0 +1 @@ +Adds update-python-version command to automate Python version updates \ No newline at end of file From 147ab2530474d9a14f693afcbf545887c71c6302 Mon Sep 17 00:00:00 2001 From: Kyle-Neale Date: Fri, 17 Oct 2025 23:13:26 -0400 Subject: [PATCH 04/10] Add tests --- .../meta/scripts/test_upgrade_py_config.py | 53 +++++ .../cli/meta/scripts/test_upgrade_python.py | 189 ++++++++++++++---- 2 files changed, 199 insertions(+), 43 deletions(-) create mode 100644 ddev/tests/cli/meta/scripts/test_upgrade_py_config.py diff --git a/ddev/tests/cli/meta/scripts/test_upgrade_py_config.py b/ddev/tests/cli/meta/scripts/test_upgrade_py_config.py new file mode 100644 index 0000000000000..28ac7aac0e485 --- /dev/null +++ b/ddev/tests/cli/meta/scripts/test_upgrade_py_config.py @@ -0,0 +1,53 @@ +# (C) Datadog, Inc. 2023-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from .conftest import NEW_PYTHON_VERSION, OLD_PYTHON_VERSION + + +def test_update_py_config(fake_repo, ddev): + constant_file = fake_repo.path / 'ddev' / 'src' / 'ddev' / 'repo' / 'constants.py' + contents = constant_file.read_text() + + assert f'PYTHON_VERSION = {OLD_PYTHON_VERSION!r}' in contents + assert f'PYTHON_VERSION = {NEW_PYTHON_VERSION!r}' not in contents + + result = ddev('meta', 'scripts', 'update-python-config', NEW_PYTHON_VERSION) + + assert result.exit_code == 0, result.output + assert result.output.endswith('Python upgrades\n\nPassed: 9\n') + + contents = constant_file.read_text() + assert f'PYTHON_VERSION = {OLD_PYTHON_VERSION!r}' not in contents + assert f'PYTHON_VERSION = {NEW_PYTHON_VERSION!r}' in contents + + ci_file = fake_repo.path / '.github' / 'workflows' / 'build-ddev.yml' + contents = ci_file.read_text() + assert f'PYTHON_VERSION: "{OLD_PYTHON_VERSION}"' not in contents + assert f'PYTHON_VERSION: "{NEW_PYTHON_VERSION}"' in contents + + hatch_file = fake_repo.path / 'dummy' / 'hatch.toml' + contents = hatch_file.read_text() + assert f'python = ["{OLD_PYTHON_VERSION}"]' not in contents + assert f'python = ["{NEW_PYTHON_VERSION}"]' in contents + + for integration in ('dummy', 'datadog_checks_dependency_provider', 'logs_only'): + pyproject_file = fake_repo.path / integration / 'pyproject.toml' + contents = pyproject_file.read_text() + assert f'Programming Language :: Python :: {OLD_PYTHON_VERSION}' not in contents + assert f'Programming Language :: Python :: {NEW_PYTHON_VERSION}' in contents + + template_file = ( + fake_repo.path + / 'datadog_checks_dev' + / 'datadog_checks' + / 'dev' + / 'tooling' + / 'templates' + / 'integration' + / 'check' + / '{check_name}' + / 'pyproject.toml' + ) + contents = template_file.read_text() + assert f'Programming Language :: Python :: {OLD_PYTHON_VERSION}' not in contents + assert f'Programming Language :: Python :: {NEW_PYTHON_VERSION}' in contents diff --git a/ddev/tests/cli/meta/scripts/test_upgrade_python.py b/ddev/tests/cli/meta/scripts/test_upgrade_python.py index 28ac7aac0e485..adc1c202a334e 100644 --- a/ddev/tests/cli/meta/scripts/test_upgrade_python.py +++ b/ddev/tests/cli/meta/scripts/test_upgrade_python.py @@ -1,53 +1,156 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from .conftest import NEW_PYTHON_VERSION, OLD_PYTHON_VERSION -def test_update_py_config(fake_repo, ddev): - constant_file = fake_repo.path / 'ddev' / 'src' / 'ddev' / 'repo' / 'constants.py' - contents = constant_file.read_text() +def setup_python_update_files(fake_repo): + """Add PYTHON_VERSION_FULL and Dockerfiles to fake_repo.""" + # Update constants.py to include PYTHON_VERSION_FULL + constants_file = fake_repo.path / 'ddev' / 'src' / 'ddev' / 'repo' / 'constants.py' + content = constants_file.read_text() + content += "PYTHON_VERSION_FULL = '3.13.7'\n" + constants_file.write_text(content) - assert f'PYTHON_VERSION = {OLD_PYTHON_VERSION!r}' in contents - assert f'PYTHON_VERSION = {NEW_PYTHON_VERSION!r}' not in contents + # Create Linux ARM64 Dockerfile + linux_aarch64_dir = fake_repo.path / '.builders' / 'images' / 'linux-aarch64' + linux_aarch64_dir.mkdir(parents=True, exist_ok=True) + (linux_aarch64_dir / 'Dockerfile').write_text("""FROM ubuntu:22.04 - result = ddev('meta', 'scripts', 'update-python-config', NEW_PYTHON_VERSION) +# Install Python +ENV PYTHON3_VERSION=3.13.7 +RUN DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \\ + VERSION="${PYTHON3_VERSION}" \\ + SHA256="6c9d80839cfa20024f34d9a6dd31ae2a9cd97ff5e980e969209746037a5153b2" \\ + bash install-from-source.sh + +CMD ["/bin/bash"] +""") + + # Create Linux x86_64 Dockerfile + linux_x86_64_dir = fake_repo.path / '.builders' / 'images' / 'linux-x86_64' + linux_x86_64_dir.mkdir(parents=True, exist_ok=True) + (linux_x86_64_dir / 'Dockerfile').write_text("""FROM ubuntu:22.04 + +# Install Python +ENV PYTHON3_VERSION=3.13.7 +RUN DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \\ + VERSION="${PYTHON3_VERSION}" \\ + SHA256="6c9d80839cfa20024f34d9a6dd31ae2a9cd97ff5e980e969209746037a5153b2" \\ + bash install-from-source.sh + +CMD ["/bin/bash"] +""") + + # Create Windows Dockerfile + windows_dockerfile_dir = fake_repo.path / '.builders' / 'images' / 'windows-x86_64' + windows_dockerfile_dir.mkdir(parents=True, exist_ok=True) + (windows_dockerfile_dir / 'Dockerfile').write_text("""FROM mcr.microsoft.com/windows/servercore:ltsc2022 + +# Install Python +ENV PYTHON_VERSION="3.13.7" +RUN powershell -Command " \\ + Invoke-WebRequest -OutFile python-$Env:PYTHON_VERSION-amd64.exe \\ + https://www.python.org/ftp/python/$Env:PYTHON_VERSION/python-$Env:PYTHON_VERSION-amd64.exe \\ + -Hash '48652a4e6af29c2f1fde2e2e6bbf3734a82ce3f577e9fd5c95c83f68e29e1eaa'" + +CMD ["powershell"] +""") + + # Create macOS workflow file + workflows_dir = fake_repo.path / '.github' / 'workflows' + workflows_dir.mkdir(parents=True, exist_ok=True) + (workflows_dir / 'resolve-build-deps.yaml').write_text("""name: Resolve build dependencies + +on: + workflow_dispatch: + +env: + PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.13.7/python-3.13.7-macos11.pkg" + +jobs: + build: + runs-on: macos-latest + steps: + - run: echo "test" +""") + + +def test_update_python_version_success(fake_repo, ddev, mocker): + """Test successful Python version update.""" + setup_python_update_files(fake_repo) + # Mock network calls + mocker.patch('ddev.cli.meta.scripts.update_python.get_latest_python_version', return_value='3.13.9') + mocker.patch( + 'ddev.cli.meta.scripts.update_python.get_python_sha256_hashes', + return_value={ + 'linux_source_sha256': 'c4c066af19c98fb7835d473bebd7e23be84f6e9874d47db9e39a68ee5d0ce35c', + 'windows_amd64_sha256': '200ddff856bbff949d2cc1be42e8807c07538abd6b6966d5113a094cf628c5c5', + }, + ) + + result = ddev('meta', 'scripts', 'update-python-version') + + assert result.exit_code == 0, result.output + assert 'Updating Python from 3.13.7 to 3.13.9' in result.output + assert 'Passed: 5' in result.output + assert 'Python version updated from 3.13.7 to 3.13.9' in result.output + + # Verify constants.py was updated + constants_file = fake_repo.path / 'ddev' / 'src' / 'ddev' / 'repo' / 'constants.py' + contents = constants_file.read_text() + assert "PYTHON_VERSION_FULL = '3.13.9'" in contents + assert "PYTHON_VERSION_FULL = '3.13.7'" not in contents + + # Verify Linux Dockerfile was updated + linux_dockerfile = fake_repo.path / '.builders' / 'images' / 'linux-aarch64' / 'Dockerfile' + contents = linux_dockerfile.read_text() + assert 'ENV PYTHON3_VERSION=3.13.9' in contents + assert 'SHA256="c4c066af19c98fb7835d473bebd7e23be84f6e9874d47db9e39a68ee5d0ce35c"' in contents + assert 'ENV PYTHON3_VERSION=3.13.7' not in contents + + # Verify Windows Dockerfile was updated + windows_dockerfile = fake_repo.path / '.builders' / 'images' / 'windows-x86_64' / 'Dockerfile' + contents = windows_dockerfile.read_text() + assert 'ENV PYTHON_VERSION="3.13.9"' in contents + assert '-Hash \'200ddff856bbff949d2cc1be42e8807c07538abd6b6966d5113a094cf628c5c5\'' in contents + assert 'ENV PYTHON_VERSION="3.13.7"' not in contents + + # Verify macOS workflow was updated + workflow_file = fake_repo.path / '.github' / 'workflows' / 'resolve-build-deps.yaml' + contents = workflow_file.read_text() + assert 'python-3.13.9-macos11.pkg' in contents + assert 'python-3.13.7-macos11.pkg' not in contents + + +def test_update_python_version_already_latest(fake_repo, ddev, mocker): + setup_python_update_files(fake_repo) + mocker.patch('ddev.cli.meta.scripts.update_python.get_latest_python_version', return_value='3.13.7') + + result = ddev('meta', 'scripts', 'update-python-version') assert result.exit_code == 0, result.output - assert result.output.endswith('Python upgrades\n\nPassed: 9\n') - - contents = constant_file.read_text() - assert f'PYTHON_VERSION = {OLD_PYTHON_VERSION!r}' not in contents - assert f'PYTHON_VERSION = {NEW_PYTHON_VERSION!r}' in contents - - ci_file = fake_repo.path / '.github' / 'workflows' / 'build-ddev.yml' - contents = ci_file.read_text() - assert f'PYTHON_VERSION: "{OLD_PYTHON_VERSION}"' not in contents - assert f'PYTHON_VERSION: "{NEW_PYTHON_VERSION}"' in contents - - hatch_file = fake_repo.path / 'dummy' / 'hatch.toml' - contents = hatch_file.read_text() - assert f'python = ["{OLD_PYTHON_VERSION}"]' not in contents - assert f'python = ["{NEW_PYTHON_VERSION}"]' in contents - - for integration in ('dummy', 'datadog_checks_dependency_provider', 'logs_only'): - pyproject_file = fake_repo.path / integration / 'pyproject.toml' - contents = pyproject_file.read_text() - assert f'Programming Language :: Python :: {OLD_PYTHON_VERSION}' not in contents - assert f'Programming Language :: Python :: {NEW_PYTHON_VERSION}' in contents - - template_file = ( - fake_repo.path - / 'datadog_checks_dev' - / 'datadog_checks' - / 'dev' - / 'tooling' - / 'templates' - / 'integration' - / 'check' - / '{check_name}' - / 'pyproject.toml' + assert 'Already at latest Python version: 3.13.7' in result.output + + +def test_update_python_version_no_new_version_found(fake_repo, ddev, mocker): + setup_python_update_files(fake_repo) + mocker.patch('ddev.cli.meta.scripts.update_python.get_latest_python_version', return_value=None) + + result = ddev('meta', 'scripts', 'update-python-version') + + assert result.exit_code == 1, result.output + assert 'Could not find latest Python version' in result.output + + +def test_update_python_version_invalid_hash_format(fake_repo, ddev, mocker): + setup_python_update_files(fake_repo) + mocker.patch('ddev.cli.meta.scripts.update_python.get_latest_python_version', return_value='3.13.9') + mocker.patch( + 'ddev.cli.meta.scripts.update_python.get_python_sha256_hashes', + return_value={'linux_source_sha256': 'not-a-valid-hash', 'windows_amd64_sha256': 'also-invalid'}, ) - contents = template_file.read_text() - assert f'Programming Language :: Python :: {OLD_PYTHON_VERSION}' not in contents - assert f'Programming Language :: Python :: {NEW_PYTHON_VERSION}' in contents + + result = ddev('meta', 'scripts', 'update-python-version') + + assert result.exit_code == 1, result.output + assert 'Invalid Linux SHA256 hash format' in result.output or 'Invalid Windows SHA256 hash format' in result.output From b3ec69abfd5f3b2d908d8e2c478f4945692ffcae Mon Sep 17 00:00:00 2001 From: Kyle-Neale Date: Sat, 18 Oct 2025 00:21:00 -0400 Subject: [PATCH 05/10] Constants refactor --- .../ddev/cli/meta/scripts/update_python.py | 69 +++++++++---------- .../cli/meta/scripts/upgrade_py_config.py | 6 +- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/ddev/src/ddev/cli/meta/scripts/update_python.py b/ddev/src/ddev/cli/meta/scripts/update_python.py index a46347e97dc49..45482c95fb43b 100644 --- a/ddev/src/ddev/cli/meta/scripts/update_python.py +++ b/ddev/src/ddev/cli/meta/scripts/update_python.py @@ -17,6 +17,26 @@ from ddev.cli.application import Application from ddev.validation.tracker import ValidationTracker +# Python.org URLs +PYTHON_FTP_URL = "https://www.python.org/ftp/python/" +PYTHON_MACOS_PKG_URL_TEMPLATE = "https://www.python.org/ftp/python/{version}/python-{version}-macos11.pkg" +PYTHON_SBOM_LINUX_URL_TEMPLATE = "https://www.python.org/ftp/python/{version}/Python-{version}.tgz.spdx.json" +PYTHON_SBOM_WINDOWS_URL_TEMPLATE = "https://www.python.org/ftp/python/{version}/python-{version}-amd64.exe.spdx.json" + +# Regex patterns for Dockerfile updates +# Linux: ENV PYTHON3_VERSION=3.13.7 (no quotes, matches version at end of line) +LINUX_VERSION_PATTERN = re.compile(r'(ENV PYTHON3_VERSION=)(\d+\.\d+\.\d+)$', re.MULTILINE) +# Windows: ENV PYTHON_VERSION="3.13.7" (with quotes) +WINDOWS_VERSION_PATTERN = re.compile(r'(ENV PYTHON_VERSION=")(\d+\.\d+\.\d+)(")', re.MULTILINE) + +# SHA256 patterns must match the Python-specific ones: +# Linux: SHA256 that comes after VERSION="${PYTHON3_VERSION}" +LINUX_SHA_PATTERN = re.compile(r'VERSION="\$\{PYTHON3_VERSION\}"[^\n]*\n[^\n]*SHA256="([0-9a-f]+)"', re.MULTILINE) +# Windows: -Hash in the same RUN block with python-$Env:PYTHON_VERSION-amd64.exe +WINDOWS_SHA_PATTERN = re.compile( + r'python-\$Env:PYTHON_VERSION-amd64\.exe[^\n]*\n[^\n]*-Hash\s+\'([0-9a-f]+)\'', re.MULTILINE +) + @click.command('update-python-version', short_help='Upgrade the Python version used in the repository.') @click.pass_obj @@ -38,7 +58,7 @@ def update_python_version(app: Application): tracker = app.create_validation_tracker('Python version updates') # Check for new version - latest_version: str | None = get_latest_python_version(app, major_minor) + latest_version = get_latest_python_version(app, major_minor) if not latest_version: app.display_error(f"Could not find latest Python version for {major_minor}") app.abort() @@ -125,19 +145,6 @@ def update_dockerfiles_python_version( tracker.error(('Dockerfiles',), message=f'Missing SHA256 hash entry: {error}') return - # Linux: ENV PYTHON3_VERSION=3.13.7 (no quotes, matches version at end of line) - # Windows: ENV PYTHON_VERSION="3.13.7" (with quotes) - linux_version_pattern = re.compile(r'(ENV PYTHON3_VERSION=)(\d+\.\d+\.\d+)$', re.MULTILINE) - windows_version_pattern = re.compile(r'(ENV PYTHON_VERSION=")(\d+\.\d+\.\d+)(")', re.MULTILINE) - - # SHA256 patterns must match the Python-specific ones: - # Linux: SHA256 that comes after VERSION="${PYTHON3_VERSION}" - # Windows: -Hash in the same RUN block with python-$Env:PYTHON_VERSION-amd64.exe - linux_sha_pattern = re.compile(r'VERSION="\$\{PYTHON3_VERSION\}"[^\n]*\n[^\n]*SHA256="([0-9a-f]+)"', re.MULTILINE) - windows_sha_pattern = re.compile( - r'python-\$Env:PYTHON_VERSION-amd64\.exe[^\n]*\n[^\n]*-Hash\s+\'([0-9a-f]+)\'', re.MULTILINE - ) - for dockerfile in dockerfiles: if not dockerfile.exists(): tracker.error((dockerfile.name,), message=f'File not found: {dockerfile}') @@ -150,8 +157,8 @@ def update_dockerfiles_python_version( continue is_windows = 'windows-x86_64' in dockerfile.parts - version_pattern = windows_version_pattern if is_windows else linux_version_pattern - sha_pattern = windows_sha_pattern if is_windows else linux_sha_pattern + version_pattern = WINDOWS_VERSION_PATTERN if is_windows else LINUX_VERSION_PATTERN + sha_pattern = WINDOWS_SHA_PATTERN if is_windows else LINUX_SHA_PATTERN target_sha = windows_sha if is_windows else linux_sha def replace_version(match: re.Match[str], _is_windows=is_windows) -> str: @@ -211,7 +218,7 @@ def update_macos_python_version(app: Application, new_version: str, tracker: Val tracker.error(('macOS workflow',), message='Could not find PYTHON3_DOWNLOAD_URL') return - new_url = f'https://www.python.org/ftp/python/{new_version}/python-{new_version}-macos11.pkg' + new_url = PYTHON_MACOS_PKG_URL_TEMPLATE.format(version=new_version) indent = target_line[: target_line.index('PYTHON3_DOWNLOAD_URL')] new_line = f'{indent}PYTHON3_DOWNLOAD_URL: "{new_url}"' @@ -270,11 +277,9 @@ def get_latest_python_version(app: Application, major_minor: str) -> str | None: Returns: Latest version string (e.g., "3.13.1") or None if not found """ - url = "https://www.python.org/ftp/python/" - try: # Explicitly verify SSL/TLS certificate - response = requests.get(url, timeout=30, verify=True) + response = requests.get(PYTHON_FTP_URL, timeout=30, verify=True) response.raise_for_status() except requests.RequestException as e: app.display_error(f"Error fetching Python versions: {e}") @@ -306,10 +311,7 @@ def get_latest_python_version(app: Application, major_minor: str) -> str | None: def get_python_sha256_hashes(app: Application, version: str) -> dict[str, str]: """ Fetch SHA256 hashes for Python release artifacts using SBOM files. - - Python.org provides SBOM (Software Bill of Materials) files in SPDX JSON format - for each release artifact. These files contain SHA256 checksums. - + Args: version: Python version string (e.g., "3.13.7") @@ -328,18 +330,13 @@ def get_python_sha256_hashes(app: Application, version: str) -> dict[str, str]: if not validate_version_string(version): raise ValueError(f"Invalid version format: {version}") - # Steps: - # 1. Construct SBOM URLs for the files we need: - # - Linux source tarball: - # https://www.python.org/ftp/python/{version}/Python-{version}.tgz.spdx.json - # - Windows AMD64 installer: - # https://www.python.org/ftp/python/{version}/python-{version}-amd64.exe.spdx.json - SBOM_URLS = [ - f"https://www.python.org/ftp/python/{version}/Python-{version}.tgz.spdx.json", - f"https://www.python.org/ftp/python/{version}/python-{version}-amd64.exe.spdx.json", + # Construct SBOM URLs for the files we need + sbom_urls = [ + PYTHON_SBOM_LINUX_URL_TEMPLATE.format(version=version), + PYTHON_SBOM_WINDOWS_URL_TEMPLATE.format(version=version), ] - # 2. Download and parse each SBOM JSON file + # Download and parse each SBOM JSON file async def get_sbom_data(client, url): try: # Explicitly verify SSL/TLS certificates @@ -358,11 +355,11 @@ async def get_sbom_data(client, url): raise RuntimeError(f'Error processing URL {url}: {e}') from e async def fetch_sbom_data(urls): - # Create client with explicit SSL verification + async with httpx.AsyncClient(verify=True) as client: return await asyncio.gather(*(get_sbom_data(client, url) for url in urls)) - sbom_packages = asyncio.run(fetch_sbom_data(SBOM_URLS)) + sbom_packages = asyncio.run(fetch_sbom_data(sbom_urls)) # Find the CPython package in the SBOM packages linux_cpython_package = next((package for package in sbom_packages[0] if package.get('name') == "CPython"), None) diff --git a/ddev/src/ddev/cli/meta/scripts/upgrade_py_config.py b/ddev/src/ddev/cli/meta/scripts/upgrade_py_config.py index 226a4aca210f8..e62627cdbb85e 100644 --- a/ddev/src/ddev/cli/meta/scripts/upgrade_py_config.py +++ b/ddev/src/ddev/cli/meta/scripts/upgrade_py_config.py @@ -19,15 +19,15 @@ @click.argument('version') @click.pass_obj def update_python_config(app: Application, version: str): - """Update Python version references in config files across all integrations. + """Upgrade the Python version of all test environments. \b - `$ ddev meta scripts update-python-config 3.11` + `$ ddev meta scripts upgrade-python 3.11` """ from ddev.repo.constants import PYTHON_VERSION as old_version - tracker = app.create_validation_tracker('Python config updates') + tracker = app.create_validation_tracker('Python upgrades') for target in integrations(app): update_hatch_file(app, target.path, version, old_version, tracker) From b5c2e765d5b740903f1fe2e5a1abda823d1a9988 Mon Sep 17 00:00:00 2001 From: Kyle Neale Date: Sat, 18 Oct 2025 00:22:27 -0400 Subject: [PATCH 06/10] Rename 21695.added to 21694.added --- ddev/changelog.d/{21695.added => 21694.added} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename ddev/changelog.d/{21695.added => 21694.added} (91%) diff --git a/ddev/changelog.d/21695.added b/ddev/changelog.d/21694.added similarity index 91% rename from ddev/changelog.d/21695.added rename to ddev/changelog.d/21694.added index 7fd26385e5825..b028e3aaf6c33 100644 --- a/ddev/changelog.d/21695.added +++ b/ddev/changelog.d/21694.added @@ -1 +1 @@ -Adds update-python-version command to automate Python version updates \ No newline at end of file +Adds update-python-version command to automate Python version updates From af01d4f4773b2ed384ba9a34a28643f26af23e15 Mon Sep 17 00:00:00 2001 From: Kyle-Neale Date: Sat, 18 Oct 2025 11:12:12 -0400 Subject: [PATCH 07/10] Small refactor and ddev test --fmt --- ddev/src/ddev/cli/meta/scripts/__init__.py | 6 +- ...grade_py_config.py => update_py_config.py} | 2 +- .../{update_python.py => upgrade_python.py} | 202 +++++++++--------- .../cli/meta/scripts/test_upgrade_python.py | 28 +-- 4 files changed, 120 insertions(+), 118 deletions(-) rename ddev/src/ddev/cli/meta/scripts/{upgrade_py_config.py => update_py_config.py} (99%) rename ddev/src/ddev/cli/meta/scripts/{update_python.py => upgrade_python.py} (70%) diff --git a/ddev/src/ddev/cli/meta/scripts/__init__.py b/ddev/src/ddev/cli/meta/scripts/__init__.py index 07bec84c4cf9f..f59de10b4fa41 100644 --- a/ddev/src/ddev/cli/meta/scripts/__init__.py +++ b/ddev/src/ddev/cli/meta/scripts/__init__.py @@ -10,8 +10,8 @@ from ddev.cli.meta.scripts.monitor import monitor from ddev.cli.meta.scripts.saved_views import sv from ddev.cli.meta.scripts.serve_openmetrics_payload import serve_openmetrics_payload -from ddev.cli.meta.scripts.update_python import update_python_version -from ddev.cli.meta.scripts.upgrade_py_config import update_python_config +from ddev.cli.meta.scripts.update_py_config import update_python_config +from ddev.cli.meta.scripts.upgrade_python import upgrade_python_version @click.group(short_help='Miscellaneous scripts that may be useful') @@ -27,6 +27,6 @@ def scripts(): scripts.add_command(remove_labels) scripts.add_command(serve_openmetrics_payload) scripts.add_command(update_python_config) -scripts.add_command(update_python_version) +scripts.add_command(upgrade_python_version) scripts.add_command(sv) scripts.add_command(monitor) diff --git a/ddev/src/ddev/cli/meta/scripts/upgrade_py_config.py b/ddev/src/ddev/cli/meta/scripts/update_py_config.py similarity index 99% rename from ddev/src/ddev/cli/meta/scripts/upgrade_py_config.py rename to ddev/src/ddev/cli/meta/scripts/update_py_config.py index e62627cdbb85e..a0075ae9f945a 100644 --- a/ddev/src/ddev/cli/meta/scripts/upgrade_py_config.py +++ b/ddev/src/ddev/cli/meta/scripts/update_py_config.py @@ -22,7 +22,7 @@ def update_python_config(app: Application, version: str): """Upgrade the Python version of all test environments. \b - `$ ddev meta scripts upgrade-python 3.11` + `$ ddev meta scripts update-python-config 3.11` """ from ddev.repo.constants import PYTHON_VERSION as old_version diff --git a/ddev/src/ddev/cli/meta/scripts/update_python.py b/ddev/src/ddev/cli/meta/scripts/upgrade_python.py similarity index 70% rename from ddev/src/ddev/cli/meta/scripts/update_python.py rename to ddev/src/ddev/cli/meta/scripts/upgrade_python.py index 45482c95fb43b..1bc9bf633dd19 100644 --- a/ddev/src/ddev/cli/meta/scripts/update_python.py +++ b/ddev/src/ddev/cli/meta/scripts/upgrade_python.py @@ -38,9 +38,9 @@ ) -@click.command('update-python-version', short_help='Upgrade the Python version used in the repository.') +@click.command('upgrade-python-version', short_help='Upgrade the Python version used in the repository.') @click.pass_obj -def update_python_version(app: Application): +def upgrade_python_version(app: Application): """Upgrade the Python version used in the repository. Automatically detects the latest Python version, fetches official SHA256 hashes, @@ -50,12 +50,12 @@ def update_python_version(app: Application): - .github/workflows/resolve-build-deps.yaml (macOS) \b - `$ ddev meta scripts update-python-version` + `$ ddev meta scripts upgrade-python-version` """ from ddev.repo.constants import PYTHON_VERSION as major_minor from ddev.repo.constants import PYTHON_VERSION_FULL as current_version - tracker = app.create_validation_tracker('Python version updates') + tracker = app.create_validation_tracker('Python version upgrades') # Check for new version latest_version = get_latest_python_version(app, major_minor) @@ -64,19 +64,13 @@ def update_python_version(app: Application): app.abort() return # Unreachable but helps type checker - # Validate version string format for security (prevent injection) - if not validate_version_string(latest_version): - app.display_error(f"Invalid version format detected: {latest_version}") - app.abort() - return # Unreachable but helps type checker - if Version(latest_version) <= Version(current_version): app.display_info(f"Already at latest Python version: {current_version}") return app.display_info(f"Updating Python from {current_version} to {latest_version}") - # Fetch hashes + # Fetch and validate hashes (validation happens inside get_python_sha256_hashes) try: new_version_hashes = get_python_sha256_hashes(app, latest_version) except Exception as e: @@ -85,21 +79,10 @@ def update_python_version(app: Application): app.abort() return # Unreachable but helps type checker - # Validate hashes - if not validate_sha256(new_version_hashes.get('linux_source_sha256', '')): - tracker.error(('SHA256 validation',), message="Invalid Linux SHA256 hash format") - if not validate_sha256(new_version_hashes.get('windows_amd64_sha256', '')): - tracker.error(('SHA256 validation',), message="Invalid Windows SHA256 hash format") - - if tracker.errors: - tracker.display() - app.abort() - return # Unreachable but helps type checker - # Perform updates - update_python_version_full_constant(app, latest_version, tracker) - update_dockerfiles_python_version(app, latest_version, new_version_hashes, tracker) - update_macos_python_version(app, latest_version, tracker) + upgrade_python_version_full_constant(app, latest_version, tracker) + upgrade_dockerfiles_python_version(app, latest_version, new_version_hashes, tracker) + upgrade_macos_python_version(app, latest_version, tracker) # Display results tracker.display() @@ -107,8 +90,9 @@ def update_python_version(app: Application): if tracker.errors: app.display_warning("Some updates failed. Please review the errors above.") app.abort() + return # Unreachable but helps type checker - app.display_success(f"Python version updated from {current_version} to {latest_version}") + app.display_success(f"Python version upgraded from {current_version} to {latest_version}") def validate_version_string(version: str) -> bool: @@ -129,7 +113,82 @@ def validate_sha256(hash_str: str) -> bool: return bool(re.match(r'^[0-9a-f]{64}$', hash_str)) -def update_dockerfiles_python_version( +def read_file_safely(file_path, file_label: str, tracker: ValidationTracker) -> str | None: + """ + Read file with error handling. + + Args: + file_path: Path to the file + file_label: Label for error messages + tracker: Validation tracker for error reporting + + Returns: + File content as string, or None if an error occurred + """ + if not file_path.exists(): + tracker.error((file_label,), message=f'File not found: {file_path}') + return None + + try: + return file_path.read_text() + except Exception as e: + tracker.error((file_label,), message=f'Failed to read: {e}') + return None + + +def write_file_safely(file_path, content: str, file_label: str, tracker: ValidationTracker) -> bool: + """ + Write file with error handling. + + Args: + file_path: Path to the file + content: Content to write + file_label: Label for error messages + tracker: Validation tracker for success/error reporting + + Returns: + True if successful, False otherwise + """ + try: + file_path.write_text(content) + tracker.success() + return True + except Exception as e: + tracker.error((file_label,), message=f'Failed to write: {e}') + return False + + +def extract_sha256_from_sbom(packages: list, platform_name: str) -> str: + """ + Extract SHA256 hash from SBOM packages. + + Args: + packages: List of packages from SBOM + platform_name: Platform name for error messages (e.g., "Linux", "Windows") + + Returns: + SHA256 hash string + + Raises: + ValueError: If package, checksum, or hash format is invalid + """ + cpython_package = next((pkg for pkg in packages if pkg.get('name') == "CPython"), None) + if cpython_package is None: + raise ValueError(f"Could not find CPython package in {platform_name} SBOM") + + checksums = cpython_package.get('checksums', []) + checksum = next((cs for cs in checksums if cs.get('algorithm') == "SHA256"), None) + if checksum is None: + raise ValueError(f"Could not find SHA256 checksum in {platform_name} SBOM") + + hash_value = checksum.get('checksumValue', '') + if not validate_sha256(hash_value): + raise ValueError(f"Invalid {platform_name} SHA256 hash format from SBOM: {hash_value}") + + return hash_value + + +def upgrade_dockerfiles_python_version( app: Application, new_version: str, hashes: dict[str, str], tracker: ValidationTracker ): dockerfiles = [ @@ -146,14 +205,8 @@ def update_dockerfiles_python_version( return for dockerfile in dockerfiles: - if not dockerfile.exists(): - tracker.error((dockerfile.name,), message=f'File not found: {dockerfile}') - continue - - try: - content = dockerfile.read_text() - except Exception as e: - tracker.error((dockerfile.name,), message=f'Failed to read: {e}') + content = read_file_safely(dockerfile, dockerfile.name, tracker) + if content is None: continue is_windows = 'windows-x86_64' in dockerfile.parts @@ -192,24 +245,14 @@ def apply_substitution(pattern: re.Pattern, replace_func, error_msg: str, _docke if apply_substitution(sha_pattern, replace_sha, 'Could not find SHA256 pattern') is None: continue - try: - dockerfile.write_text(content) - tracker.success() - except Exception as e: - tracker.error((dockerfile.name,), message=f'Failed to write: {e}') + write_file_safely(dockerfile, content, dockerfile.name, tracker) -def update_macos_python_version(app: Application, new_version: str, tracker: ValidationTracker): +def upgrade_macos_python_version(app: Application, new_version: str, tracker: ValidationTracker): macos_python_file = app.repo.path / '.github' / 'workflows' / 'resolve-build-deps.yaml' - if not macos_python_file.exists(): - tracker.error(('macOS workflow',), message=f'File not found: {macos_python_file}') - return - - try: - content = macos_python_file.read_text() - except Exception as e: - tracker.error(('macOS workflow',), message=f'Failed to read: {e}') + content = read_file_safely(macos_python_file, 'macOS workflow', tracker) + if content is None: return target_line = next((line for line in content.splitlines() if 'PYTHON3_DOWNLOAD_URL' in line), None) @@ -226,25 +269,14 @@ def update_macos_python_version(app: Application, new_version: str, tracker: Val return updated_content = content.replace(target_line, new_line, 1) - - try: - macos_python_file.write_text(updated_content) - tracker.success() - except Exception as e: - tracker.error(('macOS workflow',), message=f'Failed to write: {e}') + write_file_safely(macos_python_file, updated_content, 'macOS workflow', tracker) -def update_python_version_full_constant(app: Application, new_version: str, tracker: ValidationTracker): +def upgrade_python_version_full_constant(app: Application, new_version: str, tracker: ValidationTracker): constants_file = app.repo.path / 'ddev' / 'src' / 'ddev' / 'repo' / 'constants.py' - if not constants_file.exists(): - tracker.error(('constants.py',), message=f'File not found: {constants_file}') - return - - try: - content = constants_file.read_text() - except Exception as e: - tracker.error(('constants.py',), message=f'Failed to read: {e}') + content = read_file_safely(constants_file, 'constants.py', tracker) + if content is None: return prefix = 'PYTHON_VERSION_FULL = ' @@ -259,12 +291,7 @@ def update_python_version_full_constant(app: Application, new_version: str, trac return updated_content = content.replace(target_line, new_line, 1) - - try: - constants_file.write_text(updated_content) - tracker.success() - except Exception as e: - tracker.error(('constants.py',), message=f'Failed to write: {e}') + write_file_safely(constants_file, updated_content, 'constants.py', tracker) def get_latest_python_version(app: Application, major_minor: str) -> str | None: @@ -311,7 +338,7 @@ def get_latest_python_version(app: Application, major_minor: str) -> str | None: def get_python_sha256_hashes(app: Application, version: str) -> dict[str, str]: """ Fetch SHA256 hashes for Python release artifacts using SBOM files. - + Args: version: Python version string (e.g., "3.13.7") @@ -355,40 +382,13 @@ async def get_sbom_data(client, url): raise RuntimeError(f'Error processing URL {url}: {e}') from e async def fetch_sbom_data(urls): - async with httpx.AsyncClient(verify=True) as client: return await asyncio.gather(*(get_sbom_data(client, url) for url in urls)) sbom_packages = asyncio.run(fetch_sbom_data(sbom_urls)) - # Find the CPython package in the SBOM packages - linux_cpython_package = next((package for package in sbom_packages[0] if package.get('name') == "CPython"), None) - windows_cpython_package = next((package for package in sbom_packages[1] if package.get('name') == "CPython"), None) - - if linux_cpython_package is None: - raise ValueError("Could not find CPython package in Linux SBOM") - if windows_cpython_package is None: - raise ValueError("Could not find CPython package in Windows SBOM") - - # Find the SHA256 checksum in the CPython package checksums - linux_checksums = linux_cpython_package.get('checksums', []) - windows_checksums = windows_cpython_package.get('checksums', []) - - linux_checksum = next((checksum for checksum in linux_checksums if checksum.get('algorithm') == "SHA256"), None) - windows_checksum = next((checksum for checksum in windows_checksums if checksum.get('algorithm') == "SHA256"), None) - - if linux_checksum is None: - raise ValueError("Could not find SHA256 checksum in Linux SBOM") - if windows_checksum is None: - raise ValueError("Could not find SHA256 checksum in Windows SBOM") - - # Extract hash values and validate format - linux_hash = linux_checksum.get('checksumValue', '') - windows_hash = windows_checksum.get('checksumValue', '') - - if not validate_sha256(linux_hash): - raise ValueError(f"Invalid Linux SHA256 hash format from SBOM: {linux_hash}") - if not validate_sha256(windows_hash): - raise ValueError(f"Invalid Windows SHA256 hash format from SBOM: {windows_hash}") + # Extract SHA256 hashes from SBOM packages + linux_hash = extract_sha256_from_sbom(sbom_packages[0], 'Linux') + windows_hash = extract_sha256_from_sbom(sbom_packages[1], 'Windows') return {'linux_source_sha256': linux_hash, 'windows_amd64_sha256': windows_hash} diff --git a/ddev/tests/cli/meta/scripts/test_upgrade_python.py b/ddev/tests/cli/meta/scripts/test_upgrade_python.py index adc1c202a334e..f940afc22a16f 100644 --- a/ddev/tests/cli/meta/scripts/test_upgrade_python.py +++ b/ddev/tests/cli/meta/scripts/test_upgrade_python.py @@ -79,21 +79,21 @@ def test_update_python_version_success(fake_repo, ddev, mocker): """Test successful Python version update.""" setup_python_update_files(fake_repo) # Mock network calls - mocker.patch('ddev.cli.meta.scripts.update_python.get_latest_python_version', return_value='3.13.9') + mocker.patch('ddev.cli.meta.scripts.upgrade_python.get_latest_python_version', return_value='3.13.9') mocker.patch( - 'ddev.cli.meta.scripts.update_python.get_python_sha256_hashes', + 'ddev.cli.meta.scripts.upgrade_python.get_python_sha256_hashes', return_value={ 'linux_source_sha256': 'c4c066af19c98fb7835d473bebd7e23be84f6e9874d47db9e39a68ee5d0ce35c', 'windows_amd64_sha256': '200ddff856bbff949d2cc1be42e8807c07538abd6b6966d5113a094cf628c5c5', }, ) - result = ddev('meta', 'scripts', 'update-python-version') + result = ddev('meta', 'scripts', 'upgrade-python-version') assert result.exit_code == 0, result.output assert 'Updating Python from 3.13.7 to 3.13.9' in result.output assert 'Passed: 5' in result.output - assert 'Python version updated from 3.13.7 to 3.13.9' in result.output + assert 'Python version upgraded from 3.13.7 to 3.13.9' in result.output # Verify constants.py was updated constants_file = fake_repo.path / 'ddev' / 'src' / 'ddev' / 'repo' / 'constants.py' @@ -124,9 +124,9 @@ def test_update_python_version_success(fake_repo, ddev, mocker): def test_update_python_version_already_latest(fake_repo, ddev, mocker): setup_python_update_files(fake_repo) - mocker.patch('ddev.cli.meta.scripts.update_python.get_latest_python_version', return_value='3.13.7') + mocker.patch('ddev.cli.meta.scripts.upgrade_python.get_latest_python_version', return_value='3.13.7') - result = ddev('meta', 'scripts', 'update-python-version') + result = ddev('meta', 'scripts', 'upgrade-python-version') assert result.exit_code == 0, result.output assert 'Already at latest Python version: 3.13.7' in result.output @@ -134,9 +134,9 @@ def test_update_python_version_already_latest(fake_repo, ddev, mocker): def test_update_python_version_no_new_version_found(fake_repo, ddev, mocker): setup_python_update_files(fake_repo) - mocker.patch('ddev.cli.meta.scripts.update_python.get_latest_python_version', return_value=None) + mocker.patch('ddev.cli.meta.scripts.upgrade_python.get_latest_python_version', return_value=None) - result = ddev('meta', 'scripts', 'update-python-version') + result = ddev('meta', 'scripts', 'upgrade-python-version') assert result.exit_code == 1, result.output assert 'Could not find latest Python version' in result.output @@ -144,13 +144,15 @@ def test_update_python_version_no_new_version_found(fake_repo, ddev, mocker): def test_update_python_version_invalid_hash_format(fake_repo, ddev, mocker): setup_python_update_files(fake_repo) - mocker.patch('ddev.cli.meta.scripts.update_python.get_latest_python_version', return_value='3.13.9') + mocker.patch('ddev.cli.meta.scripts.upgrade_python.get_latest_python_version', return_value='3.13.9') + # Hash validation happens inside get_python_sha256_hashes, so it raises ValueError mocker.patch( - 'ddev.cli.meta.scripts.update_python.get_python_sha256_hashes', - return_value={'linux_source_sha256': 'not-a-valid-hash', 'windows_amd64_sha256': 'also-invalid'}, + 'ddev.cli.meta.scripts.upgrade_python.get_python_sha256_hashes', + side_effect=ValueError('Invalid Linux SHA256 hash format from SBOM: not-a-valid-hash'), ) - result = ddev('meta', 'scripts', 'update-python-version') + result = ddev('meta', 'scripts', 'upgrade-python-version') assert result.exit_code == 1, result.output - assert 'Invalid Linux SHA256 hash format' in result.output or 'Invalid Windows SHA256 hash format' in result.output + assert 'Failed to fetch' in result.output + assert 'Invalid Linux SHA256 hash format' in result.output From 807948546f8f19a3d8bd72b32a1233094e90adbc Mon Sep 17 00:00:00 2001 From: Kyle-Neale Date: Sat, 18 Oct 2025 11:39:30 -0400 Subject: [PATCH 08/10] Rename test file and add use fake_repo in tests --- ddev/tests/cli/meta/scripts/conftest.py | 25 +++++- ..._py_config.py => test_update_py_config.py} | 0 .../cli/meta/scripts/test_upgrade_python.py | 76 ------------------- 3 files changed, 24 insertions(+), 77 deletions(-) rename ddev/tests/cli/meta/scripts/{test_upgrade_py_config.py => test_update_py_config.py} (100%) diff --git a/ddev/tests/cli/meta/scripts/conftest.py b/ddev/tests/cli/meta/scripts/conftest.py index c21ddcd370335..f685d44f0c79f 100644 --- a/ddev/tests/cli/meta/scripts/conftest.py +++ b/ddev/tests/cli/meta/scripts/conftest.py @@ -1,9 +1,12 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import shutil + import pytest from ddev.repo.core import Repository +from ddev.utils.fs import Path from ddev.utils.git import GitRepository # Whenenever we bump python version, we also need to bump the python @@ -13,7 +16,7 @@ @pytest.fixture -def fake_repo(tmp_path_factory, config_file, ddev, mocker): +def fake_repo(tmp_path_factory, config_file, local_repo, ddev, mocker): repo_path = tmp_path_factory.mktemp('integrations-core') repo = Repository('integrations-core', str(repo_path)) @@ -39,6 +42,7 @@ def fake_repo(tmp_path_factory, config_file, ddev, mocker): # This is automatically maintained PYTHON_VERSION = '{OLD_PYTHON_VERSION}' +PYTHON_VERSION_FULL = '3.13.7' """, ) @@ -142,6 +146,25 @@ def fake_repo(tmp_path_factory, config_file, ddev, mocker): """, ) + # Copy actual Dockerfiles from the real repository for Python upgrade tests + dockerfiles_to_copy = [ + '.builders/images/linux-aarch64/Dockerfile', + '.builders/images/linux-x86_64/Dockerfile', + '.builders/images/windows-x86_64/Dockerfile', + ] + + for dockerfile_path in dockerfiles_to_copy: + source = Path(local_repo) / dockerfile_path + dest = repo_path / dockerfile_path + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, dest) + + # Copy actual macOS workflow file for Python upgrade tests + workflow_source = Path(local_repo) / '.github' / 'workflows' / 'resolve-build-deps.yaml' + workflow_dest = repo_path / '.github' / 'workflows' / 'resolve-build-deps.yaml' + workflow_dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(workflow_source, workflow_dest) + yield repo diff --git a/ddev/tests/cli/meta/scripts/test_upgrade_py_config.py b/ddev/tests/cli/meta/scripts/test_update_py_config.py similarity index 100% rename from ddev/tests/cli/meta/scripts/test_upgrade_py_config.py rename to ddev/tests/cli/meta/scripts/test_update_py_config.py diff --git a/ddev/tests/cli/meta/scripts/test_upgrade_python.py b/ddev/tests/cli/meta/scripts/test_upgrade_python.py index f940afc22a16f..b908f9dcd0826 100644 --- a/ddev/tests/cli/meta/scripts/test_upgrade_python.py +++ b/ddev/tests/cli/meta/scripts/test_upgrade_python.py @@ -3,81 +3,8 @@ # Licensed under a 3-clause BSD style license (see LICENSE) -def setup_python_update_files(fake_repo): - """Add PYTHON_VERSION_FULL and Dockerfiles to fake_repo.""" - # Update constants.py to include PYTHON_VERSION_FULL - constants_file = fake_repo.path / 'ddev' / 'src' / 'ddev' / 'repo' / 'constants.py' - content = constants_file.read_text() - content += "PYTHON_VERSION_FULL = '3.13.7'\n" - constants_file.write_text(content) - - # Create Linux ARM64 Dockerfile - linux_aarch64_dir = fake_repo.path / '.builders' / 'images' / 'linux-aarch64' - linux_aarch64_dir.mkdir(parents=True, exist_ok=True) - (linux_aarch64_dir / 'Dockerfile').write_text("""FROM ubuntu:22.04 - -# Install Python -ENV PYTHON3_VERSION=3.13.7 -RUN DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \\ - VERSION="${PYTHON3_VERSION}" \\ - SHA256="6c9d80839cfa20024f34d9a6dd31ae2a9cd97ff5e980e969209746037a5153b2" \\ - bash install-from-source.sh - -CMD ["/bin/bash"] -""") - - # Create Linux x86_64 Dockerfile - linux_x86_64_dir = fake_repo.path / '.builders' / 'images' / 'linux-x86_64' - linux_x86_64_dir.mkdir(parents=True, exist_ok=True) - (linux_x86_64_dir / 'Dockerfile').write_text("""FROM ubuntu:22.04 - -# Install Python -ENV PYTHON3_VERSION=3.13.7 -RUN DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \\ - VERSION="${PYTHON3_VERSION}" \\ - SHA256="6c9d80839cfa20024f34d9a6dd31ae2a9cd97ff5e980e969209746037a5153b2" \\ - bash install-from-source.sh - -CMD ["/bin/bash"] -""") - - # Create Windows Dockerfile - windows_dockerfile_dir = fake_repo.path / '.builders' / 'images' / 'windows-x86_64' - windows_dockerfile_dir.mkdir(parents=True, exist_ok=True) - (windows_dockerfile_dir / 'Dockerfile').write_text("""FROM mcr.microsoft.com/windows/servercore:ltsc2022 - -# Install Python -ENV PYTHON_VERSION="3.13.7" -RUN powershell -Command " \\ - Invoke-WebRequest -OutFile python-$Env:PYTHON_VERSION-amd64.exe \\ - https://www.python.org/ftp/python/$Env:PYTHON_VERSION/python-$Env:PYTHON_VERSION-amd64.exe \\ - -Hash '48652a4e6af29c2f1fde2e2e6bbf3734a82ce3f577e9fd5c95c83f68e29e1eaa'" - -CMD ["powershell"] -""") - - # Create macOS workflow file - workflows_dir = fake_repo.path / '.github' / 'workflows' - workflows_dir.mkdir(parents=True, exist_ok=True) - (workflows_dir / 'resolve-build-deps.yaml').write_text("""name: Resolve build dependencies - -on: - workflow_dispatch: - -env: - PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.13.7/python-3.13.7-macos11.pkg" - -jobs: - build: - runs-on: macos-latest - steps: - - run: echo "test" -""") - - def test_update_python_version_success(fake_repo, ddev, mocker): """Test successful Python version update.""" - setup_python_update_files(fake_repo) # Mock network calls mocker.patch('ddev.cli.meta.scripts.upgrade_python.get_latest_python_version', return_value='3.13.9') mocker.patch( @@ -123,7 +50,6 @@ def test_update_python_version_success(fake_repo, ddev, mocker): def test_update_python_version_already_latest(fake_repo, ddev, mocker): - setup_python_update_files(fake_repo) mocker.patch('ddev.cli.meta.scripts.upgrade_python.get_latest_python_version', return_value='3.13.7') result = ddev('meta', 'scripts', 'upgrade-python-version') @@ -133,7 +59,6 @@ def test_update_python_version_already_latest(fake_repo, ddev, mocker): def test_update_python_version_no_new_version_found(fake_repo, ddev, mocker): - setup_python_update_files(fake_repo) mocker.patch('ddev.cli.meta.scripts.upgrade_python.get_latest_python_version', return_value=None) result = ddev('meta', 'scripts', 'upgrade-python-version') @@ -143,7 +68,6 @@ def test_update_python_version_no_new_version_found(fake_repo, ddev, mocker): def test_update_python_version_invalid_hash_format(fake_repo, ddev, mocker): - setup_python_update_files(fake_repo) mocker.patch('ddev.cli.meta.scripts.upgrade_python.get_latest_python_version', return_value='3.13.9') # Hash validation happens inside get_python_sha256_hashes, so it raises ValueError mocker.patch( From 58bc862a84c3e982b844e306963eb50ec1aec968 Mon Sep 17 00:00:00 2001 From: Kyle-Neale Date: Sat, 18 Oct 2025 12:35:18 -0400 Subject: [PATCH 09/10] Remove requests and address validate_sha256 comments --- ddev/src/ddev/cli/meta/scripts/upgrade_python.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ddev/src/ddev/cli/meta/scripts/upgrade_python.py b/ddev/src/ddev/cli/meta/scripts/upgrade_python.py index 1bc9bf633dd19..3e141ea52601c 100644 --- a/ddev/src/ddev/cli/meta/scripts/upgrade_python.py +++ b/ddev/src/ddev/cli/meta/scripts/upgrade_python.py @@ -10,7 +10,6 @@ import click import httpx import orjson -import requests from packaging.version import Version if TYPE_CHECKING: @@ -110,7 +109,7 @@ def validate_version_string(version: str) -> bool: def validate_sha256(hash_str: str) -> bool: """Validate SHA256 hash format (64 hex characters).""" - return bool(re.match(r'^[0-9a-f]{64}$', hash_str)) + return bool(re.match(r'^[0-9A-Fa-f]{64}$', hash_str)) def read_file_safely(file_path, file_label: str, tracker: ValidationTracker) -> str | None: @@ -306,9 +305,9 @@ def get_latest_python_version(app: Application, major_minor: str) -> str | None: """ try: # Explicitly verify SSL/TLS certificate - response = requests.get(PYTHON_FTP_URL, timeout=30, verify=True) + response = httpx.get(PYTHON_FTP_URL, timeout=30, verify=True) response.raise_for_status() - except requests.RequestException as e: + except httpx.RequestException as e: app.display_error(f"Error fetching Python versions: {e}") return None From 9bc7e221c9f1793ca30f80bc7f61a8ac8f18f886 Mon Sep 17 00:00:00 2001 From: Kyle-Neale Date: Tue, 21 Oct 2025 11:21:30 -0400 Subject: [PATCH 10/10] Update constants last and make content variables more descriptive --- .../ddev/cli/meta/scripts/upgrade_python.py | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/ddev/src/ddev/cli/meta/scripts/upgrade_python.py b/ddev/src/ddev/cli/meta/scripts/upgrade_python.py index 3e141ea52601c..2fa7123f8ffe2 100644 --- a/ddev/src/ddev/cli/meta/scripts/upgrade_python.py +++ b/ddev/src/ddev/cli/meta/scripts/upgrade_python.py @@ -79,9 +79,9 @@ def upgrade_python_version(app: Application): return # Unreachable but helps type checker # Perform updates - upgrade_python_version_full_constant(app, latest_version, tracker) upgrade_dockerfiles_python_version(app, latest_version, new_version_hashes, tracker) upgrade_macos_python_version(app, latest_version, tracker) + upgrade_python_version_full_constant(app, latest_version, tracker) # Display results tracker.display() @@ -204,8 +204,8 @@ def upgrade_dockerfiles_python_version( return for dockerfile in dockerfiles: - content = read_file_safely(dockerfile, dockerfile.name, tracker) - if content is None: + dockerfile_content = read_file_safely(dockerfile, dockerfile.name, tracker) + if dockerfile_content is None: continue is_windows = 'windows-x86_64' in dockerfile.parts @@ -228,33 +228,33 @@ def replace_sha(match: re.Match[str], _target_sha=target_sha) -> str: return old_match.replace(old_hash, _target_sha) # Helper to apply pattern substitution with error tracking - def apply_substitution(pattern: re.Pattern, replace_func, error_msg: str, _dockerfile=dockerfile) -> str | None: - nonlocal content - content, count = pattern.subn(replace_func, content, count=1) + def apply_substitution(pattern: re.Pattern, replace_func, error_msg: str, _dockerfile=dockerfile) -> bool: + nonlocal dockerfile_content + dockerfile_content, count = pattern.subn(replace_func, dockerfile_content, count=1) if count == 0: tracker.error((_dockerfile.name,), message=error_msg) - return None - return content + return False + return True # Apply version update - if apply_substitution(version_pattern, replace_version, 'Could not find Python version pattern') is None: + if not apply_substitution(version_pattern, replace_version, 'Could not find Python version pattern'): continue # Apply SHA256 update - if apply_substitution(sha_pattern, replace_sha, 'Could not find SHA256 pattern') is None: + if not apply_substitution(sha_pattern, replace_sha, 'Could not find SHA256 pattern'): continue - write_file_safely(dockerfile, content, dockerfile.name, tracker) + write_file_safely(dockerfile, dockerfile_content, dockerfile.name, tracker) def upgrade_macos_python_version(app: Application, new_version: str, tracker: ValidationTracker): macos_python_file = app.repo.path / '.github' / 'workflows' / 'resolve-build-deps.yaml' - content = read_file_safely(macos_python_file, 'macOS workflow', tracker) - if content is None: + macos_content = read_file_safely(macos_python_file, 'macOS workflow', tracker) + if macos_content is None: return - target_line = next((line for line in content.splitlines() if 'PYTHON3_DOWNLOAD_URL' in line), None) + target_line = next((line for line in macos_content.splitlines() if 'PYTHON3_DOWNLOAD_URL' in line), None) if target_line is None: tracker.error(('macOS workflow',), message='Could not find PYTHON3_DOWNLOAD_URL') @@ -265,21 +265,22 @@ def upgrade_macos_python_version(app: Application, new_version: str, tracker: Va new_line = f'{indent}PYTHON3_DOWNLOAD_URL: "{new_url}"' if target_line == new_line: + app.display_info(f"Python version in macOS workflow is already at {new_version}") return - updated_content = content.replace(target_line, new_line, 1) + updated_content = macos_content.replace(target_line, new_line, 1) write_file_safely(macos_python_file, updated_content, 'macOS workflow', tracker) def upgrade_python_version_full_constant(app: Application, new_version: str, tracker: ValidationTracker): constants_file = app.repo.path / 'ddev' / 'src' / 'ddev' / 'repo' / 'constants.py' - content = read_file_safely(constants_file, 'constants.py', tracker) - if content is None: + constants_content = read_file_safely(constants_file, 'constants.py', tracker) + if constants_content is None: return prefix = 'PYTHON_VERSION_FULL = ' - target_line = next((line for line in content.splitlines() if line.startswith(prefix)), None) + target_line = next((line for line in constants_content.splitlines() if line.startswith(prefix)), None) if target_line is None: tracker.error(('constants.py',), message='Could not find PYTHON_VERSION_FULL constant') @@ -287,9 +288,10 @@ def upgrade_python_version_full_constant(app: Application, new_version: str, tra new_line = f"{prefix}'{new_version}'" if target_line == new_line: + app.display_info(f"Python version in constants.py is already at {new_version}") return - updated_content = content.replace(target_line, new_line, 1) + updated_content = constants_content.replace(target_line, new_line, 1) write_file_safely(constants_file, updated_content, 'constants.py', tracker)