fix(puzzle): handle self-referential extras

Co-authored-by: David Hotham <[email protected]>

Author: abn

src/poetry/puzzle/provider.py

@@ -460,18 +460,29 @@ def complete_package(
             dependency = dependency_package.dependency
             requires = package.requires
 
-        optional_dependencies = []
+        found_extras = set()
+        optional_dependencies = set()
         _dependencies = []
 
-        # If some extras/features were required, we need to
-        # add a special dependency representing the base package
-        # to the current package
         if dependency.extras:
-            for extra in dependency.extras:
-                if extra not in package.extras:
+            # Find all the optional dependencies that are wanted - taking care to allow
+            # for self-referential extras.
+            stack = list(dependency.extras)
+            while stack:
+                extra = stack.pop()
+                if extra in found_extras:
                     continue
+                found_extras.add(extra)
 
-                optional_dependencies += [d.name for d in package.extras[extra]]
+                extra_dependencies = package.extras.get(extra, [])
+                for extra_dependency in extra_dependencies:
+                    if extra_dependency.name == dependency.name:
+                        stack += list(extra_dependency.extras)
+                    else:
+                        optional_dependencies.add(extra_dependency.name)
+
+            # If some extras/features were required, we need to add a special dependency
+            # representing the base package to the current package.
 
             dependency_package = dependency_package.with_features(
                 list(dependency.extras)
@@ -507,10 +518,7 @@ def complete_package(
 
             if not package.is_root() and (
                 (dep.is_optional() and dep.name not in optional_dependencies)
-                or (
-                    dep.in_extras
-                    and not set(dep.in_extras).intersection(dependency.extras)
-                )
+                or (dep.in_extras and not set(dep.in_extras).intersection(found_extras))
             ):
                 continue
 

tests/conftest.py

@@ -21,6 +21,10 @@
 from keyring.credentials import SimpleCredential
 from keyring.errors import KeyringError
 from keyring.errors import KeyringLocked
+from packaging.utils import canonicalize_name
+from poetry.core.constraints.version import parse_constraint
+from poetry.core.packages.dependency import Dependency
+from poetry.core.version.markers import parse_marker
 from pytest import FixtureRequest
 
 from poetry.config.config import Config as BaseConfig
@@ -29,7 +33,9 @@
 from poetry.factory import Factory
 from poetry.layouts import layout
 from poetry.packages.direct_origin import _get_package_from_git
+from poetry.repositories import Repository
 from poetry.repositories import RepositoryPool
+from poetry.repositories.exceptions import PackageNotFoundError
 from poetry.repositories.installed_repository import InstalledRepository
 from poetry.utils.cache import ArtifactCache
 from poetry.utils.env import EnvManager
@@ -57,6 +63,8 @@
     from cleo.io.inputs.argument import Argument
     from cleo.io.inputs.option import Option
     from keyring.credentials import Credential
+    from packaging.utils import NormalizedName
+    from poetry.core.packages.package import Package
     from pytest import Config as PyTestConfig
     from pytest import Parser
     from pytest import TempPathFactory
@@ -66,6 +74,7 @@
     from tests.types import CommandFactory
     from tests.types import FixtureCopier
     from tests.types import FixtureDirGetter
+    from tests.types import PackageFactory
     from tests.types import ProjectFactory
     from tests.types import SetProjectContext
 
@@ -500,6 +509,80 @@ def _factory(
     return _factory
 
 
+@pytest.fixture
+def create_package(repo: Repository) -> PackageFactory:
+    """
+    This function is a pytest fixture that creates a factory function to generate
+    and customize package objects. These packages are added to the default repository
+    fixture and configured with specific versions, optional extras, and self-referenced
+    extras. This helps in setting up package dependencies for testing purposes.
+
+    :return: A factory function that can be used to create and configure packages.
+    """
+
+    def create_new_package(
+        name: str,
+        version: str | None = None,
+        dependencies: list[Dependency] | None = None,
+        extras: dict[str, list[str]] | None = None,
+    ) -> Package:
+        version = version or "1.0"
+        package = get_package(name, version)
+
+        package_extras: dict[NormalizedName, list[Dependency]] = {}
+
+        for extra, extra_dependencies in (extras or {}).items():
+            extra = canonicalize_name(extra)
+
+            if extra not in package_extras:
+                package_extras[extra] = []
+
+            for extra_dependency_spec in extra_dependencies:
+                extra_dependency = Dependency.create_from_pep_508(extra_dependency_spec)
+                extra_dependency._optional = True
+                extra_dependency.marker = extra_dependency.marker.intersect(
+                    parse_marker(f"extra == '{extra}'")
+                )
+
+                if extra_dependency.name != package.name:
+                    assert extra_dependency.constraint.allows(package.version)
+
+                    # if it is not a self-referencing dependency, make sure we add it to the repo
+                    try:
+                        pkg = repo.package(extra_dependency.name, package.version)
+                    except PackageNotFoundError:
+                        pkg = get_package(extra_dependency.name, str(package.version))
+                        repo.add_package(pkg)
+
+                    extra_dependency.constraint = parse_constraint(f"^{pkg.version}")
+
+                    # if requirement already exists in the package, update the marker
+                    for requirement in package.requires:
+                        if (
+                            requirement.name == extra_dependency.name
+                            and requirement.is_optional()
+                        ):
+                            requirement.marker = requirement.marker.union(
+                                extra_dependency.marker
+                            )
+                            break
+                    else:
+                        package.add_dependency(extra_dependency)
+
+                package_extras[extra].append(extra_dependency)
+
+        package.extras = package_extras
+
+        for dependency in dependencies or []:
+            package.add_dependency(dependency)
+
+        repo.add_package(package)
+
+        return package
+
+    return create_new_package
+
+
 @pytest.fixture(autouse=True)
 def set_simple_log_formatter() -> None:
     """

tests/installation/fixtures/with-self-referencing-extras-all-deep.test

@@ -0,0 +1,58 @@
+# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand.
+
+[[package]]
+name = "A"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.dependencies]
+download-package = {version = ">=1.0,<2.0", optional = true, markers = "extra == \"download\""}
+install-package = {version = ">=1.0,<2.0", optional = true, markers = "extra == \"install\""}
+
+[package.extras]
+all = ["a[download,install]"]
+download = ["download-package (>=1.0,<2.0)"]
+install = ["install-package (>=1.0,<2.0)"]
+nested = ["a[all]"]
+py = ["a[py310,py38]"]
+py310 = ["py310-package (>=1.0,<2.0) ; python_version > \"3.8\""]
+py38 = ["py38-package (>=1.0,<2.0) ; python_version == \"3.8\""]
+
+[[package]]
+name = "B"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.dependencies]
+a = {version = "1.0", extras = ["all"]}
+
+[[package]]
+name = "download-package"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[[package]]
+name = "install-package"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[metadata]
+lock-version = "2.1"
+python-versions = "*"
+content-hash = "123456789"

tests/installation/fixtures/with-self-referencing-extras-all-top.test

@@ -0,0 +1,46 @@
+# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand.
+
+[[package]]
+name = "A"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.dependencies]
+download-package = {version = ">=1.0,<2.0", optional = true, markers = "extra == \"download\""}
+install-package = {version = ">=1.0,<2.0", optional = true, markers = "extra == \"install\""}
+
+[package.extras]
+all = ["a[download,install]"]
+download = ["download-package (>=1.0,<2.0)"]
+install = ["install-package (>=1.0,<2.0)"]
+nested = ["a[all]"]
+py = ["a[py310,py38]"]
+py310 = ["py310-package (>=1.0,<2.0) ; python_version > \"3.8\""]
+py38 = ["py38-package (>=1.0,<2.0) ; python_version == \"3.8\""]
+
+[[package]]
+name = "download-package"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[[package]]
+name = "install-package"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[metadata]
+lock-version = "2.1"
+python-versions = "*"
+content-hash = "123456789"

tests/installation/fixtures/with-self-referencing-extras-b-markers.test

@@ -0,0 +1,42 @@
+# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand.
+
+[[package]]
+name = "A"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.dependencies]
+download-package = {version = ">=1.0,<2.0", optional = true, markers = "extra == \"download\""}
+install-package = {version = ">=1.0,<2.0", optional = true, markers = "extra == \"install\""}
+
+[package.extras]
+all = ["a[download,install] ; python_version < \"3.9\""]
+download = ["download-package (>=1.0,<2.0)"]
+install = ["install-package (>=1.0,<2.0)"]
+
+[[package]]
+name = "download-package"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[[package]]
+name = "install-package"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[metadata]
+lock-version = "2.1"
+python-versions = "*"
+content-hash = "123456789"

tests/installation/fixtures/with-self-referencing-extras-deep.test

@@ -0,0 +1,36 @@
+# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand.
+
+[[package]]
+name = "A"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.extras]
+all = ["a[download,install]"]
+download = ["download-package (>=1.0,<2.0)"]
+install = ["install-package (>=1.0,<2.0)"]
+nested = ["a[all]"]
+py = ["a[py310,py38]"]
+py310 = ["py310-package (>=1.0,<2.0) ; python_version > \"3.8\""]
+py38 = ["py38-package (>=1.0,<2.0) ; python_version == \"3.8\""]
+
+[[package]]
+name = "B"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.dependencies]
+a = "1.0"
+
+[metadata]
+lock-version = "2.1"
+python-versions = "*"
+content-hash = "123456789"