perf: filter duplicates (and equivalents) from itertools.product when building the cnf/dnf (python-poetry#851)

Author: radoering

src/poetry/core/version/markers.py

@@ -33,7 +33,9 @@
 if TYPE_CHECKING:
     from collections.abc import Callable
     from collections.abc import Iterable
+    from collections.abc import Iterator
     from collections.abc import Mapping
+    from collections.abc import Sequence
 
     from lark import Tree
 
@@ -1125,7 +1127,7 @@ def cnf(marker: BaseMarker) -> BaseMarker:
             m.markers if isinstance(m, MultiMarker) else [m] for m in cnf_markers
         ]
         return MultiMarker.of(
-            *[MarkerUnion.of(*c) for c in itertools.product(*sub_marker_lists)]
+            *[MarkerUnion.of(*c) for c in _unique_product(*sub_marker_lists)]
         )
 
     if isinstance(marker, MultiMarker):
@@ -1143,7 +1145,7 @@ def dnf(marker: BaseMarker) -> BaseMarker:
             m.markers if isinstance(m, MarkerUnion) else [m] for m in dnf_markers
         ]
         return MarkerUnion.of(
-            *[MultiMarker.of(*c) for c in itertools.product(*sub_marker_lists)]
+            *[MultiMarker.of(*c) for c in _unique_product(*sub_marker_lists)]
         )
 
     if isinstance(marker, MarkerUnion):
@@ -1225,6 +1227,21 @@ def union(*markers: BaseMarker) -> BaseMarker:
     return min(*candidates, key=lambda x: x.complexity)
 
 
+def _unique_product(
+    *sub_marker_lists: Sequence[BaseMarker],
+) -> Iterator[Sequence[BaseMarker]]:
+    """
+    Returns an itertools.product of the sub_marker_lists
+    without duplicates (and equivalents) removed while maintaining order.
+    """
+    unique_sets = set()
+    for sub_marker_list in itertools.product(*sub_marker_lists):
+        sub_marker_set = frozenset(sub_marker_list)
+        if sub_marker_set not in unique_sets:
+            unique_sets.add(sub_marker_set)
+            yield sub_marker_list
+
+
 @functools.cache
 def _merge_single_markers(
     marker1: SingleMarkerLike[SingleMarkerConstraint],

tests/version/test_markers.py

@@ -2368,6 +2368,44 @@ def test_complex_intersection() -> None:
     )
 
 
+def test_complex_intersection_with_itertools_product_duplicates() -> None:
+    """
+    Real-world example from https://github.com/python-poetry/poetry/issues/10250.
+    (Only occurs if the solver takes an unfortunate path.)
+    Takes a long time without filtering duplicates from the itertools.product()
+    in cnf/dnf early.
+    """
+    m1 = parse_marker(
+        '(python_version > "3.9" or platform_system != "Windows"'
+        ' or platform_machine != "x86") and (python_version != "3.10"'
+        ' or platform_system != "Windows" or platform_python_implementation == "PyPy")'
+    )
+    m2 = parse_marker(
+        '(platform_system != "Windows" or platform_machine != "x86"'
+        ' or python_version >= "3.10" and python_version < "3.12")'
+        ' and (sys_platform != "darwin" or platform_machine != "arm64")'
+        ' and python_version <= "3.11" and (platform_system != "Windows"'
+        ' or platform_python_implementation == "PyPy" or python_version == "3.9"'
+        ' or python_version == "3.11") and python_version >= "3.9"'
+        ' and (platform_system != "Windows" or platform_python_implementation == "PyPy"'
+        ' or platform_machine != "x86" or python_version == "3.11")'
+    )
+
+    assert str(m1.intersect(m2)) == (
+        '(python_version > "3.9" or platform_system != "Windows"'
+        ' or platform_machine != "x86") and (python_version != "3.10"'
+        ' or platform_system != "Windows" or platform_python_implementation == "PyPy")'
+        ' and (platform_system != "Windows" or platform_machine != "x86"'
+        ' or python_version >= "3.10") and python_version <= "3.11"'
+        ' and (sys_platform != "darwin" or platform_machine != "arm64") and'
+        ' (platform_system != "Windows" or platform_python_implementation == "PyPy"'
+        ' or python_version == "3.9" or python_version == "3.11")'
+        ' and python_version >= "3.9" and (platform_system != "Windows"'
+        ' or platform_python_implementation == "PyPy" or platform_machine != "x86"'
+        ' or python_version == "3.11")'
+    )
+
+
 def test_complex_union_is_deterministic() -> None:
     """
     This test might fail sporadically if marker operations are not deterministic!