[autorevert] fix handling of insufficient signals for restarts (#7209)

A bug introduced by #7204

The check that ensures that at least two failure/success signals are
available before revert could be issued was broken. This PR restores.


Here's an example of the wrong behavior:

[2025-09-23_22-44-14.html](https://github.com/user-attachments/files/22503454/2025-09-23_22-44-14.html)

---------

Co-authored-by: Nikita Shulga <[email protected]>

Author: izaitsevfb

aws/lambda/pytorch-auto-revert/pytorch_auto_revert/signal.py

@@ -368,18 +368,30 @@ def process_valid_autorevert_pattern(
         infra_check_result = partition.confirm_not_an_infra_issue()
         # note re: event_count < 2:
         # this is a confidence heuristic to detect flakiness, can adjust as needed
-        if infra_check_result == InfraCheckResult.RESTART_FAILURE or (
-            partition.failure_events_count() < 2
-            and not partition.failed[-1].has_pending
+        if (
+            infra_check_result == InfraCheckResult.RESTART_FAILURE
+            or partition.failure_events_count() < 2
         ):
-            # restarting oldest failed
-            restart_commits.add(partition.failed[-1].head_sha)
-        elif infra_check_result == InfraCheckResult.RESTART_SUCCESS or (
-            partition.success_events_count() < 2
-            and not partition.successful[0].has_pending
+            if not partition.failed[-1].has_pending:
+                # restarting oldest failed
+                restart_commits.add(partition.failed[-1].head_sha)
+            else:
+                return Ineligible(
+                    IneligibleReason.INFRA_NOT_CONFIRMED,
+                    f"waiting on pending events on suspected failure side: {partition.failed[-1].head_sha}",
+                )
+        elif (
+            infra_check_result == InfraCheckResult.RESTART_SUCCESS
+            or partition.success_events_count() < 2
         ):
-            # restarting newest successful
-            restart_commits.add(partition.successful[0].head_sha)
+            if not partition.successful[0].has_pending:
+                # restarting newest successful
+                restart_commits.add(partition.successful[0].head_sha)
+            else:
+                return Ineligible(
+                    IneligibleReason.INFRA_NOT_CONFIRMED,
+                    f"waiting on pending events on suspected success side: {partition.successful[0].head_sha}",
+                )
 
         if restart_commits:
             return RestartCommits(commit_shas=restart_commits)

aws/lambda/pytorch-auto-revert/pytorch_auto_revert/tests/test_signal.py

@@ -222,6 +222,108 @@ def test_detect_autorevert_pattern_ineligible_when_fixed(self):
         self.assertIsInstance(res, Ineligible)
         self.assertEqual(res.reason, IneligibleReason.FIXED)
 
+    def test_insufficient_failures_returns_restart_oldest_failed_when_no_pending(self):
+        # Only one failure event overall (< 2) -> suggest restart of oldest failed
+        c_failed = SignalCommit(
+            head_sha="sha_failed",
+            events=[self._ev("job", SignalStatus.FAILURE, 5)],
+        )
+        # Base successful commit with enough successes to avoid infra ambiguity
+        c_base = SignalCommit(
+            head_sha="sha_base",
+            events=[
+                self._ev("job", SignalStatus.SUCCESS, 3),
+                self._ev("job", SignalStatus.SUCCESS, 7),
+            ],
+        )
+        s = Signal(key="job", workflow_name="wf", commits=[c_failed, c_base])
+        res = s.process_valid_autorevert_pattern()
+        self.assertIsNotNone(res)
+        # With insufficient failures, we never produce an AutorevertPattern
+        self.assertNotIsInstance(res, AutorevertPattern)
+        # Should propose a restart on the suspected failure side (oldest failed)
+        self.assertTrue(hasattr(res, "commit_shas"))
+        self.assertIn("sha_failed", res.commit_shas)
+
+    def test_insufficient_failures_infra_not_confirmed_when_pending_on_failed(self):
+        # Only one failure overall, but the failed commit also has pending → ineligible awaiting confirmation
+        c_failed_pending = SignalCommit(
+            head_sha="sha_failed_pend",
+            events=[
+                self._ev("job", SignalStatus.PENDING, 4),
+                self._ev("job", SignalStatus.FAILURE, 5),
+            ],
+        )
+        c_base = SignalCommit(
+            head_sha="sha_base",
+            events=[
+                self._ev("job", SignalStatus.SUCCESS, 3),
+                self._ev("job", SignalStatus.SUCCESS, 7),
+            ],
+        )
+        s = Signal(key="job", workflow_name="wf", commits=[c_failed_pending, c_base])
+        res = s.process_valid_autorevert_pattern()
+        self.assertIsInstance(res, Ineligible)
+        self.assertEqual(res.reason, IneligibleReason.INFRA_NOT_CONFIRMED)
+        self.assertIn("sha_failed_pend", res.message)
+
+    def test_insufficient_successes_returns_restart_newest_success_when_no_pending(
+        self,
+    ):
+        # Ensure we have at least two failure events to avoid the failure-side heuristic
+        c_fail_new = SignalCommit(
+            head_sha="sha_fail_new",
+            events=[self._ev("job", SignalStatus.FAILURE, 10)],
+        )
+        c_fail_old = SignalCommit(
+            head_sha="sha_fail_old",
+            events=[self._ev("job", SignalStatus.FAILURE, 9)],
+        )
+        # Only one success event overall (< 2) → suggest restart of newest successful
+        c_success = SignalCommit(
+            head_sha="sha_success",
+            events=[self._ev("job", SignalStatus.SUCCESS, 3)],
+        )
+        s = Signal(
+            key="job",
+            workflow_name="wf",
+            commits=[c_fail_new, c_fail_old, c_success],
+        )
+        res = s.process_valid_autorevert_pattern()
+        self.assertIsNotNone(res)
+        # With insufficient successes, we never produce an AutorevertPattern
+        self.assertNotIsInstance(res, AutorevertPattern)
+        # Should propose a restart on the suspected success side (newest successful)
+        self.assertTrue(hasattr(res, "commit_shas"))
+        self.assertIn("sha_success", res.commit_shas)
+
+    def test_insufficient_successes_infra_not_confirmed_when_pending_on_success(self):
+        # Two failures present, but newest successful has pending → ineligible awaiting confirmation
+        c_fail_new = SignalCommit(
+            head_sha="sha_fail_new",
+            events=[self._ev("job", SignalStatus.FAILURE, 10)],
+        )
+        c_fail_old = SignalCommit(
+            head_sha="sha_fail_old",
+            events=[self._ev("job", SignalStatus.FAILURE, 9)],
+        )
+        c_success_pending = SignalCommit(
+            head_sha="sha_success_pend",
+            events=[
+                self._ev("job", SignalStatus.SUCCESS, 3),
+                self._ev("job", SignalStatus.PENDING, 8),
+            ],
+        )
+        s = Signal(
+            key="job",
+            workflow_name="wf",
+            commits=[c_fail_new, c_fail_old, c_success_pending],
+        )
+        res = s.process_valid_autorevert_pattern()
+        self.assertIsInstance(res, Ineligible)
+        self.assertEqual(res.reason, IneligibleReason.INFRA_NOT_CONFIRMED)
+        self.assertIn("sha_success_pend", res.message)
+
 
 if __name__ == "__main__":
     unittest.main()