Improve Summary Analysis by Relativize the metric results if there is a status quo to relativize against (facebook#4342)

Summary: Pull Request resolved: facebook#4342

Differential Revision: D82658357

Author: shrutipatel31

ax/analysis/summary.py

@@ -63,6 +63,12 @@ def compute(
         if experiment is None:
             raise UserInputError("`Summary` analysis requires an `Experiment` input")
 
+        # Determine if we should relativize based on:
+        # (1) experiment has metrics and (2) experiment has status quo
+        should_relativize = (
+            len(experiment.metrics) > 0 and experiment.status_quo is not None
+        )
+
         return self._create_analysis_card(
             title=(
                 "Summary for "
@@ -73,5 +79,6 @@ def compute(
                 trial_indices=self.trial_indices,
                 omit_empty_columns=self.omit_empty_columns,
                 trial_statuses=self.trial_statuses,
+                relativize=should_relativize,
             ),
         )

ax/analysis/tests/test_summary.py

@@ -275,3 +275,64 @@ def test_default_excludes_stale_trials(self) -> None:
         # Verify that no trials in the output have STALE status
         stale_statuses = card.df[card.df["trial_status"] == "STALE"]
         self.assertEqual(len(stale_statuses), 0)
+
+    def test_metrics_relativized_with_status_quo(self) -> None:
+        """Test that Summary relativizes metrics by default when status quo is
+        present."""
+        client = Client()
+        client.configure_experiment(
+            name="test_experiment_relativize",
+            parameters=[
+                RangeParameterConfig(
+                    name="x1",
+                    parameter_type="float",
+                    bounds=(0, 1),
+                ),
+            ],
+        )
+        client.configure_optimization(objective="metric1")
+
+        # Add status quo
+        baseline_trial_index = client.attach_baseline({"x1": 0.5})
+        client.complete_trial(
+            trial_index=baseline_trial_index, raw_data={"metric1": 90.0}
+        )
+
+        # Get trials and complete with metric data
+        client.get_next_trials(max_trials=2)
+
+        # Complete trials with different metric values
+        client.complete_trial(
+            trial_index=baseline_trial_index + 1, raw_data={"metric1": 100.0}
+        )
+        client.complete_trial(
+            trial_index=baseline_trial_index + 2, raw_data={"metric1": 80.0}
+        )
+
+        experiment = client._experiment
+
+        # Test that Summary works and produces results
+        # (relativization happens internally)
+        analysis = Summary()
+
+        card = analysis.compute(experiment=experiment)
+
+        # Verify basic structure
+        self.assertEqual(card.name, "Summary")
+        self.assertEqual(card.title, "Summary for test_experiment_relativize")
+        self.assertTrue("metric1" in card.df.columns)
+        self.assertEqual(len(card.df), 3)
+
+        # Verify all trials are present (baseline at index 0,
+        # regular trials at indices 1 and 2)
+        trial_indices = set(card.df["trial_index"].values)
+
+        self.assertEqual(trial_indices, {0, 1, 2})
+
+        # Check that metric values are present (actual relativization values depend on
+        # the underlying experiment.to_df implementation with relativize=True)
+        # Some values might be NaN due to relativization, but not all should be NaN
+        metric_values = card.df["metric1"].values
+        non_na_count = sum(~pd.isna(metric_values))
+        # At least some trials should have non-NaN metric values
+        self.assertGreater(non_na_count, 0, "All metric values are NaN")

ax/core/data.py

@@ -447,8 +447,15 @@ def relativize(
                     axis=1,
                 )
             )
+        if not dfs:
+            raise ValueError(
+                f"Relativization not possible: status quo arm '{status_quo_name}' "
+                f"not found or dataset contains no data."
+            )
         df_rel = pd.concat(dfs, axis=0)
         if include_sq:
+            # Set status quo to exactly 0 mean and 0 SEM to avoid negative zero display
+            df_rel.loc[df_rel["arm_name"] == status_quo_name, "mean"] = 0.0
             df_rel.loc[df_rel["arm_name"] == status_quo_name, "sem"] = 0.0
         return Data(df_rel)
 

ax/core/experiment.py

@@ -2039,6 +2039,7 @@ def to_df(
         trial_indices: Iterable[int] | None = None,
         trial_statuses: Sequence[TrialStatus] | None = None,
         omit_empty_columns: bool = True,
+        relativize: bool = False,
     ) -> pd.DataFrame:
         """
         High-level summary of the Experiment with one row per arm. Any values missing at
@@ -2060,10 +2061,23 @@ def to_df(
             trial_indices: If specified, only include these trial indices.
             omit_empty_columns: If True, omit columns where every value is None.
             trial_status: If specified, only include trials with this status.
+            relativize: If True and experiment has a status quo, relativize metrics
         """
 
         records = []
-        data_df = self.lookup_data(trial_indices=trial_indices).df
+        data = self.lookup_data(trial_indices=trial_indices)
+
+        # Relativize metrics if requested
+        data_df = (
+            data.relativize(
+                status_quo_name=none_throws(self.status_quo).name,
+                as_percent=True,
+                include_sq=True,
+            ).df
+            if relativize
+            else data.df
+        )
+
         trials = (
             self.get_trials_by_indices(trial_indices=trial_indices)
             if trial_indices
@@ -2123,6 +2137,7 @@ def to_df(
                 records.append(record)
 
         df = pd.DataFrame(records)
+
         if omit_empty_columns:
             df = df.loc[:, df.notnull().any()]
         return df

ax/core/tests/test_experiment.py

@@ -75,7 +75,7 @@
 )
 from ax.utils.testing.mock import mock_botorch_optimize
 from pandas.testing import assert_frame_equal
-from pyre_extensions import assert_is_instance
+from pyre_extensions import assert_is_instance, none_throws
 
 DUMMY_RUN_METADATA_KEY_1 = "test_run_metadata_key_1"
 DUMMY_RUN_METADATA_KEY_2 = "test_run_metadata_key_2"
@@ -471,7 +471,7 @@ def test_StatusQuoSetter(self) -> None:
         sq_parameters["w"] = 3.5
         self.experiment.status_quo = Arm(sq_parameters)
         self.assertEqual(self.experiment.status_quo.parameters["w"], 3.5)
-        self.assertEqual(self.experiment.status_quo.name, "status_quo_e0")
+        self.assertEqual(none_throws(self.experiment.status_quo).name, "status_quo_e0")
 
         # Verify all None values
         self.experiment.status_quo = Arm({n: None for n in sq_parameters.keys()})
@@ -1640,6 +1640,66 @@ def test_to_df(self) -> None:
         )
         self.assertTrue(df_completed.equals(expected_completed_df))
 
+    def test_to_df_with_relativize(self) -> None:
+        """Test the relativize flag in to_df method with status quo."""
+        # Create an experiment with status quo and completed trials
+        experiment = get_branin_experiment(with_status_quo=True)
+
+        # Create two completed trials
+        for _ in range(2):
+            sobol_run = get_sobol(search_space=experiment.search_space).gen(n=1)
+            trial = experiment.new_trial(generator_run=sobol_run)
+            trial.mark_running(no_runner_required=True)
+            trial.mark_completed()
+
+        # Fetch and add status quo data
+        experiment.fetch_data()
+        sq_data = Data(
+            df=pd.DataFrame(
+                [
+                    {
+                        "trial_index": i,
+                        "arm_name": "status_quo",
+                        "metric_name": "branin",
+                        "metric_signature": "branin",
+                        "mean": 10.0,
+                        "sem": 0.1,
+                    }
+                    for i in range(2)
+                ]
+            )
+        )
+        experiment.attach_data(sq_data)
+
+        # Test without relativization
+        df_no_rel = experiment.to_df(relativize=False)
+
+        # Test with relativization
+        df_with_rel = experiment.to_df(relativize=True)
+
+        # Basic structure should be the same
+        self.assertEqual(len(df_with_rel), len(df_no_rel))
+        self.assertEqual(set(df_with_rel.columns), set(df_no_rel.columns))
+
+        # Find metric columns and verify relativization occurred
+        metric_cols = [
+            col
+            for col in df_no_rel.columns
+            if col
+            not in ["trial_index", "arm_name", "trial_status", "name", "x1", "x2"]
+        ]
+
+        if metric_cols:
+            metric_name = metric_cols[0]
+            orig_values = df_no_rel[metric_name].dropna()
+            rel_values = df_with_rel[metric_name].dropna()
+
+            # Values should change for non-status-quo trials
+            non_sq_changed = any(
+                abs(o - r) > 1e-10 for o, r in zip(orig_values, rel_values) if o != 10.0
+            )
+            self.assertTrue(non_sq_changed, "Relativization should change some values")
+
 
 class ExperimentWithMapDataTest(TestCase):
     def setUp(self) -> None: