diff --git a/autosklearn/metrics/__init__.py b/autosklearn/metrics/__init__.py
index 6b241f2176..def49dbdef 100644
--- a/autosklearn/metrics/__init__.py
+++ b/autosklearn/metrics/__init__.py
@@ -125,7 +125,8 @@ def __call__(self, y_true, y_pred, sample_weight=None):
             raise ValueError("{0} format is not supported".format(y_type))
 
         if y_type == "binary":
-            y_pred = y_pred[:, 1]
+            if y_pred.ndim > 1:
+                y_pred = y_pred[:, 1]
         elif isinstance(y_pred, list):
             y_pred = np.vstack([p[:, -1] for p in y_pred]).T