Add exceptions to log file, not just stdout (automl#863)

* Add exceptions to log file, not just stdout

* Removing dummy pred as trys is not needed

Author: franchuterivera

autosklearn/automl.py

@@ -506,16 +506,21 @@ def _fit(
                 get_smac_object_callback=self._get_smac_object_callback,
                 smac_scenario_args=self._smac_scenario_args,
             )
-            self.runhistory_, self.trajectory_, self._budget_type = \
-                _proc_smac.run_smbo()
-            trajectory_filename = os.path.join(
-                self._backend.get_smac_output_directory_for_run(self._seed),
-                'trajectory.json')
-            saveable_trajectory = \
-                [list(entry[:2]) + [entry[2].get_dictionary()] + list(entry[3:])
-                 for entry in self.trajectory_]
-            with open(trajectory_filename, 'w') as fh:
-                json.dump(saveable_trajectory, fh)
+
+            try:
+                self.runhistory_, self.trajectory_, self._budget_type = \
+                    _proc_smac.run_smbo()
+                trajectory_filename = os.path.join(
+                    self._backend.get_smac_output_directory_for_run(self._seed),
+                    'trajectory.json')
+                saveable_trajectory = \
+                    [list(entry[:2]) + [entry[2].get_dictionary()] + list(entry[3:])
+                     for entry in self.trajectory_]
+                with open(trajectory_filename, 'w') as fh:
+                    json.dump(saveable_trajectory, fh)
+            except Exception as e:
+                self._logger.exception(e)
+                raise
 
         # Wait until the ensemble process is finished to avoid shutting down
         # while the ensemble builder tries to access the data

test/test_automl/test_automl.py

@@ -393,6 +393,47 @@ def test_fail_if_dummy_prediction_fails(self, ta_run_mock):
         self._tearDown(backend_api.temporary_directory)
         self._tearDown(backend_api.output_directory)
 
+    @unittest.mock.patch('autosklearn.smbo.AutoMLSMBO.run_smbo')
+    def test_exceptions_inside_log_in_smbo(self, smbo_run_mock):
+
+        # Make sure that any exception during the AutoML fit due to
+        # SMAC are properly captured in a log file
+        backend_api = self._create_backend('test_exceptions_inside_log')
+        self._tearDown(backend_api.temporary_directory)
+        self._tearDown(backend_api.output_directory)
+
+        automl = autosklearn.automl.AutoML(backend_api, 20, 5)
+
+        output_file = 'test_exceptions_inside_log.log'
+        setup_logger(output_file=output_file)
+        logger = get_logger('test_exceptions_inside_log')
+
+        # Create a custom exception to prevent other errors to slip in
+        class MyException(Exception):
+            pass
+
+        X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
+        # The first call is on dummy predictor failure
+        message = str(np.random.randint(100)) + '_run_smbo'
+        smbo_run_mock.side_effect = MyException(message)
+
+        with unittest.mock.patch('autosklearn.automl.AutoML._get_logger') as mock:
+            mock.return_value = logger
+            with self.assertRaises(MyException):
+                automl.fit(
+                    X_train,
+                    Y_train,
+                    metric=accuracy,
+                    task=MULTICLASS_CLASSIFICATION,
+                )
+            with open(output_file) as f:
+                self.assertTrue(message in f.read())
+
+        # Cleanup
+        os.unlink(output_file)
+        self._tearDown(backend_api.temporary_directory)
+        self._tearDown(backend_api.output_directory)
+
 
 if __name__ == "__main__":
     unittest.main()