diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/test_automl/test_automl.py b/test/test_automl/test_automl.py index b34a296ec5..f021279dce 100644 --- a/test/test_automl/test_automl.py +++ b/test/test_automl/test_automl.py @@ -64,9 +64,9 @@ def test_fit(dask_client): metric=accuracy, dask_client=dask_client, ) - automl.fit( - X_train, Y_train, task=MULTICLASS_CLASSIFICATION - ) + + automl.fit(X_train, Y_train, task=MULTICLASS_CLASSIFICATION) + score = automl.score(X_test, Y_test) assert score > 0.8 assert count_succeses(automl.cv_results_) > 0 @@ -109,9 +109,9 @@ def get_roar_object_callback( metric=accuracy, dask_client=dask_client_single_worker, ) - automl.fit( - X_train, Y_train, task=MULTICLASS_CLASSIFICATION, - ) + + automl.fit(X_train, Y_train, task=MULTICLASS_CLASSIFICATION) + score = automl.score(X_test, Y_test) assert score > 0.8 assert count_succeses(automl.cv_results_) > 0 @@ -224,8 +224,7 @@ def test_delete_non_candidate_models(dask_client): max_models_on_disc=3, ) - automl.fit(X, Y, task=MULTICLASS_CLASSIFICATION, - X_test=X, y_test=Y) + automl.fit(X, Y, task=MULTICLASS_CLASSIFICATION, X_test=X, y_test=Y) # Assert at least one model file has been deleted and that there were no # deletion errors @@ -271,7 +270,9 @@ def test_binary_score_and_include(dask_client): metric=accuracy, dask_client=dask_client, ) + automl.fit(X_train, Y_train, task=BINARY_CLASSIFICATION) + assert automl._task == BINARY_CLASSIFICATION # TODO, the assumption from above is not really tested here @@ -294,6 +295,7 @@ def test_automl_outputs(dask_client): dask_client=dask_client, delete_tmp_folder_after_terminate=False, ) + auto.fit( X=X_train, y=Y_train, @@ -302,6 +304,7 @@ def test_automl_outputs(dask_client): dataset_name=name, task=MULTICLASS_CLASSIFICATION, ) + data_manager_file = os.path.join( auto._backend.temporary_directory, '.auto-sklearn', @@ -624,9 +627,8 @@ def test_load_best_individual_model(metric, dask_client): # We cannot easily mock a function sent to dask # so for this test we create the whole set of models/ensembles # but prevent it to be loaded - automl.fit( - X_train, Y_train, task=MULTICLASS_CLASSIFICATION, - ) + automl.fit(X_train, Y_train, task=MULTICLASS_CLASSIFICATION) + automl._backend.load_ensemble = unittest.mock.MagicMock(return_value=None) # A memory error occurs in the ensemble construction diff --git a/test/test_automl/test_estimators.py b/test/test_automl/test_estimators.py index ac7e86cf3c..c01c6f5fe1 100644 --- a/test/test_automl/test_estimators.py +++ b/test/test_automl/test_estimators.py @@ -79,6 +79,7 @@ def __call__(self, *args, **kwargs): get_smac_object_callback=get_smac_object_wrapper_instance, max_models_on_disc=None, ) + automl.fit(X_train, Y_train) # Test that the argument is correctly passed to SMAC @@ -272,6 +273,7 @@ def test_performance_over_time_no_ensemble(tmp_dir): seed=1, initial_configurations_via_metalearning=0, ensemble_size=0,) + cls.fit(X_train, Y_train, X_test, Y_test) performance_over_time = cls.performance_over_time_ @@ -297,6 +299,7 @@ def test_cv_results(tmp_dir): original_params = copy.deepcopy(params) cls.fit(X_train, Y_train) + cv_results = cls.cv_results_ assert isinstance(cv_results, dict), type(cv_results) assert isinstance(cv_results['mean_test_score'], np.ndarray), type( @@ -382,6 +385,7 @@ def test_leaderboard( tmp_folder=tmp_dir, seed=1 ) + model.fit(X_train, Y_train) for params in params_generator: @@ -540,6 +544,7 @@ def test_can_pickle_classifier(tmp_dir, dask_client): tmp_folder=tmp_dir, dask_client=dask_client, ) + automl.fit(X_train, Y_train) initial_predictions = automl.predict(X_test) @@ -765,12 +770,14 @@ def test_autosklearn_classification_methods_returns_self(dask_client): exclude={'feature_preprocessor': ['fast_ica']}) automl_fitted = automl.fit(X_train, y_train) + assert automl is automl_fitted automl_ensemble_fitted = automl.fit_ensemble(y_train, ensemble_size=5) assert automl is automl_ensemble_fitted automl_refitted = automl.refit(X_train.copy(), y_train.copy()) + assert automl is automl_refitted @@ -801,12 +808,14 @@ def test_autosklearn2_classification_methods_returns_self(dask_client): dask_client=dask_client) automl_fitted = automl.fit(X_train, y_train) + assert automl is automl_fitted automl_ensemble_fitted = automl.fit_ensemble(y_train, ensemble_size=5) assert automl is automl_ensemble_fitted automl_refitted = automl.refit(X_train.copy(), y_train.copy()) + assert automl is automl_refitted predictions = automl_fitted.predict(X_test) @@ -824,12 +833,14 @@ def test_autosklearn2_classification_methods_returns_self_sparse(dask_client): dask_client=dask_client) automl_fitted = automl.fit(X_train, y_train) + assert automl is automl_fitted automl_ensemble_fitted = automl.fit_ensemble(y_train, ensemble_size=5) assert automl is automl_ensemble_fitted automl_refitted = automl.refit(X_train.copy(), y_train.copy()) + assert automl is automl_refitted predictions = automl_fitted.predict(X_test) @@ -933,10 +944,15 @@ def test_fit_pipeline(dask_client, task_type, resampling_strategy, disable_file_ X_test=X_test, y_test=y_test, ).get_default_configuration() - pipeline, run_info, run_value = automl.fit_pipeline(X=X_train, y=y_train, config=config, - X_test=X_test, y_test=y_test, - disable_file_output=disable_file_output, - resampling_strategy=resampling_strategy) + pipeline, run_info, run_value = automl.fit_pipeline( + X=X_train, + y=y_train, + config=config, + X_test=X_test, + y_test=y_test, + disable_file_output=disable_file_output, + resampling_strategy=resampling_strategy + ) assert isinstance(run_info.config, Configuration) assert run_info.cutoff == 30 @@ -1090,11 +1106,14 @@ def test_autosklearn_anneal(as_frame): if as_frame: # Let autosklearn calculate the feat types automl_fitted = automl.fit(X, y) + else: X_, y_ = sklearn.datasets.fetch_openml(data_id=2, return_X_y=True, as_frame=True) feat_type = ['categorical' if X_[col].dtype.name == 'category' else 'numerical' for col in X_.columns] + automl_fitted = automl.fit(X, y, feat_type=feat_type) + assert automl is automl_fitted automl_ensemble_fitted = automl.fit_ensemble(y, ensemble_size=5) diff --git a/test/test_pipeline/components/classification/test_base.py b/test/test_pipeline/components/classification/test_base.py index ddfe336b88..dda507eda4 100644 --- a/test/test_pipeline/components/classification/test_base.py +++ b/test/test_pipeline/components/classification/test_base.py @@ -9,6 +9,8 @@ import sklearn.metrics import numpy as np +from ...ignored_warnings import ignore_warnings, classifier_warnings + class BaseClassificationComponentTest(unittest.TestCase): # Magic command to not run tests on base class @@ -274,7 +276,8 @@ def is_unset_param_raw_predictions_val_error(err): + " assignment" in err.args[0]) try: - model.fit(X.copy(), y.copy()) + with ignore_warnings(classifier_warnings): + model.fit(X.copy(), y.copy()) except ValueError as e: if is_AdaBoostClassifier_error(e) or is_QDA_error(e): return None diff --git a/test/test_pipeline/components/feature_preprocessing/test_liblinear.py b/test/test_pipeline/components/feature_preprocessing/test_liblinear.py index eb4b715ce9..47465ccc8f 100644 --- a/test/test_pipeline/components/feature_preprocessing/test_liblinear.py +++ b/test/test_pipeline/components/feature_preprocessing/test_liblinear.py @@ -5,10 +5,15 @@ get_dataset import sklearn.metrics +from ...ignored_warnings import ignore_warnings, feature_preprocessing_warnings + class LiblinearComponentTest(PreprocessingTestCase): + def test_default_configuration(self): - transformation, original = _test_preprocessing(LibLinear_Preprocessor) + with ignore_warnings(feature_preprocessing_warnings): + transformation, original = _test_preprocessing(LibLinear_Preprocessor) + self.assertEqual(transformation.shape[0], original.shape[0]) self.assertFalse((transformation == 0).all()) @@ -23,7 +28,10 @@ def test_default_configuration_classify(self): for hp_name in default if default[ hp_name] is not None}) - preprocessor.fit(X_train, Y_train) + + with ignore_warnings(feature_preprocessing_warnings): + preprocessor.fit(X_train, Y_train) + X_train_trans = preprocessor.transform(X_train) X_test_trans = preprocessor.transform(X_test) @@ -35,6 +43,6 @@ def test_default_configuration_classify(self): self.assertAlmostEqual(accuracy, 0.8548876745598057, places=2) def test_preprocessing_dtype(self): - super(LiblinearComponentTest, - self)._test_preprocessing_dtype(LibLinear_Preprocessor, - test_sparse=False) + + with ignore_warnings(feature_preprocessing_warnings): + super()._test_preprocessing_dtype(LibLinear_Preprocessor, test_sparse=False) diff --git a/test/test_pipeline/components/regression/test_base.py b/test/test_pipeline/components/regression/test_base.py index 70f19c3177..b093c685c3 100644 --- a/test/test_pipeline/components/regression/test_base.py +++ b/test/test_pipeline/components/regression/test_base.py @@ -34,9 +34,12 @@ def test_default_boston(self): return for _ in range(2): - predictions, targets, n_calls = _test_regressor( - dataset="boston", Regressor=self.module - ) + + with ignore_warnings(regressor_warnings): + predictions, targets, n_calls = _test_regressor( + dataset="boston", + Regressor=self.module + ) if "default_boston_le_ge" in self.res: # Special treatment for Gaussian Process Regression @@ -73,9 +76,12 @@ def test_default_boston_iterative_fit(self): return for i in range(2): - predictions, targets, regressor = \ - _test_regressor_iterative_fit(dataset="boston", - Regressor=self.module) + with ignore_warnings(regressor_warnings): + predictions, targets, regressor = _test_regressor_iterative_fit( + dataset="boston", + Regressor=self.module + ) + score = sklearn.metrics.r2_score(targets, predictions) fixture = self.res["default_boston_iterative"] @@ -108,10 +114,12 @@ def test_default_boston_iterative_sparse_fit(self): return for i in range(2): - predictions, targets, _ = \ - _test_regressor_iterative_fit(dataset="boston", - Regressor=self.module, - sparse=True) + with ignore_warnings(regressor_warnings): + predictions, targets, _ = _test_regressor_iterative_fit( + dataset="boston", + Regressor=self.module, + sparse=True + ) self.assertAlmostEqual(self.res["default_boston_iterative_sparse"], sklearn.metrics.r2_score(targets, predictions), @@ -127,10 +135,13 @@ def test_default_boston_sparse(self): return for i in range(2): - predictions, targets, _ = \ - _test_regressor(dataset="boston", - Regressor=self.module, - sparse=True) + with ignore_warnings(regressor_warnings): + predictions, targets, _ = _test_regressor( + dataset="boston", + Regressor=self.module, + sparse=True + ) + self.assertAlmostEqual(self.res["default_boston_sparse"], sklearn.metrics.r2_score(targets, predictions), @@ -143,9 +154,11 @@ def test_default_diabetes(self): return for i in range(2): - predictions, targets, n_calls = \ - _test_regressor(dataset="diabetes", - Regressor=self.module) + with ignore_warnings(regressor_warnings): + predictions, targets, n_calls = _test_regressor( + dataset="diabetes", + Regressor=self.module + ) self.assertAlmostEqual(self.res["default_diabetes"], sklearn.metrics.r2_score(targets, @@ -165,9 +178,12 @@ def test_default_diabetes_iterative_fit(self): return for i in range(2): - predictions, targets, _ = \ - _test_regressor_iterative_fit(dataset="diabetes", - Regressor=self.module) + with ignore_warnings(regressor_warnings): + predictions, targets, _ = _test_regressor_iterative_fit( + dataset="diabetes", + Regressor=self.module + ) + self.assertAlmostEqual(self.res["default_diabetes_iterative"], sklearn.metrics.r2_score(targets, predictions), @@ -186,10 +202,13 @@ def test_default_diabetes_iterative_sparse_fit(self): return for i in range(2): - predictions, targets, regressor = \ - _test_regressor_iterative_fit(dataset="diabetes", - Regressor=self.module, - sparse=True) + with ignore_warnings(regressor_warnings): + predictions, targets, regressor = _test_regressor_iterative_fit( + dataset="diabetes", + Regressor=self.module, + sparse=True + ) + self.assertAlmostEqual(self.res["default_diabetes_iterative_sparse"], sklearn.metrics.r2_score(targets, predictions), @@ -211,10 +230,13 @@ def test_default_diabetes_sparse(self): return for i in range(2): - predictions, targets, _ = \ - _test_regressor(dataset="diabetes", - Regressor=self.module, - sparse=True) + with ignore_warnings(regressor_warnings): + predictions, targets, _ = _test_regressor( + dataset="diabetes", + Regressor=self.module, + sparse=True + ) + self.assertAlmostEqual(self.res["default_diabetes_sparse"], sklearn.metrics.r2_score(targets, predictions), @@ -264,12 +286,16 @@ def test_module_idempotent(self): # Get the parameters on the first and second fit with config params # Also compare their random state - params_first = regressor.fit(X.copy(), y.copy()).estimator.get_params() + with ignore_warnings(regressor_warnings): + params_first = regressor.fit(X.copy(), y.copy()).estimator.get_params() + if hasattr(regressor.estimator, 'random_state'): rs_1 = regressor.random_state rs_estimator_1 = regressor.estimator.random_state - params_second = regressor.fit(X.copy(), y.copy()).estimator.get_params() + with ignore_warnings(regressor_warnings): + params_second = regressor.fit(X.copy(), y.copy()).estimator.get_params() + if hasattr(regressor.estimator, 'random_state'): rs_2 = regressor.random_state rs_estimator_2 = regressor.estimator.random_state @@ -302,7 +328,7 @@ def test_fit_and_predict_with_1d_targets_as_1d( regressor: Type[RegressorChoice], X: np.ndarray, y: np.ndarray -): +) -> None: """Test that all pipelines work with 1d target types Parameters @@ -345,7 +371,7 @@ def test_fit_and_predict_with_1d_targets_as_2d( regressor: Type[RegressorChoice], X: np.ndarray, y: np.ndarray -): +) -> None: """Test that all pipelines work with 1d target types when they are wrapped as 2d Parameters @@ -394,7 +420,7 @@ def test_fit_and_predict_with_2d_targets( regressor: Type[RegressorChoice], X: np.ndarray, y: np.ndarray -): +) -> None: """Test that all pipelines work with 2d target types Parameters diff --git a/test/test_pipeline/ignored_warnings.py b/test/test_pipeline/ignored_warnings.py index 8f8203e05f..5b941281f9 100644 --- a/test/test_pipeline/ignored_warnings.py +++ b/test/test_pipeline/ignored_warnings.py @@ -2,6 +2,7 @@ from typing import List, Iterator, Tuple import warnings + from sklearn.exceptions import ConvergenceWarning @@ -68,14 +69,34 @@ r" optimization hasn't converged yet\." ) ), + ( + ConvergenceWarning, ( # From FastICA + r"FastICA did not converge\." + r" Consider increasing tolerance or the maximum number of iterations\." + ) + ), ( UserWarning, ( # From LDA (Linear Discriminant Analysis) r"Variables are collinear" ) ), + ( + UserWarning, ( + r"Clustering metrics expects discrete values but received continuous values" + r" for label, and multiclass values for target" + ) + ) +] + +feature_preprocessing_warnings = [ + ( + ConvergenceWarning, ( # From liblinear + r"Liblinear failed to converge, increase the number of iterations." + ) + ) ] -ignored_warnings = regressor_warnings + classifier_warnings +ignored_warnings = regressor_warnings + classifier_warnings + feature_preprocessing_warnings @contextmanager diff --git a/test/test_pipeline/test_classification.py b/test/test_pipeline/test_classification.py index cd0ffd9adf..237936ca74 100644 --- a/test/test_pipeline/test_classification.py +++ b/test/test_pipeline/test_classification.py @@ -174,7 +174,9 @@ def test_default_configuration(self): auto = SimpleClassificationPipeline(random_state=1) - auto = auto.fit(X_train, Y_train) + with ignore_warnings(classifier_warnings): + auto = auto.fit(X_train, Y_train) + predictions = auto.predict(X_test) acc = sklearn.metrics.accuracy_score(predictions, Y_test) @@ -197,7 +199,9 @@ def test_default_configuration_multilabel(self): default = cs.get_default_configuration() classifier.set_hyperparameters(default) - classifier = classifier.fit(X_train, Y_train) + with ignore_warnings(classifier_warnings): + classifier = classifier.fit(X_train, Y_train) + predictions = classifier.predict(X_test) acc = sklearn.metrics.accuracy_score(predictions, Y_test) @@ -221,10 +225,12 @@ def test_default_configuration_iterative_fit(self): } ) classifier.fit_transformer(X_train, Y_train) - for i in range(1, 11): - classifier.iterative_fit(X_train, Y_train) - n_estimators = classifier.steps[-1][-1].choice.estimator.n_estimators - self.assertEqual(n_estimators, i) + + with ignore_warnings(classifier_warnings): + for i in range(1, 11): + classifier.iterative_fit(X_train, Y_train) + n_estimators = classifier.steps[-1][-1].choice.estimator.n_estimators + self.assertEqual(n_estimators, i) def test_repr(self): """Test that the default pipeline can be converted to its representation and @@ -730,7 +736,9 @@ def test_predict_batched(self): # Multiclass X_train, Y_train, X_test, Y_test = get_dataset(dataset='digits') - cls.fit(X_train, Y_train) + + with ignore_warnings(classifier_warnings): + cls.fit(X_train, Y_train) X_test_ = X_test.copy() prediction_ = cls.predict_proba(X_test_) @@ -762,7 +770,8 @@ def test_predict_batched_sparse(self): # Multiclass X_train, Y_train, X_test, Y_test = get_dataset(dataset='digits', make_sparse=True) - cls.fit(X_train, Y_train) + with ignore_warnings(classifier_warnings): + cls.fit(X_train, Y_train) X_test_ = X_test.copy() prediction_ = cls.predict_proba(X_test_) @@ -791,7 +800,8 @@ def test_predict_proba_batched(self): cls = SimpleClassificationPipeline(include={'classifier': ['sgd']}) X_train, Y_train, X_test, Y_test = get_dataset(dataset='digits') - cls.fit(X_train, Y_train) + with ignore_warnings(classifier_warnings): + cls.fit(X_train, Y_train) X_test_ = X_test.copy() prediction_ = cls.predict_proba(X_test_) @@ -811,7 +821,9 @@ def test_predict_proba_batched(self): X_train, Y_train, X_test, Y_test = get_dataset(dataset='digits') Y_train = np.array(list([(list([1 if i != y else 0 for i in range(10)])) for y in Y_train])) - cls.fit(X_train, Y_train) + + with ignore_warnings(classifier_warnings): + cls.fit(X_train, Y_train) X_test_ = X_test.copy() prediction_ = cls.predict_proba(X_test_) @@ -845,7 +857,9 @@ def test_predict_proba_batched_sparse(self): X_train, Y_train, X_test, Y_test = get_dataset(dataset='digits', make_sparse=True) X_test_ = X_test.copy() - cls.fit(X_train, Y_train) + with ignore_warnings(classifier_warnings): + cls.fit(X_train, Y_train) + prediction_ = cls.predict_proba(X_test_) # The object behind the last step in the pipeline @@ -864,10 +878,13 @@ def test_predict_proba_batched_sparse(self): include={'classifier': ['lda']} ) X_train, Y_train, X_test, Y_test = get_dataset(dataset='digits', make_sparse=True) + X_test_ = X_test.copy() Y_train = np.array([[1 if i != y else 0 for i in range(10)] for y in Y_train]) - cls.fit(X_train, Y_train) + with ignore_warnings(classifier_warnings): + cls.fit(X_train, Y_train) + prediction_ = cls.predict_proba(X_test_) # The object behind the last step in the pipeline @@ -892,7 +909,9 @@ def test_pipeline_clonability(self): X_train, Y_train, X_test, Y_test = get_dataset(dataset='iris') auto = SimpleClassificationPipeline() - auto = auto.fit(X_train, Y_train) + + with ignore_warnings(classifier_warnings): + auto = auto.fit(X_train, Y_train) auto_clone = clone(auto) auto_clone_params = auto_clone.get_params() @@ -1153,12 +1172,12 @@ def test_fit_instantiates_component(self): del preprocessing_components.additional_components.components['CrashPreprocessor'] self.fail("cs={} config={} Exception={}".format(cs, config, e)) cls.set_hyperparameters(config) - with self.assertRaisesRegex( - ValueError, - "Make sure fit is called" - ): - cls.fit( - X=np.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]), - y=np.array([1, 0, 1, 1]) - ) + + with self.assertRaisesRegex(ValueError, "Make sure fit is called"): + with ignore_warnings(classifier_warnings): + cls.fit( + X=np.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]), + y=np.array([1, 0, 1, 1]) + ) + del preprocessing_components.additional_components.components['CrashPreprocessor']