From 6deaf02d6482104e7b0547926415fe755fc411f8 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 17 Jun 2022 17:25:28 +0200 Subject: [PATCH 01/16] Dummy implementation of a multi-objective ensemble. --- autosklearn/ensembles/abstract_ensemble.py | 8 +- .../multiobjective_dummy_ensemble.py | 458 ++++++++++++++++++ autosklearn/estimators.py | 44 +- 3 files changed, 498 insertions(+), 12 deletions(-) create mode 100644 autosklearn/ensembles/multiobjective_dummy_ensemble.py diff --git a/autosklearn/ensembles/abstract_ensemble.py b/autosklearn/ensembles/abstract_ensemble.py index ea14a45e90..5ee8e35d16 100644 --- a/autosklearn/ensembles/abstract_ensemble.py +++ b/autosklearn/ensembles/abstract_ensemble.py @@ -79,7 +79,7 @@ def predict( Returns ------- - array : [n_data_points] + np.ndarray """ pass @@ -97,7 +97,7 @@ def get_models_with_weights( Returns ------- - array : [(weight_1, model_1), ..., (weight_n, model_n)] + List[Tuple[float, BasePipeline]] """ @abstractmethod @@ -115,7 +115,7 @@ def get_identifiers_with_weights( Returns ------- - array : [(identifier_1, weight_1), ..., (identifier_n, weight_n)] + List[Tuple[Tuple[int, int, float], float] """ @abstractmethod @@ -141,4 +141,4 @@ def get_validation_performance(self) -> float: class AbstractMultiObjectiveEnsemble(AbstractEnsemble): def get_pareto_set(self) -> Sequence[AbstractEnsemble]: - pass + raise NotImplementedError() diff --git a/autosklearn/ensembles/multiobjective_dummy_ensemble.py b/autosklearn/ensembles/multiobjective_dummy_ensemble.py new file mode 100644 index 0000000000..cbb8d06609 --- /dev/null +++ b/autosklearn/ensembles/multiobjective_dummy_ensemble.py @@ -0,0 +1,458 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union + +import warnings + +import numpy as np + +from autosklearn.automl_common.common.utils.backend import Backend +from autosklearn.constants import TASK_TYPES +from autosklearn.data.validation import SUPPORTED_FEAT_TYPES +from autosklearn.ensemble_building.run import Run +from autosklearn.ensembles.abstract_ensemble import ( + AbstractEnsemble, + AbstractMultiObjectiveEnsemble, +) +from autosklearn.metrics import Scorer, calculate_losses +from autosklearn.pipeline.base import BasePipeline + + +class SingleModelEnsemble(AbstractEnsemble): + """Ensemble consisting of a single model. + + This class is used my the :cls:`MultiObjectiveDummyEnsemble` to represent ensembles + consisting of a single model, and this class should not be used on its own. + + Parameters + ---------- + task_type: int + An identifier indicating which task is being performed. + + metrics: Sequence[Scorer] | Scorer + The metrics used to evaluate the models. + + random_state: Optional[int | RandomState] = None + Not used. + + backend : Backend + Gives access to the backend of Auto-sklearn. Not used. + + model_index : int + Index of the model that constitutes the ensemble. This index will + be used to select the correct predictions that will be passed during + ``fit`` and ``predict``. + """ + + def __init__( + self, + task_type: int, + metrics: Sequence[Scorer] | Scorer, + random_state: int | np.random.RandomState | None, + backend: Backend, + model_index: int, + ): + # Add some default values -- at least 1 model in ensemble is assumed + self.indices_ = [model_index] + self.weights_ = [1.0] + self.task_type = task_type + self.metrics = metrics if isinstance(metrics, Sequence) else [metrics] + + def fit( + self, + base_models_predictions: np.ndarray | List[np.ndarray], + X_data: SUPPORTED_FEAT_TYPES, + true_targets: np.ndarray, + model_identifiers: List[Tuple[int, int, float]], + runs: Sequence[Run], + ) -> SingleModelEnsemble: + """Dummy implementation of the ``fit`` method. + + Actualy work of passing the model index is done in the constructor. This + method only stores the identifier of the selected model and computes it's + validation loss. + + Parameters + ---------- + base_models_predictions: np.ndarray + shape = (n_base_models, n_data_points, n_targets) + n_targets is the number of classes in case of classification, + n_targets is 0 or 1 in case of regression + + Can be a list of 2d numpy arrays as well to prevent copying all + predictions into a single, large numpy array. + + X_data : list-like or sparse data + + true_targets : array of shape [n_targets] + + model_identifiers : identifier for each base model. + Can be used for practical text output of the ensemble. + + runs: Sequence[Run] + Additional information for each run executed by SMAC that was + considered by the ensemble builder. Not used. + + Returns + ------- + self + """ + self.identifiers_ = [model_identifiers[self.indices_[0]]] + loss = calculate_losses( + solution=true_targets, + prediction=base_models_predictions[self.indices_[0]], + task_type=self.task_type, + metrics=self.metrics, + X_data=X_data, + ) + self.best_model_score_ = loss[self.metrics[0].name] + return self + + def predict(self, predictions: Union[np.ndarray, List[np.ndarray]]) -> np.ndarray: + """Select the predictions of the selected model. + + Parameters + ---------- + base_models_predictions : np.ndarray + shape = (n_base_models, n_data_points, n_targets) + Same as in the fit method. + + Returns + ------- + np.ndarray + """ + return predictions[0] + + def __str__(self) -> str: + return "Single Model:\n\tMembers: %s" "\n\tWeights: %s\n\tIdentifiers: %s" % ( + self.indices_, + self.weights_, + " ".join( + [ + str(identifier) + for idx, identifier in enumerate(self.identifiers_) + if self.weights_[idx] > 0 + ] + ), + ) + + def get_models_with_weights( + self, models: Dict[Tuple[int, int, float], BasePipeline] + ) -> List[Tuple[float, BasePipeline]]: + """Return a list of (weight, model) pairs for the model selected by this ensemble. + + Parameters + ---------- + models : dict {identifier : model object} + The identifiers are the same as the one presented to the fit() + method. Models can be used for nice printing. + + Returns + ------- + List[Tuple[float, BasePipeline]] + """ + output = [] + for i, weight in enumerate(self.weights_): + if weight > 0.0: + identifier = self.identifiers_[i] + model = models[identifier] + output.append((weight, model)) + + output.sort(reverse=True, key=lambda t: t[0]) + + return output + + def get_identifiers_with_weights( + self, + ) -> List[Tuple[Tuple[int, int, float], float]]: + """Return a (identifier, weight)-pairs for the model selected by this ensemble. + + Parameters + ---------- + models : dict {identifier : model object} + The identifiers are the same as the one presented to the fit() + method. Models can be used for nice printing. + + Returns + ------- + List[Tuple[Tuple[int, int, float], float] + """ + return list(zip(self.identifiers_, self.weights_)) + + def get_selected_model_identifiers(self) -> List[Tuple[int, int, float]]: + """Return identifier of models in the ensemble. + + This includes models which have a weight of zero! + + Returns + ------- + list + """ + output = [] + + for i, weight in enumerate(self.weights_): + identifier = self.identifiers_[i] + if weight > 0.0: + output.append(identifier) + + return output + + def get_validation_performance(self) -> float: + """Return validation performance of ensemble. + + In case of multi-objective problem, only the first metric will be returned. + + Return + ------ + float + """ + return self.best_model_score_ + + +class MultiObjectiveDummyEnsemble(AbstractMultiObjectiveEnsemble): + def __init__( + self, + task_type: int, + metrics: Sequence[Scorer] | Scorer, + random_state: Optional[Union[int, np.random.RandomState]], + backend: Backend, + ) -> None: + """A dummy implementation of a multi-objective ensemble. + + Builds ensembles that are individual models on the Pareto front each. + + Parameters + ---------- + task_type: int + An identifier indicating which task is being performed. + + metrics: Sequence[Scorer] | Scorer + The metrics used to evaluate the models. + + random_state: Optional[int | RandomState] = None + Not used. + + backend : Backend + Gives access to the backend of Auto-sklearn. Not used. + """ + self.task_type = task_type + if isinstance(metrics, Sequence): + if len(metrics) == 1: + warnings.warn( + "Passed only a single metric to a multi-objective ensemble. " + "Please use a single-objective ensemble in such cases." + ) + self.metrics = metrics + else: + self.metric = [metrics] + self.random_state = random_state + self.backend = backend + + def __getstate__(self) -> Dict[str, Any]: + # Cannot serialize a metric if + # it is user defined. + # That is, if doing pickle dump + # the metric won't be the same as the + # one in __main__. we don't use the metric + # in the EnsembleSelection so this should + # be fine + return {key: value for key, value in self.__dict__.items() if key != "metrics"} + + def fit( + self, + base_models_predictions: List[np.ndarray], + X_data: SUPPORTED_FEAT_TYPES, + true_targets: np.ndarray, + model_identifiers: List[Tuple[int, int, float]], + runs: Sequence[Run], + ) -> MultiObjectiveDummyEnsemble: + """Select dummy ensembles given predictions of base models and targets. + + Parameters + ---------- + base_models_predictions: np.ndarray + shape = (n_base_models, n_data_points, n_targets) + n_targets is the number of classes in case of classification, + n_targets is 0 or 1 in case of regression + + Can be a list of 2d numpy arrays as well to prevent copying all + predictions into a single, large numpy array. + + X_data : list-like or sparse data + + true_targets : array of shape [n_targets] + + model_identifiers : identifier for each base model. + Can be used for practical text output of the ensemble. + + runs: Sequence[Run] + Additional information for each run executed by SMAC that was + considered by the ensemble builder. Not used. + + Returns + ------- + self + """ + if self.task_type not in TASK_TYPES: + raise ValueError("Unknown task type %s." % self.task_type) + + def is_pareto_efficient_simple(costs: np.ndarray) -> np.ndarray: + """ + Plot the Pareto Front in our 2d example. + + source from: https://stackoverflow.com/a/40239615 + Find the pareto-efficient points + + Parameters + ---------- + + costs: np.ndarray + + Returns + ------- + + np.ndarray + """ + + is_efficient = np.ones(costs.shape[0], dtype=bool) + for i, c in enumerate(costs): + if is_efficient[i]: + # Keep any point with a lower cost + is_efficient[is_efficient] = np.any(costs[is_efficient] < c, axis=1) + + # And keep self + is_efficient[i] = True + return is_efficient + + all_costs = np.empty((len(base_models_predictions), len(self.metrics))) + for i, base_model_prediction in enumerate(base_models_predictions): + losses = calculate_losses( + solution=true_targets, + prediction=base_model_prediction, + task_type=self.task_type, + metrics=self.metrics, + X_data=X_data, + ) + all_costs[i] = [losses[metric.name] for metric in self.metrics] + all_costs = np.array(all_costs) + sort_by_first_metric = np.argsort(all_costs[:, 0]) + efficient_points = is_pareto_efficient_simple(all_costs) + pareto_set = [] + + for argsort_idx in sort_by_first_metric: + if not efficient_points[argsort_idx]: + continue + ensemble = SingleModelEnsemble( + task_type=self.task_type, + metrics=self.metrics, + random_state=self.random_state, + backend=self.backend, + model_index=argsort_idx, + ) + ensemble.fit( + base_models_predictions=base_models_predictions, + true_targets=true_targets, + model_identifiers=model_identifiers, + runs=runs, + X_data=X_data, + ) + pareto_set.append(ensemble) + self.pareto_set_ = pareto_set + return self + + def predict( + self, base_models_predictions: Union[np.ndarray, List[np.ndarray]] + ) -> np.ndarray: + """Predict using the ensemble which is best for the 1st metric. + + Parameters + ---------- + base_models_predictions : np.ndarray + shape = (n_base_models, n_data_points, n_targets) + Same as in the fit method. + + Returns + ------- + np.ndarray + """ + return self.pareto_set_[0].predict(base_models_predictions) + + def __str__(self) -> str: + return "MultiObjectiveDummyEnsemble:\n" + ( + "\n".join([str(ensemble) for ensemble in self.pareto_set_]) + ) + + def get_models_with_weights( + self, models: Dict[Tuple[int, int, float], BasePipeline] + ) -> List[Tuple[float, BasePipeline]]: + """Return a list of (weight, model) pairs for the ensemble that is + best for the 1st metric. + + Parameters + ---------- + models : dict {identifier : model object} + The identifiers are the same as the one presented to the fit() + method. Models can be used for nice printing. + + Returns + ------- + List[Tuple[float, BasePipeline]] + """ + output = [] + for i, weight in enumerate(self.pareto_set_[0].weights_): + if weight > 0.0: + identifier = self.pareto_set_[0].identifiers_[i] + model = models[identifier] + output.append((weight, model)) + + output.sort(reverse=True, key=lambda t: t[0]) + + return output + + def get_identifiers_with_weights( + self, + ) -> List[Tuple[Tuple[int, int, float], float]]: + """Return a (identifier, weight)-pairs for all models that were passed to the + ensemble builder based on the ensemble that is best for the 1st metric. + + Parameters + ---------- + models : dict {identifier : model object} + The identifiers are the same as the one presented to the fit() + method. Models can be used for nice printing. + + Returns + ------- + List[Tuple[Tuple[int, int, float], float] + """ + return list(zip(self.pareto_set_[0].identifiers_, self.pareto_set_[0].weights_)) + + def get_selected_model_identifiers(self) -> List[Tuple[int, int, float]]: + """Return identifiers of models in the ensemble that is best for the 1st metric. + + This includes models which have a weight of zero! + + Returns + ------- + list + """ + output = [] + + for i, weight in enumerate(self.pareto_set_[0].weights_): + identifier = self.pareto_set_[0].identifiers_[i] + if weight > 0.0: + output.append(identifier) + + return output + + def get_validation_performance(self) -> float: + """Return validation performance of the ensemble that is best for the 1st metric. + + Return + ------ + float + """ + return self.pareto_set_[0].get_validation_performance() + + def get_pareto_set(self) -> Sequence[AbstractEnsemble]: + return self.pareto_set_ diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py index 5afd8c597c..427512d301 100644 --- a/autosklearn/estimators.py +++ b/autosklearn/estimators.py @@ -36,6 +36,9 @@ ) from autosklearn.ensembles.abstract_ensemble import AbstractEnsemble from autosklearn.ensembles.ensemble_selection import EnsembleSelection +from autosklearn.ensembles.multiobjective_dummy_ensemble import ( + MultiObjectiveDummyEnsemble, +) from autosklearn.metrics import Scorer from autosklearn.pipeline.base import BasePipeline from autosklearn.util.smac_wrap import SMACCallback @@ -48,7 +51,7 @@ def __init__( per_run_time_limit=None, initial_configurations_via_metalearning=25, ensemble_size: int | None = None, - ensemble_class: Type[AbstractEnsemble] | None = EnsembleSelection, + ensemble_class: Type[AbstractEnsemble] | Literal["default"] | None = "default", ensemble_kwargs: Dict[str, Any] | None = None, ensemble_nbest=50, max_models_on_disc=50, @@ -104,12 +107,16 @@ def __init__( this argument via ``ensemble_kwargs={"ensemble_size": int}`` if you want to change the ensemble size for ensemble selection. - ensemble_class : Type[AbstractEnsemble], optional (default=EnsembleSelection) + ensemble_class : Type[AbstractEnsemble] | "default", optional (default="default") Class implementing the post-hoc ensemble algorithm. Set to ``None`` to disable ensemble building or use ``SingleBest`` to obtain only use the single best model instead of an ensemble. + If set to "default" it will use :cls:`EnsembleSelection` for + single-objective problems and :cls:`MultiObjectiveDummyEnsemble` + for multi-objective problems. + ensemble_kwargs : Dict, optional Keyword arguments that are passed to the ensemble class upon initialization. @@ -396,6 +403,14 @@ def __init__( self.initial_configurations_via_metalearning = ( initial_configurations_via_metalearning ) + + # Need to resolve the ensemble class here so we can act on it below. + if ensemble_class == "default": + ensemble_class = ( + EnsembleSelection + if isinstance(metric, Scorer) or len(metric) == 1 + else MultiObjectiveDummyEnsemble + ) self.ensemble_class = ensemble_class # User specified `ensemble_size` explicitly, warn them about deprecation @@ -610,8 +625,8 @@ def fit_ensemble( ensemble_size: int | None = None, ensemble_kwargs: Optional[Dict[str, Any]] = None, ensemble_nbest: Optional[int] = None, - ensemble_class: Optional[AbstractEnsemble] = EnsembleSelection, - metrics: Scorer | Sequence[Scorer] | None = None, + ensemble_class: Type[AbstractEnsemble] | Literal["default"] | None = "default", + metric: Scorer | Sequence[Scorer] | None = None, ): """Fit an ensemble to models trained during an optimization process. @@ -655,19 +670,32 @@ def fit_ensemble( is independent of the ``ensemble_class`` argument and this pruning step is done prior to constructing an ensemble. - ensemble_class : Type[AbstractEnsemble], optional (default=EnsembleSelection) + ensemble_class : Type[AbstractEnsemble] | "default", optional (default="default") Class implementing the post-hoc ensemble algorithm. Set to ``None`` to disable ensemble building or use ``SingleBest`` to obtain only use the single best model instead of an ensemble. - metrics: Scorer | Sequence[Scorer] | None = None + If set to "default" it will use :cls:`EnsembleSelection` for + single-objective problems and :cls:`MultiObjectiveDummyEnsemble` + for multi-objective problems. + + metric: Scorer | Sequence[Scorer] | None = None A metric or list of metrics to score the ensemble with Returns ------- self - """ + """ # noqa: E501 + + # Need to resolve the ensemble class here so we can act on it below. + if ensemble_class == "default": + ensemble_class = ( + EnsembleSelection + if isinstance(metric, Scorer) or len(metric) == 1 + else MultiObjectiveDummyEnsemble + ) + # User specified `ensemble_size` explicitly, warn them about deprecation if ensemble_size is not None: # Keep consistent behaviour @@ -720,7 +748,7 @@ def fit_ensemble( ensemble_nbest=ensemble_nbest, ensemble_class=ensemble_class, ensemble_kwargs=ensemble_kwargs, - metrics=metrics, + metrics=metric, ) return self From c720f85ee8d98a2a2ac372395742619543487906 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 17 Jun 2022 17:42:11 +0200 Subject: [PATCH 02/16] Fix bug --- autosklearn/estimators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py index 427512d301..018ffdb59e 100644 --- a/autosklearn/estimators.py +++ b/autosklearn/estimators.py @@ -408,7 +408,7 @@ def __init__( if ensemble_class == "default": ensemble_class = ( EnsembleSelection - if isinstance(metric, Scorer) or len(metric) == 1 + if metric is None or isinstance(metric, Scorer) or len(metric) == 1 else MultiObjectiveDummyEnsemble ) self.ensemble_class = ensemble_class @@ -692,7 +692,7 @@ def fit_ensemble( if ensemble_class == "default": ensemble_class = ( EnsembleSelection - if isinstance(metric, Scorer) or len(metric) == 1 + if metric is None or isinstance(metric, Scorer) or len(metric) == 1 else MultiObjectiveDummyEnsemble ) From 7a8e6df1530f9a85c7255647b9b7e933d45629b9 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 17 Jun 2022 17:54:35 +0200 Subject: [PATCH 03/16] One bugfix and suggestions from Eddie --- autosklearn/automl.py | 2 +- autosklearn/ensembles/abstract_ensemble.py | 2 +- autosklearn/ensembles/ensemble_selection.py | 2 +- .../multiobjective_dummy_ensemble.py | 46 +++++++++---------- autosklearn/ensembles/singlebest_ensemble.py | 2 +- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/autosklearn/automl.py b/autosklearn/automl.py index 8726ab8c6d..d85ea8abb5 100644 --- a/autosklearn/automl.py +++ b/autosklearn/automl.py @@ -2148,7 +2148,7 @@ def show_models(self) -> dict[int, Any]: ensemble_dict = {} - if self._ensemble_class is not None: + if self._ensemble_class is None: warnings.warn( "No models in the ensemble. Kindly provide an ensemble class." ) diff --git a/autosklearn/ensembles/abstract_ensemble.py b/autosklearn/ensembles/abstract_ensemble.py index 5ee8e35d16..9ec0fb7c3f 100644 --- a/autosklearn/ensembles/abstract_ensemble.py +++ b/autosklearn/ensembles/abstract_ensemble.py @@ -27,7 +27,7 @@ def __init__( def fit( self, base_models_predictions: np.ndarray | List[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES, + X_data: SUPPORTED_FEAT_TYPES | None, true_targets: np.ndarray, model_identifiers: List[Tuple[int, int, float]], runs: Sequence[Run], diff --git a/autosklearn/ensembles/ensemble_selection.py b/autosklearn/ensembles/ensemble_selection.py index 310f2eb6a4..0f151bd003 100644 --- a/autosklearn/ensembles/ensemble_selection.py +++ b/autosklearn/ensembles/ensemble_selection.py @@ -105,7 +105,7 @@ def __getstate__(self) -> Dict[str, Any]: def fit( self, base_models_predictions: List[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES, + X_data: SUPPORTED_FEAT_TYPES | None, true_targets: np.ndarray, model_identifiers: List[Tuple[int, int, float]], runs: Sequence[Run], diff --git a/autosklearn/ensembles/multiobjective_dummy_ensemble.py b/autosklearn/ensembles/multiobjective_dummy_ensemble.py index cbb8d06609..68f3537d22 100644 --- a/autosklearn/ensembles/multiobjective_dummy_ensemble.py +++ b/autosklearn/ensembles/multiobjective_dummy_ensemble.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional, Sequence, Tuple, Union +from typing import Any, Sequence import warnings @@ -60,10 +60,10 @@ def __init__( def fit( self, - base_models_predictions: np.ndarray | List[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES, + base_models_predictions: np.ndarray | list[np.ndarray], + X_data: SUPPORTED_FEAT_TYPES | None, true_targets: np.ndarray, - model_identifiers: List[Tuple[int, int, float]], + model_identifiers: list[tuple[int, int, float]], runs: Sequence[Run], ) -> SingleModelEnsemble: """Dummy implementation of the ``fit`` method. @@ -108,7 +108,7 @@ def fit( self.best_model_score_ = loss[self.metrics[0].name] return self - def predict(self, predictions: Union[np.ndarray, List[np.ndarray]]) -> np.ndarray: + def predict(self, predictions: np.ndarray | list[np.ndarray]) -> np.ndarray: """Select the predictions of the selected model. Parameters @@ -137,8 +137,8 @@ def __str__(self) -> str: ) def get_models_with_weights( - self, models: Dict[Tuple[int, int, float], BasePipeline] - ) -> List[Tuple[float, BasePipeline]]: + self, models: dict[tuple[int, int, float], BasePipeline] + ) -> list[tuple[float, BasePipeline]]: """Return a list of (weight, model) pairs for the model selected by this ensemble. Parameters @@ -149,7 +149,7 @@ def get_models_with_weights( Returns ------- - List[Tuple[float, BasePipeline]] + list[tuple[float, BasePipeline]] """ output = [] for i, weight in enumerate(self.weights_): @@ -164,7 +164,7 @@ def get_models_with_weights( def get_identifiers_with_weights( self, - ) -> List[Tuple[Tuple[int, int, float], float]]: + ) -> list[tuple[tuple[int, int, float], float]]: """Return a (identifier, weight)-pairs for the model selected by this ensemble. Parameters @@ -175,11 +175,11 @@ def get_identifiers_with_weights( Returns ------- - List[Tuple[Tuple[int, int, float], float] + list[tuple[tuple[int, int, float], float] """ return list(zip(self.identifiers_, self.weights_)) - def get_selected_model_identifiers(self) -> List[Tuple[int, int, float]]: + def get_selected_model_identifiers(self) -> list[tuple[int, int, float]]: """Return identifier of models in the ensemble. This includes models which have a weight of zero! @@ -214,7 +214,7 @@ def __init__( self, task_type: int, metrics: Sequence[Scorer] | Scorer, - random_state: Optional[Union[int, np.random.RandomState]], + random_state: int | np.random.RandomState | None, backend: Backend, ) -> None: """A dummy implementation of a multi-objective ensemble. @@ -248,7 +248,7 @@ def __init__( self.random_state = random_state self.backend = backend - def __getstate__(self) -> Dict[str, Any]: + def __getstate__(self) -> dict[str, Any]: # Cannot serialize a metric if # it is user defined. # That is, if doing pickle dump @@ -260,10 +260,10 @@ def __getstate__(self) -> Dict[str, Any]: def fit( self, - base_models_predictions: List[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES, + base_models_predictions: list[np.ndarray], + X_data: SUPPORTED_FEAT_TYPES | None, true_targets: np.ndarray, - model_identifiers: List[Tuple[int, int, float]], + model_identifiers: list[tuple[int, int, float]], runs: Sequence[Run], ) -> MultiObjectiveDummyEnsemble: """Select dummy ensembles given predictions of base models and targets. @@ -361,7 +361,7 @@ def is_pareto_efficient_simple(costs: np.ndarray) -> np.ndarray: return self def predict( - self, base_models_predictions: Union[np.ndarray, List[np.ndarray]] + self, base_models_predictions: np.ndarray | list[np.ndarray] ) -> np.ndarray: """Predict using the ensemble which is best for the 1st metric. @@ -383,8 +383,8 @@ def __str__(self) -> str: ) def get_models_with_weights( - self, models: Dict[Tuple[int, int, float], BasePipeline] - ) -> List[Tuple[float, BasePipeline]]: + self, models: dict[tuple[int, int, float], BasePipeline] + ) -> list[tuple[float, BasePipeline]]: """Return a list of (weight, model) pairs for the ensemble that is best for the 1st metric. @@ -396,7 +396,7 @@ def get_models_with_weights( Returns ------- - List[Tuple[float, BasePipeline]] + list[tuple[float, BasePipeline]] """ output = [] for i, weight in enumerate(self.pareto_set_[0].weights_): @@ -411,7 +411,7 @@ def get_models_with_weights( def get_identifiers_with_weights( self, - ) -> List[Tuple[Tuple[int, int, float], float]]: + ) -> list[tuple[tuple[int, int, float], float]]: """Return a (identifier, weight)-pairs for all models that were passed to the ensemble builder based on the ensemble that is best for the 1st metric. @@ -423,11 +423,11 @@ def get_identifiers_with_weights( Returns ------- - List[Tuple[Tuple[int, int, float], float] + list[tuple[tuple[int, int, float], float] """ return list(zip(self.pareto_set_[0].identifiers_, self.pareto_set_[0].weights_)) - def get_selected_model_identifiers(self) -> List[Tuple[int, int, float]]: + def get_selected_model_identifiers(self) -> list[tuple[int, int, float]]: """Return identifiers of models in the ensemble that is best for the 1st metric. This includes models which have a weight of zero! diff --git a/autosklearn/ensembles/singlebest_ensemble.py b/autosklearn/ensembles/singlebest_ensemble.py index 497ee4d495..bb021281cb 100644 --- a/autosklearn/ensembles/singlebest_ensemble.py +++ b/autosklearn/ensembles/singlebest_ensemble.py @@ -53,7 +53,7 @@ def __init__( def fit( self, base_models_predictions: np.ndarray | List[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES, + X_data: SUPPORTED_FEAT_TYPES | None, true_targets: np.ndarray, model_identifiers: List[Tuple[int, int, float]], runs: Sequence[Run], From 59e4f60f45740f1c74af02315b5a914f88879606 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 17 Jun 2022 18:57:35 +0200 Subject: [PATCH 04/16] Move single best common code into parent class --- autosklearn/automl.py | 4 +- autosklearn/ensembles/__init__.py | 8 +- .../multiobjective_dummy_ensemble.py | 192 +------- autosklearn/ensembles/singlebest_ensemble.py | 433 +++++++++++++++--- 4 files changed, 370 insertions(+), 267 deletions(-) diff --git a/autosklearn/automl.py b/autosklearn/automl.py index d85ea8abb5..4b1224c499 100644 --- a/autosklearn/automl.py +++ b/autosklearn/automl.py @@ -79,7 +79,7 @@ AbstractMultiObjectiveEnsemble, ) from autosklearn.ensembles.ensemble_selection import EnsembleSelection -from autosklearn.ensembles.singlebest_ensemble import SingleBest +from autosklearn.ensembles.singlebest_ensemble import SingleBestFromRunhistory from autosklearn.evaluation import ExecuteTaFuncWithQueue, get_cost_of_crash from autosklearn.evaluation.abstract_evaluator import _fit_and_suppress_warnings from autosklearn.evaluation.train_evaluator import TrainEvaluator, _fit_with_budget @@ -1666,7 +1666,7 @@ def _load_best_individual_model(self): return None # SingleBest contains the best model found by AutoML - ensemble = SingleBest( + ensemble = SingleBestFromRunhistory( metrics=self._metrics, task_type=self._task, seed=self._seed, diff --git a/autosklearn/ensembles/__init__.py b/autosklearn/ensembles/__init__.py index 4e1af926cf..ba262cdda6 100644 --- a/autosklearn/ensembles/__init__.py +++ b/autosklearn/ensembles/__init__.py @@ -1,10 +1,16 @@ from .abstract_ensemble import AbstractEnsemble, AbstractMultiObjectiveEnsemble from .ensemble_selection import EnsembleSelection -from .singlebest_ensemble import SingleBest +from .singlebest_ensemble import ( + SingleBest, + SingleBestFromRunhistory, + SingleModelEnsemble, +) __all__ = [ "AbstractEnsemble", "AbstractMultiObjectiveEnsemble", "EnsembleSelection", + "SingleBestFromRunhistory", "SingleBest", + "SingleModelEnsemble", ] diff --git a/autosklearn/ensembles/multiobjective_dummy_ensemble.py b/autosklearn/ensembles/multiobjective_dummy_ensemble.py index 68f3537d22..51ffba63f1 100644 --- a/autosklearn/ensembles/multiobjective_dummy_ensemble.py +++ b/autosklearn/ensembles/multiobjective_dummy_ensemble.py @@ -14,201 +14,11 @@ AbstractEnsemble, AbstractMultiObjectiveEnsemble, ) +from autosklearn.ensembles.singlebest_ensemble import SingleModelEnsemble from autosklearn.metrics import Scorer, calculate_losses from autosklearn.pipeline.base import BasePipeline -class SingleModelEnsemble(AbstractEnsemble): - """Ensemble consisting of a single model. - - This class is used my the :cls:`MultiObjectiveDummyEnsemble` to represent ensembles - consisting of a single model, and this class should not be used on its own. - - Parameters - ---------- - task_type: int - An identifier indicating which task is being performed. - - metrics: Sequence[Scorer] | Scorer - The metrics used to evaluate the models. - - random_state: Optional[int | RandomState] = None - Not used. - - backend : Backend - Gives access to the backend of Auto-sklearn. Not used. - - model_index : int - Index of the model that constitutes the ensemble. This index will - be used to select the correct predictions that will be passed during - ``fit`` and ``predict``. - """ - - def __init__( - self, - task_type: int, - metrics: Sequence[Scorer] | Scorer, - random_state: int | np.random.RandomState | None, - backend: Backend, - model_index: int, - ): - # Add some default values -- at least 1 model in ensemble is assumed - self.indices_ = [model_index] - self.weights_ = [1.0] - self.task_type = task_type - self.metrics = metrics if isinstance(metrics, Sequence) else [metrics] - - def fit( - self, - base_models_predictions: np.ndarray | list[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES | None, - true_targets: np.ndarray, - model_identifiers: list[tuple[int, int, float]], - runs: Sequence[Run], - ) -> SingleModelEnsemble: - """Dummy implementation of the ``fit`` method. - - Actualy work of passing the model index is done in the constructor. This - method only stores the identifier of the selected model and computes it's - validation loss. - - Parameters - ---------- - base_models_predictions: np.ndarray - shape = (n_base_models, n_data_points, n_targets) - n_targets is the number of classes in case of classification, - n_targets is 0 or 1 in case of regression - - Can be a list of 2d numpy arrays as well to prevent copying all - predictions into a single, large numpy array. - - X_data : list-like or sparse data - - true_targets : array of shape [n_targets] - - model_identifiers : identifier for each base model. - Can be used for practical text output of the ensemble. - - runs: Sequence[Run] - Additional information for each run executed by SMAC that was - considered by the ensemble builder. Not used. - - Returns - ------- - self - """ - self.identifiers_ = [model_identifiers[self.indices_[0]]] - loss = calculate_losses( - solution=true_targets, - prediction=base_models_predictions[self.indices_[0]], - task_type=self.task_type, - metrics=self.metrics, - X_data=X_data, - ) - self.best_model_score_ = loss[self.metrics[0].name] - return self - - def predict(self, predictions: np.ndarray | list[np.ndarray]) -> np.ndarray: - """Select the predictions of the selected model. - - Parameters - ---------- - base_models_predictions : np.ndarray - shape = (n_base_models, n_data_points, n_targets) - Same as in the fit method. - - Returns - ------- - np.ndarray - """ - return predictions[0] - - def __str__(self) -> str: - return "Single Model:\n\tMembers: %s" "\n\tWeights: %s\n\tIdentifiers: %s" % ( - self.indices_, - self.weights_, - " ".join( - [ - str(identifier) - for idx, identifier in enumerate(self.identifiers_) - if self.weights_[idx] > 0 - ] - ), - ) - - def get_models_with_weights( - self, models: dict[tuple[int, int, float], BasePipeline] - ) -> list[tuple[float, BasePipeline]]: - """Return a list of (weight, model) pairs for the model selected by this ensemble. - - Parameters - ---------- - models : dict {identifier : model object} - The identifiers are the same as the one presented to the fit() - method. Models can be used for nice printing. - - Returns - ------- - list[tuple[float, BasePipeline]] - """ - output = [] - for i, weight in enumerate(self.weights_): - if weight > 0.0: - identifier = self.identifiers_[i] - model = models[identifier] - output.append((weight, model)) - - output.sort(reverse=True, key=lambda t: t[0]) - - return output - - def get_identifiers_with_weights( - self, - ) -> list[tuple[tuple[int, int, float], float]]: - """Return a (identifier, weight)-pairs for the model selected by this ensemble. - - Parameters - ---------- - models : dict {identifier : model object} - The identifiers are the same as the one presented to the fit() - method. Models can be used for nice printing. - - Returns - ------- - list[tuple[tuple[int, int, float], float] - """ - return list(zip(self.identifiers_, self.weights_)) - - def get_selected_model_identifiers(self) -> list[tuple[int, int, float]]: - """Return identifier of models in the ensemble. - - This includes models which have a weight of zero! - - Returns - ------- - list - """ - output = [] - - for i, weight in enumerate(self.weights_): - identifier = self.identifiers_[i] - if weight > 0.0: - output.append(identifier) - - return output - - def get_validation_performance(self) -> float: - """Return validation performance of ensemble. - - In case of multi-objective problem, only the first metric will be returned. - - Return - ------ - float - """ - return self.best_model_score_ - - class MultiObjectiveDummyEnsemble(AbstractMultiObjectiveEnsemble): def __init__( self, diff --git a/autosklearn/ensembles/singlebest_ensemble.py b/autosklearn/ensembles/singlebest_ensemble.py index bb021281cb..d2473db8df 100644 --- a/autosklearn/ensembles/singlebest_ensemble.py +++ b/autosklearn/ensembles/singlebest_ensemble.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Dict, List, Sequence, Tuple, Union +from typing import Sequence import os @@ -11,18 +11,26 @@ from autosklearn.data.validation import SUPPORTED_FEAT_TYPES from autosklearn.ensemble_building.run import Run from autosklearn.ensembles.abstract_ensemble import AbstractEnsemble -from autosklearn.metrics import Scorer +from autosklearn.metrics import Scorer, calculate_losses from autosklearn.pipeline.base import BasePipeline -class SingleBest(AbstractEnsemble): - """ - In the case of a crash, this class searches - for the best individual model. +class AbstractSingleModelEnsemble(AbstractEnsemble): + """Ensemble consisting of a single model. - Such model is returned as an ensemble of a single - object, to comply with the expected interface of an - AbstractEnsemble. + Parameters + ---------- + task_type: int + An identifier indicating which task is being performed. + + metrics: Sequence[Scorer] | Scorer + The metrics used to evaluate the models. + + random_state: Optional[int | RandomState] = None + Not used. + + backend : Backend + Gives access to the backend of Auto-sklearn. Not used. """ def __init__( @@ -31,9 +39,8 @@ def __init__( metrics: Sequence[Scorer] | Scorer, random_state: int | np.random.RandomState | None, backend: Backend, - run_history: RunHistory, - seed: int, ): + self.weights_ = [1.0] self.task_type = task_type if isinstance(metrics, Sequence): self.metrics = metrics @@ -41,26 +48,358 @@ def __init__( self.metrics = [metrics] else: raise TypeError(type(metrics)) - self.seed = seed + self.random_state = random_state self.backend = backend - # Add some default values -- at least 1 model in ensemble is assumed - self.indices_ = [0] - self.weights_ = [1.0] - self.run_history = run_history - self.identifiers_ = self.get_identifiers_from_run_history() + def fit( + self, + base_models_predictions: np.ndarray | list[np.ndarray], + X_data: SUPPORTED_FEAT_TYPES | None, + true_targets: np.ndarray, + model_identifiers: list[tuple[int, int, float]], + runs: Sequence[Run], + ) -> AbstractSingleModelEnsemble: + """Fit the ensemble + + Parameters + ---------- + base_models_predictions: np.ndarray + shape = (n_base_models, n_data_points, n_targets) + n_targets is the number of classes in case of classification, + n_targets is 0 or 1 in case of regression + + Can be a list of 2d numpy arrays as well to prevent copying all + predictions into a single, large numpy array. + + X_data : list-like or sparse data + + true_targets : array of shape [n_targets] + + model_identifiers : identifier for each base model. + Can be used for practical text output of the ensemble. + + runs: Sequence[Run] + Additional information for each run executed by SMAC that was + considered by the ensemble builder. + + Returns + ------- + self + """ + return self + + def predict(self, predictions: np.ndarray | list[np.ndarray]) -> np.ndarray: + """Select the predictions of the selected model. + + Parameters + ---------- + base_models_predictions : np.ndarray + shape = (n_base_models, n_data_points, n_targets) + Same as in the fit method. + + Returns + ------- + np.ndarray + """ + return predictions[0] + + def __str__(self) -> str: + return "%s:\n\tMembers: %s" "\n\tWeights: %s\n\tIdentifiers: %s" % ( + self.__class__.__name__, + self.indices_, # type: ignore [attr-defined] + self.weights_, + " ".join( + [ + str(identifier) + for idx, identifier in enumerate(self.identifiers_) # type: ignore [attr-defined] # noqa: E501 + if self.weights_[idx] > 0 + ] + ), + ) + + def get_models_with_weights( + self, models: dict[tuple[int, int, float], BasePipeline] + ) -> list[tuple[float, BasePipeline]]: + """Return a list of (weight, model) pairs for the model selected by this ensemble. + + Parameters + ---------- + models : dict {identifier : model object} + The identifiers are the same as the one presented to the fit() + method. Models can be used for nice printing. + + Returns + ------- + list[tuple[float, BasePipeline]] + """ + output = [] + for i, weight in enumerate(self.weights_): + if weight > 0.0: + identifier = self.identifiers_[i] # type: ignore [attr-defined] + model = models[identifier] + output.append((weight, model)) + + output.sort(reverse=True, key=lambda t: t[0]) + + return output + + def get_identifiers_with_weights( + self, + ) -> list[tuple[tuple[int, int, float], float]]: + """Return a (identifier, weight)-pairs for the model selected by this ensemble. + + Parameters + ---------- + models : dict {identifier : model object} + The identifiers are the same as the one presented to the fit() + method. Models can be used for nice printing. + + Returns + ------- + list[tuple[tuple[int, int, float], float] + """ + return list(zip(self.identifiers_, self.weights_)) # type: ignore [attr-defined] # noqa: E501 + + def get_selected_model_identifiers(self) -> list[tuple[int, int, float]]: + """Return identifier of models in the ensemble. + + This includes models which have a weight of zero! + + Returns + ------- + list + """ + output = [] + + for i, weight in enumerate(self.weights_): + identifier = self.identifiers_[i] # type: ignore [attr-defined] + if weight > 0.0: + output.append(identifier) + + return output + + def get_validation_performance(self) -> float: + """Return validation performance of ensemble. + + In case of multi-objective problem, only the first metric will be returned. + + Return + ------ + float + """ + return self.best_model_score_ # type: ignore [attr-defined] + + +class SingleModelEnsemble(AbstractSingleModelEnsemble): + """Ensemble consisting of a single model. + + This class is used my the :cls:`MultiObjectiveDummyEnsemble` to represent ensembles + consisting of a single model, and this class should not be used on its own. + + Parameters + ---------- + task_type: int + An identifier indicating which task is being performed. + + metrics: Sequence[Scorer] | Scorer + The metrics used to evaluate the models. + + random_state: Optional[int | RandomState] = None + Not used. + + backend : Backend + Gives access to the backend of Auto-sklearn. Not used. + + model_index : int + Index of the model that constitutes the ensemble. This index will + be used to select the correct predictions that will be passed during + ``fit`` and ``predict``. + """ + + def __init__( + self, + task_type: int, + metrics: Sequence[Scorer] | Scorer, + random_state: int | np.random.RandomState | None, + backend: Backend, + model_index: int, + ): + super().__init__( + task_type=task_type, + metrics=metrics, + random_state=random_state, + backend=backend, + ) + self.indices_ = [model_index] + + def fit( + self, + base_models_predictions: np.ndarray | list[np.ndarray], + X_data: SUPPORTED_FEAT_TYPES | None, + true_targets: np.ndarray, + model_identifiers: list[tuple[int, int, float]], + runs: Sequence[Run], + ) -> SingleModelEnsemble: + """Dummy implementation of the ``fit`` method. + + Actualy work of passing the model index is done in the constructor. This + method only stores the identifier of the selected model and computes it's + validation loss. + + Parameters + ---------- + base_models_predictions: np.ndarray + shape = (n_base_models, n_data_points, n_targets) + n_targets is the number of classes in case of classification, + n_targets is 0 or 1 in case of regression + + Can be a list of 2d numpy arrays as well to prevent copying all + predictions into a single, large numpy array. + + X_data : list-like or sparse data + + true_targets : array of shape [n_targets] + + model_identifiers : identifier for each base model. + Can be used for practical text output of the ensemble. + + runs: Sequence[Run] + Additional information for each run executed by SMAC that was + considered by the ensemble builder. Not used. + + Returns + ------- + self + """ + self.identifiers_ = [model_identifiers[self.indices_[0]]] + loss = calculate_losses( + solution=true_targets, + prediction=base_models_predictions[self.indices_[0]], + task_type=self.task_type, + metrics=self.metrics, + X_data=X_data, + ) + self.best_model_score_ = loss[self.metrics[0].name] + return self + + +class SingleBest(AbstractSingleModelEnsemble): + """Ensemble consisting of the single best model. + + Parameters + ---------- + task_type: int + An identifier indicating which task is being performed. + + metrics: Sequence[Scorer] | Scorer + The metrics used to evaluate the models. + + random_state: Optional[int | RandomState] = None + Not used. + + backend : Backend + Gives access to the backend of Auto-sklearn. Not used. + """ + + def __init__( + self, + task_type: int, + metrics: Sequence[Scorer] | Scorer, + random_state: int | np.random.RandomState | None, + backend: Backend, + ): + super().__init__( + task_type=task_type, + metrics=metrics, + random_state=random_state, + backend=backend, + ) def fit( self, - base_models_predictions: np.ndarray | List[np.ndarray], + base_models_predictions: np.ndarray | list[np.ndarray], X_data: SUPPORTED_FEAT_TYPES | None, true_targets: np.ndarray, - model_identifiers: List[Tuple[int, int, float]], + model_identifiers: list[tuple[int, int, float]], runs: Sequence[Run], - ) -> "AbstractEnsemble": + ) -> SingleBest: + """Select the single best model. + + Parameters + ---------- + base_models_predictions: np.ndarray + shape = (n_base_models, n_data_points, n_targets) + n_targets is the number of classes in case of classification, + n_targets is 0 or 1 in case of regression + + Can be a list of 2d numpy arrays as well to prevent copying all + predictions into a single, large numpy array. + + X_data : list-like or sparse data + + true_targets : array of shape [n_targets] + + model_identifiers : identifier for each base model. + Can be used for practical text output of the ensemble. + + runs: Sequence[Run] + Additional information for each run executed by SMAC that was + considered by the ensemble builder. Not used. + + Returns + ------- + self + """ + losses = [ + calculate_losses( + solution=true_targets, + prediction=base_model_prediction, + task_type=self.task_type, + metrics=self.metrics, + X_data=X_data, + )[self.metrics[0].name] + for base_model_prediction in base_models_predictions + ] + argmin = np.argmin(losses) + self.indices_ = [argmin] + self.identifiers_ = [model_identifiers[argmin]] + self.best_model_score_ = losses[argmin] return self - def get_identifiers_from_run_history(self) -> List[Tuple[int, int, float]]: + +class SingleBestFromRunhistory(AbstractSingleModelEnsemble): + """ + In the case of a crash, this class searches + for the best individual model. + + Such model is returned as an ensemble of a single + object, to comply with the expected interface of an + AbstractEnsemble. + """ + + def __init__( + self, + task_type: int, + metrics: Sequence[Scorer] | Scorer, + random_state: int | np.random.RandomState | None, + backend: Backend, + run_history: RunHistory, + seed: int, + ): + super().__init__( + task_type=task_type, + metrics=metrics, + random_state=random_state, + backend=backend, + ) + + self.seed = seed + self.indices_ = [0] + self.weights_ = [1.0] + self.run_history = run_history + self.identifiers_ = self.get_identifiers_from_run_history() + + def get_identifiers_from_run_history(self) -> list[tuple[int, int, float]]: """Parses the run history, to identify the best performing model Populates the identifiers attribute, which is used by the backend to access @@ -115,55 +454,3 @@ def get_identifiers_from_run_history(self) -> List[Tuple[int, int, float]]: self.best_model_score_ = best_model_score return best_model_identifier - - def predict(self, predictions: Union[np.ndarray, List[np.ndarray]]) -> np.ndarray: - return predictions[0] - - def __str__(self) -> str: - return ( - "Single Model Selection:\n\tMembers: %s" - "\n\tWeights: %s\n\tIdentifiers: %s" - % ( - self.indices_, - self.weights_, - " ".join( - [ - str(identifier) - for idx, identifier in enumerate(self.identifiers_) - if self.weights_[idx] > 0 - ] - ), - ) - ) - - def get_models_with_weights( - self, models: Dict[Tuple[int, int, float], BasePipeline] - ) -> List[Tuple[float, BasePipeline]]: - output = [] - for i, weight in enumerate(self.weights_): - if weight > 0.0: - identifier = self.identifiers_[i] - model = models[identifier] - output.append((weight, model)) - - output.sort(reverse=True, key=lambda t: t[0]) - - return output - - def get_identifiers_with_weights( - self, - ) -> List[Tuple[Tuple[int, int, float], float]]: - return list(zip(self.identifiers_, self.weights_)) - - def get_selected_model_identifiers(self) -> List[Tuple[int, int, float]]: - output = [] - - for i, weight in enumerate(self.weights_): - identifier = self.identifiers_[i] - if weight > 0.0: - output.append(identifier) - - return output - - def get_validation_performance(self) -> float: - return self.best_model_score_ From 52808fc489e59789bef78d9da462dd41b3a5c4fc Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Sat, 18 Jun 2022 10:45:08 +0200 Subject: [PATCH 05/16] fix docstring --- autosklearn/ensembles/__init__.py | 2 ++ autosklearn/ensembles/abstract_ensemble.py | 4 ++-- .../ensembles/multiobjective_dummy_ensemble.py | 6 ++---- autosklearn/estimators.py | 12 ++++++------ doc/api.rst | 3 +++ 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/autosklearn/ensembles/__init__.py b/autosklearn/ensembles/__init__.py index ba262cdda6..40bb380bdb 100644 --- a/autosklearn/ensembles/__init__.py +++ b/autosklearn/ensembles/__init__.py @@ -1,5 +1,6 @@ from .abstract_ensemble import AbstractEnsemble, AbstractMultiObjectiveEnsemble from .ensemble_selection import EnsembleSelection +from .multiobjective_dummy_ensemble import MultiObjectiveDummyEnsemble from .singlebest_ensemble import ( SingleBest, SingleBestFromRunhistory, @@ -13,4 +14,5 @@ "SingleBestFromRunhistory", "SingleBest", "SingleModelEnsemble", + "MultiObjectiveDummyEnsemble", ] diff --git a/autosklearn/ensembles/abstract_ensemble.py b/autosklearn/ensembles/abstract_ensemble.py index 9ec0fb7c3f..9608c15e62 100644 --- a/autosklearn/ensembles/abstract_ensemble.py +++ b/autosklearn/ensembles/abstract_ensemble.py @@ -133,8 +133,8 @@ def get_selected_model_identifiers(self) -> List[Tuple[int, int, float]]: def get_validation_performance(self) -> float: """Return validation performance of ensemble. - Return - ------ + Returns + ------- float """ diff --git a/autosklearn/ensembles/multiobjective_dummy_ensemble.py b/autosklearn/ensembles/multiobjective_dummy_ensemble.py index 51ffba63f1..dbe5ffb707 100644 --- a/autosklearn/ensembles/multiobjective_dummy_ensemble.py +++ b/autosklearn/ensembles/multiobjective_dummy_ensemble.py @@ -115,12 +115,10 @@ def is_pareto_efficient_simple(costs: np.ndarray) -> np.ndarray: Parameters ---------- - costs: np.ndarray Returns ------- - np.ndarray """ @@ -258,8 +256,8 @@ def get_selected_model_identifiers(self) -> list[tuple[int, int, float]]: def get_validation_performance(self) -> float: """Return validation performance of the ensemble that is best for the 1st metric. - Return - ------ + Returns + ------- float """ return self.pareto_set_[0].get_validation_performance() diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py index 018ffdb59e..c39f6a6148 100644 --- a/autosklearn/estimators.py +++ b/autosklearn/estimators.py @@ -109,12 +109,12 @@ def __init__( ensemble_class : Type[AbstractEnsemble] | "default", optional (default="default") Class implementing the post-hoc ensemble algorithm. Set to - ``None`` to disable ensemble building or use ``SingleBest`` + ``None`` to disable ensemble building or use :class:`SingleBest` to obtain only use the single best model instead of an ensemble. - If set to "default" it will use :cls:`EnsembleSelection` for - single-objective problems and :cls:`MultiObjectiveDummyEnsemble` + If set to "default" it will use :class:`EnsembleSelection` for + single-objective problems and :class:`MultiObjectiveDummyEnsemble` for multi-objective problems. ensemble_kwargs : Dict, optional @@ -672,12 +672,12 @@ def fit_ensemble( ensemble_class : Type[AbstractEnsemble] | "default", optional (default="default") Class implementing the post-hoc ensemble algorithm. Set to - ``None`` to disable ensemble building or use ``SingleBest`` + ``None`` to disable ensemble building or use class:`SingleBest` to obtain only use the single best model instead of an ensemble. - If set to "default" it will use :cls:`EnsembleSelection` for - single-objective problems and :cls:`MultiObjectiveDummyEnsemble` + If set to "default" it will use :class:`EnsembleSelection` for + single-objective problems and :class:`MultiObjectiveDummyEnsemble` for multi-objective problems. metric: Scorer | Sequence[Scorer] | None = None diff --git a/doc/api.rst b/doc/api.rst index 789d2d24e3..3befb03697 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -128,3 +128,6 @@ Ensembles .. autoclass:: autosklearn.ensembles.SingleBest :members: + +.. autoclass:: autosklearn.ensembles.MultiObjectiveDummyEnsemble + :members: From 0a4dc14ab7cc1f884bc391f101cde91cc2415c86 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 8 Jul 2022 16:29:18 +0200 Subject: [PATCH 06/16] Add tests + improve docs + simplify code --- .../multiobjective_dummy_ensemble.py | 26 +--- autosklearn/ensembles/singlebest_ensemble.py | 39 ++---- autosklearn/estimators.py | 23 ++- doc/api.rst | 21 ++- .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../.auto-sklearn/runs/0_1_0.0/0.1.0.0.model | 0 .../.auto-sklearn/runs/0_2_0.0/0.2.0.0.model | 0 .../runs/0_2_0.0/predictions_test_0_2_0.0.np | Bin 160 -> 0 bytes .../runs/0_3_100.0/0.3.0.0.model | 0 .../runs/0_3_100.0/0.3.100.0.model | 0 .../test_multiobjective_dummy_ensemble.py | 73 ++++++++++ .../test_singlebest_ensemble.py | 132 ++++++++++++++++++ test/test_estimators/test_estimators.py | 14 ++ 207 files changed, 262 insertions(+), 66 deletions(-) delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model delete mode 100644 test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model create mode 100644 test/test_ensemble_builder/test_multiobjective_dummy_ensemble.py create mode 100644 test/test_ensemble_builder/test_singlebest_ensemble.py diff --git a/autosklearn/ensembles/multiobjective_dummy_ensemble.py b/autosklearn/ensembles/multiobjective_dummy_ensemble.py index dbe5ffb707..1b380010aa 100644 --- a/autosklearn/ensembles/multiobjective_dummy_ensemble.py +++ b/autosklearn/ensembles/multiobjective_dummy_ensemble.py @@ -186,9 +186,7 @@ def predict( return self.pareto_set_[0].predict(base_models_predictions) def __str__(self) -> str: - return "MultiObjectiveDummyEnsemble:\n" + ( - "\n".join([str(ensemble) for ensemble in self.pareto_set_]) - ) + return "MultiObjectiveDummyEnsemble: %d models" % len(self.pareto_set_) def get_models_with_weights( self, models: dict[tuple[int, int, float], BasePipeline] @@ -206,16 +204,7 @@ def get_models_with_weights( ------- list[tuple[float, BasePipeline]] """ - output = [] - for i, weight in enumerate(self.pareto_set_[0].weights_): - if weight > 0.0: - identifier = self.pareto_set_[0].identifiers_[i] - model = models[identifier] - output.append((weight, model)) - - output.sort(reverse=True, key=lambda t: t[0]) - - return output + return self.pareto_set_[0].get_models_with_weights(models) def get_identifiers_with_weights( self, @@ -233,7 +222,7 @@ def get_identifiers_with_weights( ------- list[tuple[tuple[int, int, float], float] """ - return list(zip(self.pareto_set_[0].identifiers_, self.pareto_set_[0].weights_)) + return self.pareto_set_[0].get_identifiers_with_weights() def get_selected_model_identifiers(self) -> list[tuple[int, int, float]]: """Return identifiers of models in the ensemble that is best for the 1st metric. @@ -244,14 +233,7 @@ def get_selected_model_identifiers(self) -> list[tuple[int, int, float]]: ------- list """ - output = [] - - for i, weight in enumerate(self.pareto_set_[0].weights_): - identifier = self.pareto_set_[0].identifiers_[i] - if weight > 0.0: - output.append(identifier) - - return output + return self.pareto_set_[0].get_selected_model_identifiers() def get_validation_performance(self) -> float: """Return validation performance of the ensemble that is best for the 1st metric. diff --git a/autosklearn/ensembles/singlebest_ensemble.py b/autosklearn/ensembles/singlebest_ensemble.py index d2473db8df..7a567ae522 100644 --- a/autosklearn/ensembles/singlebest_ensemble.py +++ b/autosklearn/ensembles/singlebest_ensemble.py @@ -104,17 +104,11 @@ def predict(self, predictions: np.ndarray | list[np.ndarray]) -> np.ndarray: return predictions[0] def __str__(self) -> str: - return "%s:\n\tMembers: %s" "\n\tWeights: %s\n\tIdentifiers: %s" % ( + return "%s:\n\tMembers: %s" "\n\tWeights: %s\n\tIdentifiers: [%s]" % ( self.__class__.__name__, self.indices_, # type: ignore [attr-defined] self.weights_, - " ".join( - [ - str(identifier) - for idx, identifier in enumerate(self.identifiers_) # type: ignore [attr-defined] # noqa: E501 - if self.weights_[idx] > 0 - ] - ), + self.identifiers_[0], # type: ignore [attr-defined] ) def get_models_with_weights( @@ -132,16 +126,7 @@ def get_models_with_weights( ------- list[tuple[float, BasePipeline]] """ - output = [] - for i, weight in enumerate(self.weights_): - if weight > 0.0: - identifier = self.identifiers_[i] # type: ignore [attr-defined] - model = models[identifier] - output.append((weight, model)) - - output.sort(reverse=True, key=lambda t: t[0]) - - return output + return [(self.weights_[0], models[self.identifiers_[0]])] # type: ignore [attr-defined] # noqa: E501 def get_identifiers_with_weights( self, @@ -169,14 +154,7 @@ def get_selected_model_identifiers(self) -> list[tuple[int, int, float]]: ------- list """ - output = [] - - for i, weight in enumerate(self.weights_): - identifier = self.identifiers_[i] # type: ignore [attr-defined] - if weight > 0.0: - output.append(identifier) - - return output + return self.identifiers_ # type: ignore [attr-defined] def get_validation_performance(self) -> float: """Return validation performance of ensemble. @@ -193,8 +171,11 @@ def get_validation_performance(self) -> float: class SingleModelEnsemble(AbstractSingleModelEnsemble): """Ensemble consisting of a single model. - This class is used my the :cls:`MultiObjectiveDummyEnsemble` to represent ensembles - consisting of a single model, and this class should not be used on its own. + This class is used by the :class:`MultiObjectiveDummyEnsemble` to represent + ensembles consisting of a single model, and this class should not be used + on its own. + + Do not use by yourself! Parameters ---------- @@ -375,6 +356,8 @@ class SingleBestFromRunhistory(AbstractSingleModelEnsemble): Such model is returned as an ensemble of a single object, to comply with the expected interface of an AbstractEnsemble. + + Do not use by yourself! """ def __init__( diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py index c39f6a6148..ee3cbd0c0e 100644 --- a/autosklearn/estimators.py +++ b/autosklearn/estimators.py @@ -405,12 +405,7 @@ def __init__( ) # Need to resolve the ensemble class here so we can act on it below. - if ensemble_class == "default": - ensemble_class = ( - EnsembleSelection - if metric is None or isinstance(metric, Scorer) or len(metric) == 1 - else MultiObjectiveDummyEnsemble - ) + ensemble_class = self.resolve_ensemble_class(ensemble_class, metric) self.ensemble_class = ensemble_class # User specified `ensemble_size` explicitly, warn them about deprecation @@ -689,12 +684,7 @@ def fit_ensemble( """ # noqa: E501 # Need to resolve the ensemble class here so we can act on it below. - if ensemble_class == "default": - ensemble_class = ( - EnsembleSelection - if metric is None or isinstance(metric, Scorer) or len(metric) == 1 - else MultiObjectiveDummyEnsemble - ) + ensemble_class = self.resolve_ensemble_class(ensemble_class, metric) # User specified `ensemble_size` explicitly, warn them about deprecation if ensemble_size is not None: @@ -752,6 +742,15 @@ def fit_ensemble( ) return self + def resolve_ensemble_class(self, ensemble_class, metric): + if ensemble_class == "default": + ensemble_class = ( + EnsembleSelection + if metric is None or isinstance(metric, Scorer) or len(metric) == 1 + else MultiObjectiveDummyEnsemble + ) + return ensemble_class + def refit(self, X, y): """Refit all models found with fit to new data. diff --git a/doc/api.rst b/doc/api.rst index 3befb03697..581c74d98d 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -117,17 +117,30 @@ Extension Interfaces Ensembles ========= -.. autoclass:: autosklearn.ensembles.AbstractEnsemble +~~~~~~~~~~~~~~~~ +Single objective +~~~~~~~~~~~~~~~~ + +.. autoclass:: autosklearn.ensembles.EnsembleSelection :members: -.. autoclass:: autosklearn.ensembles.AbstractMultiObjectiveEnsemble +Single model classes +~~~~~~~~~~~~~~~~~~~~ + +These classes wrap a single model to provide a unified interface in Auto-sklearn. + +.. autoclass:: autosklearn.ensembles.SingleBest :members: -.. autoclass:: autosklearn.ensembles.EnsembleSelection +.. autoclass:: autosklearn.ensembles.SingleModelEnsemble :members: -.. autoclass:: autosklearn.ensembles.SingleBest +.. autoclass:: autosklearn.ensembles.SingleBestFromRunhistory :members: +~~~~~~~~~~~~~~~ +Multi-objective +~~~~~~~~~~~~~~~ + .. autoclass:: autosklearn.ensembles.MultiObjectiveDummyEnsemble :members: diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np deleted file mode 100644 index fee3160c86d8995cb5ece8126aae88f13a964629..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 160 zcmbR27wQ`j$;jZwP_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= fXCxM+0{I%IItoUbItsN4aKOdLfE}QV6UzqxRmv4g diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/test_ensemble_builder/test_multiobjective_dummy_ensemble.py b/test/test_ensemble_builder/test_multiobjective_dummy_ensemble.py new file mode 100644 index 0000000000..eb639236fd --- /dev/null +++ b/test/test_ensemble_builder/test_multiobjective_dummy_ensemble.py @@ -0,0 +1,73 @@ +import numpy as np +import sklearn.metrics + +from autosklearn.constants import REGRESSION +from autosklearn.ensembles.multiobjective_dummy_ensemble import ( + MultiObjectiveDummyEnsemble, +) +from autosklearn.ensembles.singlebest_ensemble import SingleModelEnsemble +from autosklearn.metrics import MAXINT, make_scorer, root_mean_squared_error + + +def test_MultiObjectiveDummyEnsemble(backend): + negative_root_mean_squared_error = make_scorer( + "negative_root_mean_squared_error", + sklearn.metrics.mean_squared_error, + optimum=0, + worst_possible_result=-MAXINT, + greater_is_better=True, + squared=False, + ) + + ensemble = MultiObjectiveDummyEnsemble( + task_type=REGRESSION, + metrics=[root_mean_squared_error, negative_root_mean_squared_error], + random_state=0, + backend=backend, + ) + + # fit the whole thing + X_data = np.random.random(size=(100, 2)) + y_true = np.full((100), 5.5) + predictions = [] + for i in range(20, 1, -1): + pred = np.ones((100), dtype=np.float32) * i + predictions.append(pred) + identifiers = [(i, i, 0.0) for i in range(20)] + + ensemble.fit( + base_models_predictions=predictions, + X_data=X_data, + true_targets=y_true, + model_identifiers=identifiers, + runs=[], + ) + + # Because the target value is 5.5 some of the predictions have the same error + # -> not every prediction is on the Pareto front + assert len(ensemble.pareto_set_) == 15 + for sub_ensemble in ensemble.pareto_set_: + assert isinstance(sub_ensemble, SingleModelEnsemble) + + # test the prediction + ensemble_prediction = np.random.random((1, 100, 2)) + prediction = ensemble.predict(ensemble_prediction) + assert prediction.shape == (100, 2) + + assert str(ensemble) == """MultiObjectiveDummyEnsemble: 15 models""" + + # From here on everything is supposed to return the output of the best + # ensemble according to the 1st metric + models_with_weights = ensemble.get_models_with_weights( + {identifier: i for i, identifier in enumerate(identifiers)} + ) + assert models_with_weights == [(1.0, 14)] + + identifiers_with_weights = ensemble.get_identifiers_with_weights() + assert identifiers_with_weights == [((14, 14, 0.0), 1.0)] + + selected_model_identifiers = ensemble.get_selected_model_identifiers() + assert selected_model_identifiers == [(14, 14, 0.0)] + + best_model_score = ensemble.get_validation_performance() + assert best_model_score == 0.5 diff --git a/test/test_ensemble_builder/test_singlebest_ensemble.py b/test/test_ensemble_builder/test_singlebest_ensemble.py new file mode 100644 index 0000000000..915f4bc6cf --- /dev/null +++ b/test/test_ensemble_builder/test_singlebest_ensemble.py @@ -0,0 +1,132 @@ +import numpy as np + +from autosklearn.constants import REGRESSION +from autosklearn.ensembles.singlebest_ensemble import SingleBest, SingleModelEnsemble +from autosklearn.metrics import root_mean_squared_error + + +def test_SingleBest(backend): + + ensemble = SingleBest( + task_type=REGRESSION, + metrics=[root_mean_squared_error], + random_state=0, + backend=backend, + ) + + # fit the whole thing + X_data = np.random.random(size=(100, 2)) + y_true = np.full((100), 5.5) + predictions = [] + for i in range(20, 1, -1): + pred = np.ones((100), dtype=np.float32) * i + predictions.append(pred) + identifiers = [(i, i, 0.0) for i in range(20)] + + # Check that the weight isn't touched + assert ensemble.weights_ == [1.0] + + ensemble.fit( + base_models_predictions=predictions, + X_data=X_data, + true_targets=y_true, + model_identifiers=identifiers, + runs=[], + ) + + assert ensemble.weights_ == [1.0] + assert ensemble.indices_ == [14] + assert ensemble.identifiers_ == [(14, 14, 0.0)] + assert ensemble.best_model_score_ == 0.5 + + # test the prediction + ensemble_prediction = np.random.random((1, 100, 2)) + prediction = ensemble.predict(ensemble_prediction) + assert prediction.shape == (100, 2) + + # test string representation; it selects the 14th prediction (6) + # and ties with the 15th (5), but then afterwards the RMSE goes up again + assert ( + str(ensemble) + == """SingleBest: +\tMembers: [14] +\tWeights: [1.0] +\tIdentifiers: [(14, 14, 0.0)]""" + ) + + models_with_weights = ensemble.get_models_with_weights( + {identifier: i for i, identifier in enumerate(identifiers)} + ) + assert models_with_weights == [(1.0, 14)] + + identifiers_with_weights = ensemble.get_identifiers_with_weights() + assert identifiers_with_weights == [((14, 14, 0.0), 1.0)] + + selected_model_identifiers = ensemble.get_selected_model_identifiers() + assert selected_model_identifiers == [(14, 14, 0.0)] + + best_model_score = ensemble.get_validation_performance() + assert best_model_score == 0.5 + + +def test_SingleModelEnsemble(backend): + + ensemble = SingleModelEnsemble( + task_type=REGRESSION, + metrics=[root_mean_squared_error], + random_state=0, + backend=backend, + model_index=5, + ) + + # fit the whole thing + X_data = np.random.random(size=(100, 2)) + y_true = np.full((100), 5.5) + predictions = [] + for i in range(20, 1, -1): + pred = np.ones((100), dtype=np.float32) * i + predictions.append(pred) + identifiers = [(i, i, 0.0) for i in range(20)] + + # Check that the weight isn't touched + assert ensemble.weights_ == [1.0] + + ensemble.fit( + base_models_predictions=predictions, + X_data=X_data, + true_targets=y_true, + model_identifiers=identifiers, + runs=[], + ) + + assert ensemble.weights_ == [1.0] + assert ensemble.indices_ == [5] + assert ensemble.identifiers_ == [(5, 5, 0.0)] + assert ensemble.best_model_score_ == 9.5 + + # test the prediction + ensemble_prediction = np.random.random((1, 100, 2)) + prediction = ensemble.predict(ensemble_prediction) + assert prediction.shape == (100, 2) + + assert ( + str(ensemble) + == """SingleModelEnsemble: +\tMembers: [5] +\tWeights: [1.0] +\tIdentifiers: [(5, 5, 0.0)]""" + ) + + models_with_weights = ensemble.get_models_with_weights( + {identifier: i for i, identifier in enumerate(identifiers)} + ) + assert models_with_weights == [(1.0, 5)] + + identifiers_with_weights = ensemble.get_identifiers_with_weights() + assert identifiers_with_weights == [((5, 5, 0.0), 1.0)] + + selected_model_identifiers = ensemble.get_selected_model_identifiers() + assert selected_model_identifiers == [(5, 5, 0.0)] + + best_model_score = ensemble.get_validation_performance() + assert best_model_score == 9.5 diff --git a/test/test_estimators/test_estimators.py b/test/test_estimators/test_estimators.py index d0d3f28bdb..7eb3a441f0 100644 --- a/test/test_estimators/test_estimators.py +++ b/test/test_estimators/test_estimators.py @@ -29,6 +29,7 @@ from autosklearn.automl import AutoMLClassifier from autosklearn.data.validation import InputValidator from autosklearn.ensemble_building.run import Run +from autosklearn.ensembles import EnsembleSelection, MultiObjectiveDummyEnsemble from autosklearn.estimators import ( AutoSklearnClassifier, AutoSklearnEstimator, @@ -1729,3 +1730,16 @@ def test_param_dataset_compression(dataset_compression: Union[bool, Dict[str, An model = AutoSklearnClassifier(dataset_compression=dataset_compression) assert model.dataset_compression == dataset_compression + + +def test_ensemble_default_resolves(): + model = AutoSklearnClassifier( + ensemble_class="default", + metric=accuracy, + ) + assert model.ensemble_class == EnsembleSelection + model = AutoSklearnClassifier( + ensemble_class="default", + metric=[accuracy, f1_macro], + ) + assert model.ensemble_class == MultiObjectiveDummyEnsemble From 1c8a06bbc8e6de491b38a8c8a9fe3621a60dfac8 Mon Sep 17 00:00:00 2001 From: eddiebergman Date: Sat, 16 Jul 2022 15:31:31 -0400 Subject: [PATCH 07/16] Factor out pareto_front into stand alone function --- .../multiobjective_dummy_ensemble.py | 30 ++------------ autosklearn/util/multiobjective.py | 40 +++++++++++++++++++ 2 files changed, 43 insertions(+), 27 deletions(-) create mode 100644 autosklearn/util/multiobjective.py diff --git a/autosklearn/ensembles/multiobjective_dummy_ensemble.py b/autosklearn/ensembles/multiobjective_dummy_ensemble.py index 1b380010aa..0053c916e4 100644 --- a/autosklearn/ensembles/multiobjective_dummy_ensemble.py +++ b/autosklearn/ensembles/multiobjective_dummy_ensemble.py @@ -17,6 +17,7 @@ from autosklearn.ensembles.singlebest_ensemble import SingleModelEnsemble from autosklearn.metrics import Scorer, calculate_losses from autosklearn.pipeline.base import BasePipeline +from autosklearn.util.multiobjective import pareto_front class MultiObjectiveDummyEnsemble(AbstractMultiObjectiveEnsemble): @@ -106,32 +107,6 @@ def fit( if self.task_type not in TASK_TYPES: raise ValueError("Unknown task type %s." % self.task_type) - def is_pareto_efficient_simple(costs: np.ndarray) -> np.ndarray: - """ - Plot the Pareto Front in our 2d example. - - source from: https://stackoverflow.com/a/40239615 - Find the pareto-efficient points - - Parameters - ---------- - costs: np.ndarray - - Returns - ------- - np.ndarray - """ - - is_efficient = np.ones(costs.shape[0], dtype=bool) - for i, c in enumerate(costs): - if is_efficient[i]: - # Keep any point with a lower cost - is_efficient[is_efficient] = np.any(costs[is_efficient] < c, axis=1) - - # And keep self - is_efficient[i] = True - return is_efficient - all_costs = np.empty((len(base_models_predictions), len(self.metrics))) for i, base_model_prediction in enumerate(base_models_predictions): losses = calculate_losses( @@ -142,9 +117,10 @@ def is_pareto_efficient_simple(costs: np.ndarray) -> np.ndarray: X_data=X_data, ) all_costs[i] = [losses[metric.name] for metric in self.metrics] + all_costs = np.array(all_costs) sort_by_first_metric = np.argsort(all_costs[:, 0]) - efficient_points = is_pareto_efficient_simple(all_costs) + efficient_points = pareto_front(all_costs, is_loss=True) pareto_set = [] for argsort_idx in sort_by_first_metric: diff --git a/autosklearn/util/multiobjective.py b/autosklearn/util/multiobjective.py new file mode 100644 index 0000000000..84a8356c16 --- /dev/null +++ b/autosklearn/util/multiobjective.py @@ -0,0 +1,40 @@ +import operator + +import numpy as np + + +def pareto_front(values: np.ndarray, *, is_loss: bool = True) -> np.ndarray: + """Calculate the pareto front + + source from: https://stackoverflow.com/a/40239615 + + Note + ---- + Works on the assumption that every value is either something to minimize or + something to maximize, based on ``is_loss``. + + Parameters + ---------- + values: np.ndarray [n_models, n_metrics] + The value for each of the metrics + + is_loss: bool = True + Whether the metrics are a loss or a score + + Returns + ------- + np.ndarray + A boolean mask where true indicates if the model on the pareto front + """ + op = operator.lt if is_loss else operator.gt + + is_efficient = np.ones(values.shape[0], dtype=bool) + for i, c in enumerate(values): + if is_efficient[i]: + # Keep any point with a lower cost + is_efficient[is_efficient] = np.any(op(values, c), axis=1) + + # And keep self + is_efficient[i] = True + + return is_efficient From d8224a725a177a521992dc0bc406609cfd84187a Mon Sep 17 00:00:00 2001 From: eddiebergman Date: Sat, 16 Jul 2022 15:42:33 -0400 Subject: [PATCH 08/16] Make the pareto set a property By encapsulating it this way, type checker is more friendly, the @property is always available and will throw an error if not fitted. IN contrast, a non-existent property is likely to give a much more inuintive error that "attribute pareto_set_ does not exist" --- autosklearn/automl.py | 5 +-- autosklearn/ensembles/abstract_ensemble.py | 17 +++++++- .../multiobjective_dummy_ensemble.py | 40 +++++++++++-------- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/autosklearn/automl.py b/autosklearn/automl.py index 4b1224c499..cf971456e3 100644 --- a/autosklearn/automl.py +++ b/autosklearn/automl.py @@ -1693,13 +1693,12 @@ def _load_pareto_set(self) -> Sequence[VotingClassifier | VotingRegressor]: raise ValueError("Pareto set only available if ensemble can be loaded.") if isinstance(self.ensemble_, AbstractMultiObjectiveEnsemble): - pareto_set = self.ensemble_.get_pareto_set() + pareto_set = self.ensemble_.pareto_set else: self._logger.warning( "Pareto set not available for single objective ensemble " "method. The Pareto set will only include the single ensemble " - "constructed by %s", - type(self.ensemble_), + f"constructed by {type(self.ensemble_)}," ) pareto_set = [self.ensemble_] diff --git a/autosklearn/ensembles/abstract_ensemble.py b/autosklearn/ensembles/abstract_ensemble.py index 9608c15e62..38f4123a39 100644 --- a/autosklearn/ensembles/abstract_ensemble.py +++ b/autosklearn/ensembles/abstract_ensemble.py @@ -140,5 +140,18 @@ def get_validation_performance(self) -> float: class AbstractMultiObjectiveEnsemble(AbstractEnsemble): - def get_pareto_set(self) -> Sequence[AbstractEnsemble]: - raise NotImplementedError() + @property + @abstractmethod + def pareto_set(self) -> Sequence[AbstractEnsemble]: + """Get a sequence on ensembles that are on the pareto front + + Raises + ------ + SklearnNotFittedError + If ``fit`` has not been called and the pareto set does not exist yet + + Returns + ------- + Sequence[AbstractEnsemble] + """ + ... diff --git a/autosklearn/ensembles/multiobjective_dummy_ensemble.py b/autosklearn/ensembles/multiobjective_dummy_ensemble.py index 0053c916e4..04246ac722 100644 --- a/autosklearn/ensembles/multiobjective_dummy_ensemble.py +++ b/autosklearn/ensembles/multiobjective_dummy_ensemble.py @@ -5,6 +5,7 @@ import warnings import numpy as np +from sklearn.exceptions import NotFittedError from autosklearn.automl_common.common.utils.backend import Backend from autosklearn.constants import TASK_TYPES @@ -58,16 +59,13 @@ def __init__( self.metric = [metrics] self.random_state = random_state self.backend = backend + self.pareto_set_: list[SingleModelEnsemble] | None = None - def __getstate__(self) -> dict[str, Any]: - # Cannot serialize a metric if - # it is user defined. - # That is, if doing pickle dump - # the metric won't be the same as the - # one in __main__. we don't use the metric - # in the EnsembleSelection so this should - # be fine - return {key: value for key, value in self.__dict__.items() if key != "metrics"} + @property + def pareto_set(self) -> Sequence[AbstractEnsemble]: + if self.pareto_set_ is None: + raise NotFittedError("`pareto_set` not created, please call `fit()` first") + return self.pareto_set_ def fit( self, @@ -141,6 +139,7 @@ def fit( X_data=X_data, ) pareto_set.append(ensemble) + self.pareto_set_ = pareto_set return self @@ -159,10 +158,10 @@ def predict( ------- np.ndarray """ - return self.pareto_set_[0].predict(base_models_predictions) + return self.pareto_set[0].predict(base_models_predictions) def __str__(self) -> str: - return "MultiObjectiveDummyEnsemble: %d models" % len(self.pareto_set_) + return "MultiObjectiveDummyEnsemble: %d models" % len(self.pareto_set) def get_models_with_weights( self, models: dict[tuple[int, int, float], BasePipeline] @@ -180,7 +179,7 @@ def get_models_with_weights( ------- list[tuple[float, BasePipeline]] """ - return self.pareto_set_[0].get_models_with_weights(models) + return self.pareto_set[0].get_models_with_weights(models) def get_identifiers_with_weights( self, @@ -198,7 +197,7 @@ def get_identifiers_with_weights( ------- list[tuple[tuple[int, int, float], float] """ - return self.pareto_set_[0].get_identifiers_with_weights() + return self.pareto_set[0].get_identifiers_with_weights() def get_selected_model_identifiers(self) -> list[tuple[int, int, float]]: """Return identifiers of models in the ensemble that is best for the 1st metric. @@ -209,7 +208,7 @@ def get_selected_model_identifiers(self) -> list[tuple[int, int, float]]: ------- list """ - return self.pareto_set_[0].get_selected_model_identifiers() + return self.pareto_set[0].get_selected_model_identifiers() def get_validation_performance(self) -> float: """Return validation performance of the ensemble that is best for the 1st metric. @@ -218,7 +217,14 @@ def get_validation_performance(self) -> float: ------- float """ - return self.pareto_set_[0].get_validation_performance() + return self.pareto_set[0].get_validation_performance() - def get_pareto_set(self) -> Sequence[AbstractEnsemble]: - return self.pareto_set_ + def __getstate__(self) -> dict[str, Any]: + # Cannot serialize a metric if + # it is user defined. + # That is, if doing pickle dump + # the metric won't be the same as the + # one in __main__. we don't use the metric + # in the EnsembleSelection so this should + # be fine + return {key: value for key, value in self.__dict__.items() if key != "metrics"} From 04aa85051fc5b356065950aac6ce94febb1c69c6 Mon Sep 17 00:00:00 2001 From: eddiebergman Date: Sat, 16 Jul 2022 15:54:06 -0400 Subject: [PATCH 09/16] Add None defaults, fix indentation --- autosklearn/ensembles/abstract_ensemble.py | 4 +- autosklearn/ensembles/ensemble_selection.py | 28 +++--- .../multiobjective_dummy_ensemble.py | 13 +-- autosklearn/ensembles/singlebest_ensemble.py | 91 ++++++++++--------- 4 files changed, 70 insertions(+), 66 deletions(-) diff --git a/autosklearn/ensembles/abstract_ensemble.py b/autosklearn/ensembles/abstract_ensemble.py index 38f4123a39..6b9d27d575 100644 --- a/autosklearn/ensembles/abstract_ensemble.py +++ b/autosklearn/ensembles/abstract_ensemble.py @@ -18,8 +18,8 @@ def __init__( self, task_type: int, metrics: Sequence[Scorer] | Scorer, - random_state: int | np.random.RandomState | None, backend: Backend, + random_state: int | np.random.RandomState | None = None, ): pass @@ -27,10 +27,10 @@ def __init__( def fit( self, base_models_predictions: np.ndarray | List[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES | None, true_targets: np.ndarray, model_identifiers: List[Tuple[int, int, float]], runs: Sequence[Run], + X_data: SUPPORTED_FEAT_TYPES | None = None, ) -> "AbstractEnsemble": """Fit an ensemble given predictions of base models and targets. diff --git a/autosklearn/ensembles/ensemble_selection.py b/autosklearn/ensembles/ensemble_selection.py index 0f151bd003..0b36a76c60 100644 --- a/autosklearn/ensembles/ensemble_selection.py +++ b/autosklearn/ensembles/ensemble_selection.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional, Sequence, Tuple, Union +from typing import Any, Dict, List, Sequence, Tuple, Union import random import warnings @@ -23,11 +23,11 @@ def __init__( self, task_type: int, metrics: Sequence[Scorer] | Scorer, - random_state: Optional[Union[int, np.random.RandomState]], backend: Backend, ensemble_size: int = 50, bagging: bool = False, mode: str = "fast", + random_state: int | np.random.RandomState | None = None, ) -> None: """An ensemble of selected algorithms @@ -43,14 +43,6 @@ def __init__( The metric used to evaluate the models. If multiple metrics are passed, ensemble selection only optimizes for the first - random_state: Optional[int | RandomState] = None - The random_state used for ensemble selection. - - * None - Uses numpy's default RandomState object - * int - Successive calls to fit will produce the same results - * RandomState - Truly random, each call to fit will produce - different results, even with the same object. - backend : Backend Gives access to the backend of Auto-sklearn. Not used by Ensemble Selection. @@ -62,6 +54,14 @@ def __init__( * 'slow' - The original method used in Rich Caruana's ensemble selection. * 'fast' - A faster version of Rich Caruanas' ensemble selection. + random_state: int | RandomState | None = None + The random_state used for ensemble selection. + + * None - Uses numpy's default RandomState object + * int - Successive calls to fit will produce the same results + * RandomState - Truly random, each call to fit will produce + different results, even with the same object. + References ---------- | Ensemble selection from libraries of models @@ -105,10 +105,10 @@ def __getstate__(self) -> Dict[str, Any]: def fit( self, base_models_predictions: List[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES | None, true_targets: np.ndarray, model_identifiers: List[Tuple[int, int, float]], runs: Sequence[Run], + X_data: SUPPORTED_FEAT_TYPES | None = None, ) -> EnsembleSelection: self.ensemble_size = int(self.ensemble_size) if self.ensemble_size < 1: @@ -141,8 +141,8 @@ def fit( def _fit( self, predictions: List[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES, labels: np.ndarray, + X_data: SUPPORTED_FEAT_TYPES | None = None, ) -> EnsembleSelection: if self.mode == "fast": self._fast(predictions, X_data, labels) @@ -153,8 +153,8 @@ def _fit( def _fast( self, predictions: List[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES, labels: np.ndarray, + X_data: SUPPORTED_FEAT_TYPES | None = None, ) -> None: """Fast version of Rich Caruana's ensemble selection method.""" self.num_input_models_ = len(predictions) @@ -231,8 +231,8 @@ def _fast( def _slow( self, predictions: List[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES, labels: np.ndarray, + X_data: SUPPORTED_FEAT_TYPES | None = None, ) -> None: """Rich Caruana's ensemble selection method.""" self.num_input_models_ = len(predictions) diff --git a/autosklearn/ensembles/multiobjective_dummy_ensemble.py b/autosklearn/ensembles/multiobjective_dummy_ensemble.py index 04246ac722..9cea249bb0 100644 --- a/autosklearn/ensembles/multiobjective_dummy_ensemble.py +++ b/autosklearn/ensembles/multiobjective_dummy_ensemble.py @@ -26,8 +26,8 @@ def __init__( self, task_type: int, metrics: Sequence[Scorer] | Scorer, - random_state: int | np.random.RandomState | None, backend: Backend, + random_state: int | np.random.RandomState | None = None, ) -> None: """A dummy implementation of a multi-objective ensemble. @@ -41,11 +41,11 @@ def __init__( metrics: Sequence[Scorer] | Scorer The metrics used to evaluate the models. - random_state: Optional[int | RandomState] = None - Not used. - backend : Backend Gives access to the backend of Auto-sklearn. Not used. + + random_state: int | RandomState | None = None + Not used. """ self.task_type = task_type if isinstance(metrics, Sequence): @@ -70,10 +70,10 @@ def pareto_set(self) -> Sequence[AbstractEnsemble]: def fit( self, base_models_predictions: list[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES | None, true_targets: np.ndarray, model_identifiers: list[tuple[int, int, float]], runs: Sequence[Run], + X_data: SUPPORTED_FEAT_TYPES | None = None, ) -> MultiObjectiveDummyEnsemble: """Select dummy ensembles given predictions of base models and targets. @@ -87,7 +87,8 @@ def fit( Can be a list of 2d numpy arrays as well to prevent copying all predictions into a single, large numpy array. - X_data : list-like or sparse data + X_data : list-like | spmatrix | None = None + X data to give to the metric if required true_targets : array of shape [n_targets] diff --git a/autosklearn/ensembles/singlebest_ensemble.py b/autosklearn/ensembles/singlebest_ensemble.py index 7a567ae522..91ef6f572f 100644 --- a/autosklearn/ensembles/singlebest_ensemble.py +++ b/autosklearn/ensembles/singlebest_ensemble.py @@ -26,7 +26,7 @@ class AbstractSingleModelEnsemble(AbstractEnsemble): metrics: Sequence[Scorer] | Scorer The metrics used to evaluate the models. - random_state: Optional[int | RandomState] = None + random_state: int | RandomState | None = None Not used. backend : Backend @@ -37,8 +37,8 @@ def __init__( self, task_type: int, metrics: Sequence[Scorer] | Scorer, - random_state: int | np.random.RandomState | None, backend: Backend, + random_state: int | np.random.RandomState | None = None, ): self.weights_ = [1.0] self.task_type = task_type @@ -54,33 +54,33 @@ def __init__( def fit( self, base_models_predictions: np.ndarray | list[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES | None, true_targets: np.ndarray, model_identifiers: list[tuple[int, int, float]], runs: Sequence[Run], + X_data: SUPPORTED_FEAT_TYPES | None = None, ) -> AbstractSingleModelEnsemble: """Fit the ensemble - Parameters - ---------- - base_models_predictions: np.ndarray - shape = (n_base_models, n_data_points, n_targets) - n_targets is the number of classes in case of classification, - n_targets is 0 or 1 in case of regression + Parameters + ---------- + base_models_predictions: np.ndarray + shape = (n_base_models, n_data_points, n_targets) + n_targets is the number of classes in case of classification, + n_targets is 0 or 1 in case of regression - Can be a list of 2d numpy arrays as well to prevent copying all - predictions into a single, large numpy array. + Can be a list of 2d numpy arrays as well to prevent copying all + predictions into a single, large numpy array. - X_data : list-like or sparse data + X_data : list-like | spamtrix | None = None - true_targets : array of shape [n_targets] + true_targets : array of shape [n_targets] - model_identifiers : identifier for each base model. - Can be used for practical text output of the ensemble. + model_identifiers : identifier for each base model. + Can be used for practical text output of the ensemble. - runs: Sequence[Run] - Additional information for each run executed by SMAC that was - considered by the ensemble builder. + runs: Sequence[Run] + Additional information for each run executed by SMAC that was + considered by the ensemble builder. Returns ------- @@ -185,7 +185,7 @@ class SingleModelEnsemble(AbstractSingleModelEnsemble): metrics: Sequence[Scorer] | Scorer The metrics used to evaluate the models. - random_state: Optional[int | RandomState] = None + random_state: int | RandomState | None = None Not used. backend : Backend @@ -201,9 +201,9 @@ def __init__( self, task_type: int, metrics: Sequence[Scorer] | Scorer, - random_state: int | np.random.RandomState | None, backend: Backend, model_index: int, + random_state: int | np.random.RandomState | None = None, ): super().__init__( task_type=task_type, @@ -216,10 +216,10 @@ def __init__( def fit( self, base_models_predictions: np.ndarray | list[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES | None, true_targets: np.ndarray, model_identifiers: list[tuple[int, int, float]], runs: Sequence[Run], + X_data: SUPPORTED_FEAT_TYPES | None = None, ) -> SingleModelEnsemble: """Dummy implementation of the ``fit`` method. @@ -227,30 +227,31 @@ def fit( method only stores the identifier of the selected model and computes it's validation loss. - Parameters - ---------- - base_models_predictions: np.ndarray - shape = (n_base_models, n_data_points, n_targets) - n_targets is the number of classes in case of classification, - n_targets is 0 or 1 in case of regression + Parameters + ---------- + base_models_predictions: np.ndarray + shape = (n_base_models, n_data_points, n_targets) + n_targets is the number of classes in case of classification, + n_targets is 0 or 1 in case of regression - Can be a list of 2d numpy arrays as well to prevent copying all - predictions into a single, large numpy array. + Can be a list of 2d numpy arrays as well to prevent copying all + predictions into a single, large numpy array. - X_data : list-like or sparse data + X_data : list-like | spmatrix | None = None + X data to feed to a metric if it requires it - true_targets : array of shape [n_targets] + true_targets : array of shape [n_targets] - model_identifiers : identifier for each base model. - Can be used for practical text output of the ensemble. + model_identifiers : identifier for each base model. + Can be used for practical text output of the ensemble. - runs: Sequence[Run] - Additional information for each run executed by SMAC that was - considered by the ensemble builder. Not used. + runs: Sequence[Run] + Additional information for each run executed by SMAC that was + considered by the ensemble builder. Not used. - Returns - ------- - self + Returns + ------- + self """ self.identifiers_ = [model_identifiers[self.indices_[0]]] loss = calculate_losses( @@ -275,7 +276,7 @@ class SingleBest(AbstractSingleModelEnsemble): metrics: Sequence[Scorer] | Scorer The metrics used to evaluate the models. - random_state: Optional[int | RandomState] = None + random_state: int | RandomState | None = None Not used. backend : Backend @@ -286,8 +287,8 @@ def __init__( self, task_type: int, metrics: Sequence[Scorer] | Scorer, - random_state: int | np.random.RandomState | None, backend: Backend, + random_state: int | np.random.RandomState | None = None, ): super().__init__( task_type=task_type, @@ -299,10 +300,10 @@ def __init__( def fit( self, base_models_predictions: np.ndarray | list[np.ndarray], - X_data: SUPPORTED_FEAT_TYPES | None, true_targets: np.ndarray, model_identifiers: list[tuple[int, int, float]], runs: Sequence[Run], + X_data: SUPPORTED_FEAT_TYPES | None = None, ) -> SingleBest: """Select the single best model. @@ -316,7 +317,7 @@ def fit( Can be a list of 2d numpy arrays as well to prevent copying all predictions into a single, large numpy array. - X_data : list-like or sparse data + X_data : list | spmatrix | None = None true_targets : array of shape [n_targets] @@ -364,10 +365,10 @@ def __init__( self, task_type: int, metrics: Sequence[Scorer] | Scorer, - random_state: int | np.random.RandomState | None, backend: Backend, run_history: RunHistory, seed: int, + random_state: int | np.random.RandomState | None = None, ): super().__init__( task_type=task_type, @@ -376,6 +377,8 @@ def __init__( backend=backend, ) + # The seed here is seperate from RandomState and is used to indiicate a + # directory for the backend to search in self.seed = seed self.indices_ = [0] self.weights_ = [1.0] From 446b7d688369e23112fc9a00c89d34c6c1e228fe Mon Sep 17 00:00:00 2001 From: eddiebergman Date: Sat, 16 Jul 2022 16:26:35 -0400 Subject: [PATCH 10/16] Move resolve ensemble class check to init --- autosklearn/estimators.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py index ee3cbd0c0e..526365d2f1 100644 --- a/autosklearn/estimators.py +++ b/autosklearn/estimators.py @@ -405,7 +405,12 @@ def __init__( ) # Need to resolve the ensemble class here so we can act on it below. - ensemble_class = self.resolve_ensemble_class(ensemble_class, metric) + if ensemble_class == "default": + ensemble_class = ( + MultiObjectiveDummyEnsemble + if isinstance(metric, Scorer) and len(metric) > 1 + else EnsembleSelection + ) self.ensemble_class = ensemble_class # User specified `ensemble_size` explicitly, warn them about deprecation @@ -742,15 +747,6 @@ def fit_ensemble( ) return self - def resolve_ensemble_class(self, ensemble_class, metric): - if ensemble_class == "default": - ensemble_class = ( - EnsembleSelection - if metric is None or isinstance(metric, Scorer) or len(metric) == 1 - else MultiObjectiveDummyEnsemble - ) - return ensemble_class - def refit(self, X, y): """Refit all models found with fit to new data. From 18b6db487698cc21adea9674727117638d5ce41d Mon Sep 17 00:00:00 2001 From: eddiebergman Date: Sat, 16 Jul 2022 16:28:20 -0400 Subject: [PATCH 11/16] Revert "Move resolve ensemble class check to init" This reverts commit 446b7d688369e23112fc9a00c89d34c6c1e228fe. --- autosklearn/estimators.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py index 526365d2f1..ee3cbd0c0e 100644 --- a/autosklearn/estimators.py +++ b/autosklearn/estimators.py @@ -405,12 +405,7 @@ def __init__( ) # Need to resolve the ensemble class here so we can act on it below. - if ensemble_class == "default": - ensemble_class = ( - MultiObjectiveDummyEnsemble - if isinstance(metric, Scorer) and len(metric) > 1 - else EnsembleSelection - ) + ensemble_class = self.resolve_ensemble_class(ensemble_class, metric) self.ensemble_class = ensemble_class # User specified `ensemble_size` explicitly, warn them about deprecation @@ -747,6 +742,15 @@ def fit_ensemble( ) return self + def resolve_ensemble_class(self, ensemble_class, metric): + if ensemble_class == "default": + ensemble_class = ( + EnsembleSelection + if metric is None or isinstance(metric, Scorer) or len(metric) == 1 + else MultiObjectiveDummyEnsemble + ) + return ensemble_class + def refit(self, X, y): """Refit all models found with fit to new data. From 9dd6c749bbf6331e1698baeb790561848baf6e8e Mon Sep 17 00:00:00 2001 From: eddiebergman Date: Sat, 16 Jul 2022 16:46:13 -0400 Subject: [PATCH 12/16] Fix `_resolve_ensemble_class` and make it private See the comment string for the fix --- autosklearn/estimators.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py index ee3cbd0c0e..e0c429b00f 100644 --- a/autosklearn/estimators.py +++ b/autosklearn/estimators.py @@ -405,7 +405,9 @@ def __init__( ) # Need to resolve the ensemble class here so we can act on it below. - ensemble_class = self.resolve_ensemble_class(ensemble_class, metric) + if ensemble_class == "default": + ensemble_class = self._resolve_ensemble_class(metric) + self.ensemble_class = ensemble_class # User specified `ensemble_size` explicitly, warn them about deprecation @@ -683,8 +685,20 @@ def fit_ensemble( self """ # noqa: E501 - # Need to resolve the ensemble class here so we can act on it below. - ensemble_class = self.resolve_ensemble_class(ensemble_class, metric) + if ensemble_class == "default": + # Things are actually a little more nuanced here: + # * If they passed `metric=None` at init, we would infer this in automl + # during `fit` and store it in the automl instance. + # * If they passed a `metric` in init and left it `None` here, this would + # also be in the automl instance + # => We can use self.automl_ as ground truth for metric if no metrics passed + # and we have one created + if metric is None and self.automl_ is not None and self.automl_.metrics: + metric = self.automl_.metrics + + ensemble_class = self._resolve_ensemble_class(metric) + + self.ensemble_class = ensemble_class # User specified `ensemble_size` explicitly, warn them about deprecation if ensemble_size is not None: @@ -742,14 +756,15 @@ def fit_ensemble( ) return self - def resolve_ensemble_class(self, ensemble_class, metric): - if ensemble_class == "default": - ensemble_class = ( - EnsembleSelection - if metric is None or isinstance(metric, Scorer) or len(metric) == 1 - else MultiObjectiveDummyEnsemble - ) - return ensemble_class + def _resolve_ensemble_class( + self, + metric: Scorer | Sequence[Scorer] | None, + ) -> type[AbstractEnsemble]: + return ( + EnsembleSelection + if metric is None or isinstance(metric, Scorer) or len(metric) == 1 + else MultiObjectiveDummyEnsemble + ) def refit(self, X, y): """Refit all models found with fit to new data. From 0248055bf20e410dff1c779f981f77071f2b9e12 Mon Sep 17 00:00:00 2001 From: eddiebergman Date: Sat, 16 Jul 2022 20:17:21 -0400 Subject: [PATCH 13/16] Fix variable name --- autosklearn/estimators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py index e0c429b00f..fd5d602dd6 100644 --- a/autosklearn/estimators.py +++ b/autosklearn/estimators.py @@ -694,7 +694,7 @@ def fit_ensemble( # => We can use self.automl_ as ground truth for metric if no metrics passed # and we have one created if metric is None and self.automl_ is not None and self.automl_.metrics: - metric = self.automl_.metrics + metric = self.automl_._metrics ensemble_class = self._resolve_ensemble_class(metric) From 031a338291e91239b07ee42704165d57d8243150 Mon Sep 17 00:00:00 2001 From: eddiebergman Date: Sat, 16 Jul 2022 21:12:11 -0400 Subject: [PATCH 14/16] Fix missing parameter names --- autosklearn/ensembles/ensemble_selection.py | 9 +++++--- autosklearn/metrics/__init__.py | 24 ++++++++++++--------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/autosklearn/ensembles/ensemble_selection.py b/autosklearn/ensembles/ensemble_selection.py index 0b36a76c60..ef8859ac0d 100644 --- a/autosklearn/ensembles/ensemble_selection.py +++ b/autosklearn/ensembles/ensemble_selection.py @@ -142,18 +142,20 @@ def _fit( self, predictions: List[np.ndarray], labels: np.ndarray, + *, X_data: SUPPORTED_FEAT_TYPES | None = None, ) -> EnsembleSelection: if self.mode == "fast": - self._fast(predictions, X_data, labels) + self._fast(predictions=predictions, X_data=X_data, labels=labels) else: - self._slow(predictions, X_data, labels) + self._slow(predictions=predictions, X_data=X_data, labels=labels) return self def _fast( self, predictions: List[np.ndarray], labels: np.ndarray, + *, X_data: SUPPORTED_FEAT_TYPES | None = None, ) -> None: """Fast version of Rich Caruana's ensemble selection method.""" @@ -232,6 +234,7 @@ def _slow( self, predictions: List[np.ndarray], labels: np.ndarray, + *, X_data: SUPPORTED_FEAT_TYPES | None = None, ) -> None: """Rich Caruana's ensemble selection method.""" @@ -311,7 +314,7 @@ def _bagging( # Bagging a set of models indices = sorted(random.sample(range(0, n_models), bag_size)) bag = predictions[indices, :, :] - order, _ = self._fit(bag, labels) + order, _ = self._fit(predictions=bag, labels=labels) order_of_each_bag.append(order) return np.array( diff --git a/autosklearn/metrics/__init__.py b/autosklearn/metrics/__init__.py index 0485db892c..e6b2e87802 100644 --- a/autosklearn/metrics/__init__.py +++ b/autosklearn/metrics/__init__.py @@ -48,6 +48,7 @@ def __call__( self, y_true: np.ndarray, y_pred: np.ndarray, + *, X_data: Optional[SUPPORTED_XDATA_TYPES] = None, sample_weight: Optional[List[float]] = None, ) -> float: @@ -62,6 +63,7 @@ def __call__( self, y_true: np.ndarray, y_pred: np.ndarray, + *, X_data: Optional[SUPPORTED_XDATA_TYPES] = None, sample_weight: Optional[List[float]] = None, ) -> float: @@ -129,6 +131,7 @@ def __call__( self, y_true: np.ndarray, y_pred: np.ndarray, + *, X_data: Optional[SUPPORTED_XDATA_TYPES] = None, sample_weight: Optional[List[float]] = None, ) -> float: @@ -189,6 +192,7 @@ def __call__( self, y_true: np.ndarray, y_pred: np.ndarray, + *, X_data: Optional[SUPPORTED_XDATA_TYPES] = None, sample_weight: Optional[List[float]] = None, ) -> float: @@ -508,11 +512,11 @@ def calculate_scores( try: score_dict[metric_.name] = _compute_single_scorer( - metric_, - prediction, - solution, - task_type, - X_data, + metric=metric_, + prediction=prediction, + solution=solution, + task_type=task_type, + X_data=X_data, ) except ValueError as e: print(e, e.args[0]) @@ -532,11 +536,11 @@ def calculate_scores( try: score_dict[metric_.name] = _compute_single_scorer( - metric_, - prediction, - solution, - task_type, - X_data, + metric=metric_, + prediction=prediction, + solution=solution, + task_type=task_type, + X_data=X_data, ) except ValueError as e: if e.args[0] == "multiclass format is not supported": From a427b6d9fa29c7cd05c3918722f2cf4f7a75dd46 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 3 Aug 2022 11:39:46 +0200 Subject: [PATCH 15/16] Fix bug, update docs --- .../multiobjective_dummy_ensemble.py | 6 ++--- autosklearn/ensembles/singlebest_ensemble.py | 26 +++++++++---------- autosklearn/estimators.py | 2 +- .../example_parallel_manual_spawning_cli.py | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/autosklearn/ensembles/multiobjective_dummy_ensemble.py b/autosklearn/ensembles/multiobjective_dummy_ensemble.py index 9cea249bb0..1d53f83831 100644 --- a/autosklearn/ensembles/multiobjective_dummy_ensemble.py +++ b/autosklearn/ensembles/multiobjective_dummy_ensemble.py @@ -87,9 +87,6 @@ def fit( Can be a list of 2d numpy arrays as well to prevent copying all predictions into a single, large numpy array. - X_data : list-like | spmatrix | None = None - X data to give to the metric if required - true_targets : array of shape [n_targets] model_identifiers : identifier for each base model. @@ -99,6 +96,9 @@ def fit( Additional information for each run executed by SMAC that was considered by the ensemble builder. Not used. + X_data : list-like | sparse matrix | None = None + X data to give to the metric if required + Returns ------- self diff --git a/autosklearn/ensembles/singlebest_ensemble.py b/autosklearn/ensembles/singlebest_ensemble.py index 91ef6f572f..b072bcab11 100644 --- a/autosklearn/ensembles/singlebest_ensemble.py +++ b/autosklearn/ensembles/singlebest_ensemble.py @@ -26,11 +26,11 @@ class AbstractSingleModelEnsemble(AbstractEnsemble): metrics: Sequence[Scorer] | Scorer The metrics used to evaluate the models. - random_state: int | RandomState | None = None - Not used. - backend : Backend Gives access to the backend of Auto-sklearn. Not used. + + random_state: int | RandomState | None = None + Not used. """ def __init__( @@ -71,8 +71,6 @@ def fit( Can be a list of 2d numpy arrays as well to prevent copying all predictions into a single, large numpy array. - X_data : list-like | spamtrix | None = None - true_targets : array of shape [n_targets] model_identifiers : identifier for each base model. @@ -82,6 +80,8 @@ def fit( Additional information for each run executed by SMAC that was considered by the ensemble builder. + X_data : list-like | sparse matrix | None = None + Returns ------- self @@ -185,9 +185,6 @@ class SingleModelEnsemble(AbstractSingleModelEnsemble): metrics: Sequence[Scorer] | Scorer The metrics used to evaluate the models. - random_state: int | RandomState | None = None - Not used. - backend : Backend Gives access to the backend of Auto-sklearn. Not used. @@ -195,6 +192,9 @@ class SingleModelEnsemble(AbstractSingleModelEnsemble): Index of the model that constitutes the ensemble. This index will be used to select the correct predictions that will be passed during ``fit`` and ``predict``. + + random_state: int | RandomState | None = None + Not used. """ def __init__( @@ -237,9 +237,6 @@ def fit( Can be a list of 2d numpy arrays as well to prevent copying all predictions into a single, large numpy array. - X_data : list-like | spmatrix | None = None - X data to feed to a metric if it requires it - true_targets : array of shape [n_targets] model_identifiers : identifier for each base model. @@ -249,6 +246,9 @@ def fit( Additional information for each run executed by SMAC that was considered by the ensemble builder. Not used. + X_data : list-like | spmatrix | None = None + X data to feed to a metric if it requires it + Returns ------- self @@ -317,8 +317,6 @@ def fit( Can be a list of 2d numpy arrays as well to prevent copying all predictions into a single, large numpy array. - X_data : list | spmatrix | None = None - true_targets : array of shape [n_targets] model_identifiers : identifier for each base model. @@ -328,6 +326,8 @@ def fit( Additional information for each run executed by SMAC that was considered by the ensemble builder. Not used. + X_data : array-like | sparse matrix | None = None + Returns ------- self diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py index fd5d602dd6..607fb3dc2f 100644 --- a/autosklearn/estimators.py +++ b/autosklearn/estimators.py @@ -693,7 +693,7 @@ def fit_ensemble( # also be in the automl instance # => We can use self.automl_ as ground truth for metric if no metrics passed # and we have one created - if metric is None and self.automl_ is not None and self.automl_.metrics: + if metric is None and self.automl_ is not None and self.automl_._metrics: metric = self.automl_._metrics ensemble_class = self._resolve_ensemble_class(metric) diff --git a/examples/60_search/example_parallel_manual_spawning_cli.py b/examples/60_search/example_parallel_manual_spawning_cli.py index 8a1a9d5a7c..7df0907527 100644 --- a/examples/60_search/example_parallel_manual_spawning_cli.py +++ b/examples/60_search/example_parallel_manual_spawning_cli.py @@ -193,7 +193,7 @@ def cli_start_worker(scheduler_file_name): delete_tmp_folder_after_terminate=False, time_left_for_this_task=30, per_run_time_limit=10, - memory_limit=1024, + memory_limit=2048, tmp_folder=tmp_folder, seed=777, # n_jobs is ignored internally as we pass a dask client. From ffeb0e8ae2c5ce931d746f2f966665c0ade6de7d Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 3 Aug 2022 17:50:34 +0200 Subject: [PATCH 16/16] Implement requested changes, fix bug --- autosklearn/ensembles/abstract_ensemble.py | 12 +++++++++++- autosklearn/ensembles/ensemble_selection.py | 12 +----------- .../ensembles/multiobjective_dummy_ensemble.py | 15 ++------------- autosklearn/util/multiobjective.py | 2 +- 4 files changed, 15 insertions(+), 26 deletions(-) diff --git a/autosklearn/ensembles/abstract_ensemble.py b/autosklearn/ensembles/abstract_ensemble.py index 6b9d27d575..c4388b71fd 100644 --- a/autosklearn/ensembles/abstract_ensemble.py +++ b/autosklearn/ensembles/abstract_ensemble.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Dict, List, Sequence, Tuple, Union +from typing import Any, Dict, List, Sequence, Tuple, Union import numpy as np @@ -23,6 +23,16 @@ def __init__( ): pass + def __getstate__(self) -> Dict[str, Any]: + # Cannot serialize a metric if + # it is user defined. + # That is, if doing pickle dump + # the metric won't be the same as the + # one in __main__. we don't use the metric + # in the EnsembleSelection so this should + # be fine + return {key: value for key, value in self.__dict__.items() if key != "metrics"} + @abstractmethod def fit( self, diff --git a/autosklearn/ensembles/ensemble_selection.py b/autosklearn/ensembles/ensemble_selection.py index ef8859ac0d..e5e3c835b5 100644 --- a/autosklearn/ensembles/ensemble_selection.py +++ b/autosklearn/ensembles/ensemble_selection.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, List, Sequence, Tuple, Union +from typing import Dict, List, Sequence, Tuple, Union import random import warnings @@ -92,16 +92,6 @@ def __init__( # https://scikit-learn.org/stable/common_pitfalls.html#controlling-randomness self.random_state = random_state - def __getstate__(self) -> Dict[str, Any]: - # Cannot serialize a metric if - # it is user defined. - # That is, if doing pickle dump - # the metric won't be the same as the - # one in __main__. we don't use the metric - # in the EnsembleSelection so this should - # be fine - return {key: value for key, value in self.__dict__.items() if key != "metrics"} - def fit( self, base_models_predictions: List[np.ndarray], diff --git a/autosklearn/ensembles/multiobjective_dummy_ensemble.py b/autosklearn/ensembles/multiobjective_dummy_ensemble.py index 1d53f83831..8751507ecd 100644 --- a/autosklearn/ensembles/multiobjective_dummy_ensemble.py +++ b/autosklearn/ensembles/multiobjective_dummy_ensemble.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Sequence +from typing import Sequence import warnings @@ -59,11 +59,10 @@ def __init__( self.metric = [metrics] self.random_state = random_state self.backend = backend - self.pareto_set_: list[SingleModelEnsemble] | None = None @property def pareto_set(self) -> Sequence[AbstractEnsemble]: - if self.pareto_set_ is None: + if not hasattr(self, "pareto_set_"): raise NotFittedError("`pareto_set` not created, please call `fit()` first") return self.pareto_set_ @@ -219,13 +218,3 @@ def get_validation_performance(self) -> float: float """ return self.pareto_set[0].get_validation_performance() - - def __getstate__(self) -> dict[str, Any]: - # Cannot serialize a metric if - # it is user defined. - # That is, if doing pickle dump - # the metric won't be the same as the - # one in __main__. we don't use the metric - # in the EnsembleSelection so this should - # be fine - return {key: value for key, value in self.__dict__.items() if key != "metrics"} diff --git a/autosklearn/util/multiobjective.py b/autosklearn/util/multiobjective.py index 84a8356c16..f12b111153 100644 --- a/autosklearn/util/multiobjective.py +++ b/autosklearn/util/multiobjective.py @@ -32,7 +32,7 @@ def pareto_front(values: np.ndarray, *, is_loss: bool = True) -> np.ndarray: for i, c in enumerate(values): if is_efficient[i]: # Keep any point with a lower cost - is_efficient[is_efficient] = np.any(op(values, c), axis=1) + is_efficient[is_efficient] = np.any(op(values[is_efficient], c), axis=1) # And keep self is_efficient[i] = True