Issue in extending auto-sklearn with new categorical encoders

[svsaraf112]

I am trying to extend auto-sklearn with new categorical encoders like catboost encoder, target encoder etc. I am currently using the scikit-learn-contrib/category_encoder package to do so. For spot check, I am just using the catboost encoder and forbidding other encoders (OHE) to see if I get acceptable results.

Here's the script for catboost encoder (which I have added under autosklearn->pipeline->data_preprocessing->categorical-encoding)

from autosklearn.pipeline.components.base import AutoSklearnPreprocessingAlgorithm
from autosklearn.pipeline.constants import DENSE, SPARSE, UNSIGNED_DATA, INPUT
import category_encoders.cat_boost as catboost_enc

class CatBoostEncoder(AutoSklearnPreprocessingAlgorithm):
    def __init__(self, random_state=None):
        self.random_state = random_state

    def fit(self, X, y=None):
        self.preprocessor = catboost_enc()
        self.preprocessor.fit(X, y)
        return self

    def transform(self, X):
        if self.preprocessor is None:
            raise NotImplementedError()
        return self.preprocessor.transform(X)

    def fit_transform(self, X, y=None):
        return self.fit(X, y).transform(X)

    @staticmethod
    def get_properties(dataset_properties=None):
        return {'shortname': 'CatBoostEnc',
                'name': 'CatBoost Encoder',
                'handles_regression': True,
                'handles_classification': True,
                'handles_multiclass': True,
                'handles_multilabel': True,
                'handles_multioutput': True,
                # TODO find out of this is right!
                'handles_sparse': True,
                'handles_dense': True,
                'input': (DENSE, SPARSE, UNSIGNED_DATA),
                'output': (INPUT,), }

    @staticmethod
    def get_hyperparameter_search_space(dataset_properties=None):
        return ConfigurationSpace()

For spot check I print the encoders available during the run-time, which are:

But the output pipeline looks something like this:

Is there something that I am missing here? Is there a link I can refer to to add new categorical encoders? Would appreciate any help.

Thanks

[mfeurer]

Hi @svsaraf112, Thanks a lot for reporting this issue.

I just tried this myself, and found two issues:

1. Auto-sklearn did not pass y to the target algorithm, I just pushed a fix to the development branch (5f5e0db). This will be available in the next release.
2. I think import category_encoders.cat_boost as catboost_enc is wrong, as you want to import the CatBoostEncoder class. Please see the script below, which I think does what you aim for. At the end it will print the ensemble, and you should see models using the CatBoostEncoder.

# -*- encoding: utf-8 -*-
"""
=============
Feature Types
=============

In *auto-sklearn* it is possible to specify the feature types of a dataset when calling the method
:meth:`fit() <autosklearn.classification.AutoSklearnClassifier.fit>` by specifying the argument
``feat_type``. The following example demonstrates a way it can be done.

Additionally, you can provide a properly formatted pandas DataFrame, and the feature
types will be automatically inferred, as demonstrated in
`Pandas Train and Test inputs <example_pandas_train_test.html>`_.
"""

from ConfigSpace import ConfigurationSpace
import numpy as np
import sklearn.model_selection
import sklearn.datasets
import sklearn.metrics

import autosklearn.classification
from autosklearn.pipeline.components.base import AutoSklearnPreprocessingAlgorithm
from autosklearn.pipeline.constants import DENSE, SPARSE, UNSIGNED_DATA, INPUT
from category_encoders.cat_boost import CatBoostEncoder as CBE


class CatBoostEncoder(AutoSklearnPreprocessingAlgorithm):
    def __init__(self, random_state=None):
        self.random_state = random_state

    def fit(self, X, y):
        print(X, y)
        self.preprocessor = CBE(random_state=self.random_state)
        self.preprocessor.fit(X, y)
        return self

    def transform(self, X):
        if self.preprocessor is None:
            raise NotImplementedError()
        return self.preprocessor.transform(X)

    def fit_transform(self, X, y=None):
        return self.fit(X, y).transform(X)

    @staticmethod
    def get_properties(dataset_properties=None):
        return {'shortname': 'CatBoostEnc',
                'name': 'CatBoost Encoder',
                'is_deterministic': False,
                'handles_regression': True,
                'handles_classification': True,
                'handles_multiclass': True,
                'handles_multilabel': True,
                'handles_multioutput': True,
                'input': (DENSE, SPARSE, UNSIGNED_DATA),
                'output': (INPUT,), }

    @staticmethod
    def get_hyperparameter_search_space(dataset_properties=None):
        return ConfigurationSpace()


import autosklearn.pipeline.components.data_preprocessing.categorical_encoding
autosklearn.pipeline.components.data_preprocessing.categorical_encoding.add_ohe(CatBoostEncoder)


############################################################################
# Data Loading
# ============
# Load Australian dataset from https://www.openml.org/d/40981
bunch = data = sklearn.datasets.fetch_openml(data_id=40981, as_frame=True)
y = bunch['target'].to_numpy()
X = bunch['data'].to_numpy(np.float)

X_train, X_test, y_train, y_test = \
     sklearn.model_selection.train_test_split(X, y, random_state=1)

# Auto-sklearn can automatically recognize categorical/numerical data from a pandas
# DataFrame. This example highlights how the user can provide the feature types,
# when using numpy arrays, as there is no per-column dtype in this case.
# feat_type is a list that tags each column from a DataFrame/ numpy array / list
# with the case-insensitive string categorical or numerical, accordingly.
feat_type = ['Categorical' if x.name == 'category' else 'Numerical' for x in bunch['data'].dtypes]

############################################################################
# Build and fit a classifier
# ==========================

cls = autosklearn.classification.AutoSklearnClassifier(
    time_left_for_this_task=60,
    # Bellow two flags are provided to speed up calculations
    # Not recommended for a real implementation
    initial_configurations_via_metalearning=0,
)
cls.fit(X_train, y_train, X_test, y_test, feat_type=feat_type)

###########################################################################
# Get the Score of the final ensemble
# ===================================

predictions = cls.predict(X_test)
print("Accuracy score", sklearn.metrics.accuracy_score(y_test, predictions))
print(cls.show_models())

[BradKML]

I don't think CatBoost included in the default classifier atm as I can't find it in the source code, maybe a one-stop community helper script repository would be useful.