Source code for bigframes.ml.base

# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Wraps primitives for machine learning with BQML

This library is an evolving attempt to:

* implement BigQuery DataFrames API for BQML
* follow as close as possible the API design of SKLearn
    https://arxiv.org/pdf/1309.0238.pdf

"""

import abc
import typing
import warnings
from typing import Optional, TypeVar, Union

import bigframes_vendored.sklearn.base

import bigframes.exceptions as bfe
import bigframes.ml.utils as utils
import bigframes.pandas as bpd
from bigframes._tools import docs
from bigframes.ml import core


@docs.inherit_docs(bigframes_vendored.sklearn.base.BaseEstimator)
class BaseEstimator(abc.ABC):
    """
    A BigQuery DataFrames machine learning component follows sklearn API
    design Ref: https://bit.ly/3NyhKjN

    The estimator is the fundamental abstraction for all learning components. This includes learning
    algorithms, and also some preprocessing routines.

    This base class provides shared methods for inspecting parameters, and for building a consistent
    string representation of the component. By convention, the __init__ of all descendents will be
    assumed to be the list of hyperparameters.

    All descendents of this class should implement:

    .. code-block:: python

        def __init__(self, hyperparameter_1=default_1, hyperparameter_2=default_2, hyperparameter3, ...):
            '''Set hyperparameters'''
            self.hyperparameter_1 = hyperparameter_1
            self.hyperparameter_2 = hyperparameter_2
            self.hyperparameter3 = hyperparameter3
            ...

    Note: the object variable names must be exactly the same with parameter names. In order to utilize __repr__.

    fit(X, y) method is optional.

    The types of decendents of this class should be:

    1) Predictors
        These extend the interface with a .predict(self, x_test) method which predicts the target values
        according to the parameters that were calculated in .fit()

            def predict(self, x_test: Union[DataFrame, Series]) -> Union[DataFrame, Series]:
                '''Predict the target values according to the parameters that were calculated in .fit'''
                ...

    2) Transformers
        These extend the interface with .transform(self, x) and .fit_transform(x_train) methods, which
        apply data processing steps such as scaling that must be fitted to training data

            def transform(self, x: Union[DataFrame, Series]) -> Union[DataFrame, Series]:
                '''Transform the data according to the parameters that were calculated in .fit()'''
                ...

            def fit_transform(self, x_train: Union[DataFrame, Series], y_train: Union[DataFrame, Series]):
                '''Perform both fit() and transform()'''
                ...
    """

    def __init__(self):
        self._bqml_model: Optional[core.BqmlModel] = None

    def __repr__(self):
        """Print the estimator's constructor with all non-default parameter values."""

        # Estimator pretty printer adapted from Sklearn's, which is in turn an adaption of
        # the inbuilt pretty-printer in CPython
        import bigframes_vendored.cpython._pprint as adapted_pprint

        prettyprinter = adapted_pprint._EstimatorPrettyPrinter(
            compact=True, indent=1, indent_at_name=True, n_max_elements_to_show=30
        )

        return prettyprinter.pformat(self)


# TODO(garrettwu): refactor to reflect the actual property. Now the class contains .register() method.
class Predictor(BaseEstimator):
    """A BigQuery DataFrames ML Model base class that can be used to predict outputs."""

    @abc.abstractmethod
    def predict(self, X):
        pass

    _T = TypeVar("_T", bound="Predictor")

    def register(self: _T, vertex_ai_model_id: Optional[str] = None) -> _T:
        """Register the model to Vertex AI.

        After register, go to the Google Cloud console (https://console.cloud.google.com/vertex-ai/models)
        to manage the model registries.
        Refer to https://cloud.google.com/vertex-ai/docs/model-registry/introduction for more options.

        Args:
            vertex_ai_model_id (Optional[str], default None):
                Optional string id as model id in Vertex. If not set, will default to 'bigframes_{bq_model_id}'.
                Vertex Ai model id will be truncated to 63 characters due to its limitation.

        Returns:
            BigQuery DataFrames Model after register.
        """
        if not self._bqml_model:
            # TODO(garrettwu): find a more elegant way to do this.
            try:
                self._bqml_model = self._create_bqml_model()  # type: ignore
            except AttributeError:
                raise RuntimeError("A model must be trained before register.")
        self._bqml_model = typing.cast(core.BqmlModel, self._bqml_model)

        self._bqml_model.register(vertex_ai_model_id)
        return self

    @abc.abstractmethod
    def to_gbq(self, model_name, replace):
        pass


class TrainablePredictor(Predictor):
    """A BigQuery DataFrames ML Model base class that can be used to fit and predict outputs.

    Also the predictor can be attached to a pipeline with transformers."""

    @abc.abstractmethod
    def _fit(self, X, y, transforms=None):
        pass

    @abc.abstractmethod
    def score(self, X, y):
        pass


class SupervisedTrainablePredictor(TrainablePredictor):
    """A BigQuery DataFrames ML Supervised Model base class that can be used to fit and predict outputs.

    Need to provide both X and y in supervised tasks."""

    _T = TypeVar("_T", bound="SupervisedTrainablePredictor")

    def fit(
        self: _T,
        X: utils.ArrayType,
        y: utils.ArrayType,
    ) -> _T:
        return self._fit(X, y)


class SupervisedTrainableWithIdColPredictor(SupervisedTrainablePredictor):
    """Inherits from SupervisedTrainablePredictor,
    but adds an optional id_col parameter to fit()."""

    def __init__(self):
        super().__init__()
        self.id_col = None

    def _fit(
        self,
        X: utils.ArrayType,
        y: utils.ArrayType,
        transforms=None,
        id_col: Optional[utils.ArrayType] = None,
    ):
        return self

    def fit(
        self,
        X: utils.ArrayType,
        y: utils.ArrayType,
        transforms=None,
        id_col: Optional[utils.ArrayType] = None,
    ):
        self.id_col = id_col
        return self._fit(X, y, transforms=transforms, id_col=self.id_col)


class TrainableWithEvaluationPredictor(TrainablePredictor):
    """A BigQuery DataFrames ML Model base class that can be used to fit and predict outputs.

    Additional evaluation data can be provided to measure the model in the fit phase."""

    @abc.abstractmethod
    def _fit(self, X, y, transforms=None, X_eval=None, y_eval=None):
        pass

    @abc.abstractmethod
    def score(self, X, y):
        pass


class SupervisedTrainableWithEvaluationPredictor(TrainableWithEvaluationPredictor):
    """A BigQuery DataFrames ML Supervised Model base class that can be used to fit and predict outputs.

    Need to provide both X and y in supervised tasks.

    Additional X_eval and y_eval can be provided to measure the model in the fit phase.
    """

    _T = TypeVar("_T", bound="SupervisedTrainableWithEvaluationPredictor")

    def fit(
        self: _T,
        X: utils.ArrayType,
        y: utils.ArrayType,
        X_eval: Optional[utils.ArrayType] = None,
        y_eval: Optional[utils.ArrayType] = None,
    ) -> _T:
        return self._fit(X, y, X_eval=X_eval, y_eval=y_eval)


class UnsupervisedTrainablePredictor(TrainablePredictor):
    """A BigQuery DataFrames ML Unsupervised Model base class that can be used to fit and predict outputs.

    Only need to provide X (y is optional and ignored) in unsupervised tasks."""

    _T = TypeVar("_T", bound="UnsupervisedTrainablePredictor")

    def fit(
        self: _T,
        X: utils.ArrayType,
        y: Optional[utils.ArrayType] = None,
    ) -> _T:
        return self._fit(X, y)

    def fit_predict(
        self: _T,
        X: utils.ArrayType,
        y: Optional[utils.ArrayType] = None,
    ) -> _T:
        return self.fit(X).predict(X)


class RetriableRemotePredictor(BaseEstimator):
    def _predict_and_retry(
        self,
        bqml_model_predict_tvf: core.BqmlModel.TvfDef,
        X: bpd.DataFrame,
        options: dict,
        max_retries: int,
    ) -> bpd.DataFrame:
        assert self._bqml_model is not None

        df_result: Union[bpd.DataFrame, None] = None  # placeholder
        df_succ = df_fail = X
        for i in range(max_retries + 1):
            if i > 0 and df_fail.empty:
                break
            if i > 0 and df_succ.empty:
                msg = bfe.format_message("Can't make any progress, stop retrying.")
                warnings.warn(msg, category=RuntimeWarning)
                break

            df = bqml_model_predict_tvf.tvf(self._bqml_model, df_fail, options)

            success = df[bqml_model_predict_tvf.status_col].str.len() == 0
            df_succ = df[success]
            df_fail = df[~success]

            df_result = (
                bpd.concat([df_result, df_succ]) if df_result is not None else df_succ
            )

        df_result = typing.cast(
            bpd.DataFrame,
            bpd.concat([df_result, df_fail]) if df_result is not None else df_fail,
        )
        return df_result


class BaseTransformer(BaseEstimator):
    """Transformer base class."""

    @abc.abstractmethod
    def _keys(self):
        pass

    def _extract_output_names(self):
        """Extract transform output column names. Save the results to self._output_names."""
        assert self._bqml_model is not None

        output_names = []
        for transform_col in self._bqml_model._model._properties["transformColumns"]:
            transform_col_dict = typing.cast(dict, transform_col)
            # pass the columns that are not transformed
            if "transformSql" not in transform_col_dict:
                continue
            output_names.append(transform_col_dict["name"])

        self._output_names = output_names

    def __eq__(self, other) -> bool:
        return type(self) is type(other) and self._keys() == other._keys()

    def __hash__(self) -> int:
        return hash(self._keys())

    _T = TypeVar("_T", bound="BaseTransformer")

    def to_gbq(self: _T, model_name: str, replace: bool = False) -> _T:
        """Save the transformer as a BigQuery model.

        Args:
            model_name (str):
                The name of the model.
            replace (bool, default False):
                Determine whether to replace if the model already exists. Default to False.

        Returns:
            Saved transformer."""
        if not self._bqml_model:
            raise RuntimeError("A transformer must be fitted before it can be saved")

        new_model = self._bqml_model.copy(model_name, replace)
        return new_model.session.read_gbq_model(model_name)


class Transformer(BaseTransformer):
    """A BigQuery DataFrames Transformer base class that transforms data.

    Also the transformers can be attached to a pipeline with a predictor."""

    @abc.abstractmethod
    def fit(self, X, y):
        pass

    @abc.abstractmethod
    def transform(self, X):
        pass

    def fit_transform(
        self,
        X: utils.ArrayType,
        y: Optional[utils.ArrayType] = None,
    ) -> bpd.DataFrame:
        return self.fit(X, y).transform(X)


class LabelTransformer(BaseTransformer):
    """A BigQuery DataFrames Label Transformer base class that transforms data.

    Also the transformers can be attached to a pipeline with a predictor."""

    @abc.abstractmethod
    def fit(self, y):
        pass

    @abc.abstractmethod
    def transform(self, y):
        pass

    def fit_transform(
        self,
        y: utils.ArrayType,
    ) -> bpd.DataFrame:
        return self.fit(y).transform(y)