# SPDX-FileCopyrightText: skforecast team
# SPDX-FileCopyrightText: 2026 bartzbeielstein
# SPDX-License-Identifier: AGPL-3.0-or-later AND BSD-3-Clause

from __future__ import annotations
from typing import Callable
from copy import deepcopy
import warnings
import numpy as np
import pandas as pd
from joblib import Parallel, delayed, cpu_count
from tqdm.auto import tqdm

from spotforecast2_safe.forecaster.metrics import add_y_train_argument, _get_metric
from spotforecast2_safe.exceptions import (
    LongTrainingWarning,
    IgnoredArgumentWarning,
    set_skforecast_warnings,
)
from spotforecast2_safe.model_selection.split_ts_cv import TimeSeriesFold
from spotforecast2_safe.model_selection.split_one_step import OneStepAheadFold
from spotforecast2_safe.model_selection.utils_common import (
    check_backtesting_input,
    check_one_step_ahead_input,
    select_n_jobs_backtesting,
)


def _backtesting_forecaster(
    forecaster: object,
    y: pd.Series,
    cv: TimeSeriesFold,
    metric: str | Callable | list[str | Callable],
    exog: pd.Series | pd.DataFrame | None = None,
    interval: float | list[float] | tuple[float] | str | object | None = None,
    interval_method: str = "bootstrapping",
    n_boot: int = 250,
    use_in_sample_residuals: bool = True,
    use_binned_residuals: bool = True,
    random_state: int = 123,
    return_predictors: bool = False,
    n_jobs: int | str = "auto",
    verbose: bool = False,
    show_progress: bool = True,
    suppress_warnings: bool = False,
) -> tuple[pd.DataFrame, pd.DataFrame]:
    """
    Backtesting of forecaster model following the folds generated by the TimeSeriesFold
    class and using the metric(s) provided.

    If `forecaster` is already trained and `initial_train_size` is set to `None` in the
    TimeSeriesFold class, no initial train will be done and all data will be used
    to evaluate the model. However, the first `len(forecaster.last_window)` observations
    are needed to create the initial predictors, so no predictions are calculated for
    them.

    A copy of the original forecaster is created so that it is not modified during the
    process.

    Args:
        forecaster (ForecasterRecursive, ForecasterDirect, ForecasterEquivalentDate):
            Forecaster model.
        y (pd.Series): Training time series.
        cv (TimeSeriesFold): TimeSeriesFold object with the information needed to
            split the data into folds.
        metric (str | Callable | list): Metric used to quantify the goodness of fit
            of the model.

            - If `str`: {'mean_squared_error', 'mean_absolute_error',
              'mean_absolute_percentage_error', 'mean_squared_log_error',
              'mean_absolute_scaled_error', 'root_mean_squared_scaled_error'}
            - If `Callable`: Function with arguments `y_true`, `y_pred` and `y_train`
              (Optional) that returns a float.
            - If `list`: List containing multiple strings and/or Callables.
        exog (pd.Series | pd.DataFrame, optional): Exogenous variable/s included as
            predictor/s. Must have the same number of observations as `y` and should
            be aligned so that y[i] is regressed on exog[i]. Defaults to None.
        interval (float | list | tuple | str | object, optional): Specifies whether
            probabilistic predictions should be estimated and the method to use.
            The following options are supported:

            - If `float`, represents the nominal (expected) coverage (between 0 and 1).
            For instance, `interval=0.95` corresponds to `[2.5, 97.5]` percentiles.
            - If `list` or `tuple`: Sequence of percentiles to compute, each value must
              be between 0 and 100 inclusive. For example, a 95% confidence interval can
              be specified as `interval = [2.5, 97.5]` or multiple percentiles (e.g. 10,
              50 and 90) as `interval = [10, 50, 90]`.
            - If 'bootstrapping' (str): `n_boot` bootstrapping predictions will be
              generated.
            - If scipy.stats distribution object, the distribution parameters will
              be estimated for each prediction.
            - If None, no probabilistic predictions are estimated.
            Defaults to None.
        interval_method (str, optional): Technique used to estimate prediction
            intervals. Available options:

            - 'bootstrapping': Bootstrapping is used to generate prediction intervals.
            - 'conformal': Employs the conformal prediction split method for
              interval estimation.
            Defaults to 'bootstrapping'.
        n_boot (int, optional): Number of bootstrapping iterations to perform when
            estimating prediction intervals. Defaults to 250.
        use_in_sample_residuals (bool, optional): If `True`, residuals from the
            training data are used as proxy of prediction error to create predictions.
            If `False`, out of sample residuals (calibration) are used.
            Out-of-sample residuals must be precomputed using Forecaster's
            `set_out_sample_residuals()` method. Defaults to True.
        use_binned_residuals (bool, optional): If `True`, residuals are selected
            based on the predicted values (binned selection).
            If `False`, residuals are selected randomly. Defaults to True.
        random_state (int, optional): Seed for the random number generator to
            ensure reproducibility. Defaults to 123.
        return_predictors (bool, optional): If `True`, the predictors used to make
            the predictions are also returned. Defaults to False.
        n_jobs (int | str, optional): The number of jobs to run in parallel. If `-1`,
            then the number of jobs is set to the number of cores. If 'auto', `n_jobs`
            is set using the function `skforecast.utils.select_n_jobs_backtesting`.
            Defaults to 'auto'.
        verbose (bool, optional): Print number of folds and index of training and
            validation sets used for backtesting. Defaults to False.
        show_progress (bool, optional): Whether to show a progress bar.
            Defaults to True.
        suppress_warnings (bool, optional): If `True`, spotforecast warnings will be
            suppressed during the backtesting process. See
            `spotforecast.exceptions.warn_skforecast_categories` for more information.
            Defaults to False.

    Returns:
        tuple (pd.DataFrame, pd.DataFrame):
            - metric_values: Value(s) of the metric(s).
            - backtest_predictions: Value of predictions. The DataFrame includes
              the following columns:

              - fold: Indicates the fold number where the prediction was made.
              - pred: Predicted values for the corresponding series and time steps.

              If `interval` is not `None`, additional columns are included depending
              on the method:

              - For `float`: Columns `lower_bound` and `upper_bound`.
              - For `list` or `tuple` of 2 elements: Columns `lower_bound` and
                `upper_bound`.
              - For `list` or `tuple` with multiple percentiles: One column per
                percentile (e.g., `p_10`, `p_50`, `p_90`).
              - For `'bootstrapping'`: One column per bootstrapping iteration
                (e.g., `pred_boot_0`, `pred_boot_1`, ..., `pred_boot_n`).
              - For `scipy.stats` distribution objects: One column for each
                estimated parameter of the distribution (e.g., `loc`, `scale`).

              If `return_predictors` is `True`, one column per predictor is created.

              Depending on the relation between `steps` and `fold_stride`, the output
              may include repeated indexes (if `fold_stride < steps`) or gaps
              (if `fold_stride > steps`). See Notes below for more details.

    Notes:
        Note on `fold_stride` vs. `steps`:

        - If `fold_stride == steps`, test sets are placed back-to-back without overlap.
          Each observation appears only once in the output DataFrame, so the
          index is unique.
        - If `fold_stride < steps`, test sets overlap. Multiple forecasts are
          generated for the same observations and, therefore, the output
          DataFrame contains repeated indexes.
        - If `fold_stride > steps`, there are gaps between consecutive test sets.
          Some observations in the series will not have associated predictions,
          so the output DataFrame has non-contiguous indexes.

    References:
        .. [1] Forecasting: Principles and Practice (3rd ed) Rob J Hyndman and
               George Athanasopoulos. https://otexts.com/fpp3/prediction-intervals.html
        .. [2] MAPIE - Model Agnostic Prediction Interval Estimator.
               https://mapie.readthedocs.io/en/stable/theoretical_description_regression.html#the-split-method
    """

    set_skforecast_warnings(suppress_warnings, action="ignore")

    forecaster = deepcopy(forecaster)
    is_regression = forecaster.__spotforecast_tags__["forecaster_task"] == "regression"
    cv = deepcopy(cv)

    cv.set_params(
        {
            "window_size": forecaster.window_size,
            "differentiation": forecaster.differentiation_max,
            "return_all_indexes": False,
            "verbose": verbose,
        }
    )

    refit = cv.refit
    overlapping_folds = cv.overlapping_folds

    if n_jobs == "auto":
        n_jobs = select_n_jobs_backtesting(forecaster=forecaster, refit=refit)
    elif not isinstance(refit, bool) and refit != 1 and n_jobs != 1:
        warnings.warn(
            "If `refit` is an integer other than 1 (intermittent refit). `n_jobs` "
            "is set to 1 to avoid unexpected results during parallelization.",
            IgnoredArgumentWarning,
        )
        n_jobs = 1
    else:
        n_jobs = n_jobs if n_jobs > 0 else cpu_count()

    if not isinstance(metric, list):
        metrics = [
            (
                _get_metric(metric=metric)
                if isinstance(metric, str)
                else add_y_train_argument(metric)
            )
        ]
    else:
        metrics = [
            _get_metric(metric=m) if isinstance(m, str) else add_y_train_argument(m)
            for m in metric
        ]

    store_in_sample_residuals = True if use_in_sample_residuals else False
    if interval is None:
        forecaster._probabilistic_mode = False
    elif use_binned_residuals:
        forecaster._probabilistic_mode = "binned"
    else:
        forecaster._probabilistic_mode = "no_binned"

    folds = cv.split(X=y, as_pandas=False)
    initial_train_size = cv.initial_train_size
    window_size = cv.window_size
    gap = cv.gap

    if initial_train_size is not None:
        # NOTE: This allows for parallelization when `refit` is `False`. The initial
        # Forecaster fit occurs outside of the auxiliary function.
        exog_train = exog.iloc[:initial_train_size,] if exog is not None else None
        forecaster.fit(
            y=y.iloc[:initial_train_size,],
            exog=exog_train,
            store_in_sample_residuals=store_in_sample_residuals,
        )
        folds[0][5] = False

    if refit:
        n_of_fits = int(len(folds) / refit)
        if type(forecaster).__name__ != "ForecasterDirect" and n_of_fits > 50:
            warnings.warn(
                f"The forecaster will be fit {n_of_fits} times. This can take substantial"
                f" amounts of time. If not feasible, try with `refit = False`.\n",
                LongTrainingWarning,
            )
        elif (
            type(forecaster).__name__ == "ForecasterDirect"
            and n_of_fits * forecaster.max_step > 50
        ):
            warnings.warn(
                f"The forecaster will be fit {n_of_fits * forecaster.max_step} times "
                f"({n_of_fits} folds * {forecaster.max_step} estimators). This can take "
                f"substantial amounts of time. If not feasible, try with `refit = False`.\n",
                LongTrainingWarning,
            )

    if show_progress:
        folds = tqdm(folds)

    def _fit_predict_forecaster(
        fold,
        forecaster,
        y,
        exog,
        store_in_sample_residuals,
        gap,
        interval,
        interval_method,
        n_boot,
        use_in_sample_residuals,
        use_binned_residuals,
        random_state,
        return_predictors,
        is_regression,
    ) -> pd.DataFrame:
        """
        Fit the forecaster and predict `steps` ahead. This is an auxiliary
        function used to parallelize the backtesting_forecaster function.
        """

        train_iloc_start = fold[1][0]
        train_iloc_end = fold[1][1]
        last_window_iloc_start = fold[2][0]
        last_window_iloc_end = fold[2][1]
        test_iloc_start = fold[3][0]
        test_iloc_end = fold[3][1]

        if fold[5] is False:
            # When the model is not fitted, last_window must be updated to include
            # the data needed to make predictions.
            last_window_y = y.iloc[last_window_iloc_start:last_window_iloc_end]
        else:
            # The model is fitted before making predictions. If `fixed_train_size`
            # the train size doesn't increase but moves by `steps` in each iteration.
            # If `False` the train size increases by `steps` in each iteration.
            y_train = y.iloc[train_iloc_start:train_iloc_end,]
            exog_train = (
                exog.iloc[train_iloc_start:train_iloc_end,]
                if exog is not None
                else None
            )
            last_window_y = None
            forecaster.fit(
                y=y_train,
                exog=exog_train,
                store_in_sample_residuals=store_in_sample_residuals,
            )

        next_window_exog = (
            exog.iloc[test_iloc_start:test_iloc_end,] if exog is not None else None
        )

        steps = len(range(test_iloc_start, test_iloc_end))
        if type(forecaster).__name__ == "ForecasterDirect" and gap > 0:
            # Select only the steps that need to be predicted if gap > 0
            test_no_gap_iloc_start = fold[4][0]
            test_no_gap_iloc_end = fold[4][1]
            n_steps = test_no_gap_iloc_end - test_no_gap_iloc_start
            steps = list(range(gap + 1, gap + 1 + n_steps))

        preds = []
        if is_regression:
            if interval is not None:
                kwargs_interval = {
                    "steps": steps,
                    "last_window": last_window_y,
                    "exog": next_window_exog,
                    "n_boot": n_boot,
                    "use_in_sample_residuals": use_in_sample_residuals,
                    "use_binned_residuals": use_binned_residuals,
                    "random_state": random_state,
                }
                if interval_method == "bootstrapping":
                    if interval == "bootstrapping":
                        # Return all bootstrap samples
                        pred = forecaster.predict_bootstrapping(**kwargs_interval)
                    else:
                        # Use predict_interval with bootstrapping method
                        pred = forecaster.predict_interval(
                            method="bootstrapping", interval=interval, **kwargs_interval
                        )
                    preds.append(pred)
                else:
                    # Use conformal method
                    pred = forecaster.predict_interval(
                        method="conformal", interval=interval, **kwargs_interval
                    )
                    preds.append(pred)

            # NOTE: This is done after probabilistic predictions to avoid repeating
            # the same checks.
            if interval is None or interval_method != "conformal":
                pred = forecaster.predict(
                    steps=steps,
                    last_window=last_window_y,
                    exog=next_window_exog,
                    check_inputs=True if interval is None else False,
                )
                preds.insert(0, pred)
        else:
            pred = forecaster.predict_proba(
                steps=steps, last_window=last_window_y, exog=next_window_exog
            )
            preds.append(pred)

        if return_predictors:
            pred = forecaster.create_predict_X(
                steps=steps,
                last_window=last_window_y,
                exog=next_window_exog,
                check_inputs=False,
            )
            preds.append(pred)

        if len(preds) == 1:
            pred = preds[0]
        else:
            pred = pd.concat(preds, axis=1)

        if type(forecaster).__name__ != "ForecasterDirect" and gap > 0:
            pred = pred.iloc[gap:,]

        return pred

    kwargs_fit_predict_forecaster = {
        "forecaster": forecaster,
        "y": y,
        "exog": exog,
        "store_in_sample_residuals": store_in_sample_residuals,
        "gap": gap,
        "interval": interval,
        "interval_method": interval_method,
        "n_boot": n_boot,
        "use_in_sample_residuals": use_in_sample_residuals,
        "use_binned_residuals": use_binned_residuals,
        "random_state": random_state,
        "return_predictors": return_predictors,
        "is_regression": is_regression,
    }
    backtest_predictions = Parallel(n_jobs=n_jobs)(
        delayed(_fit_predict_forecaster)(fold=fold, **kwargs_fit_predict_forecaster)
        for fold in folds
    )
    fold_labels = [
        np.repeat(fold[0], backtest_predictions[i].shape[0])
        for i, fold in enumerate(folds)
    ]

    backtest_predictions = pd.concat(backtest_predictions)
    if isinstance(backtest_predictions, pd.Series):
        backtest_predictions = backtest_predictions.to_frame()

    if not is_regression:
        proba_cols = [f"{cls}_proba" for cls in forecaster.classes_]
        idx_max = backtest_predictions[proba_cols].to_numpy().argmax(axis=1)
        backtest_predictions.insert(0, "pred", np.array(forecaster.classes_)[idx_max])

    backtest_predictions.insert(0, "fold", np.concatenate(fold_labels))

    train_indexes = []
    for i, fold in enumerate(folds):
        fit_fold = fold[-1]
        if i == 0 or fit_fold:
            # NOTE: When using a scaled metric, `y_train` doesn't include the
            # first window_size observations used to create the predictors and/or
            # rolling features.
            train_iloc_start = fold[1][0] + window_size
            train_iloc_end = fold[1][1]
            train_indexes.append(np.arange(train_iloc_start, train_iloc_end))

    train_indexes = np.unique(np.concatenate(train_indexes))
    y_train = y.iloc[train_indexes]

    backtest_predictions_for_metrics = backtest_predictions
    if overlapping_folds:
        backtest_predictions_for_metrics = backtest_predictions_for_metrics.loc[
            ~backtest_predictions_for_metrics.index.duplicated(keep="last")
        ]

    y_true = y.loc[backtest_predictions_for_metrics.index]
    y_pred = backtest_predictions_for_metrics["pred"]
    metric_values = [
        [m(y_true=y_true, y_pred=y_pred, y_train=y_train) for m in metrics]
    ]

    metric_values = pd.DataFrame(
        data=metric_values, columns=[m.__name__ for m in metrics]
    )

    set_skforecast_warnings(suppress_warnings, action="default")

    return metric_values, backtest_predictions


def backtesting_forecaster_one_step(
    forecaster: object,
    y: pd.Series,
    cv: OneStepAheadFold,
    metric: str | Callable | list[str | Callable],
    exog: pd.Series | pd.DataFrame | None = None,
    interval: float | list[float] | tuple[float] | str | object | None = None,
    interval_method: str = "bootstrapping",
    n_boot: int = 250,
    use_in_sample_residuals: bool = True,
    use_binned_residuals: bool = True,
    random_state: int = 123,
    return_predictors: bool = False,
    n_jobs: int | str = "auto",
    verbose: bool = False,
    show_progress: bool = True,
    suppress_warnings: bool = False,
) -> tuple[pd.DataFrame, pd.DataFrame]:
    """
    Backtesting of forecaster model using one-step-ahead predictions.

    Args:
        forecaster (ForecasterRecursive, ForecasterDirect, ForecasterEquivalentDate):
            Forecaster model.
        y (pd.Series): Training time series.
        cv (OneStepAheadFold): OneStepAheadFold object with the information needed to
            split the data into folds.
        metric (str | Callable | list): Metric used to quantify the goodness of fit
            of the model.
        exog (pd.Series | pd.DataFrame, optional): Exogenous variable/s included as
            predictor/s. Defaults to None.
        interval (float | list | tuple | str | object, optional): Specifies whether
            probabilistic predictions should be estimated.
        interval_method (str, optional): Technique used to estimate prediction intervals.
        n_boot (int, optional): Number of bootstrapping iterations.
        use_in_sample_residuals (bool, optional): Use residuals from training data.
        use_binned_residuals (bool, optional): Use binned residuals for intervals.
        random_state (int, optional): Seed for reproducibility.
        return_predictors (bool, optional): Return predictors used for each prediction.
        n_jobs (int | str, optional): Number of jobs to run in parallel.
        verbose (bool, optional): Print information about the process.
        show_progress (bool, optional): Whether to show a progress bar.
        suppress_warnings (bool, optional): Suppress spotforecast warnings.

    Returns:
        tuple (pd.DataFrame, pd.DataFrame):
            - metric_values: Value(s) of the metric(s).
            - backtest_predictions: Value of predictions.

    Notes:
        This function is designed for one-step-ahead backtesting,
        where predictions are made for the next time step using the most recent data. The function handles the fitting and
        prediction process for each fold defined in the `OneStepAheadFold` object,
        and calculates the specified metric(s) based on the true and predicted values.
        Depending on the `interval` and `interval_method` parameters, it can also generate probabilistic predictions.

    Examples:
        >>> from spotforecast2_safe.forecaster import ForecasterRecursive
        >>> from spotforecast2_safe.model_selection.split_one_step import OneStepAheadFold
        >>> from spotforecast2_safe.model_selection.validation import backtesting_forecaster_one_step
        >>> # Create a forecaster and a one-step-ahead fold
        >>> import numpy as np
        >>> import pandas as pd
        >>> from sklearn.linear_model import LinearRegression
        >>> y = pd.Series(np.random.randn(100), name='y')
        >>> forecaster = ForecasterRecursive(estimator=LinearRegression(), lags=5)
        >>> cv = OneStepAheadFold(initial_train_size=20, window_size=5)
        >>> # Perform backtesting
        >>> metric_values, backtest_predictions = backtesting_forecaster_one_step(
        ...     forecaster=forecaster,
        ...     y=y,
        ...     cv=cv,
        ...     metric='mean_squared_error',
        ...     exog=None,
        ...     interval=0.95,
        ...     interval_method='bootstrapping',
        ...     n_boot=20,
        ...     use_in_sample_residuals=True,
        ...     use_binned_residuals=False,
        ...     random_state=42,
        ...     return_predictors=False,
        ...     n_jobs=1,
        ...     verbose=True,
        ...     show_progress=True,
        ...     suppress_warnings=False
        ... )
        # Note: For reliable bootstrapping with binned residuals, use a sufficiently large series and value spread.
        # For random data, use_binned_residuals=False.
        # TODO: Setting return_predictors=True requires ForecasterRecursive.create_predict_X().

    """

    set_skforecast_warnings(suppress_warnings, action="ignore")

    forecaster = deepcopy(forecaster)
    cv = deepcopy(cv)

    check_one_step_ahead_input(
        forecaster=forecaster,
        cv=cv,
        y=y,
        metric=metric,
        exog=exog,
        show_progress=show_progress,
        suppress_warnings=suppress_warnings,
    )

    cv.set_params(
        {
            "window_size": forecaster.window_size,
            "differentiation": forecaster.differentiation_max,
            "verbose": verbose,
        }
    )

    if not isinstance(metric, list):
        metrics = [
            (
                _get_metric(metric=metric)
                if isinstance(metric, str)
                else add_y_train_argument(metric)
            )
        ]
    else:
        metrics = [
            _get_metric(metric=m) if isinstance(m, str) else add_y_train_argument(m)
            for m in metric
        ]

    # Split data to get training set for fitting
    X_train, y_train, X_test, y_test = forecaster._train_test_split_one_step_ahead(
        y=y, initial_train_size=cv.initial_train_size, exog=exog
    )

    # Fit the forecaster
    forecaster.fit(
        y=y_train,
        exog=exog.iloc[: cv.initial_train_size] if exog is not None else None,
        store_in_sample_residuals=use_in_sample_residuals,
    )

    # Predictions
    # NOTE: For one-step-ahead, we can use the estimator directly on X_test
    # which contains all necessary lags and exog features.
    if interval is not None:
        if interval_method == "bootstrapping":
            if interval == "bootstrapping":
                backtest_predictions = forecaster.predict_bootstrapping(
                    steps=len(X_test),
                    last_window=None,
                    exog=(
                        exog.iloc[cv.initial_train_size :] if exog is not None else None
                    ),
                    n_boot=n_boot,
                    use_in_sample_residuals=use_in_sample_residuals,
                    use_binned_residuals=use_binned_residuals,
                    random_state=random_state,
                    in_sample_predict=True,  # This might be needed if supported
                )
            else:
                backtest_predictions = forecaster.predict_interval(
                    steps=len(X_test),
                    last_window=None,
                    exog=(
                        exog.iloc[cv.initial_train_size :] if exog is not None else None
                    ),
                    method="bootstrapping",
                    interval=interval,
                    n_boot=n_boot,
                    use_in_sample_residuals=use_in_sample_residuals,
                    use_binned_residuals=use_binned_residuals,
                    random_state=random_state,
                )
        else:
            backtest_predictions = forecaster.predict_interval(
                steps=len(X_test),
                last_window=None,
                exog=exog.iloc[cv.initial_train_size :] if exog is not None else None,
                method="conformal",
                interval=interval,
                n_boot=n_boot,
                use_in_sample_residuals=use_in_sample_residuals,
                use_binned_residuals=use_binned_residuals,
                random_state=random_state,
            )

        # Add 'pred' column
        if "pred" not in backtest_predictions.columns:
            pred = forecaster.estimator.predict(X_test).ravel()
            backtest_predictions.insert(0, "pred", pred)
    else:
        pred = forecaster.estimator.predict(X_test).ravel()
        backtest_predictions = pd.DataFrame({"pred": pred}, index=y_test.index)

    if return_predictors:
        X_predict = forecaster.create_predict_X(
            steps=len(X_test), last_window=None, exog=X_test
        )
        backtest_predictions = pd.concat([backtest_predictions, X_predict], axis=1)

    backtest_predictions.insert(0, "fold", 0)

    # Calculate metrics
    y_true = y_test
    # Metrics expect y_true, y_pred, y_train
    # y_train should not include first window_size observations
    y_train_metrics = y_train.iloc[forecaster.window_size :]

    metric_values = []
    for m in metrics:
        metric_values.append(
            m(
                y_true=y_true,
                y_pred=backtest_predictions["pred"],
                y_train=y_train_metrics,
            )
        )

    metric_values = pd.DataFrame(
        data=[metric_values], columns=[m.__name__ for m in metrics]
    )

    set_skforecast_warnings(suppress_warnings, action="default")

    return metric_values, backtest_predictions


def backtesting_forecaster(
    forecaster: object,
    y: pd.Series,
    cv: TimeSeriesFold,
    metric: str | Callable | list[str | Callable],
    exog: pd.Series | pd.DataFrame | None = None,
    interval: float | list[float] | tuple[float] | str | object | None = None,
    interval_method: str = "bootstrapping",
    n_boot: int = 250,
    use_in_sample_residuals: bool = True,
    use_binned_residuals: bool = True,
    random_state: int = 123,
    return_predictors: bool = False,
    n_jobs: int | str = "auto",
    verbose: bool = False,
    show_progress: bool = True,
    suppress_warnings: bool = False,
) -> tuple[pd.DataFrame, pd.DataFrame]:
    """
    Backtesting of forecaster model following the folds generated by the TimeSeriesFold
    class and using the metric(s) provided.

    If `forecaster` is already trained and `initial_train_size` is set to `None` in the
    TimeSeriesFold class, no initial train will be done and all data will be used
    to evaluate the model. However, the first `len(forecaster.last_window)` observations
    are needed to create the initial predictors, so no predictions are calculated for
    them.

    A copy of the original forecaster is created so that it is not modified during
    the process.

    Args:
        forecaster (ForecasterRecursive, ForecasterDirect, ForecasterEquivalentDate):
            Forecaster model.
        y (pd.Series): Training time series.
        cv (TimeSeriesFold): TimeSeriesFold object with the information needed to
            split the data into folds.
        metric (str | Callable | list): Metric used to quantify the goodness of fit
            of the model.

            - If `str`: {'mean_squared_error', 'mean_absolute_error',
              'mean_absolute_percentage_error', 'mean_squared_log_error',
              'mean_absolute_scaled_error', 'root_mean_squared_scaled_error'}
            - If `Callable`: Function with arguments `y_true`, `y_pred` and `y_train`
              (Optional) that returns a float.
            - If `list`: List containing multiple strings and/or Callables.
        exog (pd.Series | pd.DataFrame, optional): Exogenous variable/s included as
            predictor/s. Must have the same number of observations as `y` and should
            be aligned so that y[i] is regressed on exog[i]. Defaults to None.
        interval (float | list | tuple | str | object, optional): Specifies whether
            probabilistic predictions should be estimated and the method to use.
            The following options are supported:

            - If `float`, represents the nominal (expected) coverage (between 0 and 1).
            For instance, `interval=0.95` corresponds to `[2.5, 97.5]` percentiles.
            - If `list` or `tuple`: Sequence of percentiles to compute, each value must
              be between 0 and 100 inclusive. For example, a 95% confidence interval can
              be specified as `interval = [2.5, 97.5]` or multiple percentiles (e.g. 10,
              50 and 90) as `interval = [10, 50, 90]`.
            - If 'bootstrapping' (str): `n_boot` bootstrapping predictions will be
              generated.
            - If scipy.stats distribution object, the distribution parameters will
              be estimated for each prediction.
            - If None, no probabilistic predictions are estimated.
            Defaults to None.
        interval_method (str, optional): Technique used to estimate prediction
            intervals. Available options:

            - 'bootstrapping': Bootstrapping is used to generate prediction intervals.
            - 'conformal': Employs the conformal prediction split method for
              interval estimation.
            Defaults to 'bootstrapping'.
        n_boot (int, optional): Number of bootstrapping iterations to perform when
            estimating prediction intervals. Defaults to 250.
        use_in_sample_residuals (bool, optional): If `True`, residuals from the
            training data are used as proxy of prediction error to create predictions.
            If `False`, out of sample residuals (calibration) are used.
            Out-of-sample residuals must be precomputed using Forecaster's
            `set_out_sample_residuals()` method. Defaults to True.
        use_binned_residuals (bool, optional): If `True`, residuals are selected
            based on the predicted values (binned selection).
            If `False`, residuals are selected randomly. Defaults to True.
        random_state (int, optional): Seed for the random number generator to
            ensure reproducibility. Defaults to 123.
        return_predictors (bool, optional): If `True`, the predictors used to make
            the predictions are also returned. Defaults to False.
        n_jobs (int | str, optional): The number of jobs to run in parallel. If `-1`,
            then the number of jobs is set to the number of cores. If 'auto', `n_jobs`
            is set using the function `skforecast.utils.select_n_jobs_backtesting`.
            Defaults to 'auto'.
        verbose (bool, optional): Print number of folds and index of training and
            validation sets used for backtesting. Defaults to False.
        show_progress (bool, optional): Whether to show a progress bar.
            Defaults to True.
        suppress_warnings (bool, optional): If `True`, spotforecast warnings will be
            suppressed during the backtesting process. See
            `spotforecast.exceptions.warn_skforecast_categories` for more information.
            Defaults to False.

    Returns:
        tuple (pd.DataFrame, pd.DataFrame):
            - metric_values: Value(s) of the metric(s).
            - backtest_predictions: Value of predictions. The DataFrame includes
              the following columns:

              - fold: Indicates the fold number where the prediction was made.
              - pred: Predicted values for the corresponding series and time steps.

              If `interval` is not `None`, additional columns are included depending
              on the method:

              - For `float`: Columns `lower_bound` and `upper_bound`.
              - For `list` or `tuple` of 2 elements: Columns `lower_bound` and
                `upper_bound`.
              - For `list` or `tuple` with multiple percentiles: One column per
                percentile (e.g., `p_10`, `p_50`, `p_90`).
              - For `'bootstrapping'`: One column per bootstrapping iteration
                (e.g., `pred_boot_0`, `pred_boot_1`, ..., `pred_boot_n`).
              - For `scipy.stats` distribution objects: One column for each
                estimated parameter of the distribution (e.g., `loc`, `scale`).

              If `return_predictors` is `True`, one column per predictor is created.

              Depending on the relation between `steps` and `fold_stride`, the output
              may include repeated indexes (if `fold_stride < steps`) or gaps
              (if `fold_stride > steps`). See Notes below for more details.

    Notes:
        Note on `fold_stride` vs. `steps`:

        - If `fold_stride == steps`, test sets are placed back-to-back without overlap.
          Each observation appears only once in the output DataFrame, so the
          index is unique.
        - If `fold_stride < steps`, test sets overlap. Multiple forecasts are
          generated for the same observations and, therefore, the output
          DataFrame contains repeated indexes.
        - If `fold_stride > steps`, there are gaps between consecutive test sets.
          Some observations in the series will not have associated predictions,
          so the output DataFrame has non-contiguous indexes.

    Examples:
        >>> import pandas as pd
        >>> from sklearn.ensemble import RandomForestRegressor
        >>> from spotforecast2_safe.forecaster.recursive import ForecasterRecursive
        >>> from spotforecast2_safe.model_selection import backtesting_forecaster, TimeSeriesFold
        >>> y = pd.Series([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
        >>> forecaster = ForecasterRecursive(
        ...     estimator=RandomForestRegressor(random_state=123),
        ...     lags=5
        ... )
        >>> cv = TimeSeriesFold(
        ...     steps=2,
        ...     initial_train_size=5,
        ...     refit=False
        ... )
        >>> metric_values, backtest_predictions = backtesting_forecaster(
        ...     forecaster=forecaster,
        ...     y=y,
        ...     cv=cv,
        ...     metric='mean_squared_error'
        ... )
        >>> metric_values
           mean_squared_error
        0            0.201334
        >>> backtest_predictions
           fold  pred
        5     0  5.18
        6     0  6.10
        7     1  7.36
        8     1  8.40
        9     2  9.31
    """

    if type(cv).__name__ == "OneStepAheadFold":
        return backtesting_forecaster_one_step(
            forecaster=forecaster,
            y=y,
            cv=cv,
            metric=metric,
            exog=exog,
            interval=interval,
            interval_method=interval_method,
            n_boot=n_boot,
            use_in_sample_residuals=use_in_sample_residuals,
            use_binned_residuals=use_binned_residuals,
            random_state=random_state,
            return_predictors=return_predictors,
            n_jobs=n_jobs,
            verbose=verbose,
            show_progress=show_progress,
            suppress_warnings=suppress_warnings,
        )

    check_backtesting_input(
        forecaster=forecaster,
        cv=cv,
        y=y,
        metric=metric,
        interval=interval,
        interval_method=interval_method,
        n_boot=n_boot,
        use_in_sample_residuals=use_in_sample_residuals,
        use_binned_residuals=use_binned_residuals,
        random_state=random_state,
        return_predictors=return_predictors,
        n_jobs=n_jobs,
        show_progress=show_progress,
        suppress_warnings=suppress_warnings,
    )

    metric_values, backtest_predictions = _backtesting_forecaster(
        forecaster=forecaster,
        y=y,
        cv=cv,
        metric=metric,
        exog=exog,
        interval=interval,
        interval_method=interval_method,
        n_boot=n_boot,
        use_in_sample_residuals=use_in_sample_residuals,
        use_binned_residuals=use_binned_residuals,
        random_state=random_state,
        return_predictors=return_predictors,
        n_jobs=n_jobs,
        verbose=verbose,
        show_progress=show_progress,
        suppress_warnings=suppress_warnings,
    )

    return metric_values, backtest_predictions
