################################################################################
#                     skforecast.model_selection._utils                        #
#                                                                              #
# This work by skforecast team is licensed under the BSD 3-Clause License.     #
################################################################################
# coding=utf-8

from __future__ import annotations
from typing import Callable, Generator
import warnings
import numpy as np
import pandas as pd
from joblib import cpu_count
from tqdm.auto import tqdm
from sklearn.pipeline import Pipeline
import sklearn.linear_model
from sklearn.exceptions import NotFittedError

from ..exceptions import IgnoredArgumentWarning, OneStepAheadValidationWarning
from ..metrics import add_y_train_argument, _get_metric
from ..utils import check_interval, date_to_index_position


def initialize_lags_grid(
    forecaster: object, 
    lags_grid: (
        list[int | list[int] | np.ndarray[int] | range[int]]
        | dict[str, list[int | list[int] | np.ndarray[int] | range[int]]]
        | None
    ) = None,
) -> tuple[dict[str, int], str]:
    """
    Initialize lags grid and lags label for model selection. 

    Parameters
    ----------
    forecaster : Forecaster
        Forecaster model. ForecasterRecursive, ForecasterDirect, 
        ForecasterRecursiveMultiSeries, ForecasterDirectMultiVariate.
    lags_grid : list, dict, default None
        Lists of lags to try, containing int, lists, numpy ndarray, or range 
        objects. If `dict`, the keys are used as labels in the `results` 
        DataFrame, and the values are used as the lists of lags to try.

    Returns
    -------
    lags_grid : dict
        Dictionary with lags configuration for each iteration.
    lags_label : str
        Label for lags representation in the results object.

    """

    if not isinstance(lags_grid, (list, dict, type(None))):
        raise TypeError(
            f"`lags_grid` argument must be a list, dict or None. "
            f"Got {type(lags_grid)}."
        )

    lags_label = 'values'
    if isinstance(lags_grid, list):
        lags_grid = {f'{lags}': lags for lags in lags_grid}
    elif lags_grid is None:
        lags = [int(lag) for lag in forecaster.lags]  # Required since numpy 2.0
        lags_grid = {f'{lags}': lags}
    else:
        lags_label = 'keys'

    return lags_grid, lags_label


def check_backtesting_input(
    forecaster: object,
    cv: object,
    metric: str | Callable | list[str | Callable],
    add_aggregated_metric: bool = True,
    y: pd.Series | None = None,
    series: pd.DataFrame | dict[str, pd.Series | pd.DataFrame] = None,
    exog: pd.Series | pd.DataFrame | dict[str, pd.Series | pd.DataFrame] | None = None,
    interval: float | list[float] | tuple[float] | str | object | None = None,
    interval_method: str = 'bootstrapping',    
    alpha: float | None = None,
    n_boot: int = 250,
    use_in_sample_residuals: bool = True,
    use_binned_residuals: bool = True,
    random_state: int = 123,
    return_predictors: bool = False,
    n_jobs: int | str = 'auto',
    show_progress: bool = True,
    suppress_warnings: bool = False,
    suppress_warnings_fit: bool = False
) -> None:
    """
    This is a helper function to check most inputs of backtesting functions in 
    modules `model_selection`.

    Parameters
    ----------
    forecaster : Forecaster
        Forecaster model.
    cv : TimeSeriesFold
        TimeSeriesFold object with the information needed to split the data into folds.
    metric : str, Callable, list
        Metric used to quantify the goodness of fit of the model.
    add_aggregated_metric : bool, default True
        If `True`, the aggregated metrics (average, weighted average and pooling)
        over all levels are also returned (only multiseries).
    y : pandas Series, default None
        Training time series for uni-series forecasters.
    series : pandas DataFrame, dict, default None
        Training time series for multi-series forecasters.
    exog : pandas Series, pandas DataFrame, dict, default None
        Exogenous variables.
    interval : float, list, tuple, str, object, default None
        Specifies whether probabilistic predictions should be estimated and the 
        method to use. The following options are supported:

        - If `float`, represents the nominal (expected) coverage (between 0 and 1). 
        For instance, `interval=0.95` corresponds to `[2.5, 97.5]` percentiles.
        - If `list` or `tuple`: Sequence of percentiles to compute, each value must 
        be between 0 and 100 inclusive. For example, a 95% confidence interval can 
        be specified as `interval = [2.5, 97.5]` or multiple percentiles (e.g. 10, 
        50 and 90) as `interval = [10, 50, 90]`.
        - If 'bootstrapping' (str): `n_boot` bootstrapping predictions will be generated.
        - If scipy.stats distribution object, the distribution parameters will
        be estimated for each prediction.
        - If None, no probabilistic predictions are estimated.
    interval_method : str, default 'bootstrapping'
        Technique used to estimate prediction intervals. Available options:

        + 'bootstrapping': Bootstrapping is used to generate prediction 
        intervals.
        + 'conformal': Employs the conformal prediction split method for 
        interval estimation.
    alpha : float, default None
        The confidence intervals used in ForecasterStats are (1 - alpha) %. 
    n_boot : int, default `250`
        Number of bootstrapping iterations to perform when estimating prediction
            intervals.
    use_in_sample_residuals : bool, default True
        If `True`, residuals from the training data are used as proxy of prediction 
        error to create prediction intervals.  If `False`, out_sample_residuals 
        are used if they are already stored inside the forecaster.
    use_binned_residuals : bool, default True
        If `True`, residuals are selected based on the predicted values 
        (binned selection).
        If `False`, residuals are selected randomly.
    random_state : int, default `123`
        Seed for the random number generator to ensure reproducibility.
    return_predictors : bool, default False
        If `True`, the predictors used to make the predictions are also returned.
    n_jobs : int, 'auto', default `'auto'`
        The number of jobs to run in parallel. If `-1`, then the number of jobs is 
        set to the number of cores. If 'auto', `n_jobs` is set using the function
        skforecast.utils.select_n_jobs_fit_forecaster.
    show_progress : bool, default True
        Whether to show a progress bar.
    suppress_warnings: bool, default False
        If `True`, skforecast warnings will be suppressed during the backtesting 
        process. See skforecast.exceptions.warn_skforecast_categories for more
        information.
    suppress_warnings_fit : bool, default False
        If `True`, warnings generated during fitting will be ignored. Only 
        `ForecasterStats`.

    Returns
    -------
    None
    
    """

    forecaster_name = type(forecaster).__name__
    cv_name = type(cv).__name__

    if cv_name != "TimeSeriesFold":
        raise TypeError(f"`cv` must be a 'TimeSeriesFold' object. Got '{cv_name}'.")

    steps = cv.steps
    initial_train_size = cv.initial_train_size
    gap = cv.gap
    allow_incomplete_fold = cv.allow_incomplete_fold
    refit = cv.refit

    forecasters_uni = [
        "ForecasterRecursive",
        "ForecasterDirect",
        "ForecasterStats",
        "ForecasterEquivalentDate",
        "ForecasterRecursiveClassifier"
    ]
    forecasters_direct = [
        "ForecasterDirect",
        "ForecasterDirectMultiVariate",
        "ForecasterRnn"
    ]
    forecasters_multi_no_dict = [
        "ForecasterDirectMultiVariate",
        "ForecasterRnn",
    ]
    forecasters_multi_dict = [
        "ForecasterRecursiveMultiSeries"
    ]
    # NOTE: ForecasterStats has interval but not with bootstrapping or conformal
    forecasters_boot_conformal = [
        "ForecasterRecursive",
        "ForecasterDirect",
        "ForecasterRecursiveMultiSeries",
        "ForecasterDirectMultiVariate",
        "ForecasterEquivalentDate",
    ]
    forecasters_return_predictors = [
        "ForecasterRecursive",
        "ForecasterDirect",
        "ForecasterRecursiveMultiSeries",
        "ForecasterDirectMultiVariate",
        "ForecasterRecursiveClassifier"
    ]

    if forecaster_name in forecasters_uni:
        if not isinstance(y, pd.Series):
            raise TypeError("`y` must be a pandas Series.")
        data_name = 'y'
        data_length = len(y)

    elif forecaster_name in forecasters_multi_no_dict:
        if not isinstance(series, pd.DataFrame):
            raise TypeError("`series` must be a pandas DataFrame.")
        data_name = 'series'
        data_length = len(series)
        
    elif forecaster_name in forecasters_multi_dict:
        
        # NOTE: Checks are not need as they are done in the function 
        # `check_preprocess_series` that is used before `check_backtesting_input`
        # in the backtesting function.
        
        data_name = 'series'
        data_length = max([len(series[serie]) for serie in series])

    if exog is not None:
        if forecaster_name in forecasters_multi_dict:
            # NOTE: Checks are not need as they are done in the function 
            # `check_preprocess_exog_multiseries` that is used before 
            # `check_backtesting_input` in the backtesting function.
            pass
        else:
            if not isinstance(exog, (pd.Series, pd.DataFrame)):
                raise TypeError(
                    f"`exog` must be a pandas Series, DataFrame or None. Got {type(exog)}."
                )

    if hasattr(forecaster, 'differentiation'):
        if forecaster.differentiation_max != cv.differentiation:
            if forecaster_name == "ForecasterRecursiveMultiSeries" and isinstance(
                forecaster.differentiation, dict
            ):
                raise ValueError(
                    f"When using a dict as `differentiation` in ForecasterRecursiveMultiSeries, "
                    f"the `differentiation` included in the cv ({cv.differentiation}) must be "
                    f"the same as the maximum `differentiation` included in the forecaster "
                    f"({forecaster.differentiation_max}). Set the same value "
                    f"for both using the `differentiation` argument."
                )
            else:
                raise ValueError(
                    f"The differentiation included in the forecaster "
                    f"({forecaster.differentiation_max}) differs from the differentiation "
                    f"included in the cv ({cv.differentiation}). Set the same value "
                    f"for both using the `differentiation` argument."
                )

    if not isinstance(metric, (str, Callable, list)):
        raise TypeError(
            f"`metric` must be a string, a callable function, or a list containing "
            f"multiple strings and/or callables. Got {type(metric)}."
        )

    if forecaster_name == "ForecasterEquivalentDate" and isinstance(
        forecaster.offset, pd.tseries.offsets.DateOffset
    ):
        # NOTE: Checks when initial_train_size is not None cannot be done here
        # because the forecaster is not fitted yet and we don't know the
        # window_size since pd.DateOffset is not a fixed window size.
        if initial_train_size is None:
            raise ValueError(
                f"`initial_train_size` must be an integer greater than "
                f"the `window_size` of the forecaster ({forecaster.window_size}) "
                f"and smaller than the length of `{data_name}` ({data_length}) or "
                f"a date within this range of the index."
            )
    elif initial_train_size is not None:
        if forecaster_name in forecasters_uni:
            index = cv._extract_index(y)
        else:
            index = cv._extract_index(series)

        initial_train_size = date_to_index_position(
                                 index        = index, 
                                 date_input   = initial_train_size, 
                                 method       = 'validation',
                                 date_literal = 'initial_train_size'
                             )
        if initial_train_size < forecaster.window_size or initial_train_size >= data_length:
            raise ValueError(
                f"If `initial_train_size` is an integer, it must be greater than "
                f"the `window_size` of the forecaster ({forecaster.window_size}) "
                f"and smaller than the length of `{data_name}` ({data_length}). If "
                f"it is a date, it must be within this range of the index."
            )
        if allow_incomplete_fold:
            # At least one observation after the gap to allow incomplete fold
            if data_length <= initial_train_size + gap:
                raise ValueError(
                    f"`{data_name}` must have more than `initial_train_size + gap` "
                    f"observations to create at least one fold.\n"
                    f"    Time series length: {data_length}\n"
                    f"    Required > {initial_train_size + gap}\n"
                    f"    initial_train_size: {initial_train_size}\n"
                    f"    gap: {gap}\n"
                )
        else:
            # At least one complete fold
            if data_length < initial_train_size + gap + steps:
                raise ValueError(
                    f"`{data_name}` must have at least `initial_train_size + gap + steps` "
                    f"observations to create a minimum of one complete fold "
                    f"(allow_incomplete_fold=False).\n"
                    f"    Time series length: {data_length}\n"
                    f"    Required >= {initial_train_size + gap + steps}\n"
                    f"    initial_train_size: {initial_train_size}\n"
                    f"    gap: {gap}\n"
                    f"    steps: {steps}\n"
                )
    else:
        if forecaster_name in ['ForecasterStats', 'ForecasterEquivalentDate']:
            raise ValueError(
                f"`initial_train_size` must be an integer smaller than the "
                f"length of `{data_name}` ({data_length})."
            )
        else:
            if not forecaster.is_fitted:
                raise NotFittedError(
                    "`forecaster` must be already trained if no `initial_train_size` "
                    "is provided."
                )
            if refit:
                raise ValueError(
                    "`refit` is only allowed when `initial_train_size` is not `None`."
                )

    if forecaster_name == 'ForecasterStats' and cv.skip_folds is not None:
        raise ValueError(
            "`skip_folds` is not allowed for ForecasterStats. Set it to `None`."
        )

    if not isinstance(add_aggregated_metric, bool):
        raise TypeError("`add_aggregated_metric` must be a boolean: `True`, `False`.")
    if not isinstance(n_boot, (int, np.integer)) or n_boot < 0:
        raise TypeError(f"`n_boot` must be an integer greater than 0. Got {n_boot}.")
    if not isinstance(use_in_sample_residuals, bool):
        raise TypeError("`use_in_sample_residuals` must be a boolean: `True`, `False`.")
    if not isinstance(use_binned_residuals, bool):
        raise TypeError("`use_binned_residuals` must be a boolean: `True`, `False`.")
    if not isinstance(random_state, (int, np.integer)) or random_state < 0:
        raise TypeError(f"`random_state` must be an integer greater than 0. Got {random_state}.")
    if not isinstance(return_predictors, bool):
        raise TypeError("`return_predictors` must be a boolean: `True`, `False`.")
    if not isinstance(n_jobs, int) and n_jobs != 'auto':
        raise TypeError(f"`n_jobs` must be an integer or `'auto'`. Got {n_jobs}.")
    if not isinstance(show_progress, bool):
        raise TypeError("`show_progress` must be a boolean: `True`, `False`.")
    if not isinstance(suppress_warnings, bool):
        raise TypeError("`suppress_warnings` must be a boolean: `True`, `False`.")
    if not isinstance(suppress_warnings_fit, bool):
        raise TypeError("`suppress_warnings_fit` must be a boolean: `True`, `False`.")

    if interval is not None or alpha is not None:
        
        if forecaster_name in forecasters_boot_conformal:

            if interval_method == 'conformal':
                if not isinstance(interval, (float, list, tuple)):
                    raise TypeError(
                        f"When `interval_method` is 'conformal', `interval` must "
                        f"be a float or a list/tuple defining a symmetric interval. "
                        f"Got {type(interval)}."
                    )
            elif interval_method == 'bootstrapping':
                if (
                    not isinstance(interval, (float, list, tuple, str))
                    and (not hasattr(interval, "_pdf") or not callable(getattr(interval, "fit", None)))
                ):                
                    raise TypeError(
                        f"When `interval_method` is 'bootstrapping', `interval` "
                        f"must be a float, a list or tuple of floats, a "
                        f"scipy.stats distribution object (with methods `_pdf` and "
                        f"`fit`) or the string 'bootstrapping'. Got {type(interval)}."
                    )
                if isinstance(interval, (list, tuple)):
                    for i in interval:
                        if not isinstance(i, (int, float)):
                            raise TypeError(
                                f"`interval` must be a list or tuple of floats. "
                                f"Got {type(i)} in {interval}."
                            )
                    if len(interval) == 2:
                        check_interval(interval=interval)
                    else:
                        for q in interval:
                            if (q < 0.) or (q > 100.):
                                raise ValueError(
                                    "When `interval` is a list or tuple, all values must be "
                                    "between 0 and 100 inclusive."
                                )
                elif isinstance(interval, str):
                    if interval != 'bootstrapping':
                        raise ValueError(
                            f"When `interval` is a string, it must be 'bootstrapping'."
                            f"Got {interval}."
                        )
            else:
                raise ValueError(
                    f"`interval_method` must be 'bootstrapping' or 'conformal'. "
                    f"Got {interval_method}."
                )
        else:
            if forecaster_name == 'ForecasterRecursiveClassifier':
                raise ValueError(
                    f"`interval` is not supported for {forecaster_name}. Class "
                    f"probabilities are returned by default during backtesting, "
                    f"set `interval=None`."
                )
            check_interval(interval=interval, alpha=alpha)

    if return_predictors and forecaster_name not in forecasters_return_predictors:
        raise ValueError(
            f"`return_predictors` is only allowed for forecasters of type "
            f"{forecasters_return_predictors}. Got {forecaster_name}."
        )

    if forecaster_name in forecasters_direct and forecaster.max_step < steps + gap:
        raise ValueError(
            f"When using a {forecaster_name}, the combination of steps "
            f"+ gap ({steps + gap}) cannot be greater than the `steps` parameter "
            f"declared when the forecaster is initialized ({forecaster.max_step})."
        )


def check_one_step_ahead_input(
    forecaster: object,
    cv: object,
    metric: str | Callable | list[str | Callable],
    y: pd.Series | None = None,
    series: pd.DataFrame | dict[str, pd.Series | pd.DataFrame] = None,
    exog: pd.Series | pd.DataFrame | dict[str, pd.Series | pd.DataFrame] | None = None,
    show_progress: bool = True,
    suppress_warnings: bool = False
) -> None:
    """
    This is a helper function to check most inputs of hyperparameter tuning
    functions in modules `model_selection` when using a `OneStepAheadFold`.

    Parameters
    ----------
    forecaster : Forecaster
        Forecaster model.
    cv : OneStepAheadFold
        OneStepAheadFold object with the information needed to split the data into folds.
    metric : str, Callable, list
        Metric used to quantify the goodness of fit of the model.
    y : pandas Series, default None
        Training time series for uni-series forecasters.
    series : pandas DataFrame, dict, default None
        Training time series for multi-series forecasters.
    exog : pandas Series, pandas DataFrame, dict, default None
        Exogenous variables.
    show_progress : bool, default True
        Whether to show a progress bar.
    suppress_warnings: bool, default False
        If `True`, skforecast warnings will be suppressed during the hyperparameter 
        search. See skforecast.exceptions.warn_skforecast_categories for more
        information.

    Returns
    -------
    None
    
    """

    forecaster_name = type(forecaster).__name__
    cv_name = type(cv).__name__

    if cv_name != "OneStepAheadFold":
        raise TypeError(f"`cv` must be a 'OneStepAheadFold' object. Got '{cv_name}'.")

    initial_train_size = cv.initial_train_size

    forecasters_one_step_ahead = [
        "ForecasterRecursive",
        "ForecasterDirect",
        "ForecasterRecursiveClassifier",
        'ForecasterRecursiveMultiSeries',
        'ForecasterDirectMultiVariate'
    ]
    if forecaster_name not in forecasters_one_step_ahead:
        raise TypeError(
            f"Only forecasters of type {forecasters_one_step_ahead} are allowed "
            f"when using `cv` of type `OneStepAheadFold`. Got {forecaster_name}."
        )

    forecasters_uni = [
        "ForecasterRecursive",
        "ForecasterDirect",
        "ForecasterRecursiveClassifier"
    ]
    forecasters_multi_no_dict = [
        "ForecasterDirectMultiVariate",
    ]
    forecasters_multi_dict = [
        "ForecasterRecursiveMultiSeries"
    ]

    if forecaster_name in forecasters_uni:
        if not isinstance(y, pd.Series):
            raise TypeError(f"`y` must be a pandas Series. Got {type(y)}")
        data_name = 'y'
        data_length = len(y)

    elif forecaster_name in forecasters_multi_no_dict:
        if not isinstance(series, pd.DataFrame):
            raise TypeError(f"`series` must be a pandas DataFrame. Got {type(series)}")
        data_name = 'series'
        data_length = len(series)
    
    elif forecaster_name in forecasters_multi_dict:
        
        # NOTE: Checks are not need as they are done in the function 
        # `check_preprocess_series` that is used before `check_one_step_ahead_input`
        # in the backtesting function.
        
        data_name = 'series'
        data_length = max([len(series[serie]) for serie in series])

    if exog is not None:
        if forecaster_name in forecasters_multi_dict:
            # NOTE: Checks are not need as they are done in the function 
            # `check_preprocess_exog_multiseries` that is used before 
            # `check_backtesting_input` in the backtesting function.
            pass
        else:
            if not isinstance(exog, (pd.Series, pd.DataFrame)):
                raise TypeError(
                    f"`exog` must be a pandas Series, DataFrame or None. Got {type(exog)}."
                )

    if hasattr(forecaster, 'differentiation'):
        if forecaster.differentiation_max != cv.differentiation:
            if forecaster_name == "ForecasterRecursiveMultiSeries" and isinstance(
                forecaster.differentiation, dict
            ):
                raise ValueError(
                    f"When using a dict as `differentiation` in ForecasterRecursiveMultiSeries, "
                    f"the `differentiation` included in the cv ({cv.differentiation}) must be "
                    f"the same as the maximum `differentiation` included in the forecaster "
                    f"({forecaster.differentiation_max}). Set the same value "
                    f"for both using the `differentiation` argument."
                )
            else:
                raise ValueError(
                    f"The differentiation included in the forecaster "
                    f"({forecaster.differentiation_max}) differs from the differentiation "
                    f"included in the cv ({cv.differentiation}). Set the same value "
                    f"for both using the `differentiation` argument."
                )

    if not isinstance(metric, (str, Callable, list)):
        raise TypeError(
            f"`metric` must be a string, a callable function, or a list containing "
            f"multiple strings and/or callables. Got {type(metric)}."
        )

    if forecaster_name in forecasters_uni:
        index = cv._extract_index(y)
    else:
        index = cv._extract_index(series)

    initial_train_size = date_to_index_position(
                             index        = index, 
                             date_input   = initial_train_size, 
                             method       = 'validation',
                             date_literal = 'initial_train_size'
                         )
    if initial_train_size < forecaster.window_size or initial_train_size >= data_length:
        raise ValueError(
            f"If `initial_train_size` is an integer, it must be greater than "
            f"the `window_size` of the forecaster ({forecaster.window_size}) "
            f"and smaller than the length of `{data_name}` ({data_length}). If "
            f"it is a date, it must be within this range of the index."
        )

    if not isinstance(show_progress, bool):
        raise TypeError("`show_progress` must be a boolean: `True`, `False`.")
    if not isinstance(suppress_warnings, bool):
        raise TypeError("`suppress_warnings` must be a boolean: `True`, `False`.")
    
    if not suppress_warnings:
        warnings.warn(
            "One-step-ahead predictions are used for faster model comparison, but they "
            "may not fully represent multi-step prediction performance. It is recommended "
            "to backtest the final model for a more accurate multi-step performance "
            "estimate.", OneStepAheadValidationWarning
        )


def select_n_jobs_backtesting(
    forecaster: object,
    refit: bool | int
) -> int:
    """
    Select the optimal number of jobs to use in the backtesting process. This
    selection is based on heuristics and is not guaranteed to be optimal.

    The number of jobs is chosen as follows:

    - If `refit` is an integer, then `n_jobs = 1`. This is because parallelization doesn't 
    work with intermittent refit.
    - If forecaster is 'ForecasterRecursive' and estimator is a linear estimator, 
    then `n_jobs = 1`.
    - If forecaster is 'ForecasterRecursive' and estimator is not a linear 
    estimator then `n_jobs = cpu_count() - 1`.
    - If forecaster is 'ForecasterDirect' or 'ForecasterDirectMultiVariate'
    and `refit = True`, then `n_jobs = cpu_count() - 1`.
    - If forecaster is 'ForecasterDirect' or 'ForecasterDirectMultiVariate'
    and `refit = False`, then `n_jobs = 1`.
    - If forecaster is 'ForecasterRecursiveMultiSeries', then `n_jobs = cpu_count() - 1`.
    - If forecaster is 'ForecasterStats' or 'ForecasterEquivalentDate', 
    then `n_jobs = 1`.
    - If estimator is a `LGBMRegressor(n_jobs=1)`, then `n_jobs = cpu_count() - 1`.
    - If estimator is a `LGBMRegressor` with internal n_jobs != 1, then `n_jobs = 1`.
    This is because `lightgbm` is highly optimized for gradient boosting and
    parallelizes operations at a very fine-grained level, making additional
    parallelization unnecessary and potentially harmful due to resource contention.

    Parameters
    ----------
    forecaster : Forecaster
        Forecaster model.
    refit : bool, int
        If the forecaster is refitted during the backtesting process.

    Returns
    -------
    n_jobs : int
        The number of jobs to run in parallel.
    
    """

    forecaster_name = type(forecaster).__name__

    if isinstance(forecaster.estimator, Pipeline):
        estimator = forecaster.estimator[-1]
        estimator_name = type(estimator).__name__
    else:
        estimator = forecaster.estimator
        estimator_name = type(estimator).__name__

    linear_estimators = [
        estimator_name
        for estimator_name in dir(sklearn.linear_model)
        if not estimator_name.startswith('_')
    ]

    refit = False if refit == 0 else refit
    if not isinstance(refit, bool) and refit != 1:
        n_jobs = 1
    else:
        if forecaster_name in ['ForecasterRecursive']:
            if estimator_name in linear_estimators:
                n_jobs = 1
            elif estimator_name == 'LGBMRegressor':
                n_jobs = cpu_count() - 1 if estimator.n_jobs == 1 else 1
            else:
                n_jobs = cpu_count() - 1
        elif forecaster_name in ['ForecasterDirect', 'ForecasterDirectMultiVariate']:
            # Parallelization is applied during the fitting process.
            n_jobs = 1
        elif forecaster_name in ['ForecasterRecursiveMultiSeries']:
            if estimator_name == 'LGBMRegressor':
                n_jobs = cpu_count() - 1 if estimator.n_jobs == 1 else 1
            else:
                n_jobs = cpu_count() - 1
        elif forecaster_name in ['ForecasterStats', 'ForecasterEquivalentDate']:
            n_jobs = 1
        else:
            n_jobs = 1

    return n_jobs


def _calculate_metrics_one_step_ahead(
    forecaster: object,
    metrics: list,
    X_train: pd.DataFrame,
    y_train: pd.Series | dict[int, pd.Series],
    X_test: pd.DataFrame,
    y_test: pd.Series | dict[int, pd.Series]
) -> list:
    """
    Calculate metrics when predictions are one-step-ahead. When forecaster is
    of type ForecasterDirect only the estimator for step 1 is used.

    Parameters
    ----------
    forecaster : object
        Forecaster model.
    metrics : list
        List of metrics.
    X_train : pandas DataFrame
        Predictor values used to train the model.
    y_train : pandas Series, dict
        Target values related to each row of `X_train`.
    X_test : pandas DataFrame
        Predictor values used to test the model.
    y_test : pandas Series, dict
        Target values related to each row of `X_test`.

    Returns
    -------
    metric_values : list
        List with metric values.
    
    """

    if type(forecaster).__name__ == 'ForecasterDirect':

        step = 1  # Only the model for step 1 is optimized.
        X_train, y_train = forecaster.filter_train_X_y_for_step(
                               step    = step,
                               X_train = X_train,
                               y_train = y_train
                           )
        X_test, y_test = forecaster.filter_train_X_y_for_step(
                             step    = step,  
                             X_train = X_test,
                             y_train = y_test
                         )
        forecaster.estimators_[step].fit(X_train, y_train)
        y_pred = forecaster.estimators_[step].predict(X_test)

    else:
        forecaster.estimator.fit(X_train, y_train)
        y_pred = forecaster.estimator.predict(X_test)

    y_true = y_test.to_numpy()
    y_pred = y_pred.ravel()
    y_train = y_train.to_numpy()

    if forecaster.differentiation is not None:
        y_true = forecaster.differentiator.inverse_transform_next_window(y_true)
        y_pred = forecaster.differentiator.inverse_transform_next_window(y_pred)
        y_train = forecaster.differentiator.inverse_transform_training(y_train)

    if forecaster.transformer_y is not None:
        y_true = forecaster.transformer_y.inverse_transform(y_true.reshape(-1, 1))
        y_pred = forecaster.transformer_y.inverse_transform(y_pred.reshape(-1, 1))
        y_train = forecaster.transformer_y.inverse_transform(y_train.reshape(-1, 1))

    # NOTE: When using this metric in validation, `y_train` doesn't include
    # the first window_size observations used to create the predictors and/or
    # rolling features.
    y_true = y_true.ravel()
    y_pred = y_pred.ravel()
    y_train = y_train.ravel()
    metric_values = [
        m(y_true=y_true, y_pred=y_pred, y_train=y_train) for m in metrics
    ]

    return metric_values


def _initialize_levels_model_selection_multiseries(
    forecaster: object, 
    series:  pd.DataFrame | dict[str, pd.Series | pd.DataFrame],
    levels: str | list | None = None
) -> list:
    """
    Initialize levels for model_selection multi-series functions.

    Parameters
    ----------
    forecaster : ForecasterRecursiveMultiSeries, ForecasterDirectMultiVariate, ForecasterRnn
        Forecaster model.
    series : pandas DataFrame, dict
        Training time series.
    levels : str, list, default None
        level (`str`) or levels (`list`) at which the forecaster is optimized. 
        If `None`, all levels are taken into account. The resulting metric will be
        the average of the optimization of all levels.

    Returns
    -------
    levels : list
        List of levels to be used in model_selection multi-series functions.
    
    """

    multi_series_forecasters_with_levels = [
        'ForecasterRecursiveMultiSeries', 
        'ForecasterRnn'
    ]

    if type(forecaster).__name__ in multi_series_forecasters_with_levels  \
        and not isinstance(levels, (str, list, type(None))):
        raise TypeError(
            f"`levels` must be a `list` of column names, a `str` of a column "
            f"name or `None` when using a forecaster of type "
            f"{multi_series_forecasters_with_levels}. If the forecaster is of "
            f"type `ForecasterDirectMultiVariate`, this argument is ignored."
        )

    if type(forecaster).__name__ == 'ForecasterDirectMultiVariate':
        if levels and levels != forecaster.level and levels != [forecaster.level]:
            warnings.warn(
                f"`levels` argument have no use when the forecaster is of type "
                f"`ForecasterDirectMultiVariate`. The level of this forecaster "
                f"is '{forecaster.level}', to predict another level, change "
                f"the `level` argument when initializing the forecaster. \n",
                IgnoredArgumentWarning
            )
        levels = [forecaster.level]
    else:
        if levels is None:
            # Forecaster could be untrained, so self.series_col_names cannot be used.
            if isinstance(series, pd.DataFrame):
                levels = list(series.columns)
            else:
                levels = list(series.keys())
        else:
            if isinstance(levels, str):
                levels = [levels]
            
            if isinstance(series, pd.DataFrame):
                available_levels = list(series.columns)
            else:
                available_levels = list(series.keys())

            if not set(levels).issubset(set(available_levels)):
                raise ValueError(
                    f"Levels {levels} not found in `series`, available levels are "
                    f"{available_levels}. Review `levels` argument."
                )

    return levels


def _extract_data_folds_multiseries(
    series: pd.DataFrame | dict[str, pd.Series | pd.DataFrame],
    folds: list,
    span_index: pd.DatetimeIndex | pd.RangeIndex,
    window_size: int,
    exog: pd.Series | pd.DataFrame | dict[str, pd.Series | pd.DataFrame] | None = None,
    dropna_last_window: bool = False,
    externally_fitted: bool = False
) -> Generator[
        tuple[
            pd.DataFrame | dict[str, pd.Series | pd.DataFrame],
            pd.DataFrame,
            list[str],
            pd.Series | pd.DataFrame | dict[str, pd.Series | pd.DataFrame] | None,
            pd.Series | pd.DataFrame | dict[str, pd.Series | pd.DataFrame] | None,
            list[list[int]]
        ],
        None,
        None
    ]:
    """
    Select the data from series and exog that corresponds to each fold created 
    using the split method of the OneStepAheadFold or TimeSeriesFold objects.

    Parameters
    ----------
    series : pandas DataFrame, dict
        Time series.
    folds : list
        Folds created using the split method of the OneStepAheadFold or 
        TimeSeriesFold objects.
    span_index : pandas DatetimeIndex, pandas RangeIndex
        Full index from the minimum to the maximum index among all series.
    window_size : int
        Size of the window needed to create the predictors.
    exog : pandas Series, pandas DataFrame, dict, default None
        Exogenous variables.
    dropna_last_window : bool, default False
        If `True`, drop the columns of the last window that have NaN values.
    externally_fitted : bool, default False
        Flag indicating whether the forecaster is already trained. Only used when 
        `initial_train_size` is None and `refit` is False.

    Yield
    -----
    series_train : pandas DataFrame, dict
        Time series corresponding to the training set of the fold.
    series_last_window: pandas DataFrame
        Time series corresponding to the last window of the fold.
    levels_last_window: list
        Levels of the time series present in the last window of the fold.
    exog_train: pandas Series, pandas DataFrame, dict, None
        Exogenous variable corresponding to the training set of the fold.
    exog_test: pandas Series, pandas DataFrame, dict, None
        Exogenous variable corresponding to the test set of the fold.
    fold: list
        List containing the indexes of that fold.

    """

    is_series_dict = isinstance(series, dict)
    is_exog_dict = isinstance(exog, dict)

    for fold in folds:
        train_iloc_start       = fold[1][0]
        train_iloc_end         = fold[1][1]
        last_window_iloc_start = fold[2][0]
        last_window_iloc_end   = fold[2][1]
        test_iloc_start        = fold[3][0]
        test_iloc_end          = fold[3][1]

        if is_series_dict or is_exog_dict:
            # Subtract 1 to the iloc indexes to get the loc indexes
            train_loc_start       = span_index[train_iloc_start]
            train_loc_end         = span_index[train_iloc_end - 1]
            last_window_loc_start = span_index[last_window_iloc_start]
            last_window_loc_end   = span_index[last_window_iloc_end - 1]
            test_loc_start        = span_index[test_iloc_start]
            test_loc_end          = span_index[test_iloc_end - 1]

        if not is_series_dict:
            series_train = series.iloc[train_iloc_start:train_iloc_end, ]

            series_to_drop = []
            for col in series_train.columns:
                if series_train[col].isna().to_numpy().all():
                    series_to_drop.append(col)
                else:
                    first_valid_index = series_train[col].first_valid_index()
                    last_valid_index = series_train[col].last_valid_index()
                    if (
                        len(series_train[col].loc[first_valid_index:last_valid_index])
                        < window_size
                    ):
                        series_to_drop.append(col)

            series_last_window = series.iloc[
                last_window_iloc_start:last_window_iloc_end,
            ]
            
            series_train = series_train.drop(columns=series_to_drop)
            if not externally_fitted:
                series_last_window = series_last_window.drop(columns=series_to_drop)
        else:
            series_train = {}
            series_last_window = {}
            for k, v in series.items():
                v_train = v.loc[train_loc_start:train_loc_end]
                if not v_train.isna().to_numpy().all():
                    first_valid_index = v_train.first_valid_index()
                    last_valid_index  = v_train.last_valid_index()
                    if first_valid_index is not None and last_valid_index is not None:
                        v_train = v_train.loc[first_valid_index : last_valid_index]
                        if len(v_train) >= window_size:
                            series_train[k] = v_train
                
                v_last_window = v.loc[last_window_loc_start:last_window_loc_end]
                if ((externally_fitted or k in series_train) and len(v_last_window) >= window_size):
                    series_last_window[k] = v_last_window

            series_last_window = pd.DataFrame(series_last_window)

        if dropna_last_window:
            series_last_window = series_last_window.dropna(axis=1, how="any")
            # TODO: add the option to drop the series without minimum non NaN values.
            # Similar to how pandas does in the rolling window function.
        
        levels_last_window = list(series_last_window.columns)

        if exog is not None:
            if not is_exog_dict:
                exog_train = exog.iloc[train_iloc_start:train_iloc_end, ]
                exog_test = exog.iloc[test_iloc_start:test_iloc_end, ]
            else:
                exog_train = {}
                exog_test = {}
                for k, v in exog.items():
                    if v is None:
                        exog_train[k] = None
                        exog_test[k] = None
                    else:
                        v_train = v.loc[train_loc_start:train_loc_end]
                        if len(v_train) > 0:
                            exog_train[k] = v_train
                        if externally_fitted or k in exog_train:
                            v_test = v.loc[test_loc_start:test_loc_end]
                            if not v_test.empty:
                                exog_test[k] = v_test
        else:
            exog_train = None
            exog_test = None

        yield series_train, series_last_window, levels_last_window, exog_train, exog_test, fold


def _calculate_metrics_backtesting_multiseries(
    series: pd.DataFrame | dict[str, pd.Series | pd.DataFrame],
    predictions: pd.DataFrame,
    folds: list[list[int]] | tqdm,
    span_index: pd.DatetimeIndex | pd.RangeIndex,
    window_size: int,
    metrics: list,
    levels: list,
    add_aggregated_metric: bool = True
) -> pd.DataFrame:
    """   
    Calculate metrics for each level and also for all levels aggregated using
    average, weighted average or pooling.

    - 'average': the average (arithmetic mean) of all levels.
    - 'weighted_average': the average of the metrics weighted by the number of
    predicted values of each level.
    - 'pooling': the values of all levels are pooled and then the metric is
    calculated.

    Parameters
    ----------
    series : pandas DataFrame, dict
        Series data used for backtesting.
    predictions : pandas DataFrame
        MultiIndex DataFrame generated during the backtesting multiseries process. 
    folds : list, tqdm
        Folds created during the backtesting process.
    span_index : pandas DatetimeIndex, pandas RangeIndex
        Full index from the minimum to the maximum index among all series.
    window_size : int
        Size of the window used by the forecaster to create the predictors.
        This is used remove the first `window_size` (differentiation included) 
        values from y_train since they are not part of the training matrix.
    metrics : list
        List of metrics to calculate.
    levels : list
        Levels to calculate the metrics.
    add_aggregated_metric : bool, default True
        If `True`, and multiple series (`levels`) are predicted, the aggregated
        metrics (average, weighted average and pooled) are also returned.

        - 'average': the average (arithmetic mean) of all levels.
        - 'weighted_average': the average of the metrics weighted by the number of
        predicted values of each level.
        - 'pooling': the values of all levels are pooled and then the metric is
        calculated.

    Returns
    -------
    metrics_levels : pandas DataFrame
        Value(s) of the metric(s).
    
    """

    if not isinstance(series, (pd.DataFrame, dict)):
        raise TypeError(
            "`series` must be a pandas DataFrame or a dictionary of pandas "
            "DataFrames."
        )
    if not isinstance(predictions, pd.DataFrame):
        raise TypeError("`predictions` must be a pandas DataFrame.")
    if not isinstance(folds, (list, tqdm)):
        raise TypeError("`folds` must be a list or a tqdm object.")
    if not isinstance(span_index, (pd.DatetimeIndex, pd.RangeIndex)):
        raise TypeError("`span_index` must be a pandas DatetimeIndex or pandas RangeIndex.")
    if not isinstance(window_size, (int, np.integer)):
        raise TypeError("`window_size` must be an integer.")
    if not isinstance(metrics, list):
        raise TypeError("`metrics` must be a list.")
    if not isinstance(levels, list):
        raise TypeError("`levels` must be a list.")
    if not isinstance(add_aggregated_metric, bool):
        raise TypeError("`add_aggregated_metric` must be a boolean.")
    
    metric_names = [m.__name__ for m in metrics]
    levels_in_predictions = predictions.index.get_level_values('level').unique()

    if isinstance(series, pd.DataFrame) and not isinstance(series.index, pd.MultiIndex):
        series = series.melt(ignore_index=False, var_name='level', value_name='y_true')
        series = series.rename_axis('idx', axis=0)
        series = series.set_index('level', append=True)
        series = series.swaplevel()
    else:
        series = pd.concat(series, names = ['level', 'idx']).to_frame('y_true')
    
    predictions = predictions.swaplevel()
    predictions.columns = ['y_pred']

    y_true_y_pred = pd.merge(
        series,
        predictions,
        left_index  = True,
        right_index = True,
        how         = "inner",
    ).dropna(axis=0, how="any")

    # TODO: review list of metric that do not need y_train
    metrics_no_y_train = [
        # Regression metrics
        "mean_squared_error",
        "mean_absolute_error",
        "mean_absolute_percentage_error",
        "mean_squared_log_error",
        "median_absolute_error",
        "symmetric_mean_absolute_percentage_error",

        # Classification metrics
        "accuracy_score",
        "balanced_accuracy_score",
        "f1_score",
        "precision_score",
        "recall_score"
    ]

    train_indexes = []
    for i, fold in enumerate(folds):
        fit_fold = fold[-1]
        if i == 0 or fit_fold:
            train_iloc_start = fold[1][0]
            train_iloc_end = fold[1][1]
            train_indexes.append(np.arange(train_iloc_start, train_iloc_end))
    
    train_indexes = np.unique(np.concatenate(train_indexes))
    train_indexes = span_index[train_indexes]
    train_indexes = pd.MultiIndex.from_product([
        levels_in_predictions,
        train_indexes,
    ])
    series_train = series.loc[series.index.isin(train_indexes)]
    # NOTE: Exclude first window_size observations used to create predictors
    series_train = series_train[series_train.groupby(level="level").cumcount() >= window_size]
    ignore_y_train = all(name in metrics_no_y_train for name in metric_names)
    
    y_true_y_pred_grouped = (
        y_true_y_pred
        .reset_index(level="level")
        .groupby(by="level", sort=False, as_index=False)
    )
    series_train_grouped = (
        series_train
        .reset_index(level="level")
        .groupby(by="level", sort=False, as_index=False)
    )
    metrics_levels = []
    for level in levels:
        if level in levels_in_predictions:
            y_true = y_true_y_pred_grouped.get_group(level)['y_true']
            y_pred = y_true_y_pred_grouped.get_group(level)['y_pred']
            if not ignore_y_train:
                # NOTE: y_train includes the intercepted NaNs
                y_train = series_train_grouped.get_group(level)['y_true']
            else:
                y_train = None
            metrics_level = [
                m(y_true=y_true, y_pred=y_pred, y_train=y_train) for m in metrics
            ]
            metrics_levels.append(metrics_level)
        else:
            metrics_levels.append([None for _ in metrics])

    metrics_levels = pd.DataFrame(
                         data    = metrics_levels,
                         columns = metric_names
                     )
    metrics_levels.insert(0, 'levels', levels)

    if len(levels) < 2:
        add_aggregated_metric = False
    
    if add_aggregated_metric:

        # aggregation: average
        average = metrics_levels.drop(columns='levels').mean(skipna=True)
        average = average.to_frame().transpose()
        average['levels'] = 'average'

        # aggregation: weighted_average
        n_predictions_levels = (
            predictions
            .groupby(level="level", sort=False)["y_pred"]
            .apply(lambda x: x.notna().sum())
            .reset_index(name="n_predictions")
            .rename(columns={"level": "levels"})
        )
        metrics_levels_no_missing = (
            metrics_levels.merge(n_predictions_levels, on='levels', how='inner')
        )
        weighted_averages = {}
        for col in metric_names:
            weighted_averages[col] = np.average(
                metrics_levels_no_missing[col],
                weights = metrics_levels_no_missing['n_predictions']
            )
        weighted_average = pd.DataFrame(weighted_averages, index=[0])
        weighted_average['levels'] = 'weighted_average'

        # aggregation: pooling
        scaled_metrics = ['mean_absolute_scaled_error', 'root_mean_squared_scaled_error']

        y_true = y_true_y_pred.loc[:, 'y_true'].droplevel("level")
        y_pred = y_true_y_pred.loc[:, 'y_pred'].droplevel("level")
        y_train = series_train.loc[:, 'y_true'].droplevel("level")
        if any(name in scaled_metrics for name in metric_names):
            series_train_list = [
                group['y_true'] 
                for _, group in series_train.groupby('level', sort=False)
            ]
            
        pooled = []
        for m, m_name in zip(metrics, metric_names):
            if m_name in scaled_metrics:
                pooled.append(
                    m(y_true=y_true, y_pred=y_pred, y_train=series_train_list)
                )
            else:
                pooled.append(
                    m(y_true=y_true, y_pred=y_pred, y_train=y_train)
                )
        
        pooled = pd.DataFrame([pooled], columns=metric_names)
        pooled['levels'] = 'pooling'

        metrics_levels = pd.concat(
            [metrics_levels, average, weighted_average, pooled],
            axis=0, ignore_index=True
        )

    return metrics_levels


def _predict_and_calculate_metrics_one_step_ahead_multiseries(
    forecaster: object,
    series: pd.DataFrame | dict[str, pd.Series | pd.DataFrame],
    X_train: pd.DataFrame,
    y_train: pd.Series | dict[int, pd.Series],
    X_test: pd.DataFrame,
    y_test: pd.Series | dict[int, pd.Series],
    X_train_encoding: pd.Series,
    X_test_encoding: pd.Series,
    levels: list[str],
    metrics: list[str | Callable],
    add_aggregated_metric: bool = True
) -> tuple[pd.DataFrame, pd.DataFrame]:
    """   
    One-step-ahead predictions and metrics for each level and also for all levels
    aggregated using average, weighted average or pooling.
    Input matrices (X_train, y_train, X_train_encoding, X_test, y_test, X_test_encoding)
    should have been generated using the forecaster._train_test_split_one_step_ahead().

    - 'average': the average (arithmetic mean) of all levels.
    - 'weighted_average': the average of the metrics weighted by the number of
    predicted values of each level.
    - 'pooling': the values of all levels are pooled and then the metric is
    calculated.

    Parameters
    ----------
    forecaster : object
        Forecaster model.
    series : pandas DataFrame, dict
        Series data used to train and test the forecaster.
    X_train : pandas DataFrame
        Training matrix.
    y_train : pandas Series, dict
        Target values of the training set.
    X_test : pandas DataFrame
        Test matrix.
    y_test : pandas Series, dict
        Target values of the test set.
    X_train_encoding : pandas Series
        Series identifiers for each row of `X_train`.
    X_test_encoding : pandas Series
        Series identifiers for each row of `X_test`.
    levels : list
        Levels to calculate the metrics.
    metrics : list
        List of metrics to calculate.
    add_aggregated_metric : bool, default True
        If `True`, and multiple series (`levels`) are predicted, the aggregated
        metrics (average, weighted average and pooled) are also returned.

        - 'average': the average (arithmetic mean) of all levels.
        - 'weighted_average': the average of the metrics weighted by the number of
        predicted values of each level.
        - 'pooling': the values of all levels are pooled and then the metric is
        calculated.

    Returns
    -------
    metrics_levels : pandas DataFrame
        Value(s) of the metric(s).
    predictions : pandas DataFrame
        Value of predictions for each level.
    
    """

    if not isinstance(series, (pd.DataFrame, dict)):
        raise TypeError(
            "`series` must be a pandas DataFrame or a dictionary of pandas "
            "DataFrames."
        )
    if not isinstance(X_train, pd.DataFrame):
        raise TypeError(f"`X_train` must be a pandas DataFrame. Got: {type(X_train)}")
    if not isinstance(y_train, (pd.Series, dict)):
        raise TypeError(
            f"`y_train` must be a pandas Series or a dictionary of pandas Series. "
            f"Got: {type(y_train)}"
        )        
    if not isinstance(X_test, pd.DataFrame):
        raise TypeError(f"`X_test` must be a pandas DataFrame. Got: {type(X_test)}")
    if not isinstance(y_test, (pd.Series, dict)):
        raise TypeError(
            f"`y_test` must be a pandas Series or a dictionary of pandas Series. "
            f"Got: {type(y_test)}"
        )
    if not isinstance(X_train_encoding, pd.Series):
        raise TypeError(
            f"`X_train_encoding` must be a pandas Series. Got: {type(X_train_encoding)}"
        )
    if not isinstance(X_test_encoding, pd.Series):
        raise TypeError(
            f"`X_test_encoding` must be a pandas Series. Got: {type(X_test_encoding)}"
        )
    if not isinstance(levels, list):
        raise TypeError(f"`levels` must be a list. Got: {type(levels)}")
    if not isinstance(metrics, list):
        raise TypeError(f"`metrics` must be a list. Got: {type(metrics)}")
    if not isinstance(add_aggregated_metric, bool):
        raise TypeError(
            f"`add_aggregated_metric` must be a boolean. Got: {type(add_aggregated_metric)}"
        )

    metrics = [
        _get_metric(metric=m)
        if isinstance(m, str)
        else add_y_train_argument(m) 
        for m in metrics
    ]
    metric_names = [(m if isinstance(m, str) else m.__name__) for m in metrics]

    if isinstance(series[levels[0]].index, pd.DatetimeIndex):
        freq = series[levels[0]].index.freq
    else:
        freq = series[levels[0]].index.step

    if type(forecaster).__name__ == 'ForecasterDirectMultiVariate':
        step = 1
        X_train, y_train = forecaster.filter_train_X_y_for_step(
                               step    = step,
                               X_train = X_train,
                               y_train = y_train
                           )
        X_test, y_test = forecaster.filter_train_X_y_for_step(
                             step    = step,  
                             X_train = X_test,
                             y_train = y_test
                         )                 
        forecaster.estimators_[step].fit(X_train, y_train)
        pred = forecaster.estimators_[step].predict(X_test)
    else:
        forecaster.estimator.fit(X_train, y_train)
        pred = forecaster.estimator.predict(X_test)

    predictions_per_level = pd.DataFrame(
        {
            'y_true': y_test,
            'y_pred': pred,
            '_level_skforecast': X_test_encoding,
        },
        index=y_test.index,
    ).groupby('_level_skforecast')
    predictions_per_level = {key: group for key, group in predictions_per_level}

    y_train_per_level = pd.DataFrame(
        {"y_train": y_train, "_level_skforecast": X_train_encoding},
        index=y_train.index,
    ).groupby("_level_skforecast")

    # NOTE: Interleaved Nan values were excluded fom y_train. They are restored
    if isinstance(series[levels[0]].index, pd.DatetimeIndex):
        y_train_per_level = {
            key: group.asfreq(freq) for key, group in y_train_per_level
        }
    else:
        y_train_per_level = {
            key: group.reindex(
                pd.RangeIndex(
                    start=min(group.index), stop=max(group.index) + 1, step=freq
                )
            )
            for key, group in y_train_per_level
        }

    if forecaster.differentiation is not None:
        for level in predictions_per_level:
            differentiator = forecaster.differentiator_[level]
            if differentiator is not None:
                predictions_per_level[level]["y_true"] = (
                    differentiator.inverse_transform_next_window(
                        predictions_per_level[level]["y_true"].to_numpy()
                    )
                )
                predictions_per_level[level]["y_pred"] = (
                    differentiator.inverse_transform_next_window(
                        predictions_per_level[level]["y_pred"].to_numpy()
                    )   
                )
                y_train_per_level[level]["y_train"] = (
                    differentiator.inverse_transform_training(
                        y_train_per_level[level]["y_train"].to_numpy()
                    )
                )

    if forecaster.transformer_series is not None:
        for level in predictions_per_level:
            transformer = forecaster.transformer_series_[level]
            predictions_per_level[level]["y_true"] = transformer.inverse_transform(
                predictions_per_level[level][["y_true"]]
            )
            predictions_per_level[level]["y_pred"] = transformer.inverse_transform(
                predictions_per_level[level][["y_pred"]]
            )
            y_train_per_level[level]["y_train"] = transformer.inverse_transform(
                y_train_per_level[level][["y_train"]]
            )

    # NOTE: When using this metric in validation, `y_train` doesn't include
    # the first window_size observations used to create the predictors and/or
    # rolling features.
    metrics_no_y_train = [
        "mean_absolute_error",
        "mean_squared_error",
        "median_absolute_error",
        "mean_absolute_percentage_error",
    ]
    ignore_y_train = all(name in metrics_no_y_train for name in metric_names)

    metrics_levels = []
    for level in levels:
        if level in predictions_per_level:
            y_true = predictions_per_level[level].loc[:, 'y_true']
            y_pred = predictions_per_level[level].loc[:, 'y_pred']
            if not ignore_y_train:
                y_train = y_train_per_level[level].loc[:, 'y_train']
            else:
                y_train = None
            metrics_level = [
                m(y_true=y_true, y_pred=y_pred, y_train=y_train) for m in metrics
            ]
            metrics_levels.append(metrics_level)
        else:
            metrics_levels.append([None for _ in metrics])

    metrics_levels = pd.DataFrame(
                         data    = metrics_levels,
                         columns = metric_names
                     )
    metrics_levels.insert(0, 'levels', levels)

    if len(levels) < 2:
        add_aggregated_metric = False

    if add_aggregated_metric:

        # aggregation: average
        average = metrics_levels.drop(columns='levels').mean(skipna=True)
        average = average.to_frame().transpose()
        average['levels'] = 'average'

        # aggregation: weighted_average
        n_predictions_levels = {
            k: v['y_pred'].notna().sum()
            for k, v in predictions_per_level.items()
        }
        n_predictions_levels = pd.DataFrame(
            n_predictions_levels.items(),
            columns=['levels', 'n_predictions']
        )
        metrics_levels_no_missing = (
            metrics_levels.merge(n_predictions_levels, on='levels', how='inner')
        )
        weighted_averages = {}
        for col in metric_names:
            weighted_averages[col] = np.average(
                metrics_levels_no_missing[col],
                weights = metrics_levels_no_missing['n_predictions']
            )
        weighted_average = pd.DataFrame(weighted_averages, index=[0])
        weighted_average['levels'] = 'weighted_average'

        # aggregation: pooling
        scaled_metrics = ['mean_absolute_scaled_error', 'root_mean_squared_scaled_error']
        if any(name in scaled_metrics for name in metric_names):
            list_y_train_by_level = [
                v['y_train'].to_numpy()
                for k, v in y_train_per_level.items()
                if k in predictions_per_level
            ]
        predictions_pooled = pd.concat(predictions_per_level.values())
        y_train_pooled = pd.concat(
            [v for k, v in y_train_per_level.items() if k in predictions_per_level]
        )

        pooled = []
        y_true = predictions_pooled['y_true']
        y_pred = predictions_pooled['y_pred']
        for m, m_name in zip(metrics, metric_names):
            if m_name in scaled_metrics:
                pooled.append(
                    m(y_true=y_true, y_pred=y_pred, y_train=list_y_train_by_level)
                )
            else:
                pooled.append(
                    m(y_true=y_true, y_pred=y_pred, y_train=y_train_pooled["y_train"])
                )
        pooled = pd.DataFrame([pooled], columns=metric_names)
        pooled['levels'] = 'pooling'

        metrics_levels = pd.concat(
            [metrics_levels, average, weighted_average, pooled],
            axis=0, ignore_index=True
        )

    predictions = (
        pd.concat(predictions_per_level.values())
        .loc[:, ["y_pred", "_level_skforecast"]]
        .pivot(columns="_level_skforecast", values="y_pred")
        .rename_axis(columns=None, index=None)
    )
    if isinstance(X_test.index, pd.DatetimeIndex):
        predictions = predictions.asfreq(freq)
    else:
        predictions = predictions.reindex(
            pd.RangeIndex(
                start=min(X_test.index), stop=max(X_test.index) + 1, step=freq
            )
        )

    return metrics_levels, predictions
