# Unit test backtesting_forecaster_multiseries
# ==============================================================================
import re
import pytest
import numpy as np
import pandas as pd
from lightgbm import LGBMRegressor
from sklearn.linear_model import Ridge
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_absolute_error
from scipy.stats import norm

from skforecast.exceptions import IgnoredArgumentWarning
from skforecast.recursive import ForecasterRecursive
from skforecast.recursive import ForecasterRecursiveMultiSeries
from skforecast.direct import ForecasterDirectMultiVariate
from skforecast.model_selection import backtesting_forecaster_multiseries
from skforecast.model_selection._split import TimeSeriesFold
from skforecast.preprocessing import RollingFeatures

# Fixtures
from ....recursive.tests.tests_forecaster_recursive_multiseries.fixtures_forecaster_recursive_multiseries import expected_df_to_long_format
from ..fixtures_model_selection_multiseries import (
    series_wide_range,
    series_wide_dt,
    series_wide_dt_nans,
    series_long_dt,
    series_long_dt_nans,
    series_dict_range,
    series_dict_dt,
    series_dict_nans,
    exog_dict_nans,
    series_dict_nans_train,
    exog_dict_nans_train,
    custom_metric
)


def test_backtesting_forecaster_multiseries_TypeError_when_forecaster_not_a_forecaster_multiseries():
    """
    Test TypeError is raised in backtesting_forecaster_multiseries when 
    forecaster is not of type 'ForecasterRecursiveMultiSeries', 
    'ForecasterRnn' or 'ForecasterDirectMultiVariate'.
    """
    forecaster = ForecasterRecursive(
                     estimator = Ridge(random_state=123),
                     lags      = 2
                 )
    cv = TimeSeriesFold(
            initial_train_size = 12,
            steps              = 4,
            refit              = False,
            fixed_train_size   = False
         )
    err_msg = re.escape(
        "`forecaster` must be of type ['ForecasterRecursiveMultiSeries', "
        "'ForecasterDirectMultiVariate', 'ForecasterRnn'], for all "
        "other types of forecasters use the functions "
        "available in the `model_selection` module. "
        f"Got {type(forecaster).__name__}"
    )
    with pytest.raises(TypeError, match = err_msg):
        backtesting_forecaster_multiseries(
            forecaster            = forecaster,
            series                = series_wide_range,
            cv                    = cv,
            levels                = 'l1',
            metric                = 'mean_absolute_error',
            add_aggregated_metric = False,
            exog                  = None,
            verbose               = False
        )


# ForecasterRecursiveMultiSeries
# ======================================================================================================================
@pytest.mark.parametrize("forecaster, n_jobs", 
                         [(ForecasterRecursiveMultiSeries(estimator=Ridge(random_state=123), 
                                                          lags=2, transformer_series=None,
                                                          encoding='onehot'), -1),
                          (ForecasterRecursiveMultiSeries(estimator=Ridge(random_state=123), 
                                                          lags=2, transformer_series=None,
                                                          encoding='onehot'), 1),
                          (ForecasterRecursiveMultiSeries(estimator=Ridge(random_state=123), 
                                                          lags=2, transformer_series=None,
                                                          encoding='onehot'), 'auto')], 
                         ids=lambda fc: f'forecaster, n_jobs: {fc}')
def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_not_refit(forecaster, n_jobs):
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    without refit with mocked 
    (mocked done in Skforecast v0.5.0).
    """
    cv = TimeSeriesFold(
            initial_train_size = len(series_dict_range['l1']) - 12,
            steps              = 3,
            refit              = False,
            fixed_train_size   = False
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_dict_range,
                                               cv                    = cv,
                                               levels                = 'l1',
                                               metric                = 'mean_absolute_error',
                                               add_aggregated_metric = False,
                                               exog                  = None,
                                               verbose               = True,
                                               n_jobs                = n_jobs
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'],
                                    'mean_absolute_error': [0.20754847190853098]})
    expected_predictions = pd.DataFrame({
                               'l1': np.array([0.4978839 , 0.46288427, 0.48433446, 
                                               0.48767779, 0.477799  , 0.48523814, 
                                               0.49341916, 0.48967772, 0.48517846, 
                                               0.49868447, 0.4859614 , 0.48480032])},
                               index=pd.RangeIndex(start=38, stop=50, step=1)
                           )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_not_refit_not_initial_train_size_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    without refit and initial_train_size is None with mocked, forecaster must 
    be fitted, (mocked done in Skforecast v0.5.0).
    """

    forecaster = ForecasterRecursiveMultiSeries(
        estimator          = Ridge(random_state=123),
        lags               = 2,
        transformer_series = None,
        encoding           = "onehot",
    )
    forecaster.fit(series=series_dict_range)

    cv = TimeSeriesFold(
            initial_train_size = None,
            steps              = 1,
            refit              = False,
            fixed_train_size   = False
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_dict_range,
                                               cv                    = cv,
                                               levels                = 'l1',
                                               metric                = mean_absolute_error,
                                               add_aggregated_metric = False,
                                               exog                  = None,
                                               verbose               = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'],
                                    'mean_absolute_error': [0.18616882305307128]})
    expected_predictions = pd.DataFrame({
        'l1': np.array([0.48459053, 0.49259742, 0.51314434, 0.51387387, 0.49192289,
                        0.53266761, 0.49986433, 0.496257  , 0.49677997, 0.49641078,
                        0.52024409, 0.49255581, 0.47860725, 0.50888892, 0.51923275,
                        0.4773962 , 0.49249923, 0.51342903, 0.50350073, 0.50946515,
                        0.51912045, 0.50583902, 0.50272475, 0.51237963, 0.48600893,
                        0.49942566, 0.49056705, 0.49810661, 0.51591527, 0.47512221,
                        0.51005943, 0.5003548 , 0.50409177, 0.49838669, 0.49366925,
                        0.50348344, 0.52748975, 0.51740335, 0.49023212, 0.50969436,
                        0.47668736, 0.50262471, 0.50267211, 0.52623492, 0.47776998,
                        0.50850968, 0.53127329, 0.49010354])},
        index=pd.RangeIndex(start=2, stop=50, step=1)
    )
    expected_predictions.insert(1, 'fold', np.arange(0, 48))
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


@pytest.mark.parametrize("forecaster, n_jobs", 
                         [(ForecasterRecursiveMultiSeries(estimator=Ridge(random_state=123), 
                                                        lags=2, transformer_series=None,
                                                        encoding='onehot'), -1),
                          (ForecasterRecursiveMultiSeries(estimator=Ridge(random_state=123), 
                                                        lags=2, transformer_series=None,
                                                        encoding='onehot'), 1),
                          (ForecasterRecursiveMultiSeries(estimator=Ridge(random_state=123), 
                                                        lags=2, transformer_series=None,
                                                        encoding='onehot'), 'auto')], 
                         ids=lambda fc: f'forecaster, n_jobs: {fc}')
def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_refit_fixed_train_size_with_mocked(forecaster, n_jobs):
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with refit, fixed_train_size and custom metric with mocked 
    (mocked done in Skforecast v0.5.0).
    """
    cv = TimeSeriesFold(
            initial_train_size = len(series_dict_range['l1']) - 12,
            steps              = 3,
            refit              = True,
            fixed_train_size   = True
         )
    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_dict_range,
                                               cv                    = cv,
                                               levels                = ['l1'],
                                               metric                = custom_metric,
                                               add_aggregated_metric = False,
                                               exog                  = None,
                                               verbose               = True,
                                               n_jobs                = n_jobs
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'],
                                    'custom_metric': [0.21651617115803679]})
    expected_predictions = pd.DataFrame({
                               'l1': np.array([0.4978839 , 0.46288427, 0.48433446, 
                                               0.50853803, 0.50006415, 0.50105623,
                                               0.46764379, 0.46845675, 0.46768947, 
                                               0.48298309, 0.47778385, 0.47776533])},
                               index=pd.RangeIndex(start=38, stop=50, step=1)
                           )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


@pytest.mark.parametrize("forecaster, n_jobs", 
                         [(ForecasterRecursiveMultiSeries(estimator=Ridge(random_state=123), 
                                                        lags=2, transformer_series=None,
                                                        encoding='onehot'), -1),
                          (ForecasterRecursiveMultiSeries(estimator=Ridge(random_state=123), 
                                                        lags=2, transformer_series=None,
                                                        encoding='onehot'), 1),
                          (ForecasterRecursiveMultiSeries(estimator=Ridge(random_state=123), 
                                                        lags=2, transformer_series=None,
                                                        encoding='onehot'), 'auto')], 
                         ids=lambda fc: f'forecaster, n_jobs: {fc}')
def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_refit_with_mocked(forecaster, n_jobs):
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with refit with mocked (mocked done in Skforecast v0.5.0).
    """

    cv = TimeSeriesFold(
            initial_train_size = len(series_dict_range['l1']) - 12,
            steps              = 3,
            refit              = True,
            fixed_train_size   = False
        )
    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_dict_range,
                                               cv                    = cv,
                                               levels                = 'l1',
                                               metric                = 'mean_absolute_error',
                                               add_aggregated_metric = False,
                                               exog                  = None,
                                               verbose               = False,
                                               n_jobs                = n_jobs
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.2124129141233719]})
    expected_predictions = pd.DataFrame({
        'l1': np.array([0.4978838984103099, 0.46288426670127997, 0.48433446479429937, 
                        0.510664891759972, 0.49734477162307983, 0.5009680695304023,
                        0.48647770856843825, 0.4884651517014008, 0.48643766346259326, 
                        0.4973047492523979, 0.4899104838474172, 0.4891085370228432])},
        index=pd.RangeIndex(start=38, stop=50, step=1)
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


@pytest.mark.parametrize("series", 
                         [series_wide_range, series_dict_range],
                         ids = lambda series: f'series type: {type(series)}')
def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_refit_list_metrics_with_mocked_metrics(series):
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with refit and list of metrics with mocked and list of metrics 
    (mocked done in Skforecast v0.5.0).
    """
    if isinstance(series, pd.DataFrame):
        series = series.rename(columns={'1': 'l1', '2': 'l2'})
    
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )
    cv = TimeSeriesFold(
            initial_train_size = len(series['l1']) - 12,
            steps              = 3,
            refit              = True,
            fixed_train_size   = False
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series,
                                               cv                    = cv,
                                               levels                = 'l1',
                                               metric                = ['mean_absolute_error', mean_absolute_error],
                                               add_aggregated_metric = False,
                                               exog                  = None,
                                               verbose               = False
                                           )
    
    expected_metric = pd.DataFrame(
        data    = [['l1', 0.2124129141233719, 0.2124129141233719]],
        columns = ['levels', 'mean_absolute_error', 'mean_absolute_error']
    )
    expected_predictions = pd.DataFrame({
        'l1': np.array([0.4978838984103099, 0.46288426670127997, 0.48433446479429937, 
                        0.510664891759972, 0.49734477162307983, 0.5009680695304023,
                        0.48647770856843825, 0.4884651517014008, 0.48643766346259326, 
                        0.4973047492523979, 0.4899104838474172, 0.4891085370228432])},
        index=pd.RangeIndex(start=38, stop=50, step=1)
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_no_refit_levels_metrics_remainder_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with no refit, remainder, multiple levels and metrics with mocked 
    (mocked done in Skforecast v0.5.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )

    cv = TimeSeriesFold(
            initial_train_size = len(series_dict_range['l1']) - 12,
            steps              = 5,
            refit              = False,
            fixed_train_size   = False
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_dict_range,
                                               cv                    = cv,
                                               levels                = None,
                                               metric                = ['mean_absolute_error', mean_absolute_error],
                                               add_aggregated_metric = False,
                                               exog                  = None,
                                               verbose               = False
                                           )
    
    expected_metric = pd.DataFrame(
        data    = [['l1', 0.21143995953996186, 0.21143995953996186],
                   ['l2', 0.2194174144550234, 0.2194174144550234]],
        columns = ['levels', 'mean_absolute_error', 'mean_absolute_error']
    )
    expected_predictions = pd.DataFrame({
        'l1': np.array([0.4978839 , 0.46288427, 0.48433446, 0.48677605, 0.48562473,
                        0.50259242, 0.49536197, 0.48478881, 0.48496106, 0.48555902,
                        0.49673897, 0.4576795 ]),
        'l2': np.array([0.50266337, 0.53045945, 0.50527774, 0.50315834, 0.50452649,
                        0.47372756, 0.51226827, 0.50650107, 0.50420766, 0.50448097,
                        0.52211914, 0.51092531])},
        index=pd.RangeIndex(start=38, stop=50, step=1)
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_refit_levels_metrics_remainder_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with refit, remainder, multiple levels and metrics with mocked 
    (mocked done in Skforecast v0.5.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )

    cv = TimeSeriesFold(
            initial_train_size = len(series_dict_range['l1']) - 12,
            steps              = 5,
            refit              = True,
            fixed_train_size   = False
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_dict_range,
                                               cv                    = cv,
                                               levels                = None,
                                               metric                = ['mean_absolute_error', mean_absolute_error],
                                               add_aggregated_metric = False,
                                               exog                  = None,
                                               verbose               = False
                                           )
    
    expected_metric = pd.DataFrame(
        data    = [['l1', 0.20809130188099298, 0.20809130188099298],
                   ['l2', 0.22082212805693338, 0.22082212805693338]],
        columns = ['levels', 'mean_absolute_error', 'mean_absolute_error']
    )
    expected_predictions = pd.DataFrame({
        'l1': np.array([0.4978839 , 0.46288427, 0.48433446, 0.48677605, 0.48562473,
                        0.49724331, 0.4990606 , 0.4886555 , 0.48776085, 0.48830266,
                        0.52381728, 0.47432451]),
        'l2': np.array([0.50266337, 0.53045945, 0.50527774, 0.50315834, 0.50452649,
                        0.46847508, 0.5144631 , 0.51135241, 0.50842259, 0.50838289,
                        0.52555989, 0.51801796])},
        index=pd.RangeIndex(start=38, stop=50, step=1)
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_not_refit_exog_interval_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    without refit with mocked using exog and intervals (mocked done in Skforecast v0.5.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )

    cv = TimeSeriesFold(
            initial_train_size = len(series_dict_range['l1']) - 12,
            steps              = 3,
            refit              = False,
            fixed_train_size   = False
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_dict_range,
                                               cv                      = cv,
                                               levels                  = ['l1'],
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_dict_range['l1'].rename('exog_1'),
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 500,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.14238176570382063]})
    expected_predictions = pd.DataFrame(
                               data = np.array([[0.64371728, 0.4255348 , 0.8903798 ],
                                                [0.47208179, 0.19965087, 0.71918954],
                                                [0.52132498, 0.2490416 , 0.79113351],
                                                [0.3685079 , 0.15032541, 0.61517042],
                                                [0.42192697, 0.14949604, 0.66903471],
                                                [0.46785602, 0.19557264, 0.73766456],
                                                [0.61543694, 0.39725445, 0.86209946],
                                                [0.41627752, 0.1438466 , 0.66338527],
                                                [0.4765156 , 0.20423222, 0.74632413],
                                                [0.65858347, 0.44040098, 0.90524599],
                                                [0.49986428, 0.22743336, 0.74697203],
                                                [0.51750994, 0.24522656, 0.78731848]]),
                               columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
                               index = pd.RangeIndex(start=38, stop=50, step=1)
                           )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_refit_fixed_train_size_exog_interval_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with refit and fixed_train_size with mocked using exog and intervals 
    (mocked done in Skforecast v0.5.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )

    cv = TimeSeriesFold(
            initial_train_size = len(series_dict_range['l1']) - 12,
            steps              = 3,
            refit              = True,
            fixed_train_size   = True
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_dict_range,
                                               cv                      = cv,
                                               levels                  = 'l1',
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_dict_range['l1'].rename('exog_1'),
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 500,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.1509587543248219]})
    expected_predictions = pd.DataFrame(
                               data = np.array([[0.64371728, 0.4255348 , 0.8903798 ],
                                                [0.47208179, 0.19965087, 0.71918954],
                                                [0.52132498, 0.2490416 , 0.79113351],
                                                [0.38179014, 0.0984125 , 0.64923866],
                                                [0.43343713, 0.16869732, 0.70945522],
                                                [0.4695322 , 0.1874821 , 0.72854573],
                                                [0.57891069, 0.29994629, 0.8852084 ],
                                                [0.41212578, 0.1370795 , 0.68793307],
                                                [0.46851038, 0.19263402, 0.75231849],
                                                [0.63190066, 0.35803673, 0.92190809],
                                                [0.49132695, 0.23368778, 0.76411531],
                                                [0.51665452, 0.24370733, 0.80453074]]),
                               columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
                               index = pd.RangeIndex(start=38, stop=50, step=1)
                           )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_no_refit_exog_interval_gap_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with no refit and gap with mocked using exog and intervals 
    (mocked done in Skforecast v0.5.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )

    cv = TimeSeriesFold(
            initial_train_size = len(series_dict_range['l1']) - 20,
            steps              = 5,
            gap                = 3,
            refit              = False,
            fixed_train_size   = False
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_dict_range,
                                               cv                      = cv,
                                               levels                  = None,
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_dict_range['l1'].rename('exog_1'),
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 150,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )

    expected_metric = pd.DataFrame(
        {'levels': ['l1', 'l2'],
         'mean_absolute_error': [0.12454132173965092, 0.27469835002689663]}
    )
    expected_predictions = pd.DataFrame(
        data={
            "level": [
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
                "l1",
                "l2",
            ],
            "fold": [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                3,
                3,
                3,
                3,
            ],
            "pred": [
                0.49730036243537784,
                0.4998402577520187,
                0.47420842953795883,
                0.47674469563507416,
                0.4353776720245559,
                0.4379138356939922,
                0.4743644399072573,
                0.4769006315978118,
                0.6336613001675475,
                0.6361974926695857,
                0.6507444028348909,
                0.6533103400863456,
                0.4989678150521124,
                0.5015041166715075,
                0.5403001666320686,
                0.542836128765268,
                0.3684756420744971,
                0.3710118333555373,
                0.43668441377416667,
                0.43922060783578604,
                0.4712609487756162,
                0.4737907296720655,
                0.624431491108326,
                0.6269679682275641,
                0.41464205849840485,
                0.41717830059257244,
                0.49248162880625845,
                0.49501781891590835,
                0.665206915274275,
                0.6677431071682638,
                0.5060918367467624,
                0.5086115456199943,
                0.5364289684440446,
                0.5389649676269659,
            ],
            "lower_bound": [
                0.28665806709134994,
                0.0016639673904948753,
                0.2627403857990317,
                -0.08198462790327045,
                0.14792272535427037,
                -0.01721379281157473,
                0.26492951126379966,
                -0.022945795607171405,
                0.38057452866433383,
                0.18163384112960576,
                0.4401021074908629,
                0.15513404972482184,
                0.28749977131318527,
                -0.05722520686683701,
                0.252845219961783,
                0.08770850025970103,
                0.15904071343103957,
                -0.12883459384944596,
                0.1835976422709529,
                -0.015343043704193848,
                0.26061865343158824,
                -0.024385560689458324,
                0.41296344736939894,
                0.06823864468921947,
                0.12718711182811931,
                -0.03794932791299452,
                0.2830467001628009,
                -0.004828608289074926,
                0.4121201437710612,
                0.21317945562828403,
                0.29544954140273444,
                0.05191842245524596,
                0.3249609247051175,
                0.04175542357060692,
            ],
            "upper_bound": [
                0.7766114097999868,
                0.873459347707239,
                0.6716035154929846,
                0.8474974824443811,
                0.6681203376544204,
                0.8137785233427846,
                0.7072111453567272,
                0.8549721590887374,
                0.8647342609948074,
                1.0110735673917544,
                0.9300554501994998,
                1.026929430041566,
                0.6963629010071382,
                0.8722569034808145,
                0.773042832261933,
                0.9187008164140603,
                0.6013223475239671,
                0.7490833608464628,
                0.6677573746014266,
                0.8140966825579546,
                0.7505719961402251,
                0.8474098196272858,
                0.8218265770633518,
                0.997720755036871,
                0.6473847241282694,
                0.7930429882413648,
                0.7253283342557284,
                0.8730893464068338,
                0.8962798761015348,
                1.0426191818904327,
                0.7854028841113713,
                0.8861773289228698,
                0.7338240543990704,
                0.9178744245002909,
            ],
        },
        columns=["level", "fold", "pred", "lower_bound", "upper_bound"],
        index=pd.Index(
            [
                33,
                33,
                34,
                34,
                35,
                35,
                36,
                36,
                37,
                37,
                38,
                38,
                39,
                39,
                40,
                40,
                41,
                41,
                42,
                42,
                43,
                43,
                44,
                44,
                45,
                45,
                46,
                46,
                47,
                47,
                48,
                48,
                49,
                49,
            ]
        ),
    )

    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_refit_exog_interval_gap_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with refit, gap, allow_incomplete_fold False with mocked using exog and intervals 
    (mocked done in Skforecast v0.5.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )

    cv = TimeSeriesFold(
            initial_train_size = len(series_dict_range['l1']) - 20,
            steps              = 5,
            gap                = 3,
            refit              = True,
            fixed_train_size   = False,
            allow_incomplete_fold = False,
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_dict_range,
                                               cv                      = cv,
                                               levels                  = 'l1',
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_dict_range['l1'].rename('exog_1'),
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 150,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.136188772278576]})
    expected_predictions = pd.DataFrame(
                               data = np.array([[0.49730036, 0.28665807, 0.77661141],
                                                [0.47420843, 0.26274039, 0.67160352],
                                                [0.43537767, 0.14792273, 0.66812034],
                                                [0.47436444, 0.26492951, 0.70721115],
                                                [0.6336613 , 0.38057453, 0.86473426],
                                                [0.64530446, 0.44046218, 0.81510852],
                                                [0.48649557, 0.27908693, 0.65686118],
                                                [0.5353094 , 0.29070659, 0.74110867],
                                                [0.34635064, 0.11318645, 0.55271673],
                                                [0.42546209, 0.18120408, 0.6633966 ],
                                                [0.47082006, 0.22911629, 0.74899899],
                                                [0.61511518, 0.38584201, 0.88291368],
                                                [0.42930559, 0.17763009, 0.68681774],
                                                [0.4836202 , 0.20329772, 0.76514001],
                                                [0.6539521 , 0.36508359, 0.93752186]]),
                               columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
                               index = pd.RangeIndex(start=33, stop=48, step=1)
                           )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_refit_exog_fixed_train_size_interval_gap_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with refit, fixed_train_size, gap, with mocked using exog and intervals 
    (mocked done in Skforecast v0.5.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )

    cv = TimeSeriesFold(
            initial_train_size = len(series_long_dt.loc['l1']) - 20,
            steps              = 5,
            gap                = 5,
            refit              = True,
            fixed_train_size   = True,
            allow_incomplete_fold = False,
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_long_dt,
                                               cv                      = cv,
                                               levels                  = 'l1',
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_dict_dt['l1'].rename('exog_1'),
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 150,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.13924838761692487]})
    expected_predictions = pd.DataFrame(
        data = np.array([[0.43537767, 0.14792273, 0.66812034],
                         [0.47436444, 0.26492951, 0.70721115],
                         [0.6336613 , 0.38057453, 0.86473426],
                         [0.6507473 , 0.43794985, 0.88087126],
                         [0.49896782, 0.21084521, 0.73020093],
                         [0.52865114, 0.31806214, 0.72153007],
                         [0.33211557, 0.09961948, 0.51874788],
                         [0.4205642 , 0.16355891, 0.60846567],
                         [0.44468828, 0.21231039, 0.63715056],
                         [0.61301873, 0.35329202, 0.80571696],
                         [0.41671844, 0.13272987, 0.72577332],
                         [0.47385345, 0.19808844, 0.77943379],
                         [0.62360146, 0.36771827, 0.89062182],
                         [0.49407875, 0.20961775, 0.79262603],
                         [0.51234652, 0.28959935, 0.77646864]]),
        columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
        index = pd.date_range(start='2020-02-05', periods=15, freq='D')
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_no_refit_different_lengths_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with no refit and gap with mocked using exog and intervals and series 
    with different lengths (mocked done in Skforecast v0.5.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )

    exog_long_dt = series_wide_dt['l1'].copy().rename('exog_1').to_frame()
    exog_long_dt.index.name = "datetime"
    exog_long_dt = [exog_long_dt.assign(series_id=f"l{i}") for i in range(1, 3)]
    exog_long_dt = pd.concat(exog_long_dt)
    exog_long_dt = exog_long_dt.set_index(["series_id", exog_long_dt.index])

    cv = TimeSeriesFold(
            initial_train_size = len(series_long_dt.loc['l1']) - 20,
            steps              = 5,
            gap                = 3,
            refit              = False,
            fixed_train_size   = False,
            allow_incomplete_fold = True,
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_long_dt_nans,
                                               cv                      = cv,
                                               levels                  = 'l1',
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = exog_long_dt,
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 150,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.11243765852459384]})
    expected_predictions = pd.DataFrame(
        data = np.array([[0.49746847, 0.29550116, 0.74555927],
                        [0.46715961, 0.2507391 , 0.64732398],
                        [0.42170236, 0.16610213, 0.64208341],
                        [0.47242371, 0.29346814, 0.68859381],
                        [0.66450589, 0.42917887, 0.87949769],
                        [0.67311291, 0.47114561, 0.92120371],
                        [0.48788629, 0.27146579, 0.66805066],
                        [0.55141437, 0.29581413, 0.77179541],
                        [0.33369285, 0.15473728, 0.54986294],
                        [0.43287852, 0.19755151, 0.64787033],
                        [0.46643379, 0.26446648, 0.71452459],
                        [0.6535986 , 0.43717809, 0.83376297],
                        [0.38328273, 0.12768249, 0.60366377],
                        [0.49931045, 0.32035488, 0.71548055],
                        [0.70119564, 0.46586862, 0.91618744],
                        [0.49290276, 0.29093545, 0.74099356],
                        [0.54653561, 0.3301151 , 0.72669998]]),
        columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
        index = pd.date_range(start='2020-02-03', periods=17, freq='D')
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_refit_different_lengths_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with refit, gap, allow_incomplete_fold False with mocked using exog and intervals 
    (mocked done in Skforecast v0.5.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )
    exog_wide_dt_nans = series_wide_dt_nans['l1'].copy().rename('exog_1')

    cv = TimeSeriesFold(
            initial_train_size = len(series_long_dt.loc['l1']) - 20,
            steps              = 5,
            gap                = 3,
            refit              = True,
            fixed_train_size   = False,
            allow_incomplete_fold = False,
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_long_dt_nans,
                                               cv                      = cv,
                                               levels                  = 'l1',
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = exog_wide_dt_nans,
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 150,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.12472922864372042]})
    expected_predictions = pd.DataFrame(
                               data = np.array([[0.49746847, 0.29550116, 0.74555927],
                                                [0.46715961, 0.2507391 , 0.64732398],
                                                [0.42170236, 0.16610213, 0.64208341],
                                                [0.47242371, 0.29346814, 0.68859381],
                                                [0.66450589, 0.42917887, 0.87949769],
                                                [0.66852682, 0.47070803, 0.84676133],
                                                [0.47796299, 0.28588773, 0.64102533],
                                                [0.5477168 , 0.33322731, 0.7398035 ],
                                                [0.31418572, 0.11553706, 0.50797078],
                                                [0.42390085, 0.21039766, 0.64473339],
                                                [0.46927962, 0.25424061, 0.74034771],
                                                [0.63294789, 0.42481451, 0.88398601],
                                                [0.41355599, 0.17601853, 0.66530356],
                                                [0.48437311, 0.21791903, 0.74775272],
                                                [0.67731074, 0.40792678, 0.94040138]]),
                               columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
                               index = pd.date_range(start='2020-02-03', periods=15, freq='D')
                           )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_refit_fixed_train_size_different_lengths_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with refit, fixed_train_size, gap, with mocked using exog and intervals 
    (mocked done in Skforecast v0.5.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )
    exog_wide_dt_nans = series_wide_dt_nans['l1'].copy().rename('exog_1')

    cv = TimeSeriesFold(
            initial_train_size = len(series_long_dt.loc['l1']) - 20,
            steps              = 5,
            gap                = 5,
            refit              = True,
            fixed_train_size   = True,
            allow_incomplete_fold = False,
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_long_dt_nans,
                                               cv                      = cv,
                                               levels                  = 'l1',
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = exog_wide_dt_nans,
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 150,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.1355099897175138]})
    expected_predictions = pd.DataFrame(
        data = np.array([[0.42170236, 0.16610213, 0.64208341],
                         [0.47242371, 0.29346814, 0.68859381],
                         [0.66450589, 0.42917887, 0.87949769],
                         [0.67311586, 0.46594862, 0.88258655],
                         [0.487886  , 0.24037371, 0.69096301],
                         [0.52759202, 0.31779086, 0.72667332],
                         [0.33021382, 0.09341694, 0.51993219],
                         [0.4226274 , 0.17011294, 0.61466733],
                         [0.44647715, 0.21145026, 0.648546  ],
                         [0.61435678, 0.35925221, 0.81778071],
                         [0.41671844, 0.13272987, 0.72577332],
                         [0.47385345, 0.19808844, 0.77943379],
                         [0.62360146, 0.36771827, 0.89062182],
                         [0.49407875, 0.20961775, 0.79262603],
                         [0.51234652, 0.28959935, 0.77646864]]),
        columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
        index = pd.date_range(start='2020-02-05', periods=15, freq='D')
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_refit_int_interval_yes_exog_yes_remainder_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with refit int, interval, gap, with mocked using exog and intervals 
    (mocked done in Skforecast v0.9.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )

    cv = TimeSeriesFold(
            initial_train_size = len(series_dict_range['l1']) - 20,
            steps              = 2,
            gap                = 0,
            refit              = 2,
            fixed_train_size   = True,
            allow_incomplete_fold = False,
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_dict_range,
                                               cv                      = cv,
                                               levels                  = None,
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_dict_range['l1'].rename('exog_1'),
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 150,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame(
        {'levels': ['l1', 'l2'], 
         'mean_absolute_error': [0.13341641696298234, 0.2532943228025243]}
    )
    expected_predictions = pd.DataFrame(
        data = np.array([[ 3.62121311e-01,  7.42133126e-02,  5.16491440e-01,
                           3.65807102e-01, -1.32713926e-01,  7.41872674e-01],
                         [ 4.75603912e-01,  2.64999832e-01,  7.55479197e-01,
                           4.77660870e-01, -8.83834647e-04,  8.53705295e-01],
                         [ 4.78856340e-01,  1.90948342e-01,  6.33226469e-01,
                           4.79828611e-01, -1.86924167e-02,  8.55894183e-01],
                         [ 4.97626059e-01,  2.87021979e-01,  7.77501344e-01,
                           4.98181879e-01,  1.96371748e-02,  8.74226305e-01],
                         [ 4.57502853e-01,  1.78521668e-01,  7.53381877e-01,
                           4.93394245e-01,  7.68545677e-03,  8.60345618e-01],
                         [ 4.19762089e-01,  1.91477424e-01,  6.84027755e-01,
                           4.37345761e-01, -7.41413436e-02,  8.03815999e-01],
                         [ 4.64704145e-01,  1.85722960e-01,  7.60583169e-01,
                           4.95977483e-01,  1.02686947e-02,  8.62928856e-01],
                         [ 6.27792451e-01,  3.99507787e-01,  8.92058118e-01,
                           6.80737238e-01,  1.69250133e-01,  1.04720748e+00],
                         [ 5.87334375e-01,  3.81482355e-01,  8.69376004e-01,
                           6.65280778e-01,  1.23472110e-01,  1.07074278e+00],
                         [ 4.53164888e-01,  1.99428684e-01,  6.63429971e-01,
                           5.17357914e-01, -1.97873941e-02,  9.24306617e-01],
                         [ 4.91347698e-01,  2.85495678e-01,  7.73389327e-01,
                           5.56066550e-01,  1.42578818e-02,  9.61528557e-01],
                         [ 3.58133021e-01,  1.04396816e-01,  5.68398103e-01,
                           3.94861192e-01, -1.42284117e-01,  8.01809894e-01],
                         [ 4.23074894e-01,  1.64994380e-01,  7.17782657e-01,
                           4.56570627e-01, -8.17000863e-02,  9.13597456e-01],
                         [ 4.77639455e-01,  2.66741895e-01,  7.85095439e-01,
                           4.52788025e-01, -7.06830401e-02,  8.79697509e-01],
                         [ 5.90866263e-01,  3.32785749e-01,  8.85574027e-01,
                           6.06068550e-01,  6.77978370e-02,  1.06309538e+00],
                         [ 4.29943139e-01,  2.19045579e-01,  7.37399123e-01,
                           4.22379461e-01, -1.01091604e-01,  8.49288945e-01],
                         [ 4.71297777e-01,  1.72980646e-01,  7.32960612e-01,
                           5.16307783e-01, -5.04225194e-02,  8.72417518e-01],
                         [ 6.45316619e-01,  3.55842080e-01,  9.16796124e-01,
                           6.51701762e-01,  1.00592142e-01,  1.01902812e+00],
                         [ 5.42727946e-01,  2.44410815e-01,  8.04390781e-01,
                           5.10069712e-01, -5.66605897e-02,  8.66179448e-01],
                         [ 5.30915933e-01,  2.41441395e-01,  8.02395438e-01,
                           5.43494604e-01, -7.61501624e-03,  9.10820963e-01]]),
        columns = ['l1', 'l1_lower_bound', 'l1_upper_bound', 'l2', 'l2_lower_bound', 'l2_upper_bound'],
        index = pd.RangeIndex(start=30, stop=50, step=1)
    )
    expected_predictions.insert(1, 'fold', [
        0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9
    ])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


@pytest.mark.parametrize("initial_train_size", 
                         [len(series_dict_dt['l1']) - 20, "2020-01-30 00:00:00"],
                         ids=lambda init: f'initial_train_size: {init}')
def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_refit_int_interval_yes_exog_not_allow_remainder_gap_with_mocked(initial_train_size):
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with refit int, interval, gap, with mocked using exog and intervals 
    (mocked done in Skforecast v0.9.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )

    cv = TimeSeriesFold(
            initial_train_size = initial_train_size,
            steps              = 4,
            gap                = 3,
            refit              = 3,
            fixed_train_size   = False,
            allow_incomplete_fold = False,
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_dict_dt,
                                               cv                      = cv,
                                               levels                  = ['l2'],
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_dict_dt['l1'].rename('exog_1').copy(),
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 100,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame(
        {'levels': ['l2'], 
         'mean_absolute_error': [0.2791832310123065]}
    )
    expected_predictions = pd.DataFrame(
        data = np.array([[0.49984026, -0.00291582, 0.87744884],
                         [0.4767447 ,  0.01828895, 0.85327192],
                         [0.43791384, -0.0206504 , 0.8127147 ],
                         [0.47690063, -0.02575759, 0.85123134],
                         [0.63619086,  0.13343478, 1.01379943],
                         [0.65328344,  0.1948277 , 1.02981067],
                         [0.50150406,  0.04293982, 0.87630492],
                         [0.54283634,  0.04017811, 0.91716704],
                         [0.37097633, -0.13177975, 0.74858491],
                         [0.43922059, -0.01923515, 0.81574781],
                         [0.47379722,  0.01523298, 0.84859808],
                         [0.62696782,  0.12430959, 1.00129852],
                         [0.45263688, -0.03481576, 0.83425956],
                         [0.5005012 ,  0.01682382, 0.96458419],
                         [0.66342359,  0.09678738, 1.08072056],
                         [0.53547991, -0.00707326, 1.06275125]]),
        columns = ['l2', 'l2_lower_bound', 'l2_upper_bound'],
        index = pd.date_range(start='2020-02-03', periods=16, freq='D')
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_series_and_exog_dict_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    when series and exog are dictionaries (mocked done in Skforecast v0.12.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=LGBMRegressor(
            n_estimators=2, random_state=123, verbose=-1, max_depth=2
        ),
        lags=14,
        encoding='ordinal',
        dropna_from_series=False,
        transformer_series=StandardScaler(),
        transformer_exog=StandardScaler(),
    )
    cv = TimeSeriesFold(
             initial_train_size    = len(series_dict_nans_train['id_1000']),
             steps                 = 24,
             refit                 = False,
             fixed_train_size      = True,
             allow_incomplete_fold = True,
         )
    
    metrics, predictions = backtesting_forecaster_multiseries(
        forecaster            = forecaster,
        series                = series_dict_nans,
        exog                  = exog_dict_nans,
        cv                    = cv,
        metric                = 'mean_absolute_error',
        add_aggregated_metric = False,
        n_jobs                = 'auto',
        verbose               = False,
        show_progress         = False,
        suppress_warnings     = True
    )

    expected_metrics = pd.DataFrame(
        data={
        'levels': ['id_1000', 'id_1001', 'id_1002', 'id_1003', 'id_1004'],
        'mean_absolute_error': [286.6227398656757, 1364.7345740769094,
                                np.nan, 237.4894217124842, 1267.85941538558]
        },
        columns=['levels', 'mean_absolute_error']
    )
    expected_predictions = pd.DataFrame(
        data=np.array([
            [1438.14154717, 2090.79352613, 2166.9832933 , 7285.52781428],
            [1438.14154717, 2089.11038884, 2074.55994929, 7488.18398744],
            [1438.14154717, 2089.11038884, 2035.99448247, 7488.18398744],
            [1403.93625654, 2089.11038884, 2035.99448247, 7488.18398744],
            [1403.93625654, 2089.11038884, 2035.99448247, 7488.18398744],
            [1403.93625654, 2076.10228838, 2035.99448247, 7250.69119259],
            [1403.93625654, 2076.10228838,        np.nan, 7085.32315355],
            [1403.93625654, 2000.42985714,        np.nan, 7285.52781428],
            [1403.93625654, 2013.4379576 ,        np.nan, 7285.52781428],
            [1403.93625654, 2013.4379576 ,        np.nan, 7285.52781428]
        ]),
        index=pd.date_range('2016-08-01', periods=10, freq='D'),
        columns=['id_1000', 'id_1001', 'id_1003', 'id_1004']
    )
    expected_predictions.insert(1, 'fold', [0] * 10)
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )

    pd.testing.assert_frame_equal(metrics, expected_metrics)
    pd.testing.assert_frame_equal(predictions.head(40), expected_predictions)


@pytest.mark.parametrize("initial_train_size", 
                         [len(series_dict_nans_train['id_1000']), "2016-07-31 00:00:00"],
                         ids=lambda init: f'initial_train_size: {init}')
def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_series_and_exog_dict_with_window_features(initial_train_size):
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    when series and exog are dictionaries and window features
    (mocked done in Skforecast v0.14.0).
    """
    window_features = RollingFeatures(
        stats = ['mean', 'std', 'min', 'max', 'sum', 'median', 'ratio_min_max', 'coef_variation'],
        window_sizes = 10,
    )
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=LGBMRegressor(
            n_estimators=20, random_state=123, verbose=-1
        ),
        lags=14,
        window_features=window_features,
        encoding='ordinal',
        dropna_from_series=False,
        transformer_series=StandardScaler(),
        transformer_exog=StandardScaler(),
    )
    cv = TimeSeriesFold(
             initial_train_size    = initial_train_size,
             steps                 = 24,
             refit                 = False,
             fixed_train_size      = True,
             allow_incomplete_fold = True,
         )
    
    metrics, predictions = backtesting_forecaster_multiseries(
        forecaster            = forecaster,
        series                = series_dict_nans,
        exog                  = exog_dict_nans,
        cv                    = cv,
        metric                = 'mean_absolute_error',
        add_aggregated_metric = False,
        n_jobs                = 'auto',
        verbose               = False,
        show_progress         = False,
        suppress_warnings     = True
    )

    expected_metrics = pd.DataFrame(
        data={
        'levels': ['id_1000', 'id_1001', 'id_1002', 'id_1003', 'id_1004'],
        'mean_absolute_error': [203.9468527, 1105.40996716,
                                np.nan, 253.89651216, 792.72364223]
        },
        columns=['levels', 'mean_absolute_error']
    )
    expected_predictions = pd.DataFrame(
        data=np.array([
            [1298.57461037, 2667.47526192, 2542.97100542, 8411.2157276 ],
            [1304.25805669, 2487.352452  , 2229.55427308, 8500.56915202],
            [1344.1446645 , 2636.37058144, 2199.05843636, 8440.88224657],
            [1379.38364674, 2544.32608586, 2236.41755734, 8116.99208078],
            [1338.09366426, 2314.51247115, 2187.52526646, 8190.43433684],
            [1151.17626597, 1901.63671417, 2181.09966124, 6262.31416544],
            [ 948.94468539, 1683.47114395,        np.nan, 5937.21928644],
            [1348.71328768, 1948.01948249,        np.nan, 8391.00440651],
            [1384.41932734, 2515.20885091,        np.nan, 8542.34427233],
            [1402.03041386, 2542.16541225,        np.nan, 8359.17989861]
        ]),
        index=pd.date_range('2016-08-01', periods=10, freq='D'),
        columns=['id_1000', 'id_1001', 'id_1003', 'id_1004']
    )
    expected_predictions.insert(1, 'fold', [0] * 10)
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )

    pd.testing.assert_frame_equal(metrics, expected_metrics)
    pd.testing.assert_frame_equal(predictions.head(40), expected_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_aggregate_metrics_true():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    with no refit, remainder, multiple levels and add_aggregated_metric.
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=Ridge(random_state=123), lags=2, transformer_series=None, encoding='onehot'
    )

    cv = TimeSeriesFold(
             initial_train_size = len(series_dict_range['l1']) - 12,
             steps              = 5,
             refit              = False,
             fixed_train_size   = False,
         )

    metrics, backtest_predictions = backtesting_forecaster_multiseries(
                                        forecaster            = forecaster,
                                        series                = series_dict_range,
                                        cv                    = cv,
                                        levels                = None,
                                        metric                = ['mean_absolute_error', 'mean_absolute_scaled_error'],
                                        add_aggregated_metric = True,
                                        exog                  = None,
                                        verbose               = False
                                    )

    expected_metrics = pd.DataFrame(
        {
            "levels": ["l1", "l2", "average", "weighted_average", "pooling"],
            "mean_absolute_error": [
                0.21143995953996186,
                0.2194174144550234,
                0.21542868699749262,
                0.21542868699749262,
                0.21542868699749262,
            ],
            "mean_absolute_scaled_error": [
                0.9265126347211138,
                0.7326447294149787,
                0.8295786820680462,
                0.8295786820680462,
                0.8164857848143883,
            ],
        }
    )
    expected_predictions = pd.DataFrame(
        {
            "l1": [
                0.4978839,
                0.46288427,
                0.48433446,
                0.48677605,
                0.48562473,
                0.50259242,
                0.49536197,
                0.48478881,
                0.48496106,
                0.48555902,
                0.49673897,
                0.4576795,
            ],
            "l2": [
                0.50266337,
                0.53045945,
                0.50527774,
                0.50315834,
                0.50452649,
                0.47372756,
                0.51226827,
                0.50650107,
                0.50420766,
                0.50448097,
                0.52211914,
                0.51092531,
            ],
        },
        index=pd.RangeIndex(start=38, stop=50, step=1),
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )

    pd.testing.assert_frame_equal(expected_metrics, metrics)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_series_and_exog_dict_with_mocked_multiple_aggregated_metrics():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    when series and exog are dictionaries and multiple aggregated metrics are calculated.
    (mocked done in Skforecast v0.12.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=LGBMRegressor(
            n_estimators=2, random_state=123, verbose=-1, max_depth=2
        ),
        lags=14,
        encoding='ordinal',
        dropna_from_series=False,
        transformer_series=StandardScaler(),
        transformer_exog=StandardScaler(),
    )
    cv = TimeSeriesFold(
            initial_train_size = len(series_dict_nans_train['id_1000']),
            steps              = 24,
            refit              = False,
            fixed_train_size   = True,
         )
    
    metrics, predictions = backtesting_forecaster_multiseries(
        forecaster            = forecaster,
        series                = series_dict_nans,
        exog                  = exog_dict_nans,
        cv                    = cv,
        metric                = ['mean_absolute_error', 'mean_squared_error'],
        add_aggregated_metric = True,
        n_jobs                = 'auto',
        verbose               = False,
        show_progress         = False,
        suppress_warnings     = True
    )

    expected_metrics = pd.DataFrame({
        'levels': {0: 'id_1000',
            1: 'id_1001',
            2: 'id_1002',
            3: 'id_1003',
            4: 'id_1004',
            5: 'average',
            6: 'weighted_average',
            7: 'pooling'},
        'mean_absolute_error': {0: 286.6227398656757,
            1: 1364.7345740769094,
            2: np.nan,
            3: 237.4894217124842,
            4: 1267.85941538558,
            5: 789.1765377601623,
            6: 745.7085483145497,
            7: 745.7085483145497},
        'mean_squared_error': {0: 105816.86051259708,
            1: 2175934.9583102698,
            2: np.nan,
            3: 95856.72602398091,
            4: 2269796.338792736,
            5: 1161851.2209098958,
            6: 1024317.153152019,
            7: 1024317.1531520189}
    })
    
    expected_predictions = pd.DataFrame(
        data=np.array([[1438.14154717, 2090.79352613, 2166.9832933, 7285.52781428],
                       [1438.14154717, 2089.11038884, 2074.55994929, 7488.18398744],
                       [1438.14154717, 2089.11038884, 2035.99448247, 7488.18398744],
                       [1403.93625654, 2089.11038884, 2035.99448247, 7488.18398744],
                       [1403.93625654, 2089.11038884, 2035.99448247, 7488.18398744],
                       [1403.93625654, 2076.10228838, 2035.99448247, 7250.69119259],
                       [1403.93625654, 2076.10228838, np.nan, 7085.32315355],
                       [1403.93625654, 2000.42985714, np.nan, 7285.52781428],
                       [1403.93625654, 2013.4379576, np.nan, 7285.52781428],
                       [1403.93625654, 2013.4379576, np.nan, 7285.52781428]]),
        columns=['id_1000', 'id_1001', 'id_1003', 'id_1004'],
        index=pd.date_range('2016-08-01', periods=10, freq='D')
    )
    expected_predictions.insert(1, 'fold', [0] * 10)
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )

    pd.testing.assert_frame_equal(metrics, expected_metrics)
    pd.testing.assert_frame_equal(predictions.head(40), expected_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_series_and_exog_dict_with_mocked_skip_folds_2():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    when series and exog are dictionaries (mocked done in Skforecast v0.12.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=LGBMRegressor(
            n_estimators=2, random_state=123, verbose=-1, max_depth=2
        ),
        lags=14,
        encoding='ordinal',
        dropna_from_series=False,
        transformer_series=StandardScaler(),
        transformer_exog=StandardScaler(),
    )
    cv = TimeSeriesFold(
            initial_train_size = len(series_dict_nans_train['id_1000']),
            steps              = 5,
            refit              = False,
            fixed_train_size   = True,
            gap                = 0,
            skip_folds         = 2
        )
    
    metrics, predictions = backtesting_forecaster_multiseries(
        forecaster        = forecaster,
        series            = series_dict_nans,
        exog              = exog_dict_nans,
        cv                = cv,
        metric            = 'mean_absolute_error',
        n_jobs            = 'auto',
        verbose           = True,
        show_progress     = True,
        suppress_warnings = True
    )

    expected_metrics = pd.DataFrame(
        data={
        'levels': ['id_1000', 'id_1001', 'id_1002', 'id_1003', 'id_1004',
                   'average', 'weighted_average', 'pooling'],
        'mean_absolute_error': [258.53959034, 1396.68222457, np.nan, 253.72012285,
                                1367.69574404, 819.1594204522052, 753.0251104677614,
                                753.0251104677616]
        },
        columns=['levels', 'mean_absolute_error']
    )
    expected_predictions = pd.DataFrame(
        data=np.array([
            [1438.14154717, 2090.79352613, 2166.9832933 , 7285.52781428],
            [1438.14154717, 2089.11038884, 2074.55994929, 7488.18398744],
            [1438.14154717, 2089.11038884, 2035.99448247, 7488.18398744],
            [1403.93625654, 2089.11038884, 2035.99448247, 7488.18398744],
            [1403.93625654, 2089.11038884, 2035.99448247, 7488.18398744],
            [1432.18965392, 2013.4379576 ,        np.nan, 7488.18398744],
            [1403.93625654, 2013.4379576 ,        np.nan, 7488.18398744],
            [1403.93625654, 2136.7144822 ,        np.nan, 7250.69119259],
            [1403.93625654, 2136.7144822 ,        np.nan, 7085.32315355],
            [1438.14154717, 2013.4379576 ,        np.nan, 7285.52781428]]),
        index=pd.DatetimeIndex([
                '2016-08-01', '2016-08-02', '2016-08-03', '2016-08-04',
                '2016-08-05', '2016-08-11', '2016-08-12', '2016-08-13',
                '2016-08-14', '2016-08-15'],
                dtype='datetime64[ns]', freq=None),
        columns=['id_1000', 'id_1001', 'id_1003', 'id_1004']
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 2, 2, 2, 2, 2])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )

    pd.testing.assert_frame_equal(metrics, expected_metrics)
    pd.testing.assert_frame_equal(predictions.head(40), expected_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_series_and_exog_dict_encoding_None_differentiation():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    when series and exog are dictionaries, encoding=None, and differentiation=1 
    (mocked done in Skforecast v0.13.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=LGBMRegressor(
            n_estimators=30, random_state=123, verbose=-1, max_depth=4
        ),
        lags=[1, 7, 14],
        encoding=None,
        differentiation=1,
        dropna_from_series=False,
        transformer_series=None,
        transformer_exog=StandardScaler(),
    )
    
    cv = TimeSeriesFold(
             initial_train_size = len(series_dict_nans_train['id_1000']),
             steps              = 24,
             refit              = False,
             fixed_train_size   = True,
             gap                = 0,
             differentiation    = forecaster.differentiation_max
         )
    
    metrics, predictions = backtesting_forecaster_multiseries(
        forecaster        = forecaster,
        series            = series_dict_nans,
        exog              = exog_dict_nans,
        cv                = cv,
        metric            = ['mean_absolute_error', 'mean_absolute_scaled_error'],
        n_jobs            = 'auto',
        verbose           = True,
        show_progress     = True,
        suppress_warnings = True
    )

    expected_metrics = pd.DataFrame(
        data={
            'levels': [
                'id_1000', 'id_1001', 'id_1002', 'id_1003', 'id_1004',
                'average', 'weighted_average', 'pooling'
            ],
            'mean_absolute_error': [
                234.51032919, 1145.83513569, np.nan, 1342.06986733,
                1025.76779699, 937.0457823, 818.85514869, 818.85514869
            ],
            'mean_absolute_scaled_error': [
                1.08353766, 3.16716567, np.nan, 5.4865466 ,
                0.8435216 , 2.64519288, 2.55539804, 1.98180886
            ]
        }
    )
    expected_predictions = pd.DataFrame(
        data=np.array([
            [1351.44874045, 3267.13419659, 3843.06135374, 7220.15909652],
            [1411.73786344, 3537.21728977, 3823.777024  , 7541.54119992],
            [1367.56233886, 3595.37617537, 3861.96220571, 7599.93110962],
            [1349.98221742, 3730.36025071, 3900.14738742, 7997.86353537],
            [1330.33321825, 3752.7395927 , 3850.39111741, 7923.44398373],
            [1045.2013386 , 2652.15007539, 4227.38385608, 6651.39787084],
            [ 801.6084919 , 2059.57604816,        np.nan, 5625.61804572],
            [1473.95692547, 3159.99921219,        np.nan, 6740.53028097],
            [1596.51499989, 3449.40467366,        np.nan, 7172.21235605],
            [1572.31317919, 3507.56355926,        np.nan, 7506.35124681]]),
        index=pd.date_range('2016-08-01', periods=10, freq='D'),
        columns=['id_1000', 'id_1001', 'id_1003', 'id_1004']
    )
    expected_predictions.insert(1, 'fold', [0] * 10)
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )

    pd.testing.assert_frame_equal(metrics, expected_metrics)
    pd.testing.assert_frame_equal(predictions.head(40), expected_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_series_and_exog_dict_differentiation_dict():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    when series and exog are dictionaries, encoding='ordinal', and differentiation as dict.
    (mocked done in Skforecast v0.15.0).
    """
    differentiation = {
        'id_1000': 1, 'id_1001': 2, 'id_1002': None, 'id_1003': 2, 'id_1004': 1, '_unknown_level': 1
    }

    forecaster = ForecasterRecursiveMultiSeries(
        estimator=LGBMRegressor(
            n_estimators=30, random_state=123, verbose=-1, max_depth=4
        ),
        lags=[1, 7, 14],
        encoding='ordinal',
        differentiation=differentiation,
        dropna_from_series=False,
        transformer_series=None,
        transformer_exog=StandardScaler(),
    )

    cv = TimeSeriesFold(
             initial_train_size = len(series_dict_nans_train['id_1000']),
             steps              = 24,
             refit              = False,
             fixed_train_size   = True,
             gap                = 0,
             differentiation    = forecaster.differentiation_max
         )
    
    metrics, predictions = backtesting_forecaster_multiseries(
        forecaster        = forecaster,
        series            = series_dict_nans,
        exog              = exog_dict_nans,
        cv                = cv,
        metric            = ['mean_absolute_error', 'mean_absolute_scaled_error'],
        n_jobs            = 'auto',
        verbose           = True,
        show_progress     = True,
        suppress_warnings = True
    )

    expected_metrics = pd.DataFrame(
        {
            "levels": {
                0: "id_1000",
                1: "id_1001",
                2: "id_1002",
                3: "id_1003",
                4: "id_1004",
                5: "average",
                6: "weighted_average",
                7: "pooling",
            },
            "mean_absolute_error": {
                0: 709.932118979991,
                1: 18124.927301272775,
                2: np.nan,
                3: 6434.862937930988,
                4: 677.5474280183303,
                5: 6486.817446550521,
                6: 8270.334566642234,
                7: 8270.334566642236,
            },
            "mean_absolute_scaled_error": {
                0: 3.2915229401688673,
                1: 49.625963480324536,
                2: np.nan,
                3: 26.186189742022798,
                4: 0.5544820306613829,
                5: 19.914539548294393,
                6: 24.408235797583576,
                7: 20.00576968474758,
            },
        }
    )
    expected_predictions = pd.DataFrame(
        data=np.array([
            [1548.71621914,  3897.51607168,  6148.05420402,  8566.05067444],
            [ 1536.05313173,  5635.69476485,  7726.49471767,  9178.12103644],
            [ 1553.93326121,  6886.39340049,  9808.83531995,  9061.45721677],
            [ 1477.30370685,  8185.08084028, 12024.19439089,  9077.12418461],
            [ 1503.35710688,  9446.39566238, 14498.93483814,  8462.20628033],
            [ 1270.65730287,  9966.339615  , 17062.71265964,  6924.19872059],
            [ 1153.69012476, 11367.60163467,         np.nan,  5961.34534784],
            [ 1768.39356058, 13476.42137718,         np.nan,  7960.28068817],
            [ 1902.92132925, 16618.90352301,         np.nan,  8691.77459165],
            [ 1930.50552359, 18784.85993513,         np.nan,  8759.63500202]]),
        index=pd.date_range('2016-08-01', periods=10, freq='D'),
        columns=['id_1000', 'id_1001', 'id_1003', 'id_1004']
    )
    expected_predictions.insert(1, 'fold', [0] * 10)
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )

    pd.testing.assert_frame_equal(metrics, expected_metrics)
    pd.testing.assert_frame_equal(predictions.head(40), expected_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_series_and_exog_dict_interval_bootstrapping():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    when series and exog are dictionaries, encoding='ordinal', and interval as 5
    bootstrapping iterations (mocked done in Skforecast v0.15.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=LGBMRegressor(
            n_estimators=30, random_state=123, verbose=-1, max_depth=4
        ),
        lags=[1, 7, 14],
        encoding='ordinal',
        dropna_from_series=False,
        transformer_series=None,
        transformer_exog=StandardScaler(),
    )

    cv = TimeSeriesFold(
             initial_train_size = len(series_dict_nans_train['id_1000']),
             steps              = 24,
             refit              = False
         )
    
    metrics, predictions = backtesting_forecaster_multiseries(
        forecaster              = forecaster,
        series                  = series_dict_nans,
        exog                    = exog_dict_nans,
        cv                      = cv,
        metric                  = ['mean_absolute_error', 'mean_absolute_scaled_error'],
        interval                = "bootstrapping",
        interval_method         = "bootstrapping",
        n_boot                  = 5,
        use_in_sample_residuals = True,
        use_binned_residuals    = False,
        n_jobs                  = 'auto',
        verbose                 = False,
        show_progress           = True,
        suppress_warnings       = True
    )

    expected_metrics = pd.DataFrame(
        {
            "levels": {
                0: "id_1000",
                1: "id_1001",
                2: "id_1002",
                3: "id_1003",
                4: "id_1004",
                5: "average",
                6: "weighted_average",
                7: "pooling",
            },
            "mean_absolute_error": {
                0: 177.94640447766702,
                1: 1451.3480109896332,
                2: np.nan,
                3: 277.78113362955673,
                4: 993.6769068120083,
                5: 725.1881139772163,
                6: 724.9604804988818,
                7: 724.960480498882,
            },
            "mean_absolute_scaled_error": {
                0: 0.8178593233613526,
                1: 4.1364664709651064,
                2: np.nan,
                3: 1.1323827428361022,
                4: 0.8271748048818786,
                5: 1.72847083551111,
                6: 2.0965105153721213,
                7: 1.760615501057647,
            },
        }
    )
    expected_predictions = pd.DataFrame(
        {
            "level": [
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
            ],
            "fold": [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
            ],
            "pred": [
                1559.6918278739745,
                2934.363291873329,
                3392.6095502790795,
                7097.054479229451,
                1572.8044765344362,
                3503.7475024125847,
                3118.0493908325134,
                8301.53364484904,
                1537.6749468304602,
                3354.2752034001587,
            ],
            "pred_boot_0": [
                1244.2487779414616,
                3249.3099580454227,
                3026.660773746065,
                7283.009034196002,
                1379.325669602402,
                2700.01667124113,
                3034.5049394996236,
                8133.9504544000665,
                1645.746552367544,
                2161.096995835602,
            ],
            "pred_boot_1": [
                1465.818251789675,
                1749.4308557013514,
                3209.353725050357,
                7596.364313849582,
                1549.9990717200433,
                2456.5235020742034,
                2727.686547196392,
                7934.2179814613755,
                1482.0354924313588,
                2118.013348030759,
            ],
            "pred_boot_2": [
                1426.9105599999878,
                2154.9574614057674,
                3300.752739211812,
                8581.874365472368,
                1472.404037564884,
                2138.1376247083017,
                2965.978799872021,
                9417.261118654871,
                1504.0231525311342,
                1838.1870081278496,
            ],
            "pred_boot_3": [
                1694.8630142768532,
                2630.418618988223,
                3433.808066331928,
                8810.79839032039,
                1704.69978628218,
                3090.1184264796143,
                3208.1917726246043,
                7948.341186101785,
                1460.1402826616131,
                2169.342767228181,
            ],
            "pred_boot_4": [
                1366.5475543161697,
                2562.3887885124836,
                3316.595598427939,
                7283.009034196002,
                1487.411991726873,
                2138.1376247083017,
                3225.084699103416,
                7880.217194939906,
                1504.0231525311342,
                2150.5287792591175,
            ],
        },
        index=pd.DatetimeIndex(
            [
                "2016-08-01",
                "2016-08-01",
                "2016-08-01",
                "2016-08-01",
                "2016-08-02",
                "2016-08-02",
                "2016-08-02",
                "2016-08-02",
                "2016-08-03",
                "2016-08-03",
            ],
            dtype="datetime64[ns]",
            freq=None,
        ),
        columns=['level', 'fold', 'pred', 'pred_boot_0', 'pred_boot_1', 
                 'pred_boot_2', 'pred_boot_3', 'pred_boot_4'],
    )

    pd.testing.assert_frame_equal(metrics, expected_metrics)
    pd.testing.assert_frame_equal(predictions.head(10), expected_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_series_and_exog_dict_interval_percentiles():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    when series and exog are dictionaries, encoding='ordinal', and interval as percentiles.
    (mocked done in Skforecast v0.15.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=LGBMRegressor(
            n_estimators=30, random_state=123, verbose=-1, max_depth=4
        ),
        lags=[1, 7, 14],
        encoding='ordinal',
        dropna_from_series=False,
        transformer_series=None,
        transformer_exog=StandardScaler(),
    )

    cv = TimeSeriesFold(
             initial_train_size = len(series_dict_nans_train['id_1000']),
             steps              = 24,
             refit              = False
         )
    
    metrics, predictions = backtesting_forecaster_multiseries(
        forecaster              = forecaster,
        series                  = series_dict_nans,
        exog                    = exog_dict_nans,
        cv                      = cv,
        metric                  = ['mean_absolute_error', 'mean_absolute_scaled_error'],
        interval                = [10, 50, 90],
        interval_method         = "bootstrapping",
        n_boot                  = 25,
        use_in_sample_residuals = True,
        use_binned_residuals    = False,
        n_jobs                  = 'auto',
        verbose                 = False,
        show_progress           = True,
        suppress_warnings       = True
    )

    expected_metrics = pd.DataFrame(
        {
            "levels": {
                0: "id_1000",
                1: "id_1001",
                2: "id_1002",
                3: "id_1003",
                4: "id_1004",
                5: "average",
                6: "weighted_average",
                7: "pooling",
            },
            "mean_absolute_error": {
                0: 177.94640447766702,
                1: 1451.3480109896332,
                2: np.nan,
                3: 277.78113362955673,
                4: 993.6769068120083,
                5: 725.1881139772163,
                6: 724.9604804988818,
                7: 724.960480498882,
            },
            "mean_absolute_scaled_error": {
                0: 0.8178593233613526,
                1: 4.1364664709651064,
                2: np.nan,
                3: 1.1323827428361022,
                4: 0.8271748048818786,
                5: 1.72847083551111,
                6: 2.0965105153721213,
                7: 1.760615501057647,
            },
        }
    )
    expected_predictions = pd.DataFrame(
        {
            "level": [
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
            ],
            "fold": [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
            ],
            "pred": [
                1559.6918278739745,
                2934.363291873329,
                3392.6095502790795,
                7097.054479229451,
                1572.8044765344362,
                3503.7475024125847,
                3118.0493908325134,
                8301.53364484904,
                1537.6749468304602,
                3354.2752034001587,
            ],
            "p_10": [
                1254.3937600463285,
                2250.0470173812155,
                2985.320707618097,
                5354.055089308734,
                1403.5717169677177,
                1951.5020122145777,
                2632.727668573485,
                6939.072959904811,
                1366.3621900495914,
                1932.690821216753,
            ],
            "p_50": [
                1467.3747504982232,
                2677.246109137839,
                3329.152734925742,
                6929.471288780478,
                1484.163094840302,
                2700.01667124113,
                2959.318453557907,
                7973.74849672378,
                1456.154435701382,
                2669.958928908045,
            ],
            "p_90": [
                1662.1929830180393,
                3379.480213187932,
                3507.050519419739,
                7765.275126610016,
                1554.52561973504,
                3358.054988829244,
                3257.4005331397716,
                8875.197956517535,
                1544.7755982424908,
                3799.3921247147614,
            ],
        },
        index=pd.DatetimeIndex(
            [
                "2016-08-01",
                "2016-08-01",
                "2016-08-01",
                "2016-08-01",
                "2016-08-02",
                "2016-08-02",
                "2016-08-02",
                "2016-08-02",
                "2016-08-03",
                "2016-08-03",
            ],
            dtype="datetime64[ns]",
            freq=None,
        )
    )

    pd.testing.assert_frame_equal(metrics, expected_metrics)
    pd.testing.assert_frame_equal(predictions.head(10), expected_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_series_and_exog_dict_interval_distribution():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    when series and exog are dictionaries, encoding='ordinal', and interval as a
    scipy.stats norm distribution. (mocked done in Skforecast v0.15.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=LGBMRegressor(
            n_estimators=30, random_state=123, verbose=-1, max_depth=4
        ),
        lags=[1, 7, 14],
        encoding='ordinal',
        dropna_from_series=False,
        transformer_series=None,
        transformer_exog=StandardScaler(),
    )

    cv = TimeSeriesFold(
             initial_train_size = len(series_dict_nans_train['id_1000']),
             steps              = 24,
             refit              = False
         )
    
    metrics, predictions = backtesting_forecaster_multiseries(
        forecaster              = forecaster,
        series                  = series_dict_nans,
        exog                    = exog_dict_nans,
        cv                      = cv,
        metric                  = ['mean_absolute_error', 'mean_absolute_scaled_error'],
        interval                = norm,
        interval_method         = "bootstrapping",
        n_boot                  = 25,
        use_in_sample_residuals = True,
        use_binned_residuals    = False,
        n_jobs                  = 'auto',
        verbose                 = False,
        show_progress           = True,
        suppress_warnings       = True
    )

    expected_metrics = pd.DataFrame(
        {
            "levels": {
                0: "id_1000",
                1: "id_1001",
                2: "id_1002",
                3: "id_1003",
                4: "id_1004",
                5: "average",
                6: "weighted_average",
                7: "pooling",
            },
            "mean_absolute_error": {
                0: 177.94640447766702,
                1: 1451.3480109896332,
                2: np.nan,
                3: 277.78113362955673,
                4: 993.6769068120083,
                5: 725.1881139772163,
                6: 724.9604804988818,
                7: 724.960480498882,
            },
            "mean_absolute_scaled_error": {
                0: 0.8178593233613526,
                1: 4.1364664709651064,
                2: np.nan,
                3: 1.1323827428361022,
                4: 0.8271748048818786,
                5: 1.72847083551111,
                6: 2.0965105153721213,
                7: 1.760615501057647,
            },
        }
    )
    expected_predictions = pd.DataFrame(
        {
            "level": [
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
            ],
            "fold": [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
            ],
            "pred": [
                1559.6918278739745,
                2934.363291873329,
                3392.6095502790795,
                7097.054479229451,
                1572.8044765344362,
                3503.7475024125847,
                3118.0493908325134,
                8301.53364484904,
                1537.6749468304602,
                3354.2752034001587,
            ],
            "loc": [
                1446.9358198599687,
                2738.992298449951,
                3285.495122563763,
                6901.155995994967,
                1481.355117305214,
                2676.9190965330286,
                2945.420322544557,
                8004.388241447931,
                1453.751765541335,
                2732.5658183877763,
            ],
            "scale": [
                171.66343357536252,
                476.1301874517562,
                254.33164031416072,
                930.8659121040267,
                75.0170147519068,
                595.8040735431155,
                434.0324080210821,
                905.0370379547618,
                84.72947549974545,
                678.2575115584594,
            ],
        },
        index=pd.DatetimeIndex(
            [
                "2016-08-01",
                "2016-08-01",
                "2016-08-01",
                "2016-08-01",
                "2016-08-02",
                "2016-08-02",
                "2016-08-02",
                "2016-08-02",
                "2016-08-03",
                "2016-08-03",
            ],
            dtype="datetime64[ns]",
            freq=None,
        )
    )

    pd.testing.assert_frame_equal(metrics, expected_metrics)
    pd.testing.assert_frame_equal(predictions.head(10), expected_predictions)


@pytest.mark.parametrize("interval", 
                         [0.90, (5, 95)], 
                         ids = lambda value: f'interval: {value}')
def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_series_and_exog_dict_interval_conformal(interval):
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    when series and exog are dictionaries, encoding='ordinal', and interval as 
    conformal with binned residuals (mocked done in Skforecast v0.15.0).
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=LGBMRegressor(
            n_estimators=30, random_state=123, verbose=-1, max_depth=4
        ),
        lags=[1, 7, 14],
        encoding='ordinal',
        dropna_from_series=False,
        transformer_series=None,
        transformer_exog=StandardScaler(),
    )

    cv = TimeSeriesFold(
             initial_train_size = len(series_dict_nans_train['id_1000']),
             steps              = 24,
             refit              = False
         )

    metrics, predictions = backtesting_forecaster_multiseries(
        forecaster              = forecaster,
        series                  = series_dict_nans,
        exog                    = exog_dict_nans,
        cv                      = cv,
        metric                  = ['mean_absolute_error', 'mean_absolute_scaled_error'],
        interval                = interval,
        interval_method         = "conformal",
        use_in_sample_residuals = True,
        use_binned_residuals    = True,
        n_jobs                  = 'auto',
        verbose                 = False,
        show_progress           = True,
        suppress_warnings       = True
    )

    expected_metrics = pd.DataFrame(
        {
            "levels": {
                0: "id_1000",
                1: "id_1001",
                2: "id_1002",
                3: "id_1003",
                4: "id_1004",
                5: "average",
                6: "weighted_average",
                7: "pooling",
            },
            "mean_absolute_error": {
                0: 177.94640447766702,
                1: 1451.3480109896332,
                2: np.nan,
                3: 277.78113362955673,
                4: 993.6769068120083,
                5: 725.1881139772163,
                6: 724.9604804988818,
                7: 724.960480498882,
            },
            "mean_absolute_scaled_error": {
                0: 0.8178593233613526,
                1: 4.1364664709651064,
                2: np.nan,
                3: 1.1323827428361022,
                4: 0.8271748048818786,
                5: 1.72847083551111,
                6: 2.0965105153721213,
                7: 1.760615501057647,
            },
        }
    )
    expected_predictions = pd.DataFrame(
        {
            "level": [
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
            ],
            "fold": [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
            ],
            "pred": [
                1559.6918278739745,
                2934.363291873329,
                3392.6095502790795,
                7097.054479229451,
                1572.8044765344362,
                3503.7475024125847,
                3118.0493908325134,
                8301.53364484904,
                1537.6749468304602,
                3354.2752034001587,
            ],
            "lower_bound": [
                1336.857463880777,
                2569.1917715600575,
                2556.6487812039504,
                5093.465893848657,
                1349.9701125412387,
                3138.575982099313,
                2282.0886217573843,
                7895.559245312443,
                1314.8405828372627,
                2989.103683086887,
            ],
            "upper_bound": [
                1782.526191867172,
                3299.534812186601,
                4228.570319354209,
                9100.643064610245,
                1795.6388405276336,
                3868.9190227258564,
                3954.0101599076424,
                8707.508044385637,
                1760.5093108236576,
                3719.4467237134304,
            ],
        },
        index=pd.DatetimeIndex(
            [
                "2016-08-01",
                "2016-08-01",
                "2016-08-01",
                "2016-08-01",
                "2016-08-02",
                "2016-08-02",
                "2016-08-02",
                "2016-08-02",
                "2016-08-03",
                "2016-08-03",
            ],
            dtype="datetime64[ns]",
            freq=None,
        )
    )

    pd.testing.assert_frame_equal(metrics, expected_metrics)
    pd.testing.assert_frame_equal(predictions.head(10), expected_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_return_predictors_same_predictions_as_predict():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    when series and exog are dictionaries, predictions from 
    _backtesting_forecaster predictors are the same as predictions from 
    predict method.
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=LGBMRegressor(
            n_estimators=30, random_state=123, verbose=-1, max_depth=4
        ),
        lags=[1, 7, 14],
        encoding='ordinal',
        dropna_from_series=False,
        transformer_series=None,
        transformer_exog=StandardScaler(),
    )

    cv = TimeSeriesFold(
             initial_train_size = len(series_dict_nans_train['id_1000']),
             steps              = 24,
             refit              = False
         )

    metrics, predictions = backtesting_forecaster_multiseries(
        forecaster        = forecaster,
        series            = series_dict_nans,
        exog              = exog_dict_nans,
        cv                = cv,
        metric            = ['mean_absolute_error', 'mean_absolute_scaled_error'],
        return_predictors = True,
        n_jobs            = 'auto',
        verbose           = False,
        show_progress     = True,
        suppress_warnings = True
    )

    expected_metrics = pd.DataFrame(
        {
            "levels": {
                0: "id_1000",
                1: "id_1001",
                2: "id_1002",
                3: "id_1003",
                4: "id_1004",
                5: "average",
                6: "weighted_average",
                7: "pooling",
            },
            "mean_absolute_error": {
                0: 177.94640447766702,
                1: 1451.3480109896332,
                2: np.nan,
                3: 277.78113362955673,
                4: 993.6769068120083,
                5: 725.1881139772163,
                6: 724.9604804988818,
                7: 724.960480498882,
            },
            "mean_absolute_scaled_error": {
                0: 0.8178593233613526,
                1: 4.1364664709651064,
                2: np.nan,
                3: 1.1323827428361022,
                4: 0.8271748048818786,
                5: 1.72847083551111,
                6: 2.0965105153721213,
                7: 1.760615501057647,
            },
        }
    )

    forecaster.fit(series=series_dict_nans_train, exog=exog_dict_nans_train)
    expected_predictions = forecaster.estimator.predict(
        predictions[forecaster.X_train_features_names_out_]
    )
    nan_predictions_index = predictions['pred'].isna()
    expected_predictions[nan_predictions_index] = np.nan

    pd.testing.assert_frame_equal(metrics, expected_metrics)
    np.testing.assert_array_almost_equal(
        expected_predictions, 
        predictions['pred'].to_numpy()
    )


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_series_and_exog_dict_fold_stride():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    when series and exog are dictionaries, encoding='ordinal', and fold_stride is used.
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=LGBMRegressor(
            n_estimators=100, random_state=123, verbose=-1, max_depth=4
        ),
        lags=[1, 7, 14],
        encoding='ordinal',
        dropna_from_series=False,
        transformer_series=None,
        transformer_exog=StandardScaler(),
    )

    cv = TimeSeriesFold(
             initial_train_size = len(series_dict_nans_train['id_1000']),
             steps              = 6,
             fold_stride        = 3,
             refit              = False
         )

    metrics, predictions = backtesting_forecaster_multiseries(
        forecaster        = forecaster,
        series            = series_dict_nans,
        exog              = exog_dict_nans,
        cv                = cv,
        metric            = ['mean_absolute_error', 'mean_absolute_scaled_error'],
        n_jobs            = 'auto',
        verbose           = False,
        show_progress     = True,
        suppress_warnings = True
    )

    expected_metrics = pd.DataFrame(
        {
            "levels": {
                0: "id_1000",
                1: "id_1001",
                2: "id_1002",
                3: "id_1003",
                4: "id_1004",
                5: "average",
                6: "weighted_average",
                7: "pooling",
            },
            "mean_absolute_error": {
                0: 137.63640920084475,
                1: 1125.7738019343692,
                2: np.nan,
                3: 271.59479275890055,
                4: 684.2844296219656,
                5: 554.82235837902,
                6: 566.5279689272918,
                7: 566.527968927292,
            },
            "mean_absolute_scaled_error": {
                0: 0.6325905872013097,
                1: 3.208552015320668,
                2: np.nan,
                3: 1.1071639471904091,
                4: 0.5696246291686657,
                5: 1.379482794720263,
                6: 1.664091840444422,
                7: 1.3758514439155465,
            },
        }
    )
    expected_predictions = pd.DataFrame(
        {
            "level": [
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
            ],
            "fold": [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
            ],
            "pred": [
                1496.6826971645523,
                3072.3764982469875,
                3299.191522356802,
                7072.533812609552,
                1487.9714641896444,
                3426.4972780704393,
                3053.711435102844,
                8601.965836378662,
                1452.8419344856684,
                3252.2993473033084,
                3004.6608026442464,
                8885.898621753877,
                1396.5202568353473,
                3642.356885360602,
                2659.034134670032,
                8808.775425272173,
                1396.5202568353473,
                3290.0239566848713,
                1864.6225324730788,
                8934.33149885238,
                1107.9667392445235,
                2484.4858907270695,
                2071.549356973606,
                6347.631033092048,
                1396.5202568353473,
                2741.458201980781,
                1885.108314225759,
                8808.775425272173,
                1396.5202568353473,
                2732.557991608278,
                1928.4526199581364,
                8934.33149885238,
                1107.9667392445235,
                2808.4765752368116,
                2152.478850510751,
                6347.631033092048,
                999.3757166023669,
                1996.2149991056203,
                np.nan,
                5049.975665851924,
                1394.6081500274586,
                3313.4205475549293,
                np.nan,
                7082.0444371062185,
                1456.8667963770772,
                3278.088830635534,
                np.nan,
                8669.715220549642,
                999.3757166023669,
                1996.2149991056203,
                np.nan,
                5979.040595708178,
                1394.6081500274586,
                3313.4205475549293,
                np.nan,
                6727.9915808740325,
                1456.8667963770772,
                3278.088830635534,
                np.nan,
                7496.565716814842,
            ],
        },
        index=pd.DatetimeIndex(
            [
                "2016-08-01",
                "2016-08-01",
                "2016-08-01",
                "2016-08-01",
                "2016-08-02",
                "2016-08-02",
                "2016-08-02",
                "2016-08-02",
                "2016-08-03",
                "2016-08-03",
                "2016-08-03",
                "2016-08-03",
                "2016-08-04",
                "2016-08-04",
                "2016-08-04",
                "2016-08-04",
                "2016-08-05",
                "2016-08-05",
                "2016-08-05",
                "2016-08-05",
                "2016-08-06",
                "2016-08-06",
                "2016-08-06",
                "2016-08-06",
                "2016-08-04",
                "2016-08-04",
                "2016-08-04",
                "2016-08-04",
                "2016-08-05",
                "2016-08-05",
                "2016-08-05",
                "2016-08-05",
                "2016-08-06",
                "2016-08-06",
                "2016-08-06",
                "2016-08-06",
                "2016-08-07",
                "2016-08-07",
                "2016-08-07",
                "2016-08-07",
                "2016-08-08",
                "2016-08-08",
                "2016-08-08",
                "2016-08-08",
                "2016-08-09",
                "2016-08-09",
                "2016-08-09",
                "2016-08-09",
                "2016-08-07",
                "2016-08-07",
                "2016-08-07",
                "2016-08-07",
                "2016-08-08",
                "2016-08-08",
                "2016-08-08",
                "2016-08-08",
                "2016-08-09",
                "2016-08-09",
                "2016-08-09",
                "2016-08-09",
            ],
            dtype="datetime64[ns]",
            freq=None,
        ),
    )

    pd.testing.assert_frame_equal(metrics, expected_metrics)
    pd.testing.assert_frame_equal(predictions.head(60), expected_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterRecursiveMultiSeries_series_and_exog_dict_fold_stride_greater_than_steps():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterRecursiveMultiSeries 
    when series and exog are dictionaries, encoding='ordinal', and fold_stride is used 
    and greater than steps.
    """
    forecaster = ForecasterRecursiveMultiSeries(
        estimator=LGBMRegressor(
            n_estimators=100, random_state=123, verbose=-1, max_depth=4
        ),
        lags=[1, 7, 14],
        encoding='ordinal',
        dropna_from_series=False,
        transformer_series=None,
        transformer_exog=StandardScaler(),
    )

    cv = TimeSeriesFold(
             initial_train_size = len(series_dict_nans_train['id_1000']),
             steps              = 6,
             fold_stride        = 9,
             refit              = False
         )

    metrics, predictions = backtesting_forecaster_multiseries(
        forecaster        = forecaster,
        series            = series_dict_nans,
        exog              = exog_dict_nans,
        cv                = cv,
        metric            = ['mean_absolute_error', 'mean_absolute_scaled_error'],
        n_jobs            = 'auto',
        verbose           = False,
        show_progress     = True,
        suppress_warnings = True
    )

    expected_metrics = pd.DataFrame(
        {
            "levels": {
                0: "id_1000",
                1: "id_1001",
                2: "id_1002",
                3: "id_1003",
                4: "id_1004",
                5: "average",
                6: "weighted_average",
                7: "pooling",
            },
            "mean_absolute_error": {
                0: 148.6020332440131,
                1: 1227.9321998720077,
                2: np.nan,
                3: 328.265097171603,
                4: 811.1618005336145,
                5: 628.9902827053096,
                6: 628.4941665601436,
                7: 628.4941665601435,
            },
            "mean_absolute_scaled_error": {
                0: 0.6829896828532042,
                1: 3.4997122226567505,
                2: np.nan,
                3: 1.3381820653387495,
                4: 0.6752422235882412,
                5: 1.5490315486092363,
                6: 1.8348312671758897,
                7: 1.5263405409473236,
            },
        }
    )
    expected_predictions = pd.DataFrame(
        {
            "level": [
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
                "id_1000",
                "id_1001",
                "id_1003",
                "id_1004",
            ],
            "fold": [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                1,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
                2,
            ],
            "pred": [
                1496.6826971645523,
                3072.3764982469875,
                3299.191522356802,
                7072.533812609552,
                1487.9714641896444,
                3426.4972780704393,
                3053.711435102844,
                8601.965836378662,
                1452.8419344856684,
                3252.2993473033084,
                3004.6608026442464,
                8885.898621753877,
                1396.5202568353473,
                3642.356885360602,
                2659.034134670032,
                8808.775425272173,
                1396.5202568353473,
                3290.0239566848713,
                1864.6225324730788,
                8934.33149885238,
                1107.9667392445235,
                2484.4858907270695,
                2071.549356973606,
                6347.631033092048,
                1414.8753525505379,
                1970.7895327686674,
                np.nan,
                8576.324768078144,
                1396.5202568353473,
                2371.7852605492076,
                np.nan,
                8715.720335897213,
                1298.5494009891943,
                2850.6145922844453,
                np.nan,
                8909.95666176813,
                1089.7553408426302,
                2849.631084588996,
                np.nan,
                6093.69986064051,
                999.3757166023669,
                2632.2649665399126,
                np.nan,
                4431.156825796141,
                1486.7183596300656,
                2724.5824532737747,
                np.nan,
                6780.925854683978,
                1415.5343547988482,
                2014.8866860318237,
                np.nan,
                8409.993066869763,
                1107.9667392445235,
                2585.9232047473556,
                np.nan,
                6520.869613289696,
                999.3757166023669,
                2878.480311718611,
                np.nan,
                5821.115715072534,
            ],
        },
        index=pd.DatetimeIndex(
            [
                "2016-08-01",
                "2016-08-01",
                "2016-08-01",
                "2016-08-01",
                "2016-08-02",
                "2016-08-02",
                "2016-08-02",
                "2016-08-02",
                "2016-08-03",
                "2016-08-03",
                "2016-08-03",
                "2016-08-03",
                "2016-08-04",
                "2016-08-04",
                "2016-08-04",
                "2016-08-04",
                "2016-08-05",
                "2016-08-05",
                "2016-08-05",
                "2016-08-05",
                "2016-08-06",
                "2016-08-06",
                "2016-08-06",
                "2016-08-06",
                "2016-08-10",
                "2016-08-10",
                "2016-08-10",
                "2016-08-10",
                "2016-08-11",
                "2016-08-11",
                "2016-08-11",
                "2016-08-11",
                "2016-08-12",
                "2016-08-12",
                "2016-08-12",
                "2016-08-12",
                "2016-08-13",
                "2016-08-13",
                "2016-08-13",
                "2016-08-13",
                "2016-08-14",
                "2016-08-14",
                "2016-08-14",
                "2016-08-14",
                "2016-08-15",
                "2016-08-15",
                "2016-08-15",
                "2016-08-15",
                "2016-08-19",
                "2016-08-19",
                "2016-08-19",
                "2016-08-19",
                "2016-08-20",
                "2016-08-20",
                "2016-08-20",
                "2016-08-20",
                "2016-08-21",
                "2016-08-21",
                "2016-08-21",
                "2016-08-21",
            ],
            dtype="datetime64[ns]",
            freq=None,
        ),
    )

    pd.testing.assert_frame_equal(metrics, expected_metrics)
    pd.testing.assert_frame_equal(predictions.head(60), expected_predictions)


# ======================================================================================================================
# ======================================================================================================================
# ForecasterDirectMultiVariate
# ======================================================================================================================
# ======================================================================================================================


@pytest.mark.parametrize("n_jobs", [1, -1, 'auto'],
                         ids=lambda n: f'n_jobs: {n}')
def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_not_refit_with_mocked(n_jobs):
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate without refit
    with mocked (mocked done in Skforecast v0.6.0).
    """

    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = 2,
                     steps              = 3,
                     transformer_series = None
                 )
    cv = TimeSeriesFold(
             initial_train_size = len(series_wide_range) - 12,
             steps              = 3,
             refit              = False,
             fixed_train_size   = False,
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_wide_range,
                                               cv                    = cv,
                                               levels                = 'l1',
                                               metric                = 'mean_absolute_error',
                                               add_aggregated_metric = False,
                                               exog                  = None,
                                               n_jobs                = n_jobs,
                                               verbose               = True
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'],
                                    'mean_absolute_error': [0.2056686186667702]})
    expected_predictions = pd.DataFrame({
                               'l1': np.array([0.55397908, 0.48026456, 0.52368724,
                                               0.48490132, 0.46928502, 0.52511441, 
                                               0.46529858, 0.45430583, 0.51706306, 
                                               0.50561424, 0.47109786, 0.45568319])},
                               index=pd.RangeIndex(start=38, stop=50, step=1)
                           )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_not_refit_not_initial_train_size_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    without refit and initial_train_size is None with mocked, forecaster must be fitted,
    (mocked done in Skforecast v0.6.0).
    """

    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = 2,
                     steps              = 3,
                     transformer_series = None
                 )
    forecaster.fit(series=series_wide_range)
    cv = TimeSeriesFold(
             initial_train_size = None,
             steps              = 1,
             refit              = False,
             fixed_train_size   = False,
         )
    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_wide_range,
                                               cv                    = cv,
                                               levels                = ['l1'],
                                               metric                = mean_absolute_error,
                                               add_aggregated_metric = False,
                                               exog                  = None,
                                               verbose               = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'],
                                    'mean_absolute_error': [0.17959810844511925]})
    expected_predictions = pd.DataFrame({
        'l1': np.array([0.42102839, 0.47138953, 0.52252844, 0.54594276, 0.51253167,
                        0.57557448, 0.45455673, 0.42240387, 0.48555804, 0.46974027,
                        0.52264755, 0.45674877, 0.43440543, 0.47187135, 0.59653789,
                        0.41804686, 0.53408781, 0.58853203, 0.49880785, 0.57834799,
                        0.47345798, 0.46890693, 0.45765737, 0.59034503, 0.46198262,
                        0.49384858, 0.54212837, 0.56867955, 0.5095804 , 0.47751184,
                        0.50402253, 0.48993588, 0.52583999, 0.4306855 , 0.42782129,
                        0.52841356, 0.62570147, 0.55585762, 0.48719966, 0.48508799,
                        0.37122115, 0.53115279, 0.47119561, 0.52734455, 0.41557646,
                        0.57546277, 0.57700474, 0.50898628])},
        index=pd.RangeIndex(start=2, stop=50, step=1)
    )
    expected_predictions.insert(1, 'fold', np.arange(0, 48))
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


@pytest.mark.parametrize("n_jobs", [1, -1, 'auto'],
                         ids=lambda n: f'n_jobs: {n}')
def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_refit_fixed_train_size_with_mocked(n_jobs):
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate with refit,
    fixed_train_size and custom metric with mocked (mocked done in Skforecast v0.6.0).
    """

    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l2',
                     lags               = 2,
                     steps              = 3,
                     transformer_series = None
                 )
    cv = TimeSeriesFold(
             initial_train_size = len(series_wide_range) - 12,
             steps              = 3,
             refit              = True,
             fixed_train_size   = True,
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_wide_range,
                                               cv                    = cv,
                                               levels                = None,
                                               metric                = custom_metric,
                                               add_aggregated_metric = False,
                                               exog                  = None,
                                               n_jobs                = n_jobs,
                                               verbose               = True
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l2'],
                                    'custom_metric': [0.2326510995879597]})
    expected_predictions = pd.DataFrame({
                               'l2': np.array([0.58478895, 0.56729494, 0.54469663,
                                               0.50326485, 0.53339207, 0.50892268, 
                                               0.46841857, 0.48498214, 0.52778775,
                                               0.51476103, 0.48480385, 0.53470992])},
                               index=pd.RangeIndex(start=38, stop=50, step=1)
                           )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


@pytest.mark.parametrize("n_jobs", [1, -1, 'auto'],
                         ids=lambda n: f'n_jobs: {n}')
@pytest.mark.parametrize("initial_train_size", [38, '2020-02-07 00:00:00'],
                         ids=lambda init: f'initial_train_size: {init}')
def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_window_features(n_jobs, initial_train_size):
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate with refit,
    fixed_train_size with window features with mocked 
    (mocked done in Skforecast v0.14.0).
    """
    window_features = RollingFeatures(
        stats = ['mean', 'std', 'min', 'max', 'sum', 'median', 'ratio_min_max', 'coef_variation'],
        window_sizes = 3,
    )
    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     steps              = 3,
                     level              = 'l2',
                     lags               = 2,
                     window_features    = window_features,
                     transformer_series = None
                 )
    cv = TimeSeriesFold(
             initial_train_size = initial_train_size,
             steps              = 3,
             refit              = True,
             fixed_train_size   = True,
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_wide_dt,
                                               cv                    = cv,
                                               levels                = None,
                                               metric                = 'mean_absolute_error',
                                               add_aggregated_metric = False,
                                               exog                  = None,
                                               n_jobs                = n_jobs,
                                               verbose               = True
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l2'],
                                    'mean_absolute_error': [0.23856556]})
    expected_predictions = pd.DataFrame({
        'l2': np.array([0.55657737, 0.37891865, 0.43460762, 0.58417451, 0.59388689,
                        0.55074551, 0.3267003 , 0.33008016, 0.40918253, 0.5313121 ,
                        0.43536832, 0.5907553])},
        index=pd.DatetimeIndex(
                    ['2020-02-08', '2020-02-09', '2020-02-10', '2020-02-11', 
                     '2020-02-12', '2020-02-13', '2020-02-14', '2020-02-15', 
                     '2020-02-16', '2020-02-17', '2020-02-18', '2020-02-19'],
                    dtype='datetime64[ns]', freq=None)
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions.asfreq('D'), backtest_predictions)


@pytest.mark.parametrize("n_jobs", [1, -1, 'auto'],
                         ids=lambda n: f'n_jobs: {n}')
def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_refit_with_mocked(n_jobs):
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    with refit with mocked (mocked done in Skforecast v0.6.0).
    """

    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = 2,
                     steps              = 3,
                     transformer_series = None
                 )
    cv = TimeSeriesFold(
            initial_train_size = len(series_wide_range) - 12,
            steps              = 3,
            refit              = True,
            fixed_train_size   = False,
        )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_wide_range,
                                               cv                    = cv,
                                               levels                = 'l1',
                                               metric                = 'mean_absolute_error',
                                               add_aggregated_metric = False,
                                               exog                  = None,
                                               n_jobs                = n_jobs,
                                               verbose               = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.20733067815663564]})
    expected_predictions = pd.DataFrame({
                               'l1': np.array([0.55397908, 0.48026456, 0.52368724, 
                                               0.49816586, 0.48470807, 0.54162611,
                                               0.45270749, 0.47194035, 0.53386908,
                                               0.55296942, 0.53498642, 0.44772825])},
                               index=pd.RangeIndex(start=38, stop=50, step=1)
                           )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_refit_list_metrics_with_mocked_metrics():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    with refit and list of metrics with mocked and list of metrics 
    (mocked done in Skforecast v0.6.0).
    """

    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = 2,
                     steps              = 3,
                     transformer_series = None
                 )
    cv = TimeSeriesFold(
            initial_train_size = len(series_wide_range) - 12,
            steps              = 3,
            refit              = True,
            fixed_train_size   = False,
        )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_wide_range,
                                               cv                    = cv,
                                               levels                = 'l1',
                                               metric                = ['mean_absolute_error', mean_absolute_error],
                                               add_aggregated_metric = False,
                                               exog                  = None,
                                               verbose               = False
                                           )
    
    expected_metric = pd.DataFrame(
        data    = [['l1', 0.20733067815663564, 0.20733067815663564]],
        columns = ['levels', 'mean_absolute_error', 'mean_absolute_error']
    )
    expected_predictions = pd.DataFrame({
                               'l1': np.array([0.55397908, 0.48026456, 0.52368724, 
                                               0.49816586, 0.48470807, 0.54162611, 
                                               0.45270749, 0.47194035, 0.53386908, 
                                               0.55296942, 0.53498642, 0.44772825])},
                               index=pd.RangeIndex(start=38, stop=50, step=1)
                           )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-predict'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_not_refit_exog_interval_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    without refit with mocked using exog and intervals 
    (mocked done in Skforecast v0.7.0).
    """

    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = {'l1': 3, 'l2': None},
                     steps              = 3,
                     transformer_series = None
                 )

    cv = TimeSeriesFold(
            initial_train_size = len(series_wide_range) - 12,
            steps              = 3,
            refit              = False,
            fixed_train_size   = False,
        )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_wide_range,
                                               cv                      = cv,
                                               levels                  = ['l1'],
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_wide_range['l1'].rename('exog_1'),
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 500,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.0849883385916299]})
    expected_predictions = pd.DataFrame(
        data = np.array([[0.76991301, 0.62064319, 0.88703848],
                        [0.4856994 , 0.33642958, 0.63146337],
                        [0.59391504, 0.43575145, 0.73470559],
                        [0.2924861 , 0.14321628, 0.40961157],
                        [0.37961054, 0.23034071, 0.5253745 ],
                        [0.42813627, 0.26997268, 0.56892681],
                        [0.69375682, 0.544487  , 0.81088229],
                        [0.3375219 , 0.18825207, 0.48328587],
                        [0.49705665, 0.33889306, 0.63784719],
                        [0.80370716, 0.65443734, 0.92083262],
                        [0.49265658, 0.34338675, 0.63842054],
                        [0.5482819 , 0.39011831, 0.68907245]]),
        columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
        index = pd.RangeIndex(start=38, stop=50, step=1)
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_refit_fixed_train_size_exog_interval_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    with refit and fixed_train_size with mocked using exog and intervals 
    (mocked done in Skforecast v0.7.0).
    """

    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = {'l1': None, 'l2': [1, 3]},
                     steps              = 3,
                     transformer_series = None
                 )
    cv = TimeSeriesFold(
            initial_train_size = len(series_wide_range) - 12,
            steps              = 3,
            refit              = True,
            fixed_train_size   = True,
        )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_wide_range,
                                               cv                      = cv,
                                               levels                  = 'l1',
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_wide_range['l1'].rename('exog_1'),
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 500,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.08191170312799796]})
    expected_predictions = pd.DataFrame(
        data = np.array([[0.76727126, 0.62448381, 0.87733835],
                        [0.48402505, 0.33143897, 0.63172374],
                        [0.55855356, 0.40055037, 0.70625225],
                        [0.23916462, 0.10047205, 0.37521533],
                        [0.38545642, 0.24278893, 0.53781831],
                        [0.45551238, 0.31681981, 0.61066531],
                        [0.72213794, 0.58748583, 0.86890672],
                        [0.33389321, 0.1992411 , 0.4778306 ],
                        [0.4767024 , 0.34677964, 0.62049076],
                        [0.81570257, 0.683558  , 0.95545438],
                        [0.5040059 , 0.37137789, 0.64264426],
                        [0.54173456, 0.40958998, 0.6770899 ]]),
        columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
        index = pd.RangeIndex(start=38, stop=50, step=1)
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_no_refit_exog_interval_gap_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    with no refit and gap with mocked using exog and intervals 
    (mocked done in Skforecast v0.5.0).
    """
    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = {'l1': 2, 'l2': [1, 3]},
                     steps              = 8,
                     transformer_series = None
                 )
    
    cv = TimeSeriesFold(
            initial_train_size = len(series_wide_range) - 20,
            steps              = 5,
            gap                = 3,
            refit              = False,
            fixed_train_size   = False,
        )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_wide_range,
                                               cv                      = cv,
                                               levels                  = 'l1',
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_wide_range['l1'].rename('exog_1'),
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 150,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.11791887332493929]})
    expected_predictions = pd.DataFrame(
        data = np.array([[0.55880533, 0.38060874, 0.73874531],
                        [0.46285725, 0.25809849, 0.62184013],
                        [0.35358667, 0.17182584, 0.50598197],
                        [0.44404948, 0.23582063, 0.59338206],
                        [0.64659616, 0.46058367, 0.84125392],
                        [0.70306475, 0.52486815, 0.88300472],
                        [0.48677757, 0.28201881, 0.64576044],
                        [0.49848981, 0.31672898, 0.65088511],
                        [0.31544893, 0.10722008, 0.46478151],
                        [0.4450306 , 0.25901811, 0.63968836],
                        [0.50164877, 0.32345217, 0.68158874],
                        [0.62883248, 0.42407372, 0.78781535],
                        [0.33387601, 0.15211518, 0.48627131],
                        [0.45961408, 0.25138523, 0.60894666],
                        [0.63726975, 0.45125726, 0.83192751],
                        [0.54013414, 0.36193754, 0.72007411],
                        [0.52550978, 0.32075102, 0.68449266]]),
        columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
        index = pd.RangeIndex(start=33, stop=50, step=1)
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_refit_exog_interval_gap_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    with refit, gap, allow_incomplete_fold False with mocked using exog and 
    intervals (mocked done in Skforecast v0.5.0).
    """
    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = 2,
                     steps              = 8,
                     transformer_series = None
                 )
    cv = TimeSeriesFold(
            initial_train_size    = len(series_wide_range) - 20,
            steps                 = 5,
            refit                 = True,
            fixed_train_size      = False,
            allow_incomplete_fold = False,
            gap                   = 3,
        )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_wide_range,
                                               cv                      = cv,
                                               levels                  = 'l1',
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_wide_range['l1'].rename('exog_1'),
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 150,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.09906557188440204]})
    expected_predictions = pd.DataFrame(
                               data = np.array([[0.5137416 , 0.33718724, 0.66241188],
                                                [0.51578881, 0.33778405, 0.68539556],
                                                [0.38569703, 0.20626662, 0.59274175],
                                                [0.41784889, 0.23032996, 0.59483184],
                                                [0.69352545, 0.50694426, 0.86815556],
                                                [0.73709708, 0.56225855, 0.88218337],
                                                [0.49379408, 0.31947225, 0.62537925],
                                                [0.54129634, 0.38926854, 0.70779241],
                                                [0.2889167 , 0.14435288, 0.43658242],
                                                [0.41762991, 0.26652975, 0.59977787],
                                                [0.4281944 , 0.2903754 , 0.56310062],
                                                [0.6800295 , 0.5437135 , 0.82824156],
                                                [0.3198172 , 0.18103062, 0.45765294],
                                                [0.45269985, 0.33388826, 0.57987974],
                                                [0.76214215, 0.61792676, 0.89239812]]),
                               columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
                               index = pd.RangeIndex(start=33, stop=48, step=1)
                           )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_refit_fixed_train_size_exog_interval_gap_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    with refit, fixed_train_size, gap, with mocked using exog and intervals 
    (mocked done in Skforecast v0.5.0).
    """
    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = 2,
                     steps              = 10,
                     transformer_series = None
                 )
    cv = TimeSeriesFold(
             initial_train_size    = len(series_wide_dt) - 20,
             steps                 = 5,
             refit                 = True,
             fixed_train_size      = True,
             allow_incomplete_fold = False,
             gap                   = 5,
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_wide_dt,
                                               cv                      = cv,
                                               levels                  = 'l1',
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_wide_dt['l1'].rename('exog_1'),
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 150,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.12073089072357064]})
    expected_predictions = pd.DataFrame(
        data = np.array([[0.40010139, 0.20836912, 0.57962111],
                        [0.42717215, 0.25573292, 0.55497709],
                        [0.68150611, 0.51006688, 0.85585729],
                        [0.66167269, 0.47474991, 0.82699029],
                        [0.48946946, 0.29338102, 0.6346203 ],
                        [0.52955424, 0.35505547, 0.71187994],
                        [0.30621348, 0.13348582, 0.43983428],
                        [0.44140106, 0.25814985, 0.61072336],
                        [0.41294246, 0.25914736, 0.58112186],
                        [0.66798745, 0.4936962 , 0.84638439],
                        [0.35206832, 0.20487179, 0.54023327],
                        [0.41828203, 0.23993991, 0.61293962],
                        [0.6605119 , 0.492704  , 0.84794069],
                        [0.49225437, 0.31519742, 0.68294248],
                        [0.52528842, 0.33889897, 0.7220394 ]]),
        columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
        index = pd.date_range(start='2020-02-05', periods=15, freq='D')
    )
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
    expected_predictions = expected_predictions.asfreq('D')
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_refit_int_interval_yes_exog_yes_remainder_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    with refit int, interval, gap, with mocked using exog and intervals 
    (mocked done in Skforecast v0.9.0).
    """
    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = 2,
                     steps              = 2,
                     transformer_series = None
                 )
    cv = TimeSeriesFold(
             initial_train_size    = len(series_wide_range) - 20,
             steps                 = 2,
             refit                 = 2,
             fixed_train_size      = True,
             allow_incomplete_fold = False,
             gap                   = 0,
         )

    warn_msg = re.escape(
        "If `refit` is an integer other than 1 (intermittent refit). `n_jobs` "
        "is set to 1 to avoid unexpected results during parallelization."
    )
    with pytest.warns(IgnoredArgumentWarning, match = warn_msg):
        metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                                   forecaster              = forecaster,
                                                   series                  = series_wide_range,
                                                   cv                      = cv,
                                                   levels                  = 'l1',
                                                   metric                  = 'mean_absolute_error',
                                                   add_aggregated_metric   = False,                                            
                                                   exog                    = series_wide_range['l1'].rename('exog_1'),
                                                   interval                = [5, 95],
                                                   interval_method         = "bootstrapping",
                                                   n_boot                  = 100,
                                                   random_state            = 123,
                                                   use_in_sample_residuals = True,
                                                   use_binned_residuals    = False,
                                                   verbose                 = False,
                                                   n_jobs                  = 2
                                               )
    
    expected_metric = pd.DataFrame(
        {'levels': ['l1'], 
         'mean_absolute_error': [0.09237017311770493]}
    )
    expected_predictions = pd.DataFrame(
        data = np.array([[0.2983264 , 0.14944839, 0.40498101],
                        [0.45367569, 0.33634679, 0.60319301],
                        [0.4802857 , 0.33140768, 0.58694031],
                        [0.48384823, 0.36651933, 0.63336556],
                        [0.44991269, 0.27875359, 0.62870524],
                        [0.38062305, 0.24498469, 0.52404441],
                        [0.42817363, 0.25701453, 0.60696618],
                        [0.68209515, 0.54645679, 0.82551651],
                        [0.72786565, 0.58678633, 0.86276723],
                        [0.45147711, 0.27887235, 0.56756344],
                        [0.52904599, 0.38796667, 0.66394758],
                        [0.28657426, 0.1139695 , 0.40266059],
                        [0.36296117, 0.21805307, 0.52321437],
                        [0.46902166, 0.32411356, 0.63850226],
                        [0.68519422, 0.54028611, 0.84544742],
                        [0.35197602, 0.20706791, 0.52145662],
                        [0.47338638, 0.33907898, 0.63357264],
                        [0.74059646, 0.56765939, 0.90167033],
                        [0.55112717, 0.41681977, 0.71131344],
                        [0.55778417, 0.3848471 , 0.71885804]]),
        columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
        index = pd.RangeIndex(start=30, stop=50, step=1)
    )
    expected_predictions.insert(1, 'fold', [
        0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9
    ])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_refit_int_interval_yes_exog_not_allow_remainder_gap_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    with refit int, interval, gap, with mocked using exog and intervals 
    (mocked done in Skforecast v0.9.0).
    """
    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = 2,
                     steps              = 7,
                     transformer_series = None
                 )
    cv = TimeSeriesFold(
             initial_train_size    = len(series_wide_dt) - 30,
             steps                 = 4,
             refit                 = 3,
             fixed_train_size      = False,
             allow_incomplete_fold = False,
             gap                   = 3,
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_wide_dt,
                                               cv                      = cv,
                                               levels                  = ['l1'],
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,                                            
                                               exog                    = series_wide_dt['l1'].rename('exog_1'),
                                               interval                = [5, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 100,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame(
        {'levels': ['l1'], 
         'mean_absolute_error': [0.10066454067329249]}
    )
    expected_predictions = pd.DataFrame(
        data = np.array([[0.53203113, 0.30246275, 0.80217901],
                        [0.62129188, 0.36244991, 0.86033622],
                        [0.40145922, 0.18215073, 0.67365328],
                        [0.38821275, 0.13409599, 0.58621711],
                        [0.37636995, 0.14680156, 0.64651783],
                        [0.36599243, 0.10715046, 0.60503677],
                        [0.47654862, 0.25724013, 0.74874268],
                        [0.33725755, 0.08314079, 0.53526191],
                        [0.46122163, 0.23165324, 0.73136951],
                        [0.47541934, 0.21657737, 0.71446368],
                        [0.50601999, 0.28671151, 0.77821406],
                        [0.39293452, 0.13881776, 0.59093888],
                        [0.43793405, 0.29094361, 0.63929639],
                        [0.52600886, 0.37014241, 0.7224745 ],
                        [0.70882239, 0.55311365, 0.8623405 ],
                        [0.70063755, 0.51906805, 0.83714135],
                        [0.48631217, 0.33932174, 0.68767451],
                        [0.52248417, 0.36661772, 0.71894981],
                        [0.30010035, 0.1443916 , 0.45361845],
                        [0.43342987, 0.25186036, 0.56993366],
                        [0.42479569, 0.27780526, 0.62615803],
                        [0.68566159, 0.52979514, 0.88212723],
                        [0.34632377, 0.19061503, 0.49984188],
                        [0.44695116, 0.26538166, 0.58345495]]),
        columns = ['l1', 'l1_lower_bound', 'l1_upper_bound'],
        index = pd.date_range(start='2020-01-24', periods=24, freq='D')
    )
    expected_predictions.insert(1, 'fold', [
        0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5])
    expected_predictions = expected_df_to_long_format(
        expected_predictions, method='backtesting-interval'
    )
    expected_predictions = expected_predictions.asfreq('D')
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_no_refit_exog_interval_percentiles_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    with no refit and gap with mocked using exog and intervals as percentiles
    (mocked done in Skforecast v0.15.0).
    """
    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = {'l1': 2, 'l2': [1, 3]},
                     steps              = 8,
                     transformer_series = None
                 )
    
    cv = TimeSeriesFold(
            initial_train_size = len(series_wide_range) - 20,
            steps              = 5,
            gap                = 3,
            refit              = False
        )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_wide_range,
                                               cv                      = cv,
                                               levels                  = 'l1',
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_wide_range['l1'].rename('exog_1'),
                                               interval                = [5, 50, 95],
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 150,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.11791887332493929]})
    expected_predictions = pd.DataFrame(
        data = np.array([[0.55880533, 0.38060874, 0.5629389 , 0.73874531],
                        [0.46285725, 0.25809849, 0.4598901 , 0.62184013],
                        [0.35358667, 0.17182584, 0.35061952, 0.50598197],
                        [0.44404948, 0.23582063, 0.43595463, 0.59338206],
                        [0.64659616, 0.46058367, 0.63092664, 0.84125392],
                        [0.70306475, 0.52486815, 0.70719832, 0.88300472],
                        [0.48677757, 0.28201881, 0.48381042, 0.64576044],
                        [0.49848981, 0.31672898, 0.49552265, 0.65088511],
                        [0.31544893, 0.10722008, 0.30735409, 0.46478151],
                        [0.4450306 , 0.25901811, 0.42936108, 0.63968836],
                        [0.50164877, 0.32345217, 0.50578234, 0.68158874],
                        [0.62883248, 0.42407372, 0.62586533, 0.78781535],
                        [0.33387601, 0.15211518, 0.33090885, 0.48627131],
                        [0.45961408, 0.25138523, 0.45151923, 0.60894666],
                        [0.63726975, 0.45125726, 0.62160023, 0.83192751],
                        [0.54013414, 0.36193754, 0.54426771, 0.72007411],
                        [0.52550978, 0.32075102, 0.52254263, 0.68449266]]),
        columns = ['pred', 'p_5', 'p_50', 'p_95'],
        index = pd.RangeIndex(start=33, stop=50, step=1)
    )
    expected_predictions.insert(0, 'level', np.tile(['l1'], len(expected_predictions)))
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3])
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_no_refit_exog_interval_distribution_with_mocked():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    with no refit and gap with mocked using exog and intervals as scipy.stats.norm 
    distribution (mocked done in Skforecast v0.15.0).
    """
    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = {'l1': 2, 'l2': [1, 3]},
                     steps              = 8,
                     transformer_series = None
                 )
    
    cv = TimeSeriesFold(
            initial_train_size = len(series_wide_range) - 20,
            steps              = 5,
            gap                = 3,
            refit              = False
        )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_wide_range,
                                               cv                      = cv,
                                               levels                  = 'l1',
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_wide_range['l1'].rename('exog_1'),
                                               interval                = norm,
                                               interval_method         = "bootstrapping",
                                               n_boot                  = 150,
                                               random_state            = 123,
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = False,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.11791887332493929]})
    expected_predictions = pd.DataFrame(
        data = np.array([[0.55880533, 0.57494787, 0.10929407],
                        [0.46285725, 0.45630135, 0.1044358 ],
                        [0.35358667, 0.34987494, 0.10721041],
                        [0.44404948, 0.43109115, 0.11140753],
                        [0.64659616, 0.64976095, 0.11444684],
                        [0.70306475, 0.71920729, 0.10929407],
                        [0.48677757, 0.48022166, 0.1044358 ],
                        [0.49848981, 0.49477808, 0.10721041],
                        [0.31544893, 0.30249061, 0.11140753],
                        [0.4450306 , 0.44819539, 0.11444684],
                        [0.50164877, 0.51779131, 0.10929407],
                        [0.62883248, 0.62227657, 0.1044358 ],
                        [0.33387601, 0.33016428, 0.10721041],
                        [0.45961408, 0.44665575, 0.11140753],
                        [0.63726975, 0.64043454, 0.11444684],
                        [0.54013414, 0.55627668, 0.10929407],
                        [0.52550978, 0.51895388, 0.1044358 ]]),
        columns = ['pred', 'loc', 'scale'],
        index = pd.RangeIndex(start=33, stop=50, step=1)
    )
    expected_predictions.insert(0, 'level', np.tile(['l1'], len(expected_predictions)))
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3])

    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


@pytest.mark.parametrize("interval", 
                         [0.90, (5, 95)], 
                         ids = lambda value: f'interval: {value}')
def test_output_backtesting_forecaster_interval_conformal_and_binned_with_mocked_ForecasterDirectMultiVariate(interval):
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    with no refit with mocked using exog and binned conformal intervals 
    (mocked done in Skforecast v0.15.0).
    """
    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = 3,
                     steps              = 5,
                     transformer_series = None,
                     binner_kwargs      = {'n_bins': 10}
                 )
    
    cv = TimeSeriesFold(
             initial_train_size = len(series_wide_range) - 20,
             steps              = 5,
             refit              = False
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster              = forecaster,
                                               series                  = series_wide_range,
                                               cv                      = cv,
                                               levels                  = 'l1',
                                               metric                  = 'mean_absolute_error',
                                               add_aggregated_metric   = False,
                                               exog                    = series_wide_range['l1'].rename('exog_1'),
                                               interval                = interval,
                                               interval_method         = 'conformal',
                                               use_in_sample_residuals = True,
                                               use_binned_residuals    = True,
                                               random_state            = 123,
                                               verbose                 = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.11916192143057436]})
    expected_predictions = pd.DataFrame(
        data = np.array([[0.36268205, 0.20433986, 0.52102424],
                         [0.51787041, 0.43620381, 0.599537  ],
                         [0.52472422, 0.42645324, 0.6229952 ],
                         [0.59903598, 0.46654146, 0.7315305 ],
                         [0.50738853, 0.42572194, 0.58905513],
                         [0.42622724, 0.3313424 , 0.52111209],
                         [0.47163992, 0.38997332, 0.55330651],
                         [0.70873165, 0.48281375, 0.93464954],
                         [0.71565768, 0.48973979, 0.94157557],
                         [0.49000773, 0.40834113, 0.57167433],
                         [0.61296286, 0.48046834, 0.74545738],
                         [0.4031027 , 0.30821786, 0.49798755],
                         [0.39728743, 0.30240258, 0.49217227],
                         [0.46010284, 0.37843624, 0.54176944],
                         [0.63022554, 0.49773102, 0.76272006],
                         [0.39528807, 0.30040323, 0.49017292],
                         [0.47347775, 0.39181115, 0.55514435],
                         [0.76236728, 0.53644939, 0.98828517],
                         [0.53407054, 0.43579957, 0.63234152],
                         [0.5014202 , 0.4197536 , 0.5830868 ]]),
        columns = ['pred', 'lower_bound', 'upper_bound'],
        index = pd.RangeIndex(start=30, stop=50, step=1)
    )
    expected_predictions.insert(0, 'level', np.tile(['l1'], len(expected_predictions)))
    expected_predictions.insert(1, 'fold', [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3])

    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_multiseries_ForecasterDirectMultiVariate_return_predictors_same_predictions_as_predict():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    when series and exog are dictionaries, predictions from 
    _backtesting_forecaster predictors are the same as predictions from 
    predict method.
    """
    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = 2,
                     steps              = 3,
                     transformer_series = None
                 )
    cv = TimeSeriesFold(
             initial_train_size = len(series_wide_range) - 12,
             steps              = 3,
             refit              = False,
             fixed_train_size   = False,
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster         = forecaster,
                                               series             = series_wide_range,
                                               exog               = series_wide_range['l1'].rename('exog_1'),
                                               cv                 = cv,
                                               levels             = 'l1',
                                               metric             = 'mean_absolute_error',
                                               return_predictors  = True,
                                               verbose            = True
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'],
                                    'mean_absolute_error': [0.080981301131163]})
    
    forecaster.fit(
        series = series_wide_range.iloc[:len(series_wide_range) - 12],
        exog   = series_wide_range.iloc[:len(series_wide_range) - 12]['l1'].rename('exog_1')
    )
    estimators = [1, 2, 3] * 4
    len_predictions = len(backtest_predictions)
    results = np.full(shape=len_predictions, fill_value=np.nan, dtype=float)
    for i, step in enumerate(estimators):
        results[i] = forecaster.estimators_[step].predict(
            backtest_predictions.iloc[[i]][
                ['l1_lag_1', 'l1_lag_2', 'l2_lag_1', 'l2_lag_2', 'exog_1']
            ]
        )
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    np.testing.assert_array_almost_equal(results, backtest_predictions['pred'].to_numpy())


def test_output_backtesting_forecaster_ForecasterDirectMultiVariate_fold_stride():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    with no refit with mocked using fold stride.
    (mocked done in Skforecast v0.18.0).
    """
    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = 3,
                     steps              = 5,
                     transformer_series = None,
                 )
    
    cv = TimeSeriesFold(
             initial_train_size = len(series_wide_range) - 20,
             steps              = 5,
             fold_stride        = 3,
             refit              = False
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_wide_range,
                                               cv                    = cv,
                                               levels                = 'l1',
                                               metric                = 'mean_absolute_error',
                                               add_aggregated_metric = False,
                                               exog                  = series_wide_range['l1'].rename('exog_1'),
                                               random_state          = 123,
                                               verbose               = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.1110500102960009]})
    expected_predictions = pd.DataFrame(
        data = np.array([
            [0.        , 0.36268205],
            [0.        , 0.51787041],
            [0.        , 0.52472422],
            [0.        , 0.59903598],
            [0.        , 0.50738853],
            [1.        , 0.48881923],
            [1.        , 0.48843457],
            [1.        , 0.43042214],
            [1.        , 0.53517054],
            [1.        , 0.67818829],
            [2.        , 0.45435034],
            [2.        , 0.68926351],
            [2.        , 0.69673039],
            [2.        , 0.51511   ],
            [2.        , 0.51222045],
            [3.        , 0.55179877],
            [3.        , 0.62436174],
            [3.        , 0.35535408],
            [3.        , 0.41023648],
            [3.        , 0.44663944],
            [4.        , 0.39436005],
            [4.        , 0.47991353],
            [4.        , 0.65933721],
            [4.        , 0.38570842],
            [4.        , 0.45739258],
            [5.        , 0.39528807],
            [5.        , 0.47347775],
            [5.        , 0.76236728],
            [5.        , 0.53407054],
            [5.        , 0.5014202 ],
            [6.        , 0.55466973],
            [6.        , 0.55769425]]),
        columns = ['fold', 'pred'],
        index = pd.Index([
                    30, 31, 32, 33, 34, 33, 34, 35, 36, 37, 36, 37, 38, 39, 40, 39, 40, 41,
                    42, 43, 42, 43, 44, 45, 46, 45, 46, 47, 48, 49, 48, 49
                ])
    ).astype({'fold': int})
    expected_predictions.insert(0, 'level', np.tile(['l1'], len(expected_predictions)))
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)


def test_output_backtesting_forecaster_ForecasterDirectMultiVariate_fold_stride_greater_than_steps():
    """
    Test output of backtesting_forecaster_multiseries in ForecasterDirectMultiVariate 
    with no refit with mocked using fold stride greater than steps.
    (mocked done in Skforecast v0.18.0).
    """
    forecaster = ForecasterDirectMultiVariate(
                     estimator          = Ridge(random_state=123),
                     level              = 'l1',
                     lags               = 3,
                     steps              = 5,
                     transformer_series = None,
                 )
    
    cv = TimeSeriesFold(
             initial_train_size = len(series_wide_range) - 20,
             steps              = 5,
             fold_stride        = 10,
             refit              = False
         )

    metrics_levels, backtest_predictions = backtesting_forecaster_multiseries(
                                               forecaster            = forecaster,
                                               series                = series_wide_range,
                                               cv                    = cv,
                                               levels                = 'l1',
                                               metric                = 'mean_absolute_error',
                                               add_aggregated_metric = False,
                                               exog                  = series_wide_range['l1'].rename('exog_1'),
                                               random_state          = 123,
                                               verbose               = False
                                           )
    
    expected_metric = pd.DataFrame({'levels': ['l1'], 
                                    'mean_absolute_error': [0.12953535188286774]})
    expected_predictions = pd.DataFrame(
        data = np.array([
            [0.        , 0.36268205],
            [0.        , 0.51787041],
            [0.        , 0.52472422],
            [0.        , 0.59903598],
            [0.        , 0.50738853],
            [1.        , 0.61296286],
            [1.        , 0.4031027 ],
            [1.        , 0.39728743],
            [1.        , 0.46010284],
            [1.        , 0.63022554]]),
        columns = ['fold', 'pred'],
        index = pd.Index([
                    30, 31, 32, 33, 34, 40, 41, 42, 43, 44
                ])
    ).astype({'fold': int})
    expected_predictions.insert(0, 'level', np.tile(['l1'], len(expected_predictions)))
                                   
    pd.testing.assert_frame_equal(expected_metric, metrics_levels)
    pd.testing.assert_frame_equal(expected_predictions, backtest_predictions)
