{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c1b25ef-d375-4ae3-abc6-4dd26d335505",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import os\n",
    "# from google.colab import drive\n",
    "# drive.mount('/content/drive')\n",
    "# os.chdir('./drive/MyDrive/nixtlats')\n",
    "# print(os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e509e2fc-5032-40e2-9bf7-5dc76eb43061",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install pytorch-lightning\n",
    "# !pip install torchinfo\n",
    "# !pip install fastcore\n",
    "# !pip install s3fs\n",
    "# !pip install patool\n",
    "# !pip install --upgrade pandas==1.2.4\n",
    "# !pip install --upgrade requests==2.25.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "008b33ea-667a-46de-a7aa-dc039371f8a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#default_exp models.nbeats.ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbda3893-48b0-4e29-b891-7316616fcf0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "408fb921-12f5-4201-87bc-58e67b39b5e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from dataclasses import dataclass\n",
    "from itertools import product\n",
    "from pathlib import Path\n",
    "import shutil\n",
    "from typing import Callable, Dict, Iterable, Union, List\n",
    "from tqdm import tqdm\n",
    "import pylab as plt\n",
    "from pylab import rcParams\n",
    "import math\n",
    "from IPython.display import clear_output\n",
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pytorch_lightning as pl\n",
    "from pytorch_lightning.loggers import TensorBoardLogger\n",
    "\n",
    "from nixtlats.models.nbeats.nbeats import NBEATS\n",
    "from nixtlats.data.datasets.m4 import M4Info, M4, M4Evaluation\n",
    "from nixtlats.data.tsdataset import WindowsDataset\n",
    "from nixtlats.data.tsloader import TimeSeriesLoader\n",
    "from nixtlats.experiments.utils import create_datasets, get_mask_dfs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "105e8682-817f-4e27-a2d9-dec82fe035ae",
   "metadata": {},
   "source": [
    "# M4 Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afb2cda2-1fcc-4096-83d3-ab707faaffc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def _parameter_grid(grid):\n",
    "    specs_list = list(product(*list(grid.values())))\n",
    "    model_specs_df = pd.DataFrame(specs_list, columns=list(grid.keys()))\n",
    "    \n",
    "    return model_specs_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90470d19-9b25-414f-a669-ef5d5cffc3a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "common_grid = {}\n",
    "\n",
    "# Architecture parameters\n",
    "common_grid['activation'] = ['ReLU'] # Oreshkin\n",
    "common_grid['n_x'] = [0] # No exogenous variables\n",
    "common_grid['n_s'] = [0] # No static variables\n",
    "common_grid['n_x_hidden'] = [0] # No exogenous variables\n",
    "common_grid['n_s_hidden'] = [0] # No static variables\n",
    "common_grid['stack_types'] = [['trend', 'seasonality']] # NBEATS-I original architecture\n",
    "common_grid['n_blocks'] = [[3, 3]] # Trend blocks, Seasonal blocks - Oreshkin\n",
    "common_grid['n_layers'] = [[4, 4]] # Trend-block layers, Seasonal-block - Oreshkin\n",
    "common_grid['shared_weights'] = [True] # Oreshkin\n",
    "common_grid['n_harmonics'] = [1] # Oreshkin\n",
    "common_grid['n_polynomials'] = [2] # Trend polynomial degree\n",
    "common_grid['n_theta_hidden'] = [[common_grid['n_layers'][0][0] * [256],\n",
    "                                  common_grid['n_layers'][0][1] * [2048]]] # Oreshkin\n",
    "common_grid['initialization'] = ['lecun_normal'] # Arbitrary\n",
    "\n",
    "# Optimization parameters\n",
    "common_grid['learning_rate'] = [0.001] # Oreshkin\n",
    "common_grid['lr_decay'] = [0] # No lr_decay in the original implementation\n",
    "common_grid['lr_decay_step_size'] = [1_000] # No lr_decay in the original implementation\n",
    "common_grid['loss_val'] = ['SMAPE']\n",
    "common_grid['dropout_prob_theta'] = [0] # No dropout in the original implementation\n",
    "common_grid['weight_decay'] = [0] # # No weight_decay in the original implementation\n",
    "common_grid['batch_size'] = [1024] # Oreshkin\n",
    "common_grid['batch_normalization'] = [False] # No batch_normalization in the original implementation\n",
    "\n",
    "# Data Parameters\n",
    "common_grid['complete_inputs'] = [False] # ???\n",
    "common_grid['mode'] = ['simple'] # ???\n",
    "lookbacks = list(range(2, 8)) # Change to range(2, 8). Oreshkin\n",
    "\n",
    "ensemble_grid = {'loss_train': ['MAPE', 'SMAPE', 'MASE'],\n",
    "                #  'loss_train': ['MAPE', 'SMAPE', 'MASE'],\n",
    "                 'n_steps': [260],\n",
    "                 'random_seed': list(range(1))}\n",
    "\n",
    "@dataclass\n",
    "class Yearly:\n",
    "    group = M4Info['Yearly']\n",
    "\n",
    "    grid_freq = {}\n",
    "    grid_freq['n_time_in'] = [M4Info['Yearly'].horizon * i for i in lookbacks]\n",
    "    grid_freq['n_time_out'] = [group.horizon]\n",
    "    grid_freq['train_sample_freq'] = [1] # ???\n",
    "    grid_freq['frequency'] = ['Y'] # ???\n",
    "    grid_freq['seasonality'] = [1] # ???\n",
    "    grid_freq['l_h'] = [1.5] # Oreshkin\n",
    "\n",
    "    grid = {**common_grid,\n",
    "            **grid_freq}\n",
    "    ensemble_grid = ensemble_grid\n",
    "\n",
    "@dataclass\n",
    "class Quarterly:\n",
    "    group = M4Info['Quarterly']\n",
    "\n",
    "    grid_freq = {}\n",
    "    grid_freq['n_time_in'] = [M4Info['Quarterly'].horizon * i for i in lookbacks]\n",
    "    grid_freq['n_time_out'] = [group.horizon]\n",
    "    grid_freq['train_sample_freq'] = [1] # ???\n",
    "    grid_freq['frequency'] = ['Q'] # ???\n",
    "    grid_freq['seasonality'] = [4] # ???\n",
    "    grid_freq['l_h'] = [1.5] # Oreshkin\n",
    "\n",
    "    grid = {**common_grid,\n",
    "            **grid_freq}\n",
    "    ensemble_grid = ensemble_grid\n",
    "\n",
    "@dataclass\n",
    "class Monthly:\n",
    "    group = M4Info['Monthly']\n",
    "\n",
    "    grid_freq = {}\n",
    "    grid_freq['n_time_in'] = [M4Info['Monthly'].horizon * i for i in lookbacks]\n",
    "    grid_freq['n_time_out'] = [group.horizon]\n",
    "    grid_freq['train_sample_freq'] = [1] # ???\n",
    "    grid_freq['frequency'] = ['M'] # ???\n",
    "    grid_freq['seasonality'] = [12] # ???\n",
    "    grid_freq['l_h'] = [1.5] # Oreshkin\n",
    "\n",
    "    grid = {**common_grid,\n",
    "            **grid_freq}\n",
    "    ensemble_grid = ensemble_grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9efd66cc-7dcd-4d40-bbfe-afc5091f62c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def print_models_list(frequencies: list, table_width: int):\n",
    "    for freq in frequencies:\n",
    "        freq_grid_table = pd.Series({**freq.grid, **freq.ensemble_grid})\n",
    "        freq_table_header  = f'\\n{freq.group.name} '\n",
    "        freq_table_header += 'grid (# of different model configurations = '\n",
    "        freq_table_header += f'{len(_parameter_grid({**freq.grid, **freq.ensemble_grid}))}):\\n'      \n",
    "        print(f'{freq_table_header}{table_width*\"=\"}\\n{freq_grid_table}\\n{table_width*\"=\"}\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9784422-c0e6-4d2b-8836-561cc50b7e86",
   "metadata": {},
   "source": [
    "# Loaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "921a4201-c89a-42a6-9edb-92b66c186ca1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def create_loaders_M4(Y_df, S_df, hparams, num_workers):\n",
    "\n",
    "    print(f'Instantiating loaders (n_time_in = {hparams[\"n_time_in\"]})...', end=' ')\n",
    "    \n",
    "    train_mask_df, valid_mask_df, _ = get_mask_dfs(Y_df=Y_df,\n",
    "                                                   ds_in_test=0,\n",
    "                                                   ds_in_val=hparams['n_time_out'])\n",
    "\n",
    "    train_dataset = WindowsDataset(Y_df=Y_df, S_df=S_df,\n",
    "                                   mask_df=train_mask_df,\n",
    "                                   input_size=hparams['n_time_in'],\n",
    "                                   output_size=hparams['n_time_out'],\n",
    "                                   sample_freq=hparams['train_sample_freq'],\n",
    "                                   complete_windows=hparams['complete_inputs'])\n",
    "    \n",
    "    valid_dataset = WindowsDataset(Y_df=Y_df, S_df=S_df,\n",
    "                                   mask_df=valid_mask_df,\n",
    "                                   input_size=hparams['n_time_in'],\n",
    "                                   output_size=hparams['n_time_out'],\n",
    "                                   sample_freq=hparams['train_sample_freq'],\n",
    "                                   complete_windows=hparams['complete_inputs'],\n",
    "                                   last_window=True)\n",
    "                    \n",
    "    train_loader = TimeSeriesLoader(dataset=train_dataset,\n",
    "                                    batch_size=int(hparams['batch_size']),\n",
    "                                    eq_batch_size=True,\n",
    "                                    num_workers=num_workers,\n",
    "                                    shuffle=True)\n",
    "\n",
    "    valid_loader = TimeSeriesLoader(dataset=valid_dataset,\n",
    "                                    batch_size=int(hparams['batch_size']),\n",
    "                                    eq_batch_size=False,\n",
    "                                    num_workers=num_workers,\n",
    "                                    shuffle=False)\n",
    "    \n",
    "    print('Data loaders ready.\\n')\n",
    "    \n",
    "    del train_dataset, valid_dataset\n",
    "    \n",
    "    return train_loader, valid_loader"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3e7e9de-fe07-4862-85f0-c7fef0370b36",
   "metadata": {},
   "source": [
    "# NBEATS Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d576201-d247-4dba-9950-d8ba7bf07986",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def NBEATS_instantiate(hparams):\n",
    "\n",
    "    model = NBEATS(n_time_in=int(hparams['n_time_in']),\n",
    "                   n_time_out=int(hparams['n_time_out']),\n",
    "                   n_x=hparams['n_x'],\n",
    "                   n_s=hparams['n_s'],\n",
    "                   n_s_hidden=int(hparams['n_s_hidden']),\n",
    "                   n_x_hidden=int(hparams['n_x_hidden']),\n",
    "                   shared_weights=hparams['shared_weights'],\n",
    "                   initialization=hparams['initialization'],\n",
    "                   activation=hparams['activation'],\n",
    "                   stack_types=hparams['stack_types'],\n",
    "                   n_blocks=hparams['n_blocks'],\n",
    "                   n_layers=hparams['n_layers'],\n",
    "                   n_theta_hidden=hparams['n_theta_hidden'],\n",
    "                   n_harmonics=int(hparams['n_harmonics']),\n",
    "                   n_polynomials=int(hparams['n_polynomials']),\n",
    "                   batch_normalization = hparams['batch_normalization'],\n",
    "                   dropout_prob_theta=hparams['dropout_prob_theta'],\n",
    "                   learning_rate=float(hparams['learning_rate']),\n",
    "                   lr_decay=float(hparams['lr_decay']),\n",
    "                   lr_decay_step_size=float(hparams['lr_decay_step_size']),\n",
    "                   weight_decay=hparams['weight_decay'],\n",
    "                   loss_train=hparams['loss_train'],\n",
    "                   loss_hypar=int(hparams['seasonality']),\n",
    "                   loss_valid=hparams['loss_val'],\n",
    "                   frequency=hparams['frequency'],\n",
    "                   seasonality=int(hparams['seasonality']),\n",
    "                   random_seed=int(hparams['random_seed']))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0347761-831c-4e46-8a55-1703465d63f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def show_tensorboard(logs_path, model_path):\n",
    "    logs_model_path = f'{logs_path}/{model_path}'\n",
    "    # %load_ext tensorboard\n",
    "    # %tensorboard --logdir $logs_model_path\n",
    "    os.system('load_ext tensorboard')\n",
    "    os.system('tensorboard --logdir $logs_model_path')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57602b80-a6da-413d-8390-20e91dbb7f2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class NBEATSEnsemble:\n",
    "\n",
    "    def __init__(self, use_gpus: bool=False, gpus: int=None, auto_select_gpus: bool=False):\n",
    "        \n",
    "        if use_gpus:\n",
    "            assert isinstance(gpus, (int, list, str)), \\\n",
    "                f'if use_gpus == True, gpus must be {int}, {list} or {str}, not {type(gpus)}.'\n",
    "            if (isinstance(gpus, int)):\n",
    "                assert gpus > 0 or gpus == -1, \\\n",
    "                    f'if gpus is of type {int}, it must be either a positive integer or equal to -1.'\n",
    "        else:\n",
    "            assert gpus == None, f'if use_gpus == False, gpus must be {None}, not {type(gpus)}.'\n",
    "        \n",
    "        self.gpus = gpus\n",
    "        self.auto_select_gpus = auto_select_gpus\n",
    "\n",
    "    def fit(self,\n",
    "            frequencies: List[type],\n",
    "            loader: callable,\n",
    "            val_freq_steps: int,\n",
    "            tensorboard_logs: bool,\n",
    "            logs_path: str,\n",
    "            num_workers: int):\n",
    "        \n",
    "        results = {}\n",
    "\n",
    "        for freq in frequencies:\n",
    "            idx_ensemble = 0\n",
    "\n",
    "            Y_df, _, S_df = M4.load(directory='data', group=freq.group.name)\n",
    "            freq_grid = _parameter_grid(freq.grid)\n",
    "            forecasts = []\n",
    "\n",
    "            if tensorboard_logs and Path(f'{logs_path}/{freq.group.name}').exists():\n",
    "                shutil.rmtree(f'{logs_path}/{freq.group.name}')\n",
    "                show_tensorboard(logs_path=logs_path, model_path=freq.group.name)\n",
    "\n",
    "            for idx_hparams, row_hparams in freq_grid.iterrows():                             \n",
    "                hparams = row_hparams.to_dict()\n",
    "                train_loader, test_loader = create_loaders_M4(Y_df=Y_df, \n",
    "                                                              S_df=S_df, \n",
    "                                                              hparams=hparams, \n",
    "                                                              num_workers=num_workers)\n",
    "\n",
    "                ensemble_grid = _parameter_grid(freq.ensemble_grid)\n",
    "\n",
    "                for idx_ensemble_hparams, row_ensemble_hparams in ensemble_grid.iterrows():\n",
    "                    clear_output(wait=True)\n",
    "                    idx_ensemble += 1\n",
    "                    hparams_ensemble = {**hparams, **row_ensemble_hparams.to_dict()}\n",
    "\n",
    "                    model = NBEATS_instantiate(hparams_ensemble)\n",
    "                    self.print_model_version(freq, hparams_ensemble, idx_ensemble)\n",
    "\n",
    "                    if tensorboard_logs: logger = self.create_logger(freq, \n",
    "                                                                     hparams_ensemble, \n",
    "                                                                     logs_path)\n",
    "                    else: logger = False\n",
    "\n",
    "                    trainer = pl.Trainer(max_steps=hparams_ensemble['n_steps'],\n",
    "                                         gradient_clip_val=0,\n",
    "                                         progress_bar_refresh_rate=50, \n",
    "                                         gpus=self.gpus,\n",
    "                                         auto_select_gpus=self.auto_select_gpus, \n",
    "                                         check_val_every_n_epoch=val_freq_steps,\n",
    "                                         logger=logger)             \n",
    "                    trainer.fit(model, train_dataloader=train_loader, val_dataloaders=train_loader)\n",
    "                    outputs = trainer.predict(model, test_loader)\n",
    "\n",
    "                    outputs_df = self.outputs_to_df(outputs, idx_ensemble)\n",
    "                    forecasts.append(outputs_df.copy())\n",
    "\n",
    "                    del trainer, model, outputs, outputs_df\n",
    "\n",
    "                del train_loader, test_loader\n",
    "\n",
    "            forecasts = pd.concat(forecasts).groupby('unique_id').median(0)\n",
    "            forecasts.reset_index(inplace=True) \n",
    "\n",
    "            results[freq.group.name] = forecasts.copy()\n",
    "            \n",
    "            del forecasts, Y_df, _, S_df\n",
    " \n",
    "        return results\n",
    "\n",
    "    def outputs_to_df(self, outputs, idx_ensemble):\n",
    "        outputs_df = torch.vstack([outputs[i][1] \\\n",
    "                                   for i in range(len(outputs))]).detach().cpu().numpy()\n",
    "        outputs_df = pd.DataFrame(outputs_df)\n",
    "        outputs_df.insert(0, 'unique_id', np.arange(outputs_df.shape[0]))\n",
    "        outputs_df.insert(1, 'model', f'm_{idx_ensemble}')\n",
    "\n",
    "        return outputs_df\n",
    "\n",
    "    def create_logger(self, freq, hparams, logs_path):\n",
    "        name = freq.group.name\n",
    "        version  = f'loss-{hparams[\"loss_train\"]}_'\n",
    "        version += f'lbl-{hparams[\"n_time_in\"] // freq.group.horizon}_'\n",
    "        version += f'_rs-{hparams[\"random_seed\"]}'\n",
    "\n",
    "        logger = TensorBoardLogger(logs_path, name=name, version=version, default_hp_metric=False)\n",
    "\n",
    "        return logger\n",
    "\n",
    "    def print_model_version(self, freq, hparams, idx_ensemble):\n",
    "        n_models = len(freq.ensemble_grid['loss_train']) * \\\n",
    "                   len(freq.grid['n_time_in']) * \\\n",
    "                   len(freq.ensemble_grid['random_seed'])\n",
    "        model_version  = f'\\n{freq.group.name} ({idx_ensemble}/{n_models}) - '\n",
    "        model_version += f'loss: {hparams[\"loss_train\"]}, ' \n",
    "        model_version += f'lookback length: {hparams[\"n_time_in\"] // freq.group.horizon}, '\n",
    "        model_version += f'random_seed: {hparams[\"random_seed\"]}'\n",
    "        print(model_version)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd7addfe-0395-479a-bf44-ddae33da871c",
   "metadata": {},
   "source": [
    "# Ensemble Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "050f8c28-b81f-4ace-8445-c7a63b947f8d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Yearly grid (# of different model configurations = 18):\n",
      "===========================================================================\n",
      "activation                                                        [ReLU]\n",
      "n_x                                                                  [0]\n",
      "n_s                                                                  [0]\n",
      "n_x_hidden                                                           [0]\n",
      "n_s_hidden                                                           [0]\n",
      "stack_types                                       [[trend, seasonality]]\n",
      "n_blocks                                                        [[3, 3]]\n",
      "n_layers                                                        [[4, 4]]\n",
      "shared_weights                                                    [True]\n",
      "n_harmonics                                                          [1]\n",
      "n_polynomials                                                        [2]\n",
      "n_theta_hidden         [[[256, 256, 256, 256], [2048, 2048, 2048, 204...\n",
      "initialization                                            [lecun_normal]\n",
      "learning_rate                                                    [0.001]\n",
      "lr_decay                                                             [0]\n",
      "lr_decay_step_size                                                [1000]\n",
      "loss_val                                                         [SMAPE]\n",
      "dropout_prob_theta                                                   [0]\n",
      "weight_decay                                                         [0]\n",
      "batch_size                                                        [1024]\n",
      "batch_normalization                                              [False]\n",
      "complete_inputs                                                  [False]\n",
      "mode                                                            [simple]\n",
      "n_time_in                                       [12, 18, 24, 30, 36, 42]\n",
      "n_time_out                                                           [6]\n",
      "train_sample_freq                                                    [1]\n",
      "frequency                                                            [Y]\n",
      "seasonality                                                          [1]\n",
      "l_h                                                                [1.5]\n",
      "loss_train                                           [MAPE, SMAPE, MASE]\n",
      "n_steps                                                            [260]\n",
      "random_seed                                                          [0]\n",
      "dtype: object\n",
      "===========================================================================\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# LOGS_PATH = Path('lightning_logs')\n",
    "# val_freq_steps = 2\n",
    "# tensorboard_logs = True\n",
    "# NUM_WORKERS = 4\n",
    "\n",
    "# frequencies = [Yearly]\n",
    "# print_models_list(frequencies=frequencies, table_width=75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8795fa39-d39d-44d8-bdb4-861ed52cb7b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Output hidden; open in https://colab.research.google.com to view."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ensemble = NBEATSEnsemble(use_gpus=True if torch.cuda.is_available() else False, \n",
    "#                           gpus=-1 if torch.cuda.is_available() else None, \n",
    "#                           auto_select_gpus=True if torch.cuda.is_available() else False)\n",
    "\n",
    "# forecasts = ensemble.fit(frequencies=frequencies,\n",
    "#                          loader=TimeSeriesLoader,  \n",
    "#                          val_freq_steps=val_freq_steps,\n",
    "#                          tensorboard_logs=tensorboard_logs,\n",
    "#                          logs_path=LOGS_PATH,\n",
    "#                          num_workers=NUM_WORKERS)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "AN5x6RvTNR_e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SMAPE</th>\n",
       "      <th>MASE</th>\n",
       "      <th>OWA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Yearly</th>\n",
       "      <td>17.448574</td>\n",
       "      <td>5.438234</td>\n",
       "      <td>1.218016</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            SMAPE      MASE       OWA\n",
       "Yearly  17.448574  5.438234  1.218016"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# M4Evaluation.evaluate('data', 'Yearly', forecasts['Yearly'].drop('unique_id', axis=1).to_numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ID67FE2vm57Q",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
