{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#default_exp data.datasets.m4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# M4 dataset\n",
    "\n",
    "> Download and evaluate the M4 dataset.\n",
    "\n",
    "The M4 competition extended on the previous three M competitions. The dataset of the contest included 100,000 time series 95,000 of which on yearly, quarterly and monthly frequencies, and the rest in weekly, daily and hourly higher frequencies. This competition received 61 different forecasting methods, notably a neural network model outperformed the rest of the competitors, for the first time in contrast with the previous M forecasting competitions.\n",
    "\n",
    "[Spyros  Makridakis,  Evangelos  Spiliotis, and  Vassilios Assimakopoulos. The  M4  competition: 100,000  time  series and 61 forecasting methods. International Journal of Forecasting, 36(1):54–74, 2020. ISSN  0169-2070.](https://www.sciencedirect.com/science/article/pii/S0169207019301128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev import *\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import os\n",
    "from dataclasses import dataclass\n",
    "from typing import Dict, List, Optional, Tuple, Union\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from nixtlats.data.datasets.utils import download_file, Info\n",
    "from nixtlats.losses.numpy import smape, mase"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## M4 meta information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@dataclass\n",
    "class Yearly:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 6\n",
    "    freq: str = 'Y'\n",
    "    name: str = 'Yearly'\n",
    "    n_ts: int = 23_000\n",
    "\n",
    "@dataclass\n",
    "class Quarterly:\n",
    "    seasonality: int = 4\n",
    "    horizon: int = 8\n",
    "    freq: str = 'Q'\n",
    "    name: str = 'Quarterly'\n",
    "    n_ts: int = 24_000\n",
    "\n",
    "@dataclass\n",
    "class Monthly:\n",
    "    seasonality: int = 12\n",
    "    horizon: int = 18\n",
    "    freq: str = 'M'\n",
    "    name: str = 'Monthly'\n",
    "    n_ts: int = 48_000\n",
    "\n",
    "@dataclass\n",
    "class Weekly:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 13\n",
    "    freq: str = 'W'\n",
    "    name: str = 'Weekly'\n",
    "    n_ts: int = 359\n",
    "        \n",
    "@dataclass\n",
    "class Daily:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 14\n",
    "    freq: str = 'D'\n",
    "    name: str = 'Daily'\n",
    "    n_ts: int = 4_227\n",
    "\n",
    "@dataclass\n",
    "class Hourly:\n",
    "    seasonality: int = 24\n",
    "    horizon: int = 48\n",
    "    freq: str = 'H'\n",
    "    name: str = 'Hourly'\n",
    "    n_ts: int = 414\n",
    "\n",
    "        \n",
    "@dataclass\n",
    "class Other:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 8\n",
    "    freq: str = 'D'\n",
    "    name: str = 'Other'\n",
    "    n_ts: int = 5_000\n",
    "    included_groups: Tuple = ('Weekly', 'Daily', 'Hourly')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "M4Info = Info(groups=('Yearly', 'Quarterly', 'Monthly', 'Weekly', 'Daily', 'Hourly', 'Other'),\n",
    "              class_groups=(Yearly, Quarterly, Monthly, Weekly, Daily, Hourly, Other))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download data class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@dataclass\n",
    "class M4:\n",
    "    \n",
    "    source_url: str = 'https://raw.githubusercontent.com/Mcompetitions/M4-methods/master/Dataset/'\n",
    "    naive2_forecast_url: str = 'https://github.com/Nixtla/m4-forecasts/raw/master/forecasts/submission-Naive2.zip'\n",
    "\n",
    "    @staticmethod\n",
    "    def load(directory: str,\n",
    "             group: str,\n",
    "             cache: bool = True) -> Tuple[pd.DataFrame, \n",
    "                                          Optional[pd.DataFrame], \n",
    "                                          Optional[pd.DataFrame]]:\n",
    "        \"\"\"Downloads and loads M4 data.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        directory: str\n",
    "            Directory where data will be downloaded.\n",
    "        group: str\n",
    "            Group name.\n",
    "            Allowed groups: 'Yearly', 'Quarterly', 'Monthly', \n",
    "                            'Weekly', 'Daily', 'Hourly'.\n",
    "        cache: bool\n",
    "            If `True` saves and loads \n",
    "            \n",
    "        Notes\n",
    "        -----\n",
    "        [1] Returns train+test sets.\n",
    "        \"\"\"\n",
    "        path = f'{directory}/m4/datasets'\n",
    "        file_cache = f'{path}/{group}.p'\n",
    "        \n",
    "        if os.path.exists(file_cache) and cache:\n",
    "            df, X_df, S_df = pd.read_pickle(file_cache)\n",
    "            \n",
    "            return df, X_df, S_df\n",
    "        \n",
    "        if group == 'Other':\n",
    "            #Special case.\n",
    "            included_dfs = [M4.load(directory, gr) \\\n",
    "                            for gr in M4Info['Other'].included_groups]\n",
    "            df, *_ = zip(*included_dfs)\n",
    "            df = pd.concat(df)\n",
    "        else:\n",
    "            M4.download(directory)\n",
    "            path = f'{directory}/m4/datasets'\n",
    "            class_group = M4Info[group]\n",
    "            S_df = pd.read_csv(f'{directory}/m4/datasets/M4-info.csv', \n",
    "                               usecols=['M4id','category'])\n",
    "            S_df['category'] = S_df['category'].astype('category').cat.codes\n",
    "            S_df.rename({'M4id': 'unique_id'}, axis=1, inplace=True)\n",
    "            S_df = S_df[S_df['unique_id'].str.startswith(class_group.name[0])]\n",
    "\n",
    "            def read_and_melt(file):\n",
    "                df = pd.read_csv(file)\n",
    "                df.columns = ['unique_id'] + list(range(1, df.shape[1]))\n",
    "                df = pd.melt(df, id_vars=['unique_id'], var_name='ds', value_name='y')\n",
    "                df = df.dropna()\n",
    "\n",
    "                return df\n",
    "\n",
    "            df_train = read_and_melt(file=f'{path}/{group}-train.csv')\n",
    "            df_test = read_and_melt(file=f'{path}/{group}-test.csv')\n",
    "\n",
    "            len_train = df_train.groupby('unique_id').agg({'ds': 'max'}).reset_index()\n",
    "            len_train.columns = ['unique_id', 'len_serie']\n",
    "            df_test = df_test.merge(len_train, on=['unique_id'])\n",
    "            df_test['ds'] = df_test['ds'] + df_test['len_serie']\n",
    "            df_test.drop('len_serie', axis=1, inplace=True)\n",
    "\n",
    "            df = pd.concat([df_train, df_test])\n",
    "            df = df.sort_values(['unique_id', 'ds']).reset_index(drop=True)\n",
    "            \n",
    "            S_df = S_df.sort_values('unique_id').reset_index(drop=True)\n",
    "        \n",
    "        X_df = None\n",
    "        if cache:\n",
    "            pd.to_pickle((df, X_df, S_df), file_cache)\n",
    "            \n",
    "        return df, None, S_df\n",
    "\n",
    "    @staticmethod\n",
    "    def download(directory: str) -> None:\n",
    "        \"\"\"Download M4 Dataset.\"\"\"\n",
    "        path = f'{directory}/m4/datasets/'\n",
    "        if not os.path.exists(path):\n",
    "            for group in M4Info.groups:\n",
    "                download_file(path, f'{M4.source_url}/Train/{group}-train.csv')\n",
    "                download_file(path, f'{M4.source_url}/Test/{group}-test.csv')\n",
    "            download_file(path, f'{M4.source_url}/M4-info.csv')\n",
    "            download_file(path, M4.naive2_forecast_url, decompress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group: Hourly n_series: 414\n"
     ]
    }
   ],
   "source": [
    "for group, meta in M4Info:\n",
    "    if group != 'Hourly':\n",
    "        continue\n",
    "    df, *_ = M4.load(directory='data', group=group)\n",
    "    n_series = len(np.unique(df.unique_id.values))\n",
    "\n",
    "    display_str  = f'Group: {group} '\n",
    "    display_str += f'n_series: {n_series}'\n",
    "    print(display_str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class M4Evaluation:\n",
    "    \n",
    "    @staticmethod\n",
    "    def load_benchmark(directory: str, group: str,\n",
    "                       source_url: Optional[str] = None) -> np.ndarray:\n",
    "        \"\"\"Downloads and loads a bechmark forecasts.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        directory: str\n",
    "            Directory where data will be downloaded.\n",
    "        group: str\n",
    "            Group name.\n",
    "            Allowed groups: 'Yearly', 'Quarterly', 'Monthly', \n",
    "                            'Weekly', 'Daily', 'Hourly'.\n",
    "        source_url: str, optional\n",
    "            Optional benchmark url obtained from \n",
    "            https://github.com/Nixtla/m4-forecasts/tree/master/forecasts.\n",
    "            If `None` returns Naive2. \n",
    "        \n",
    "        Returns\n",
    "        -------\n",
    "        benchmark: numpy array\n",
    "            Numpy array of shape (n_series, horizon).\n",
    "        \"\"\"\n",
    "        path = f'{directory}/m4/datasets'\n",
    "        initial = group[0]\n",
    "        if source_url is not None:\n",
    "            filename = source_url.split('/')[-1].replace('.rar', '.csv')\n",
    "            filepath = f'{path}/{filename}'\n",
    "            if not os.path.exists(filepath):\n",
    "                download_file(path, source_url, decompress=True)\n",
    "            \n",
    "        else:\n",
    "            filepath = f'{path}/submission-Naive2.csv'\n",
    "        \n",
    "        benchmark = pd.read_csv(filepath)\n",
    "        benchmark = benchmark[benchmark['id'].str.startswith(initial)]\n",
    "        benchmark = benchmark.set_index('id').dropna(1)\n",
    "        benchmark = benchmark.sort_values('id').values\n",
    "        \n",
    "        return benchmark\n",
    "    \n",
    "    @staticmethod\n",
    "    def evaluate(directory: str, group: str, \n",
    "                 y_hat: Union[np.ndarray, str]) -> pd.DataFrame:\n",
    "        \"\"\"Evaluates y_hat according to M4 methodology.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        directory: str\n",
    "            Directory where data will be downloaded.\n",
    "        group: str\n",
    "            Group name.\n",
    "            Allowed groups: 'Yearly', 'Quarterly', 'Monthly', \n",
    "                            'Weekly', 'Daily', 'Hourly'.\n",
    "        y_hat: numpy array, str\n",
    "            Group forecasts as numpy array or\n",
    "            benchmark url from\n",
    "            https://github.com/Nixtla/m4-forecasts/tree/master/forecasts.\n",
    "            \n",
    "        Returns\n",
    "        -------\n",
    "        evaluation: pandas dataframe\n",
    "            DataFrame with columns OWA, SMAPE, MASE\n",
    "            and group as index.\n",
    "        \"\"\"\n",
    "        if isinstance(y_hat, str):\n",
    "            y_hat = M4Evaluation.load_benchmark(directory, group, y_hat)\n",
    "        \n",
    "        initial = group[0]\n",
    "        class_group = M4Info[group]\n",
    "        horizon = class_group.horizon\n",
    "        seasonality = class_group.seasonality\n",
    "        path = f'{directory}/m4/datasets'\n",
    "        y_df, *_ = M4.load(directory, group)\n",
    "        \n",
    "        y_train = y_df.groupby('unique_id')['y'] \n",
    "        y_train = y_train.apply(lambda x: x.head(-horizon).values)\n",
    "        y_train = y_train.values\n",
    "        \n",
    "        y_test = y_df.groupby('unique_id')['y']\n",
    "        y_test = y_test.tail(horizon)\n",
    "        y_test = y_test.values.reshape(-1, horizon)    \n",
    "\n",
    "        naive2 = M4Evaluation.load_benchmark(directory, group)\n",
    "        smape_y_hat = smape(y_test, y_hat)\n",
    "        smape_naive2 = smape(y_test, naive2)\n",
    "        \n",
    "        mase_y_hat = np.mean([mase(y_test[i], y_hat[i], y_train[i], seasonality)\n",
    "                              for i in range(class_group.n_ts)])\n",
    "        mase_naive2 = np.mean([mase(y_test[i], naive2[i], y_train[i], seasonality)\n",
    "                               for i in range(class_group.n_ts)])\n",
    "        \n",
    "        owa = .5 * (mase_y_hat / mase_naive2 + smape_y_hat / smape_naive2)\n",
    "        \n",
    "        evaluation = pd.DataFrame({'SMAPE': smape_y_hat,\n",
    "                                   'MASE': mase_y_hat,\n",
    "                                   'OWA': owa},\n",
    "                                   index=[group])\n",
    "        \n",
    "        return evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### URL-based evaluation\n",
    "\n",
    "The method `evaluate` from the class `M4Evaluation` can receive a url of a [benchmark uploaded to the M4 competiton](https://github.com/Mcompetitions/M4-methods/tree/master/Point%20Forecasts).  \n",
    "\n",
    "The results compared to the on-the-fly evaluation were obtained from the [official evaluation](https://github.com/Mcompetitions/M4-methods/blob/master/Evaluation%20and%20Ranks.xlsx)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SMAPE</th>\n",
       "      <th>MASE</th>\n",
       "      <th>OWA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Hourly</th>\n",
       "      <td>9.328443</td>\n",
       "      <td>0.893046</td>\n",
       "      <td>0.440163</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           SMAPE      MASE       OWA\n",
       "Hourly  9.328443  0.893046  0.440163"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "esrnn_url = 'https://github.com/Nixtla/m4-forecasts/raw/master/forecasts/submission-118.zip'\n",
    "esrnn_evaluation = M4Evaluation.evaluate('data', 'Hourly', esrnn_url)\n",
    "# Test of the same evaluation as the original one\n",
    "test_close(esrnn_evaluation['SMAPE'].item(), 9.328, eps=1e-3)\n",
    "test_close(esrnn_evaluation['MASE'].item(), 0.893, eps=1e-3)\n",
    "test_close(esrnn_evaluation['OWA'].item(), 0.440, eps=1e-3)\n",
    "esrnn_evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Numpy-based evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also the method `evaluate` can recevie a numpy array of forecasts.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SMAPE</th>\n",
       "      <th>MASE</th>\n",
       "      <th>OWA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Hourly</th>\n",
       "      <td>11.505702</td>\n",
       "      <td>0.818598</td>\n",
       "      <td>0.483841</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            SMAPE      MASE       OWA\n",
       "Hourly  11.505702  0.818598  0.483841"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fforma_url = 'https://github.com/Nixtla/m4-forecasts/raw/master/forecasts/submission-245.zip'\n",
    "fforma_forecasts = M4Evaluation.load_benchmark('data', 'Hourly', fforma_url)\n",
    "fforma_evaluation = M4Evaluation.evaluate('data', 'Hourly', fforma_forecasts)\n",
    "# Test of the same evaluation as the original one\n",
    "test_close(fforma_evaluation['SMAPE'].item(), 11.506, eps=1e-3)\n",
    "test_close(fforma_evaluation['MASE'].item(), 0.819, eps=1e-3)\n",
    "test_close(fforma_evaluation['OWA'].item(), 0.484, eps=1e-3)\n",
    "fforma_evaluation"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nixtla",
   "language": "python",
   "name": "nixtla"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
