{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#default_exp data.datasets.m3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# M3 dataset\n",
    "\n",
    "> Download the M3 dataset.\n",
    "\n",
    "The [M3 forecasting competition](https://forecasters.org/resources/time-series-data/m3-competition/) of 1998 was organized by Makridakis and Hibon(2000), as a response to the controversy generated by the previous M competitions.\n",
    "The dataset used for this event corresponded to a total of 3003 business and economic series of which 645 are annual, 756 are quarterly and 1428 are monthly and 174 on other frequencies.\n",
    "\n",
    "[Spyros Makridakis and Michele Hibon. The M3-competition: results, conclusions and implications. International Journal of Forecasting, 16(4):451–476, 2000. ISSN 0169-2070.](https://www.sciencedirect.com/science/article/abs/pii/S0169207000000571?via%3Dihub)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import os\n",
    "from dataclasses import dataclass\n",
    "from typing import Dict, List, Optional, Tuple, Union\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from nixtlats.data.datasets.utils import download_file, Info, TimeSeriesDataclass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "M3 meta information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@dataclass\n",
    "class Yearly:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 6\n",
    "    freq: str = 'Y'\n",
    "    sheet_name: str = 'M3Year'\n",
    "    name: str = 'Yearly'\n",
    "    n_ts: int = 645\n",
    "\n",
    "@dataclass\n",
    "class Quarterly:\n",
    "    seasonality: int = 4\n",
    "    horizon: int = 8\n",
    "    freq: str = 'Q'\n",
    "    sheet_name: str = 'M3Quart'\n",
    "    name: str = 'Quarterly'\n",
    "    n_ts: int = 756\n",
    "\n",
    "@dataclass\n",
    "class Monthly:\n",
    "    seasonality: int = 12\n",
    "    horizon: int = 18\n",
    "    freq: str = 'M'\n",
    "    sheet_name: str = 'M3Month'\n",
    "    name: str = 'Monthly'\n",
    "    n_ts: int = 1428\n",
    "\n",
    "@dataclass\n",
    "class Other:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 8\n",
    "    freq: str = 'D'\n",
    "    sheet_name: str = 'M3Other'\n",
    "    name: str = 'Other'\n",
    "    n_ts: int = 174"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "M3Info = Info(groups=('Yearly', 'Quarterly', 'Monthly', 'Other'),\n",
    "              class_groups=(Yearly, Quarterly, Monthly, Other))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#exporti\n",
    "def _return_year(ts):\n",
    "    year = ts.iloc[0]\n",
    "    year = year if year != 0 else 1970\n",
    "\n",
    "    return year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@dataclass\n",
    "class M3(TimeSeriesDataclass):\n",
    "    \n",
    "    source_url = 'https://forecasters.org/data/m3comp/M3C.xls'\n",
    "\n",
    "    @staticmethod\n",
    "    def load(directory: str,\n",
    "             group: str) -> Tuple[pd.DataFrame, \n",
    "                                  Optional[pd.DataFrame], \n",
    "                                  Optional[pd.DataFrame]]:\n",
    "        \"\"\"\n",
    "        Downloads and loads M3 data.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        directory: str\n",
    "            Directory where data will be downloaded.\n",
    "        group: str\n",
    "            Group name.\n",
    "            Allowed groups: 'Yearly', 'Quarterly', 'Monthly', 'Other'.\n",
    "            \n",
    "        Notes\n",
    "        -----\n",
    "        [1] Returns train+test sets.\n",
    "        [2] There are monthly time series without start year. \n",
    "            This time series will start with 1970.\n",
    "        [3] Other time series have no start date.\n",
    "            This time series will start with 1970.\n",
    "        \"\"\"\n",
    "        M3.download(directory)\n",
    "        \n",
    "        path = f'{directory}/m3/datasets/'\n",
    "        file = f'{path}/M3C.xls'\n",
    "\n",
    "        class_group = M3Info.get_group(group)\n",
    "\n",
    "        df = pd.read_excel(file, sheet_name=class_group.sheet_name)\n",
    "        df = df.rename(columns={'Series': 'unique_id',\n",
    "                                'Category': 'category',\n",
    "                                'Starting Year': 'year',\n",
    "                                'Starting Month': 'month'})\n",
    "        df['unique_id'] = [class_group.name[0] + str(i + 1) for i in range(len(df))]\n",
    "        S = df.filter(items=['unique_id', 'category'])\n",
    "\n",
    "        id_vars = list(df.columns[:6])\n",
    "        df = pd.melt(df, id_vars=id_vars, var_name='ds', value_name='y')\n",
    "        df = df.dropna().sort_values(['unique_id', 'ds']).reset_index(drop=True)\n",
    "\n",
    "        freq = pd.tseries.frequencies.to_offset(class_group.freq)\n",
    "\n",
    "        if group == 'Other':\n",
    "            df['year'] = 1970\n",
    "\n",
    "        df['ds'] = df.groupby('unique_id')['year'] \\\n",
    "                     .transform(lambda df: pd.date_range(f'{_return_year(df)}-01-01',\n",
    "                                                         periods=df.shape[0],\n",
    "                                                         freq=freq))\n",
    "        df = df.filter(items=['unique_id', 'ds', 'y'])\n",
    "        \n",
    "        return df, None, None\n",
    "\n",
    "    @staticmethod\n",
    "    def download(directory: str) -> None:\n",
    "        \"\"\"Download M3 Dataset.\"\"\"\n",
    "        path = f'{directory}/m3/datasets/'\n",
    "        if not os.path.exists(path):\n",
    "            download_file(path, M3.source_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for group, meta in M3Info:\n",
    "    data, *_ = M3.load(directory='data', group=group)\n",
    "    unique_elements = data.groupby(['unique_id', 'ds']).size()\n",
    "    unique_ts = data.groupby('unique_id').size()\n",
    "\n",
    "    assert (unique_elements != 1).sum() == 0, f'Duplicated records found: {group}'\n",
    "    assert unique_ts.shape[0] == meta.n_ts, f'Number of time series not match: {group}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
