{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "established-marketing",
   "metadata": {},
   "outputs": [],
   "source": [
    "#default_exp data.datasets.favorita"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "saving-company",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style> table {float:left} </style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%html\n",
    "<style> table {float:left} </style>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "recorded-mixture",
   "metadata": {},
   "source": [
    "# Kaggle-Competition-Favorita\n",
    "\n",
    "The 2018 Kaggle competition was organized by Corporación Favorita, a major Ecuatorian grocery retailer. The large dataset is comprised of item sales history and promotions information, with additional information of items and stores, regional and national holidays among other covariates. The original task consisted on point forecasting sixteen days for the log-sales of each item store combination, for a total of 135K series.\n",
    "\n",
    "[Corporación Favorita. Corporación favorita grocery sales forecasting. Kaggle Competition, 2018.](https://www.kaggle.com/c/favorita-grocery-sales-forecasting/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "boxed-acrylic",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import os\n",
    "import re\n",
    "import gc\n",
    "import time\n",
    "import logging\n",
    "\n",
    "import shutil\n",
    "from py7zr import unpack_7zarchive\n",
    "\n",
    "from dataclasses import dataclass\n",
    "from typing import Dict, List, Optional, Tuple, Union\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "from nixtlats.data.datasets.utils import (\n",
    "    download_file, \n",
    "    Info, \n",
    "    TimeSeriesDataclass,\n",
    "    create_calendar_variables,\n",
    "    create_us_holiday_distance_variables,\n",
    ")\n",
    "from nixtlats.data.tsdataset import TimeSeriesDataset\n",
    "\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "#shutil.register_unpack_format('7zip', ['.7z'], unpack_7zarchive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "improved-princeton",
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def check_nans(df):\n",
    "    \"\"\" For data wrangling logs \"\"\"\n",
    "    n_rows = len(df)\n",
    "    \n",
    "    check_df = {'col': [], 'dtype': [], 'nan_prc': []}\n",
    "    for col in df.columns:\n",
    "        check_df['col'].append(col)\n",
    "        check_df['dtype'].append(df[col].dtype)\n",
    "        check_df['nan_prc'].append(df[col].isna().sum()/n_rows)\n",
    "    \n",
    "    check_df = pd.DataFrame(check_df)\n",
    "    print(\"\\n\")\n",
    "    print(f\"dataframe n_rows {n_rows}\")\n",
    "    print(check_df)\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "greenhouse-attribute",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class Favorita:\n",
    "    \n",
    "    # original data available from Kaggle directly\n",
    "    # pip install kaggle --upgrade\n",
    "    # kaggle competitions download -c favorita-grocery-sales-forecasting\n",
    "    source_url = 'https://www.dropbox.com/s/fe4y1hnphb4ykpy/favorita-grocery-sales-forecasting.zip?dl=1'\n",
    "    files = ['holidays_events.csv.7z', 'items.csv.7z', 'oil.csv.7z', 'sample_submission.csv.7z',\n",
    "             'stores.csv.7z', 'test.csv.7z', 'train.csv.7z', 'transactions.csv.7z']\n",
    "    \n",
    "    @staticmethod\n",
    "    def unzip(path):\n",
    "        # Unzip Load, Price, Solar and Wind data\n",
    "        # shutil.register_unpack_format('7zip', ['.7z'], unpack_7zarchive)\n",
    "        for file in Favorita.files:\n",
    "            filepath = f'{path}/{file}'\n",
    "            #Archive(filepath).extractall(path)\n",
    "            shutil.unpack_archive(filepath, path)\n",
    "            logger.info(f'Successfully decompressed {filepath}')\n",
    "        \n",
    "    @staticmethod\n",
    "    def download(directory: str) -> None:\n",
    "        \"\"\"Downloads Favorita Competition Dataset.\"\"\"\n",
    "        path = f'{directory}/favorita'\n",
    "        if not os.path.exists(path):            \n",
    "            download_file(directory=path, \n",
    "                          source_url=Favorita.source_url,\n",
    "                          decompress=True)\n",
    "            Favorita.unzip(path)\n",
    "            \n",
    "    @staticmethod\n",
    "    def read_raw_data(directory):\n",
    "        path = f'{directory}/favorita'\n",
    "\n",
    "        #------------------------------ Read Data ------------------------------#\n",
    "        # We avoid memory-intensive task of infering dtypes\n",
    "        dtypes_dict = {'id': 'int32',\n",
    "                       'date': 'str',\n",
    "                       'item_nbr': 'int32',\n",
    "                       'store_nbr': 'int8', # there are only 54 stores\n",
    "                       'unit_sales': 'float32', # values beyond are f32 outliers\n",
    "                       'onpromotion': 'boolean'}\n",
    "\n",
    "        # We read once from csv then from feather (much faster)\n",
    "        if not os.path.exists(f'{path}/train.feather'):\n",
    "            train_df = pd.read_csv(f'{path}/train.csv',\n",
    "                                   dtype=dtypes_dict,\n",
    "                                   parse_dates=['date'])\n",
    "            del train_df['id']\n",
    "            train_df.reset_index(drop=True, inplace=True)\n",
    "            train_df.to_feather(f'{path}/train.feather')\n",
    "            print(\"saved train.csv to train.feather for fast access\")\n",
    "            \n",
    "        items_df = pd.read_csv(f'{path}/items.csv')\n",
    "        stores_df = pd.read_csv(f'{path}/stores.csv')\n",
    "        \n",
    "        # Test is avoided because y_true is unavailable\n",
    "        train_df = pd.read_feather(f'{path}/train.feather')        \n",
    "        oil_df = pd.read_csv(f'{path}/oil.csv', parse_dates=['date'])\n",
    "        holidays_df = pd.read_csv(f'{path}/holidays_events.csv', parse_dates=['date'])\n",
    "        transactions_df = pd.read_csv(f'{path}/transactions.csv', parse_dates=['date'])\n",
    "\n",
    "        return train_df, oil_df, items_df, stores_df, holidays_df, transactions_df\n",
    "    \n",
    "    @staticmethod\n",
    "    def load(directory):\n",
    "        \n",
    "        # To prioritize comparability with benchmark models \n",
    "        # We replicate the data processing from:\n",
    "        # https://github.com/google-research/google-research/blob/master/tft/script_download_data.py\n",
    "        \n",
    "        Favorita.download(directory)\n",
    "        temporal, oil, items, stores, holidays, transactions \\\n",
    "                                                = Favorita.read_raw_data(directory)\n",
    "\n",
    "        #------------------------------ S Data Wrangling -----------------------------#\n",
    "        start = time.time()\n",
    "\n",
    "        # Transform the static variable strings to int for later fast one_hot\n",
    "        encoder = LabelEncoder()\n",
    "        items['family'] = encoder.fit_transform(items['family'].values)\n",
    "        items['class']  = encoder.fit_transform(items['class'].values)\n",
    "        items['perishable'] = encoder.fit_transform(items['perishable'].values)\n",
    "\n",
    "        #stores['city']  = encoder.fit_transform(stores['city'].values)\n",
    "        #stores['state'] = encoder.fit_transform(stores['state'].values)\n",
    "        stores['type']  = encoder.fit_transform(stores['type'].values)\n",
    "        stores['cluster']  = encoder.fit_transform(stores['cluster'].values)\n",
    "        stores_statecity = stores[['store_nbr', 'city', 'state']]\n",
    "\n",
    "        # Declare static variables dataframe and unique_id\n",
    "        S_df = temporal[['item_nbr', 'store_nbr']].copy()\n",
    "        S_df.drop_duplicates(inplace=True)\n",
    "\n",
    "        S_df = S_df.merge(items,  on='item_nbr',  how='left')\n",
    "        S_df = S_df.merge(stores, on='store_nbr', how='left')\n",
    "        S_df['traj_id'] = S_df.index\n",
    "\n",
    "        # Create one_hot variables,\n",
    "        store_nbr_orig = S_df['store_nbr'].values # used to merge with holidays\n",
    "        S_df = pd.get_dummies(data=S_df, \n",
    "                              columns=['family', 'class', 'perishable', \n",
    "                                       'store_nbr', 'city', 'state', 'type', 'cluster'])\n",
    "        S_df['store_nbr'] = store_nbr_orig\n",
    "\n",
    "        # # Avoid item_nbr dummies because length would be 4096\n",
    "        # # Substitute item_nbr/store_nbr the mean log(unit_sales) per item_nbr/store_nbr\n",
    "        # y_item = temporal.groupby('item_nbr', as_index=False).y.mean() \\\n",
    "        #                                             .rename(columns={'y':'y_item'})\n",
    "        # y_store = temporal.groupby('store_nbr', as_index=False).y.mean() \\\n",
    "        #                                             .rename(columns={'y':'y_store'})\n",
    "        # y_itemstore = temporal.groupby(['item_nbr', 'store_nbr'], as_index=False).y.mean() \\\n",
    "        #                                                 .rename(columns={'y':'y_itemstore'})\n",
    "\n",
    "        # S_df = S_df.merge(y_item, on='item_nbr', how='left')\n",
    "        # S_df = S_df.merge(y_store, on='store_nbr', how='left')\n",
    "        # S_df = S_df.merge(y_itemstore, on=['item_nbr', 'store_nbr'], how='left')\n",
    "\n",
    "        # S_df['y_item'].fillna(S_df['y_store'], inplace=True)\n",
    "        # S_df['y_itemstore'].fillna(S_df['y_item'], inplace=True)\n",
    "\n",
    "        del items, stores\n",
    "        gc.collect()\n",
    "\n",
    "        print(f'S wrangle time {time.time()-start :.2f} segs \\n')\n",
    "        check_nans(S_df)\n",
    "\n",
    "\n",
    "\n",
    "        # Avoid computation if temporal data is already created\n",
    "        temporal_path = f'{directory}/favorita/temporal.feather'\n",
    "        if not os.path.exists(temporal_path):\n",
    "            # if promotion is unknown it is set to False\n",
    "            temporal['onpromotion'] = temporal['onpromotion'].fillna(False) \n",
    "            temporal['open'] = 1 # flag where sales data is known\n",
    "            check_nans(temporal)\n",
    "\n",
    "            #------------------------- Filter/Balance Temporal Data ----------------------#\n",
    "            # Extract only a subset of data to save/process for efficiency (according to GR)\n",
    "            start = time.time()\n",
    "            start_date = datetime(2015, 1, 1)\n",
    "            end_date = datetime(2016, 6, 1)\n",
    "            if start_date is not None: temporal = temporal[(temporal['date'] >= start_date)]\n",
    "            if end_date is not None: temporal = temporal[(temporal['date'] < end_date)]\n",
    "\n",
    "            # Add trajectory identifier\n",
    "            temporal = temporal.merge(S_df[['item_nbr', 'store_nbr', 'traj_id']], \n",
    "                                      on=['item_nbr', 'store_nbr'], how='left')\n",
    "\n",
    "            # Remove all IDs with negative returns\n",
    "            start = time.time()\n",
    "            min_returns = temporal['unit_sales'].groupby(temporal['traj_id']).min()\n",
    "            valid_ids = set(min_returns[min_returns >= 0].index)\n",
    "            new_temporal = temporal[temporal['traj_id'].isin(valid_ids)].copy()\n",
    "\n",
    "            del temporal, S_df['store_nbr']\n",
    "            gc.collect()\n",
    "            temporal = new_temporal\n",
    "\n",
    "            print(f'Filter and Removing returns data {time.time()-start :.2f} segs')\n",
    "            check_nans(temporal)\n",
    "\n",
    "            # Resampling\n",
    "            print(\"Resampling\")\n",
    "            start = time.time()\n",
    "            resampled_dfs = []\n",
    "            for traj_id, raw_sub_df in temporal.groupby('traj_id'):\n",
    "\n",
    "                #print('Resampling', traj_id)\n",
    "                sub_df = raw_sub_df.set_index('date', drop=True).copy()\n",
    "\n",
    "                #sub_df = sub_df.resample('1d').last()\n",
    "                #sub_df['date'] = sub_df.index\n",
    "                #sub_df[['store_nbr', 'item_nbr', 'onpromotion']] \\\n",
    "                #            = sub_df[['store_nbr', 'item_nbr', 'onpromotion']].fillna(method='ffill')\n",
    "                #sub_df['open'] = sub_df['open'].fillna(0)    # flag where sales data is unknown\n",
    "\n",
    "                index = pd.date_range(sub_df.index[0], sub_df.index[-1], freq=\"D\") \n",
    "\n",
    "                ffill_vars = ['store_nbr', 'item_nbr', 'traj_id', 'onpromotion']\n",
    "                sub_df1 = sub_df[ffill_vars].reindex(index, method='ffill')\n",
    "                sub_df1['date'] = sub_df1.index\n",
    "\n",
    "                zfill_vars = ['unit_sales', 'open']\n",
    "                sub_df2 = sub_df[zfill_vars].reindex(index, fill_value=0)\n",
    "\n",
    "                sub_df = pd.concat([sub_df1, sub_df2], axis=1)\n",
    "                resampled_dfs.append(sub_df.reset_index(drop=True))\n",
    "\n",
    "            new_temporal = pd.concat(resampled_dfs, axis=0)\n",
    "            del temporal\n",
    "            gc.collect()\n",
    "            temporal = new_temporal\n",
    "            temporal['log_sales'] = np.log1p(temporal['unit_sales'])\n",
    "\n",
    "            check_nans(temporal)\n",
    "            print(f'Resampling to regular grid {time.time()-start} segs')\n",
    "\n",
    "\n",
    "            #--------------------------- Temporal Data Augmentation ---------------------------#\n",
    "            dates = temporal['date'].unique()\n",
    "\n",
    "            # Transactions variable\n",
    "            start = time.time()\n",
    "            temporal = temporal.merge(transactions,\n",
    "                                      on=['date', 'store_nbr'],\n",
    "                                      #left_on=['date', 'store_nbr'],\n",
    "                                      #right_on=['date', 'store_nbr'],\n",
    "                                      how='left')\n",
    "            temporal['transactions'] = temporal['transactions'].fillna(-1)\n",
    "            print(f'Transactions variable {time.time()-start :.2f} segs')\n",
    "\n",
    "            # Oil variable\n",
    "            start = time.time()\n",
    "            oil = oil[oil['date'].isin(dates)].fillna(method='ffill')\n",
    "            oil.rename(columns={\"dcoilwtico\": \"oil\"}, inplace=True)\n",
    "            temporal = temporal.merge(oil, on=['date'], how='left')\n",
    "            temporal['oil'] = temporal['oil'].fillna(-1)\n",
    "            print(f'Oil variables {time.time()-start :.2f} segs')\n",
    "\n",
    "            # Calendar variables\n",
    "            start = time.time()\n",
    "            calendar = pd.DataFrame({'date': dates})\n",
    "            calendar['day_of_week'] = calendar['date'].dt.dayofweek\n",
    "            calendar['day_of_month'] = calendar['date'].dt.day\n",
    "            calendar['month'] = calendar['date'].dt.month\n",
    "            temporal = temporal.merge(calendar, on=['date'], how='left')\n",
    "            print(f'Calendar variables {time.time()-start :.2f} segs')\n",
    "\n",
    "            # Holiday variables\n",
    "            start = time.time()\n",
    "            holidays = holidays[holidays['transferred']==False].copy()\n",
    "            holidays.rename(columns={'type': 'holiday_type'}, inplace=True)\n",
    "\n",
    "            national_holidays = holidays[holidays['locale']=='National']\n",
    "            local_holidays    = holidays[holidays['locale']=='Local']\n",
    "            regional_holidays = holidays[holidays['locale']=='Regional']\n",
    "\n",
    "            temporal['national_hol'] = temporal.merge(national_holidays, \n",
    "                                                      left_on=['date'], \n",
    "                                                      right_on=['date'],\n",
    "                                                      how='left')['description'].fillna('')\n",
    "\n",
    "            temporal = temporal.merge(stores_statecity, on=['store_nbr'], how='left')\n",
    "            temporal['regional_hol'] = temporal.merge(regional_holidays,\n",
    "                                                      left_on=['state', 'date'],\n",
    "                                                      right_on=['locale_name', 'date'],\n",
    "                                                      how='left')['description'].fillna('')\n",
    "            temporal['local_hol'] = temporal.merge(local_holidays,\n",
    "                                                   left_on=['city', 'date'],\n",
    "                                                   right_on=['locale_name', 'date'],\n",
    "                                                   how='left')['description'].fillna('')\n",
    "            del temporal['state'], temporal['city']\n",
    "\n",
    "            print(f'Holiday variables {time.time()-start :.2f} segs')\n",
    "            check_nans(temporal)\n",
    "\n",
    "            print('Saving processed file to {}'.format(temporal_path))\n",
    "            temporal.to_feather(temporal_path)\n",
    "\n",
    "        else:\n",
    "            temporal = pd.read_feather(temporal_path)\n",
    "\n",
    "        #------------------------------ X, Y Data Wrangling -----------------------------#\n",
    "\n",
    "        Y_df = temporal[['traj_id', 'date', 'log_sales']]\n",
    "        X_df = temporal[['traj_id', 'date',\n",
    "                         'onpromotion', 'open', 'transactions', 'oil', \n",
    "                         'day_of_week', 'day_of_month', 'month']]\n",
    "        # 'national_hol', 'regional_hol', 'local_hol'\n",
    "        del temporal\n",
    "        gc.collect()\n",
    "        \n",
    "        Y_df.rename(columns={'traj_id': 'unique_id', 'date': 'ds'}, inplace=True)\n",
    "        X_df.rename(columns={'traj_id': 'unique_id', 'date': 'ds'}, inplace=True)\n",
    "\n",
    "        return S_df, Y_df, X_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "distant-flesh",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 480M/480M [00:15<00:00, 31.5MiB/s] \n",
      "INFO:nixtla.data.datasets.utils:Successfully downloaded favorita-grocery-sales-forecasting.zip?dl=1, 480014675, bytes.\n",
      "INFO:nixtla.data.datasets.utils:Decompressing zip file...\n",
      "INFO:nixtla.data.datasets.utils:Successfully decompressed data/favorita/favorita-grocery-sales-forecasting.zip?dl=1\n",
      "INFO:__main__:Successfully decompressed ./data/favorita/holidays_events.csv.7z\n",
      "INFO:__main__:Successfully decompressed ./data/favorita/items.csv.7z\n",
      "INFO:__main__:Successfully decompressed ./data/favorita/oil.csv.7z\n",
      "INFO:__main__:Successfully decompressed ./data/favorita/sample_submission.csv.7z\n",
      "INFO:__main__:Successfully decompressed ./data/favorita/stores.csv.7z\n",
      "INFO:__main__:Successfully decompressed ./data/favorita/test.csv.7z\n",
      "INFO:__main__:Successfully decompressed ./data/favorita/train.csv.7z\n",
      "INFO:__main__:Successfully decompressed ./data/favorita/transactions.csv.7z\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved train.csv to train.feather for fast access\n",
      "S wrangle time 8.49 segs \n",
      "\n",
      "\n",
      "\n",
      "dataframe n_rows 174685\n",
      "            col  dtype  nan_prc\n",
      "0      item_nbr  int32      0.0\n",
      "1       traj_id  int64      0.0\n",
      "2      family_0  uint8      0.0\n",
      "3      family_1  uint8      0.0\n",
      "4      family_2  uint8      0.0\n",
      "..          ...    ...      ...\n",
      "481  cluster_13  uint8      0.0\n",
      "482  cluster_14  uint8      0.0\n",
      "483  cluster_15  uint8      0.0\n",
      "484  cluster_16  uint8      0.0\n",
      "485   store_nbr   int8      0.0\n",
      "\n",
      "[486 rows x 3 columns]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "dataframe n_rows 125497040\n",
      "           col           dtype  nan_prc\n",
      "0         date  datetime64[ns]      0.0\n",
      "1    store_nbr            int8      0.0\n",
      "2     item_nbr           int32      0.0\n",
      "3   unit_sales         float32      0.0\n",
      "4  onpromotion         boolean      0.0\n",
      "5         open           int64      0.0\n",
      "\n",
      "\n",
      "Filter and Removing returns data 4.39 segs\n",
      "\n",
      "\n",
      "dataframe n_rows 41518905\n",
      "           col           dtype  nan_prc\n",
      "0         date  datetime64[ns]      0.0\n",
      "1    store_nbr            int8      0.0\n",
      "2     item_nbr           int32      0.0\n",
      "3   unit_sales         float32      0.0\n",
      "4  onpromotion         boolean      0.0\n",
      "5         open           int64      0.0\n",
      "6      traj_id           int64      0.0\n",
      "\n",
      "\n",
      "Resampling\n",
      "\n",
      "\n",
      "dataframe n_rows 58106641\n",
      "           col           dtype  nan_prc\n",
      "0    store_nbr            int8      0.0\n",
      "1     item_nbr           int32      0.0\n",
      "2      traj_id           int64      0.0\n",
      "3  onpromotion         boolean      0.0\n",
      "4         date  datetime64[ns]      0.0\n",
      "5   unit_sales         float32      0.0\n",
      "6         open           int64      0.0\n",
      "7    log_sales         float32      0.0\n",
      "\n",
      "\n",
      "Resampling to regular grid 803.1067633628845 segs\n",
      "Transactions variable 12.72 segs\n",
      "Oil variables 12.66 segs\n",
      "Calendar variables 12.76 segs\n",
      "Holiday variables 106.95 segs\n",
      "\n",
      "\n",
      "dataframe n_rows 58106641\n",
      "             col           dtype  nan_prc\n",
      "0      store_nbr            int8      0.0\n",
      "1       item_nbr           int32      0.0\n",
      "2        traj_id           int64      0.0\n",
      "3    onpromotion         boolean      0.0\n",
      "4           date  datetime64[ns]      0.0\n",
      "5     unit_sales         float32      0.0\n",
      "6           open           int64      0.0\n",
      "7      log_sales         float32      0.0\n",
      "8   transactions         float64      0.0\n",
      "9            oil         float64      0.0\n",
      "10   day_of_week           int64      0.0\n",
      "11  day_of_month           int64      0.0\n",
      "12         month           int64      0.0\n",
      "13  national_hol          object      0.0\n",
      "14  regional_hol          object      0.0\n",
      "15     local_hol          object      0.0\n",
      "\n",
      "\n",
      "Saving processed file to ./data/favorita/temporal.feather\n"
     ]
    }
   ],
   "source": [
    "# S_df, Y_df, X_df = Favorita.load(directory='./data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "raised-electron",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_nbr</th>\n",
       "      <th>traj_id</th>\n",
       "      <th>family_0</th>\n",
       "      <th>family_1</th>\n",
       "      <th>family_2</th>\n",
       "      <th>family_3</th>\n",
       "      <th>family_4</th>\n",
       "      <th>family_5</th>\n",
       "      <th>family_6</th>\n",
       "      <th>family_7</th>\n",
       "      <th>...</th>\n",
       "      <th>cluster_7</th>\n",
       "      <th>cluster_8</th>\n",
       "      <th>cluster_9</th>\n",
       "      <th>cluster_10</th>\n",
       "      <th>cluster_11</th>\n",
       "      <th>cluster_12</th>\n",
       "      <th>cluster_13</th>\n",
       "      <th>cluster_14</th>\n",
       "      <th>cluster_15</th>\n",
       "      <th>cluster_16</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>103665</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>105574</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>105575</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>108079</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>108701</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 485 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   item_nbr  traj_id  family_0  family_1  family_2  family_3  family_4  \\\n",
       "0    103665        0         0         0         0         0         0   \n",
       "1    105574        1         0         0         0         0         0   \n",
       "2    105575        2         0         0         0         0         0   \n",
       "3    108079        3         0         0         0         0         0   \n",
       "4    108701        4         0         0         0         0         0   \n",
       "\n",
       "   family_5  family_6  family_7  ...  cluster_7  cluster_8  cluster_9  \\\n",
       "0         1         0         0  ...          0          0          0   \n",
       "1         0         0         0  ...          0          0          0   \n",
       "2         0         0         0  ...          0          0          0   \n",
       "3         0         0         0  ...          0          0          0   \n",
       "4         0         0         0  ...          0          0          0   \n",
       "\n",
       "   cluster_10  cluster_11  cluster_12  cluster_13  cluster_14  cluster_15  \\\n",
       "0           0           0           0           0           0           0   \n",
       "1           0           0           0           0           0           0   \n",
       "2           0           0           0           0           0           0   \n",
       "3           0           0           0           0           0           0   \n",
       "4           0           0           0           0           0           0   \n",
       "\n",
       "   cluster_16  \n",
       "0           0  \n",
       "1           0  \n",
       "2           0  \n",
       "3           0  \n",
       "4           0  \n",
       "\n",
       "[5 rows x 485 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# S_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "expensive-popularity",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>unique_id</th>\n",
       "      <th>ds</th>\n",
       "      <th>log_sales</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2015-01-02</td>\n",
       "      <td>2.302585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2015-01-03</td>\n",
       "      <td>1.386294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>1.098612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2015-01-05</td>\n",
       "      <td>0.693147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2015-01-06</td>\n",
       "      <td>1.098612</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   unique_id         ds  log_sales\n",
       "0          1 2015-01-02   2.302585\n",
       "1          1 2015-01-03   1.386294\n",
       "2          1 2015-01-04   1.098612\n",
       "3          1 2015-01-05   0.693147\n",
       "4          1 2015-01-06   1.098612"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Y_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "printable-metro",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>unique_id</th>\n",
       "      <th>ds</th>\n",
       "      <th>onpromotion</th>\n",
       "      <th>open</th>\n",
       "      <th>transactions</th>\n",
       "      <th>oil</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2015-01-02</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>3114.0</td>\n",
       "      <td>52.72</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2015-01-03</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>2495.0</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>999.0</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2015-01-05</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>981.0</td>\n",
       "      <td>50.05</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2015-01-06</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>827.0</td>\n",
       "      <td>47.98</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   unique_id         ds  onpromotion  open  transactions    oil  day_of_week  \\\n",
       "0          1 2015-01-02        False     1        3114.0  52.72            4   \n",
       "1          1 2015-01-03        False     1        2495.0  -1.00            5   \n",
       "2          1 2015-01-04        False     1         999.0  -1.00            6   \n",
       "3          1 2015-01-05        False     1         981.0  50.05            0   \n",
       "4          1 2015-01-06        False     1         827.0  47.98            1   \n",
       "\n",
       "   day_of_month  month  \n",
       "0             2      1  \n",
       "1             3      1  \n",
       "2             4      1  \n",
       "3             5      1  \n",
       "4             6      1  "
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# X_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "liquid-apparel",
   "metadata": {},
   "source": [
    "# Kaggle-Competition-Favorita References\n",
    "\n",
    "The evaluation metric of the Favorita Kaggle competition was the normalized weighted root mean squared logarithmic error (NWRMSLE).\n",
    "Perishable items have a score weight of 1.25; otherwise, the weight is 1.0.\n",
    "\n",
    "$$ NWRMSLE = \\sqrt{\\frac{\\sum^{n}_{i=1} w_{i}\\left(log(\\hat{y}_{i}+1)  - log(y_{i}+1)\\right)^{2}}{\\sum^{n}_{i=1} w_{i}}}$$\n",
    "\n",
    "Kaggle Competition Forecasting Methods                                                              | 16D ahead NWRMSLE\n",
    ":-------------------------------------------------------------------------------------------------: | :-------: \n",
    "[LGBM](https://www.kaggle.com/shixw125/1st-place-lgb-model-public-0-506-private-0-511/comments) [1] | 0.5091   | \n",
    "[Seq2Seq WaveNet](https://arxiv.org/abs/1803.04037) [2]                                             | 0.5129   |\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "breeding-salmon",
   "metadata": {},
   "source": [
    "1.\t[Corporación Favorita. Corporación favorita grocery sales forecasting. Kaggle Competition Leaderboard, 2018.](https://www.kaggle.com/c/favorita-grocery-sales-forecasting/leaderboard)\n",
    "2.\t[Glib Kechyn, Lucius Yu, Yangguang Zang, and Svyatoslav Kechyn.  Sales forecasting using wavenet within the framework of the Favorita Kaggle competition. Computing Research Repository, abs/1803.04037, 2018](https://arxiv.org/abs/1803.04037)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "chemical-programming",
   "metadata": {},
   "source": [
    "# Favorita Extra References\n",
    "\n",
    "Given that the test set is unavailable, some papers have chosen a different data partition to work on the dataset. The following table uses the favorita dataset with 90 days for training and predicts 30 days into the future for the logarithm of sales. The evaluation metric used in other papers was the quantile risk for P50 (normalized quantile loss):\n",
    "\n",
    "$$ QL-risk = \\frac{2 \\sum_{y_{t} \\in \\Omega } \\sum^{\\tau_{max}}_{\\tau=t} q (y_{t+\\tau}-\\hat{y}^{(q)}_{t+\\tau})_{+} + (1-q) (\\hat{y}^{(q)}_{t+\\tau}-y_{t+\\tau})_{+} }{\\sum_{y_{t} \\in \\Omega } \\sum^{\\tau_{max}}_{\\tau=1} | y_{t} | }$$\n",
    "\n",
    "\n",
    "Forecasting Methods                                                            | 30D ahead P50 QL-risk\n",
    ":-----------------------------------------------------------------------------------------------: | :-------: \n",
    "[MQTransformer](https://arxiv.org/abs/2009.14799) [4]                                             | 0.323     |\n",
    "[TFT](https://arxiv.org/abs/1912.09363) [3]                                                       | 0.354     | \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "caroline-contrast",
   "metadata": {},
   "source": [
    "3.\t[Bryan Lim, Sercan O. Arik, Nicolas Loeff, and Tomas Pfister. Temporal fusion transformers for interpretable multi-horizon time series forecasting. Computing Research Repository, abs/1912.09363, 2020](https://arxiv.org/abs/1912.09363).\n",
    "4.\t[Carson Eisenach, Yagna Patel, and Dhruv Madeka. MQtransformer: Multi-horizon forecasts with context dependent and feedback-aware attention. Computing Research Repository, abs/2009.14799, 2020](https://arxiv.org/abs/2009.14799)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "taken-episode",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f889c75-c01d-4c27-a108-3be2de660157",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nixtla",
   "language": "python",
   "name": "nixtla"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
