{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp datasets.aotm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AOTM dataset\n",
    "> AOTM dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from typing import List, Optional, Callable, Union, Any, Tuple\n",
    "\n",
    "import os\n",
    "import os.path as osp\n",
    "from collections.abc import Sequence\n",
    "import sys\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datetime import timezone, datetime, timedelta\n",
    "import time\n",
    "\n",
    "from recohut.datasets.bases.common import SessionDataset\n",
    "from recohut.utils.common_utils import download_url, extract_zip, makedirs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class AOTMDataset(SessionDataset):\n",
    "    url = 'https://github.com/RecoHut-Datasets/aotm/raw/v1/aotm.zip'\n",
    "    \n",
    "    def __init__(self, root, process_method, min_session_length=2, min_item_support=2,\n",
    "                 num_slices=5, days_offset=0, days_shift=95, days_train=90, days_test=5):\n",
    "        min_date = session_length = None\n",
    "        super().__init__(root, process_method, min_date, session_length, \n",
    "                         min_session_length, min_item_support, num_slices, days_offset, \n",
    "                         days_shift, days_train, days_test)\n",
    "    \n",
    "    @property\n",
    "    def raw_file_names(self) -> str:\n",
    "        return 'playlists-aotm.csv'\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'dataset.pkl'\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        from shutil import move, rmtree\n",
    "        move(osp.join(self.raw_dir, 'aotm', 'raw', 'playlists-aotm.csv'),\n",
    "             osp.join(self.raw_dir, 'playlists-aotm.csv'))\n",
    "        rmtree(osp.join(self.raw_dir, 'aotm'))\n",
    "        os.unlink(path)\n",
    "\n",
    "    def load(self):\n",
    "        #load csv\n",
    "        data = pd.read_csv(osp.join(self.raw_dir,self.raw_file_names), sep='\\t')\n",
    "        data.sort_values(by=['SessionId','Time'], inplace=True)\n",
    "        \n",
    "        #output\n",
    "        data_start = datetime.fromtimestamp(data.Time.min(), timezone.utc)\n",
    "        data_end = datetime.fromtimestamp(data.Time.max(), timezone.utc)\n",
    "        \n",
    "        print('Loaded data set\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}\\n\\tSpan: {} / {}\\n\\n'.\n",
    "            format(len(data), data.SessionId.nunique(), data.ItemId.nunique(), data_start.date().isoformat(), data_end.date().isoformat()))\n",
    "    \n",
    "        self.data = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading https://github.com/RecoHut-Datasets/aotm/raw/v1/aotm.zip\n",
      "Extracting /content/aotm/raw/aotm.zip\n",
      "Processing...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded data set\n",
      "\tEvents: 1821241\n",
      "\tSessions: 93313\n",
      "\tItems: 765790\n",
      "\tSpan: 2016-01-02 / 2016-12-30\n",
      "\n",
      "\n",
      "Filtered data set\n",
      "\tEvents: 1192938\n",
      "\tSessions: 87654\n",
      "\tItems: 138815\n",
      "\tSpan: 2016-01-02 / 2016-12-30\n",
      "\n",
      "\n",
      "Full train set\n",
      "\tEvents: 1189593\n",
      "\tSessions: 87410\n",
      "\tItems: 138815\n",
      "Test set\n",
      "\tEvents: 3345\n",
      "\tSessions: 244\n",
      "\tItems: 3105\n",
      "Train set\n",
      "\tEvents: 1185992\n",
      "\tSessions: 87145\n",
      "\tItems: 138814\n",
      "Validation set\n",
      "\tEvents: 3600\n",
      "\tSessions: 265\n",
      "\tItems: 3363\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    }
   ],
   "source": [
    "!rm -r /content/aotm\n",
    "aotmdata = AOTMDataset(root='/content/aotm', process_method='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34m/content/aotm\u001b[00m\n",
      "├── [ 83M]  \u001b[01;34mprocessed\u001b[00m\n",
      "│   ├── [120K]  events_test.txt\n",
      "│   ├── [ 42M]  events_train_full.txt\n",
      "│   ├── [ 41M]  events_train_tr.txt\n",
      "│   └── [129K]  events_train_valid.txt\n",
      "└── [ 65M]  \u001b[01;34mraw\u001b[00m\n",
      "    └── [ 65M]  playlists-aotm.csv\n",
      "\n",
      " 149M used in 2 directories, 5 files\n"
     ]
    }
   ],
   "source": [
    "!tree --du -h -C /content/aotm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading https://github.com/RecoHut-Datasets/aotm/raw/v1/aotm.zip\n",
      "Extracting /content/aotm/raw/aotm.zip\n",
      "Processing...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded data set\n",
      "\tEvents: 1821241\n",
      "\tSessions: 93313\n",
      "\tItems: 765790\n",
      "\tSpan: 2016-01-02 / 2016-12-30\n",
      "\n",
      "\n",
      "Filtered data set\n",
      "\tEvents: 1192938\n",
      "\tSessions: 87654\n",
      "\tItems: 138815\n",
      "\tSpan: 2016-01-02 / 2016-12-30\n",
      "\n",
      "\n",
      "Full train set\n",
      "\tEvents: 1176744\n",
      "\tSessions: 86474\n",
      "\tItems: 138786\n",
      "Test set\n",
      "\tEvents: 16138\n",
      "\tSessions: 1179\n",
      "\tItems: 12841\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    }
   ],
   "source": [
    "!rm -r /content/aotm\n",
    "aotmdata = AOTMDataset(root='/content/aotm', process_method='days_test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34m/content/aotm\u001b[00m\n",
      "├── [ 42M]  \u001b[01;34mprocessed\u001b[00m\n",
      "│   ├── [578K]  events_test.txt\n",
      "│   └── [ 41M]  events_train_full.txt\n",
      "└── [ 65M]  \u001b[01;34mraw\u001b[00m\n",
      "    └── [ 65M]  playlists-aotm.csv\n",
      "\n",
      " 107M used in 2 directories, 3 files\n"
     ]
    }
   ],
   "source": [
    "!tree --du -h -C /content/aotm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded data set\n",
      "\tEvents: 1821241\n",
      "\tSessions: 93313\n",
      "\tItems: 765790\n",
      "\tSpan: 2016-01-02 / 2016-12-30\n",
      "\n",
      "\n",
      "Filtered data set\n",
      "\tEvents: 1192938\n",
      "\tSessions: 87654\n",
      "\tItems: 138815\n",
      "\tSpan: 2016-01-02 / 2016-12-30\n",
      "\n",
      "\n",
      "Full data set 0\n",
      "\tEvents: 1192938\n",
      "\tSessions: 87654\n",
      "\tItems: 138815\n",
      "\tSpan: 2016-01-02T23:00:00+00:00 / 2016-12-30T23:00:42+00:00\n",
      "Slice data set 0\n",
      "\tEvents: 315346\n",
      "\tSessions: 23197\n",
      "\tItems: 95205\n",
      "\tSpan: 2016-01-02 / 2016-04-01 / 2016-04-06\n",
      "Train set 0\n",
      "\tEvents: 298510\n",
      "\tSessions: 21960\n",
      "\tItems: 92778\n",
      "\tSpan: 2016-01-02 / 2016-04-01\n",
      "Test set 0\n",
      "\tEvents: 14333\n",
      "\tSessions: 1213\n",
      "\tItems: 10851\n",
      "\tSpan: 2016-04-01 / 2016-04-06 \n",
      "\n",
      "\n",
      "Full data set 1\n",
      "\tEvents: 1192938\n",
      "\tSessions: 87654\n",
      "\tItems: 138815\n",
      "\tSpan: 2016-01-02T23:00:00+00:00 / 2016-12-30T23:00:42+00:00\n",
      "Slice data set 1\n",
      "\tEvents: 310422\n",
      "\tSessions: 22778\n",
      "\tItems: 94240\n",
      "\tSpan: 2016-04-06 / 2016-07-05 / 2016-07-10\n",
      "Train set 1\n",
      "\tEvents: 294246\n",
      "\tSessions: 21594\n",
      "\tItems: 91873\n",
      "\tSpan: 2016-04-06 / 2016-07-05\n",
      "Test set 1\n",
      "\tEvents: 13733\n",
      "\tSessions: 1163\n",
      "\tItems: 10441\n",
      "\tSpan: 2016-07-05 / 2016-07-10 \n",
      "\n",
      "\n",
      "Full data set 2\n",
      "\tEvents: 1192938\n",
      "\tSessions: 87654\n",
      "\tItems: 138815\n",
      "\tSpan: 2016-01-02T23:00:00+00:00 / 2016-12-30T23:00:42+00:00\n",
      "Slice data set 2\n",
      "\tEvents: 309023\n",
      "\tSessions: 22710\n",
      "\tItems: 94366\n",
      "\tSpan: 2016-07-10 / 2016-10-08 / 2016-10-13\n",
      "Train set 2\n",
      "\tEvents: 292756\n",
      "\tSessions: 21528\n",
      "\tItems: 91935\n",
      "\tSpan: 2016-07-10 / 2016-10-08\n",
      "Test set 2\n",
      "\tEvents: 13747\n",
      "\tSessions: 1155\n",
      "\tItems: 10553\n",
      "\tSpan: 2016-10-08 / 2016-10-13 \n",
      "\n",
      "\n",
      "Full data set 3\n",
      "\tEvents: 1192938\n",
      "\tSessions: 87654\n",
      "\tItems: 138815\n",
      "\tSpan: 2016-01-02T23:00:00+00:00 / 2016-12-30T23:00:42+00:00\n",
      "Slice data set 3\n",
      "\tEvents: 258147\n",
      "\tSessions: 18969\n",
      "\tItems: 86140\n",
      "\tSpan: 2016-10-13 / 2017-01-11 / 2017-01-16\n",
      "Train set 3\n",
      "\tEvents: 258147\n",
      "\tSessions: 18969\n",
      "\tItems: 86140\n",
      "\tSpan: 2016-10-13 / 2017-01-11\n",
      "Test set 3\n",
      "\tEvents: 0\n",
      "\tSessions: 0\n",
      "\tItems: 0\n",
      "\tSpan: 2017-01-11 / 2017-01-16 \n",
      "\n",
      "\n",
      "Full data set 4\n",
      "\tEvents: 1192938\n",
      "\tSessions: 87654\n",
      "\tItems: 138815\n",
      "\tSpan: 2016-01-02T23:00:00+00:00 / 2016-12-30T23:00:42+00:00\n",
      "Slice data set 4\n",
      "\tEvents: 0\n",
      "\tSessions: 0\n",
      "\tItems: 0\n",
      "\tSpan: 2017-01-16 / 2017-04-16 / 2017-04-21\n",
      "Train set 4\n",
      "\tEvents: 0\n",
      "\tSessions: 0\n",
      "\tItems: 0\n",
      "\tSpan: 2017-01-16 / 2017-04-16\n",
      "Test set 4\n",
      "\tEvents: 0\n",
      "\tSessions: 0\n",
      "\tItems: 0\n",
      "\tSpan: 2017-04-16 / 2017-04-21 \n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    }
   ],
   "source": [
    "!rm -r /content/aotm/processed/*\n",
    "aotmdata = AOTMDataset(root='/content/aotm', process_method='slice')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34m/content/aotm\u001b[00m\n",
      "├── [ 41M]  \u001b[01;34mprocessed\u001b[00m\n",
      "│   ├── [513K]  events_test.0.txt\n",
      "│   ├── [491K]  events_test.1.txt\n",
      "│   ├── [491K]  events_test.2.txt\n",
      "│   ├── [  38]  events_test.3.txt\n",
      "│   ├── [  38]  events_test.4.txt\n",
      "│   ├── [ 10M]  events_train_full.0.txt\n",
      "│   ├── [ 10M]  events_train_full.1.txt\n",
      "│   ├── [ 10M]  events_train_full.2.txt\n",
      "│   ├── [9.0M]  events_train_full.3.txt\n",
      "│   └── [  38]  events_train_full.4.txt\n",
      "└── [ 65M]  \u001b[01;34mraw\u001b[00m\n",
      "    └── [ 65M]  playlists-aotm.csv\n",
      "\n",
      " 107M used in 2 directories, 11 files\n"
     ]
    }
   ],
   "source": [
    "!tree --du -h -C /content/aotm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-22 09:23:49\n",
      "\n",
      "recohut: 0.0.5\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.144+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "pandas : 1.1.5\n",
      "IPython: 5.5.0\n",
      "numpy  : 1.19.5\n",
      "sys    : 3.7.12 (default, Sep 10 2021, 00:21:48) \n",
      "[GCC 7.5.0]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d -p recohut"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
