{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"https://colab.research.google.com/github/RecoHut-Projects/recohut/blob/master/tutorials/preprocessing/T341694_Preprocessing_of_Music_Sequential_Dataset.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing of Music Sequential Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import datetime\n",
    "import calendar\n",
    "from collections import Counter\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_test_sequences(test_data, given_k):\n",
    "    # we can run evaluation only over sequences longer than abs(LAST_K)\n",
    "    test_sequences = test_data.loc[test_data['sequence'].map(len) > abs(given_k), 'sequence'].values\n",
    "    return test_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_test_sequences_and_users(test_data, given_k, train_users):\n",
    "    # we can run evaluation only over sequences longer than abs(LAST_K)\n",
    "    mask = test_data['sequence'].map(len) > abs(given_k)\n",
    "    mask &= test_data['user_id'].isin(train_users)\n",
    "    test_sequences = test_data.loc[mask, 'sequence'].values\n",
    "    test_users = test_data.loc[mask, 'user_id'].values\n",
    "    return test_sequences, test_users"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from io import BytesIO\n",
    "from urllib.request import urlopen\n",
    "from zipfile import ZipFile\n",
    "zipurl = 'https://github.com/RecoHut-Datasets/30music/raw/v2/sessions.zip'\n",
    "with urlopen(zipurl) as zipresp:\n",
    "    with ZipFile(BytesIO(zipresp.read())) as zfile:\n",
    "        zfile.extractall('datasets')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_path = 'datasets/sessions.csv'\n",
    "# load this sample if you experience a severe slowdown with the previous dataset\n",
    "dataset_path = 'datasets/sessions_sample_10.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_and_adapt(path, last_months=0):\n",
    "    file_ext = os.path.splitext(path)[-1]\n",
    "    if file_ext == '.csv':\n",
    "        data = pd.read_csv(path, header=0)\n",
    "    elif file_ext == '.hdf':\n",
    "        data = pd.read_hdf(path)\n",
    "    else:\n",
    "        raise ValueError('Unsupported file {} having extension {}'.format(path, file_ext))\n",
    "\n",
    "    col_names = ['session_id', 'user_id', 'item_id', 'ts'] + data.columns.values.tolist()[4:]\n",
    "    data.columns = col_names\n",
    "\n",
    "    if last_months > 0:\n",
    "        def add_months(sourcedate, months):\n",
    "            month = sourcedate.month - 1 + months\n",
    "            year = int(sourcedate.year + month / 12)\n",
    "            month = month % 12 + 1\n",
    "            day = min(sourcedate.day, calendar.monthrange(year, month)[1])\n",
    "            return datetime.date(year, month, day)\n",
    "\n",
    "        lastdate = datetime.datetime.fromtimestamp(data.ts.max())\n",
    "        firstdate = add_months(lastdate, -last_months)\n",
    "        initial_unix = time.mktime(firstdate.timetuple())\n",
    "\n",
    "        # filter out older interactions\n",
    "        data = data[data['ts'] >= initial_unix]\n",
    "\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_seq_db_filter_top_k(path, topk=0, last_months=0):\n",
    "    file = load_and_adapt(path, last_months=last_months)\n",
    "\n",
    "    c = Counter(list(file['item_id']))\n",
    "\n",
    "    if topk > 1:\n",
    "        keeper = set([x[0] for x in c.most_common(topk)])\n",
    "        file = file[file['item_id'].isin(keeper)]\n",
    "\n",
    "    # group by session id and concat song_id\n",
    "    groups = file.groupby('session_id')\n",
    "\n",
    "    # convert item ids to string, then aggregate them to lists\n",
    "    aggregated = groups['item_id'].agg(sequence = lambda x: list(map(str, x)))\n",
    "    init_ts = groups['ts'].min()\n",
    "    users = groups['user_id'].min()  # it's just fast, min doesn't actually make sense\n",
    "\n",
    "    result = aggregated.join(init_ts).join(users)\n",
    "    result.reset_index(inplace=True)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for the sake of speed, let's keep only the top-1k most popular items in the last month\n",
    "dataset = create_seq_db_filter_top_k(path=dataset_path, topk=1000, last_months=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>ts</th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>357</td>\n",
       "      <td>[793, 3489]</td>\n",
       "      <td>1421003874</td>\n",
       "      <td>4296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>359</td>\n",
       "      <td>[1762]</td>\n",
       "      <td>1421018535</td>\n",
       "      <td>4296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>394</td>\n",
       "      <td>[1256]</td>\n",
       "      <td>1421007470</td>\n",
       "      <td>30980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4127</td>\n",
       "      <td>[1948, 1364, 2060, 1115, 6488, 2060]</td>\n",
       "      <td>1421416896</td>\n",
       "      <td>28117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6400</td>\n",
       "      <td>[687, 1394]</td>\n",
       "      <td>1420807778</td>\n",
       "      <td>35247</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   session_id                              sequence          ts  user_id\n",
       "0         357                           [793, 3489]  1421003874     4296\n",
       "1         359                                [1762]  1421018535     4296\n",
       "2         394                                [1256]  1421007470    30980\n",
       "3        4127  [1948, 1364, 2060, 1115, 6488, 2060]  1421416896    28117\n",
       "4        6400                           [687, 1394]  1420807778    35247"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of items: 1000\n",
      "Number of users: 4165\n",
      "Number of sessions: 6765\n",
      "\n",
      "Session length:\n",
      "\tAverage: 4.29\n",
      "\tMedian: 3.0\n",
      "\tMin: 1\n",
      "\tMax: 148\n",
      "Sessions per user:\n",
      "\tAverage: 1.62\n",
      "\tMedian: 1.0\n",
      "\tMin: 1\n",
      "\tMax: 13\n",
      "Most popular items: [('443', 207), ('1065', 155), ('67', 146), ('2308', 138), ('658', 131)]\n"
     ]
    }
   ],
   "source": [
    "cnt = Counter()\n",
    "dataset.sequence.map(cnt.update);\n",
    "\n",
    "sequence_length = dataset.sequence.map(len).values\n",
    "n_sessions_per_user = dataset.groupby('user_id').size()\n",
    "\n",
    "print('Number of items: {}'.format(len(cnt)))\n",
    "print('Number of users: {}'.format(dataset.user_id.nunique()))\n",
    "print('Number of sessions: {}'.format(len(dataset)) )\n",
    "\n",
    "print('\\nSession length:\\n\\tAverage: {:.2f}\\n\\tMedian: {}\\n\\tMin: {}\\n\\tMax: {}'.format(\n",
    "    sequence_length.mean(), \n",
    "    np.quantile(sequence_length, 0.5), \n",
    "    sequence_length.min(), \n",
    "    sequence_length.max()))\n",
    "\n",
    "print('Sessions per user:\\n\\tAverage: {:.2f}\\n\\tMedian: {}\\n\\tMin: {}\\n\\tMax: {}'.format(\n",
    "    n_sessions_per_user.mean(), \n",
    "    np.quantile(n_sessions_per_user, 0.5), \n",
    "    n_sessions_per_user.min(), \n",
    "    n_sessions_per_user.max()))\n",
    "\n",
    "print('Most popular items: {}'.format(cnt.most_common(5)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Splitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_holdout(dataset, perc=0.8, seed=1234):\n",
    "    \"\"\"\n",
    "    Split sequence dataset randomly\n",
    "    :param dataset: the sequence dataset\n",
    "    :param perc: the training percentange\n",
    "    :param seed: the random seed\n",
    "    :return: the training and test splits\n",
    "    \"\"\"\n",
    "    dataset = dataset.sample(frac=1, random_state=seed)\n",
    "    nseqs = len(dataset)\n",
    "    train_size = int(nseqs * perc)\n",
    "    # split data according to the shuffled index and the holdout size\n",
    "    train_split = dataset[:train_size]\n",
    "    test_split = dataset[train_size:]\n",
    "\n",
    "    return train_split, test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def temporal_holdout(dataset, ts_threshold):\n",
    "    \"\"\"\n",
    "    Split sequence dataset using timestamps\n",
    "    :param dataset: the sequence dataset\n",
    "    :param ts_threshold: the timestamp from which test sequences will start\n",
    "    :return: the training and test splits\n",
    "    \"\"\"\n",
    "    train = dataset.loc[dataset['ts'] < ts_threshold]\n",
    "    test = dataset.loc[dataset['ts'] >= ts_threshold]\n",
    "    train, test = clean_split(train, test)\n",
    "\n",
    "    return train, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def last_session_out_split(data,\n",
    "                           user_key='user_id',\n",
    "                           session_key='session_id',\n",
    "                           time_key='ts'):\n",
    "    \"\"\"\n",
    "    Assign the last session of every user to the test set and the remaining ones to the training set\n",
    "    \"\"\"\n",
    "    sessions = data.sort_values(by=[user_key, time_key]).groupby(user_key)[session_key]\n",
    "    last_session = sessions.last()\n",
    "    train = data[~data.session_id.isin(last_session.values)].copy()\n",
    "    test = data[data.session_id.isin(last_session.values)].copy()\n",
    "    train, test = clean_split(train, test)\n",
    "    return train, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_split(train, test):\n",
    "    \"\"\"\n",
    "    Remove new items from the test set.\n",
    "    :param train: The training set.\n",
    "    :param test: The test set.\n",
    "    :return: The cleaned training and test sets.\n",
    "    \"\"\"\n",
    "    train_items = set()\n",
    "    train['sequence'].apply(lambda seq: train_items.update(set(seq)))\n",
    "    test['sequence'] = test['sequence'].apply(lambda seq: [it for it in seq if it in train_items])\n",
    "    return train, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def balance_dataset(x, y):\n",
    "    number_of_elements = y.shape[0]\n",
    "    nnz = set(find(y)[0])\n",
    "    zero = set(range(number_of_elements)).difference(nnz)\n",
    "\n",
    "    max_samples = min(len(zero), len(nnz))\n",
    "\n",
    "    nnz_indices = random.sample(nnz, max_samples)\n",
    "    zero_indeces = random.sample(zero, max_samples)\n",
    "    indeces = nnz_indices + zero_indeces\n",
    "\n",
    "    return x[indeces, :], y[indeces, :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For simplicity, let's split the dataset by assigning the last session of every user to the test set, and all the previous ones to the training set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train sessions: 2600 - Test sessions: 4165\n"
     ]
    }
   ],
   "source": [
    "train_data, test_data = last_session_out_split(dataset)\n",
    "print(\"Train sessions: {} - Test sessions: {}\".format(len(train_data), len(test_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>ts</th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>357</td>\n",
       "      <td>[793, 3489]</td>\n",
       "      <td>1421003874</td>\n",
       "      <td>4296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9795</td>\n",
       "      <td>[1719]</td>\n",
       "      <td>1420700645</td>\n",
       "      <td>5612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>9796</td>\n",
       "      <td>[1065, 3812, 4012]</td>\n",
       "      <td>1420713509</td>\n",
       "      <td>5612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>9797</td>\n",
       "      <td>[532, 93]</td>\n",
       "      <td>1420724661</td>\n",
       "      <td>5612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>9798</td>\n",
       "      <td>[6932, 3437, 1829, 335]</td>\n",
       "      <td>1420728275</td>\n",
       "      <td>5612</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    session_id                 sequence          ts  user_id\n",
       "0          357              [793, 3489]  1421003874     4296\n",
       "9         9795                   [1719]  1420700645     5612\n",
       "10        9796       [1065, 3812, 4012]  1420713509     5612\n",
       "11        9797                [532, 93]  1420724661     5612\n",
       "12        9798  [6932, 3437, 1829, 335]  1420728275     5612"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2600 entries, 0 to 6747\n",
      "Data columns (total 4 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   session_id  2600 non-null   int64 \n",
      " 1   sequence    2600 non-null   object\n",
      " 2   ts          2600 non-null   int64 \n",
      " 3   user_id     2600 non-null   int64 \n",
      "dtypes: int64(3), object(1)\n",
      "memory usage: 101.6+ KB\n"
     ]
    }
   ],
   "source": [
    "train_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>ts</th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>359</td>\n",
       "      <td>[1762]</td>\n",
       "      <td>1421018535</td>\n",
       "      <td>4296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>394</td>\n",
       "      <td>[1256]</td>\n",
       "      <td>1421007470</td>\n",
       "      <td>30980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4127</td>\n",
       "      <td>[1948, 1364, 2060, 1115, 6488, 2060]</td>\n",
       "      <td>1421416896</td>\n",
       "      <td>28117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6400</td>\n",
       "      <td>[687, 1394]</td>\n",
       "      <td>1420807778</td>\n",
       "      <td>35247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6930</td>\n",
       "      <td>[630, 631, 632, 633, 634, 635]</td>\n",
       "      <td>1420538498</td>\n",
       "      <td>950</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   session_id                              sequence          ts  user_id\n",
       "1         359                                [1762]  1421018535     4296\n",
       "2         394                                [1256]  1421007470    30980\n",
       "3        4127  [1948, 1364, 2060, 1115, 6488, 2060]  1421416896    28117\n",
       "4        6400                           [687, 1394]  1420807778    35247\n",
       "5        6930        [630, 631, 632, 633, 634, 635]  1420538498      950"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 4165 entries, 1 to 6764\n",
      "Data columns (total 4 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   session_id  4165 non-null   int64 \n",
      " 1   sequence    4165 non-null   object\n",
      " 2   ts          4165 non-null   int64 \n",
      " 3   user_id     4165 non-null   int64 \n",
      "dtypes: int64(3), object(1)\n",
      "memory usage: 162.7+ KB\n"
     ]
    }
   ],
   "source": [
    "test_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-05 05:13:49\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.104+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "IPython: 5.5.0\n",
      "numpy  : 1.19.5\n",
      "pandas : 1.1.5\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!pip install -q watermark\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**END**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
