{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp datasets.tianchi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tianchi dataset\n",
    "> Tianchi dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import os\n",
    "import os.path as osp\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from recohut.datasets.bases.common import Dataset\n",
    "from recohut.utils.common_utils import download_url, extract_zip, makedirs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class TianchiDataset(Dataset):\n",
    "    data_id = '16XaYNZHn4Sb33sAI7J3N_QkScVNmvYGa'\n",
    "    \n",
    "    def __init__(self, root):\n",
    "        super().__init__(root)\n",
    "\n",
    "        self._process()\n",
    "    \n",
    "    @property\n",
    "    def raw_file_names(self) -> str:\n",
    "        return ['user.csv', 'item.csv', 'user_behavior.csv']\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        return 'tianchi.csv'\n",
    "\n",
    "    def download(self):\n",
    "        from google_drive_downloader import GoogleDriveDownloader as gdd\n",
    "        from shutil import move, rmtree\n",
    "\n",
    "        path = osp.join(self.raw_dir, 'tianchi_raw.zip')\n",
    "        gdd.download_file_from_google_drive(self.data_id, path)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "        for f in self.raw_file_names:\n",
    "            move(osp.join(self.raw_dir, 'ECommAI_EUIR_round2_train_20190816', f),\n",
    "                 osp.join(self.raw_dir, f))\n",
    "        rmtree(osp.join(self.raw_dir, 'ECommAI_EUIR_round2_train_20190816'))\n",
    "        os.unlink(path)\n",
    "\n",
    "    @staticmethod\n",
    "    def bucket_age(age):\n",
    "        if age < 30:\n",
    "            return 1\n",
    "        elif age < 40:\n",
    "            return 2\n",
    "        elif age < 50:\n",
    "            return 3\n",
    "        else:\n",
    "            return 4\n",
    "\n",
    "    def process(self):\n",
    "        # 1. loading the data into memory\n",
    "\n",
    "        user_feat = pd.read_csv(self.raw_paths[0], header=None,\n",
    "                                names=[\"user\", \"sex\", \"age\", \"pur_power\"])\n",
    "        item_feat = pd.read_csv(self.raw_paths[1], header=None,\n",
    "                                names=[\"item\", \"category\", \"shop\", \"brand\"])\n",
    "        behavior = pd.read_csv(self.raw_paths[2], header=None,\n",
    "                            names=[\"user\", \"item\", \"behavior\", \"time\"])\n",
    "        \n",
    "        # 2. sorting values chronologically and dropping duplicate records\n",
    "\n",
    "        behavior = behavior.sort_values(by=\"time\").reset_index(drop=True)\n",
    "        behavior = behavior.drop_duplicates(subset=[\"user\", \"item\", \"behavior\"])\n",
    "\n",
    "        # 3. Choosing 60K random users with short journey and 20K with long journey\n",
    "        user_counts = behavior.groupby(\"user\")[[\"user\"]].count().rename(\n",
    "            columns={\"user\": \"count_user\"}\n",
    "        ).sort_values(\"count_user\", ascending=False)\n",
    "\n",
    "        short_users = np.array(\n",
    "            user_counts[\n",
    "                (user_counts.count_user > 5) & (user_counts.count_user <= 50)\n",
    "            ].index\n",
    "        )\n",
    "        long_users = np.array(\n",
    "            user_counts[\n",
    "                (user_counts.count_user > 50) & (user_counts.count_user <= 200)\n",
    "            ].index\n",
    "        )\n",
    "        short_chosen_users = np.random.choice(short_users, 60000, replace=False)\n",
    "        long_chosen_users = np.random.choice(long_users, 20000, replace=False)\n",
    "        chosen_users = np.concatenate([short_chosen_users, long_chosen_users])\n",
    "\n",
    "        behavior = behavior[behavior.user.isin(chosen_users)]\n",
    "        print(f\"n_users: {behavior.user.nunique()}, \"\n",
    "            f\"n_items: {behavior.item.nunique()}, \"\n",
    "            f\"behavior length: {len(behavior)}\")\n",
    "\n",
    "        # 4. merge with all features, bucketizing the age and saving the processed data\n",
    "        behavior = behavior.merge(user_feat, on=\"user\")\n",
    "        behavior = behavior.merge(item_feat, on=\"item\")\n",
    "        behavior[\"age\"] = behavior[\"age\"].apply(self.bucket_age)\n",
    "        behavior = behavior.sort_values(by=\"time\").reset_index(drop=True)\n",
    "        behavior.to_csv(self.processed_paths[0], header=None, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_users: 80000, n_items: 1046825, behavior length: 3232761\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    }
   ],
   "source": [
    "ds = TianchiDataset(root='/content/tianchi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34m/content/tianchi\u001b[00m\n",
      "├── [160M]  \u001b[01;34mprocessed\u001b[00m\n",
      "│   └── [160M]  tianchi.csv\n",
      "└── [2.1G]  \u001b[01;34mraw\u001b[00m\n",
      "    ├── [114M]  item.csv\n",
      "    ├── [2.0G]  user_behavior.csv\n",
      "    └── [ 19M]  user.csv\n",
      "\n",
      " 2.3G used in 2 directories, 4 files\n"
     ]
    }
   ],
   "source": [
    "!tree --du -h -C /content/tianchi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-23 17:37:47\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.144+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "pandas : 1.1.5\n",
      "IPython: 5.5.0\n",
      "numpy  : 1.19.5\n",
      "sys    : 3.7.12 (default, Sep 10 2021, 00:21:48) \n",
      "[GCC 7.5.0]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
