{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp datasets.weeplaces"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Weeplaces\n",
    "> Implementation of Weeplaces dataset.\n",
    "\n",
    "This dataset is collected from Weeplaces, a website that aims to visualize users’ check-in activities in location-based social networks (LBSN). It is now integrated with the APIs of other location-based social networking services, e.g., Facebook Places, Foursquare, and Gowalla. Users can login Weeplaces using their LBSN accounts and connect with their friends in the same LBSN who have also used this application. All the crawled data is originally generated in Foursquare. This dataset contains 7,658,368 check-ins generated by 15,799 users over 971,309 locations. In the data collection, we can’t get the original Foursquare IDs of the Weeplaces users. We can only get their check-in history, their friends who also use Weeplaces, and other additional information about the locations.\n",
    "\n",
    "You can download this dataset from **[here](https://drive.google.com/file/d/0BzpKyxX1dqTYYzRmUXRZMWloblU/view?usp=sharing)** (about 140 MB). Note that this dataset is released solely for research purpose."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy.sparse as sp\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "import torch\n",
    "from torch.utils import data\n",
    "\n",
    "import recohut\n",
    "from recohut.datasets.bases import common as base\n",
    "from recohut.utils.common_utils import download_url, extract_zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class WeeplacesDataset(base.Dataset, data.Dataset):\n",
    "    url = \"https://github.com/RecoHut-Datasets/weeplaces/raw/v2/data.zip\"\n",
    "\n",
    "    def __init__(self, root, datatype='train', is_group=False, n_items=None, negs_per_group=None, padding_idx=None, verbose=True):\n",
    "        super().__init__(root)\n",
    "        self.datatype = datatype\n",
    "        self.n_items = n_items\n",
    "        self.negs_per_group = negs_per_group\n",
    "        self.is_group = is_group\n",
    "        self.padding_idx = padding_idx\n",
    "        if is_group:\n",
    "            if datatype=='train':\n",
    "                self.user_data = self.load_user_data_train()\n",
    "                self.group_data, self.group_users = self.load_group_data_train()\n",
    "                self.group_inputs = [self.user_data[self.group_users[g]] for g in self.groups_list]\n",
    "            else:\n",
    "                self.eval_groups_list = []\n",
    "                self.user_data = self.load_user_data_tr_te(datatype)\n",
    "                self.eval_group_data, self.eval_group_users = self.load_group_data_tr_te(datatype)\n",
    "        else:\n",
    "            if datatype=='train':\n",
    "                self.train_data_ui = self.load_ui_train()\n",
    "                self.user_list = list(range(self.n_users))\n",
    "            else:\n",
    "                self.data_tr, self.data_te = self.load_ui_tr_te(datatype)\n",
    "    \n",
    "    def __len__(self):\n",
    "        if self.is_group:\n",
    "            if self.datatype=='train':\n",
    "                return len(self.groups_list)\n",
    "            return len(self.eval_groups_list)\n",
    "        return len(self.user_list)\n",
    "\n",
    "    def __train__(self, index):\n",
    "        \"\"\" load user_id, binary vector over items \"\"\"\n",
    "        user = self.user_list[index]\n",
    "        user_items = torch.from_numpy(self.train_data_ui[user, :].toarray()).squeeze()  # [I]\n",
    "        return torch.from_numpy(np.array([user], dtype=np.int32)), user_items\n",
    "\n",
    "    def __test__(self, index):\n",
    "        \"\"\" load user_id, fold-in items, held-out items \"\"\"\n",
    "        user = self.user_list[index]\n",
    "        fold_in, held_out = self.data_tr[user, :].toarray(), self.data_te[user, :].toarray()  # [I], [I]\n",
    "        return user, torch.from_numpy(fold_in).squeeze(), held_out.squeeze()  # user, fold-in items, fold-out items.\n",
    "\n",
    "    def __train_group__(self, index):\n",
    "        \"\"\" load group_id, padded group users, mask, group items, group member items, negative user items \"\"\"\n",
    "        group = self.groups_list[index]\n",
    "        user_ids = torch.from_numpy(np.array(self.group_users[group], np.int32))  # [G] group member ids\n",
    "        group_items = torch.from_numpy(self.group_data[group].toarray().squeeze())  # [I] items per group\n",
    "\n",
    "        corrupted_group = self.get_corrupted_users(group)  # [# negs]\n",
    "        corrupted_user_items = torch.from_numpy(self.user_data[corrupted_group].toarray().squeeze())  # [# negs, I]\n",
    "\n",
    "        # group mask to create fixed-size padded groups.\n",
    "        group_length = self.max_group_size - list(user_ids).count(self.padding_idx)\n",
    "        group_mask = torch.from_numpy(np.concatenate([np.zeros(group_length, dtype=np.float32), (-1) * np.inf *\n",
    "                                                      np.ones(self.max_group_size - group_length,\n",
    "                                                              dtype=np.float32)]))  # [G]\n",
    "\n",
    "        user_items = torch.from_numpy(self.group_inputs[group].toarray())  # [G, |I|] group member items\n",
    "\n",
    "        return torch.tensor([group]), user_ids, group_mask, group_items, user_items, corrupted_user_items\n",
    "\n",
    "    def __test_group__(self, index):\n",
    "        \"\"\" load group_id, padded group users, mask, group items, group member items \"\"\"\n",
    "        group = self.eval_groups_list[index]\n",
    "        user_ids = self.eval_group_users[group]  # [G]\n",
    "        length = self.max_gsize - list(user_ids).count(self.padding_idx)\n",
    "        mask = torch.from_numpy(np.concatenate([np.zeros(length, dtype=np.float32), (-1) * np.inf *\n",
    "                                                np.ones(self.max_gsize - length, dtype=np.float32)]))  # [G]\n",
    "        group_items = torch.from_numpy(self.eval_group_data[group].toarray().squeeze())  # [I]\n",
    "        user_items = torch.from_numpy(self.user_data[user_ids].toarray().squeeze())  # [G, I]\n",
    "\n",
    "        return torch.tensor([group]), torch.tensor(user_ids), mask, group_items, user_items\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        if self.is_group:\n",
    "            if self.datatype=='train':\n",
    "                return self.__train_group__(index)\n",
    "            return self.__test_group__(index)\n",
    "        else:\n",
    "            if self.datatype=='train':\n",
    "                return self.__train__(index)\n",
    "            return self.__test__(index)\n",
    "\n",
    "    @property\n",
    "    def raw_file_names(self) -> str:\n",
    "        return ['train_ui.csv',\n",
    "                'val_ui_te.csv',\n",
    "                'group_users.csv',\n",
    "                'data.zip',\n",
    "                'train_gi.csv',\n",
    "                'test_ui_tr.csv',\n",
    "                'val_ui_tr.csv',\n",
    "                'test_ui_te.csv',\n",
    "                'val_gi.csv',\n",
    "                'test_gi.csv']\n",
    "\n",
    "    def download(self):\n",
    "        path = download_url(self.url, self.raw_dir)\n",
    "        extract_zip(path, self.raw_dir)\n",
    "\n",
    "    @property\n",
    "    def processed_file_names(self) -> str:\n",
    "        pass\n",
    "\n",
    "    def process(self):\n",
    "        pass\n",
    "\n",
    "    def load_ui_train(self):\n",
    "        \"\"\" load training user-item interactions as a sparse matrix \"\"\"\n",
    "        path_ui = [p for p in self.raw_paths if \"train_ui\" in p][0]\n",
    "        df_ui = pd.read_csv(path_ui)\n",
    "        self.n_users, self.n_items = df_ui['user'].max() + 1, df_ui['item'].max() + 1\n",
    "        rows_ui, cols_ui = df_ui['user'], df_ui['item']\n",
    "        data_ui = sp.csr_matrix((np.ones_like(rows_ui), (rows_ui, cols_ui)), dtype='float32',\n",
    "                                shape=(self.n_users, self.n_items))  # [# train users, I] sparse matrix\n",
    "        print(\"# train users\", self.n_users, \"# items\", self.n_items)\n",
    "        return data_ui\n",
    "\n",
    "    def load_ui_tr_te(self, datatype='val'):\n",
    "        \"\"\" load user-item interactions of val/test user sets as two sparse matrices of fold-in and held-out items \"\"\"\n",
    "        ui_tr_path = [p for p in self.raw_paths if '{}_ui_tr.csv'.format(datatype) in p][0]\n",
    "\n",
    "        ui_te_path = [p for p in self.raw_paths if '{}_ui_te.csv'.format(datatype) in p][0]\n",
    "\n",
    "        ui_df_tr, ui_df_te = pd.read_csv(ui_tr_path), pd.read_csv(ui_te_path)\n",
    "\n",
    "        start_idx = min(ui_df_tr['user'].min(), ui_df_te['user'].min())\n",
    "        end_idx = max(ui_df_tr['user'].max(), ui_df_te['user'].max())\n",
    "\n",
    "        rows_tr, cols_tr = ui_df_tr['user'] - start_idx, ui_df_tr['item']\n",
    "        rows_te, cols_te = ui_df_te['user'] - start_idx, ui_df_te['item']\n",
    "        self.user_list = list(range(0, end_idx - start_idx + 1))\n",
    "\n",
    "        ui_data_tr = sp.csr_matrix((np.ones_like(rows_tr), (rows_tr, cols_tr)), dtype='float32',\n",
    "                                   shape=(end_idx - start_idx + 1, self.n_items))  # [# eval users, I] sparse matrix\n",
    "        ui_data_te = sp.csr_matrix((np.ones_like(rows_te), (rows_te, cols_te)), dtype='float32',\n",
    "                                   shape=(end_idx - start_idx + 1, self.n_items))  # [# eval users, I] sparse matrix\n",
    "        return ui_data_tr, ui_data_te\n",
    "\n",
    "    def get_corrupted_users(self, group):\n",
    "        \"\"\" negative user sampling per group (eta balances item-biased and random sampling) \"\"\"\n",
    "        eta = 0.5\n",
    "        p = np.ones(self.n_users + 1)\n",
    "        p[self.group_users[group]] = 0\n",
    "        p = normalize([p], norm='l1')[0]\n",
    "        item_biased = normalize(self.user_data[:, self.group_data[group].indices].sum(1).squeeze(), norm='l1')[0]\n",
    "        p = eta * item_biased + (1 - eta) * p\n",
    "        negative_users = torch.multinomial(torch.from_numpy(p), self.negs_per_group)\n",
    "        return negative_users\n",
    "\n",
    "    def load_user_data_train(self):\n",
    "        \"\"\" load user-item interactions of all users that appear in training groups, as a sparse matrix \"\"\"\n",
    "        df_ui = pd.DataFrame()\n",
    "        train_path_ui = [p for p in self.raw_paths if 'train_ui.csv' in p][0]\n",
    "        df_train_ui = pd.read_csv(train_path_ui)\n",
    "        df_ui = df_ui.append(df_train_ui)\n",
    "\n",
    "        # include users from the (fold-in item set) of validation and test sets of user-item data.\n",
    "        val_path_ui = [p for p in self.raw_paths if 'val_ui_tr.csv' in p][0]\n",
    "        df_val_ui = pd.read_csv(val_path_ui)\n",
    "        df_ui = df_ui.append(df_val_ui)\n",
    "\n",
    "        test_path_ui = [p for p in self.raw_paths if 'test_ui_tr.csv' in p][0]\n",
    "        df_test_ui = pd.read_csv(test_path_ui)\n",
    "        df_ui = df_ui.append(df_test_ui)\n",
    "\n",
    "        self.n_users = df_ui['user'].max() + 1\n",
    "        self.padding_idx = self.n_users  # padding idx for user when creating groups of fixed size.\n",
    "        assert self.n_items == df_ui['item'].max() + 1\n",
    "        rows_ui, cols_ui = df_ui['user'], df_ui['item']\n",
    "\n",
    "        data_ui = sp.csr_matrix((np.ones_like(rows_ui), (rows_ui, cols_ui)), dtype='float32',\n",
    "                                shape=(self.n_users + 1, self.n_items))  # [U, I] sparse matrix\n",
    "        return data_ui\n",
    "\n",
    "    def load_user_data_tr_te(self, datatype):\n",
    "        \"\"\" load all user-item interactions of users that occur in val/test groups, as a sparse matrix \"\"\"\n",
    "        df_ui = pd.DataFrame()\n",
    "        train_path_ui = [p for p in self.raw_paths if 'train_ui.csv' in p][0]\n",
    "        df_train_ui = pd.read_csv(train_path_ui)\n",
    "        df_ui = df_ui.append(df_train_ui)\n",
    "\n",
    "        val_path_ui = [p for p in self.raw_paths if 'val_ui_tr.csv' in p][0]\n",
    "        df_val_ui = pd.read_csv(val_path_ui)\n",
    "        df_ui = df_ui.append(df_val_ui)\n",
    "\n",
    "        if datatype == 'val' or datatype == 'test':\n",
    "            # include eval user set (tr) items (since they might occur in evaluation set)\n",
    "            test_path_ui = [p for p in self.raw_paths if 'test_ui_tr.csv' in p][0]\n",
    "            df_test_ui = pd.read_csv(test_path_ui)\n",
    "            df_ui = df_ui.append(df_test_ui)\n",
    "\n",
    "        n_users = df_ui['user'].max() + 1\n",
    "        assert self.n_items == df_ui['item'].max() + 1\n",
    "        rows_ui, cols_ui = df_ui['user'], df_ui['item']\n",
    "        data_ui = sp.csr_matrix((np.ones_like(rows_ui), (rows_ui, cols_ui)), dtype='float32',\n",
    "                                shape=(n_users + 1, self.n_items))  # [# users, I] sparse matrix\n",
    "        return data_ui\n",
    "\n",
    "    def load_group_data_train(self):\n",
    "        \"\"\" load training group-item interactions as a sparse matrix and user-group memberships \"\"\"\n",
    "        path_ug = [p for p in self.raw_paths if 'group_users.csv' in p][0]\n",
    "        path_gi = [p for p in self.raw_paths if 'train_gi.csv' in p][0]\n",
    "\n",
    "        df_gi = pd.read_csv(path_gi)  # load training group-item interactions.\n",
    "        start_idx, end_idx = df_gi['group'].min(), df_gi['group'].max()\n",
    "        self.n_groups = end_idx - start_idx + 1\n",
    "        rows_gi, cols_gi = df_gi['group'] - start_idx, df_gi['item']\n",
    "\n",
    "        data_gi = sp.csr_matrix((np.ones_like(rows_gi), (rows_gi, cols_gi)), dtype='float32',\n",
    "                                shape=(self.n_groups, self.n_items))  # [# groups,  I] sparse matrix.\n",
    "\n",
    "        df_ug = pd.read_csv(path_ug).astype(int)  # load user-group memberships.\n",
    "        df_ug_train = df_ug[df_ug.group.isin(range(start_idx, end_idx + 1))]\n",
    "        df_ug_train = df_ug_train.sort_values('group')  # sort in ascending order of group ids.\n",
    "        self.max_group_size = df_ug_train.groupby('group').size().max()  # max group size denoted by G\n",
    "\n",
    "        g_u_list_train = df_ug_train.groupby('group')['user'].apply(list).reset_index()\n",
    "        g_u_list_train['user'] = list(map(lambda x: x + [self.padding_idx] * (self.max_group_size - len(x)),\n",
    "                                          g_u_list_train.user))\n",
    "        data_gu = np.squeeze(np.array(g_u_list_train[['user']].values.tolist()))  # [# groups, G] with padding.\n",
    "        self.groups_list = list(range(0, end_idx - start_idx + 1))\n",
    "\n",
    "        assert len(df_ug_train['group'].unique()) == self.n_groups\n",
    "        print(\"# training groups: {}, # max train group size: {}\".format(self.n_groups, self.max_group_size))\n",
    "\n",
    "        return data_gi, data_gu\n",
    "\n",
    "    def load_group_data_tr_te(self, datatype):\n",
    "        \"\"\" load val/test group-item interactions as a sparse matrix and user-group memberships \"\"\"\n",
    "        path_ug = [p for p in self.raw_paths if 'group_users.csv' in p][0]\n",
    "        path_gi = [p for p in self.raw_paths if '{}_gi.csv'.format(datatype) in p][0]\n",
    "\n",
    "        df_gi = pd.read_csv(path_gi)  # load group-item interactions\n",
    "        start_idx, end_idx = df_gi['group'].min(), df_gi['group'].max()\n",
    "        self.n_groups = end_idx - start_idx + 1\n",
    "        rows_gi, cols_gi = df_gi['group'] - start_idx, df_gi['item']\n",
    "        data_gi = sp.csr_matrix((np.ones_like(rows_gi), (rows_gi, cols_gi)), dtype='float32',\n",
    "                                shape=(self.n_groups, self.n_items))  # [# eval groups, I] sparse matrix\n",
    "\n",
    "        df_ug = pd.read_csv(path_ug)  # load user-group memberships\n",
    "        df_ug_eval = df_ug[df_ug.group.isin(range(start_idx, end_idx + 1))]\n",
    "        df_ug_eval = df_ug_eval.sort_values('group')  # sort in ascending order of group ids\n",
    "        self.max_gsize = df_ug_eval.groupby('group').size().max()  # max group size denoted by G\n",
    "        g_u_list_eval = df_ug_eval.groupby('group')['user'].apply(list).reset_index()\n",
    "        g_u_list_eval['user'] = list(map(lambda x: x + [self.padding_idx] * (self.max_gsize - len(x)),\n",
    "                                         g_u_list_eval.user))\n",
    "        data_gu = np.squeeze(np.array(g_u_list_eval[['user']].values.tolist(), dtype=np.int32))  # [# groups, G]\n",
    "        self.eval_groups_list = list(range(0, end_idx - start_idx + 1))\n",
    "        return data_gi, data_gu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading https://github.com/RecoHut-Datasets/weeplaces/raw/v2/data.zip\n",
      "Extracting ./data/raw/data.zip\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# train users 6050 # items 25081\n",
      "# training groups: 15913, # max train group size: 22\n"
     ]
    }
   ],
   "source": [
    "root = './data'\n",
    "\n",
    "# Define train/val/test datasets on user interactions.\n",
    "train_dataset = WeeplacesDataset(root, is_group=False, datatype='train')  # train dataset for user-item interactions.\n",
    "n_users, n_items = train_dataset.n_users, train_dataset.n_items\n",
    "val_dataset = WeeplacesDataset(root, is_group=False, datatype='val', n_items=n_items)\n",
    "test_dataset = WeeplacesDataset(root, is_group=False, datatype='test', n_items=n_items)\n",
    "\n",
    "# Define train/val/test datasets on group and user interactions.\n",
    "train_group_dataset = WeeplacesDataset(root, is_group=True, datatype='train', negs_per_group=5, n_items=n_items)\n",
    "padding_idx = train_group_dataset.padding_idx\n",
    "val_group_dataset = WeeplacesDataset(root, is_group=True, datatype='val', n_items=n_items, padding_idx=padding_idx)\n",
    "test_group_dataset = WeeplacesDataset(root, is_group=True, datatype='test', n_items=n_items, padding_idx=padding_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34m./data\u001b[00m\n",
      "└── [9.1M]  \u001b[01;34mraw\u001b[00m\n",
      "    ├── [2.5M]  \u001b[01;31mdata.zip\u001b[00m\n",
      "    ├── [794K]  group_users.csv\n",
      "    ├── [156K]  test_gi.csv\n",
      "    ├── [433K]  test_ui_te.csv\n",
      "    ├── [635K]  test_ui_tr.csv\n",
      "    ├── [498K]  train_gi.csv\n",
      "    ├── [3.5M]  train_ui.csv\n",
      "    ├── [ 72K]  val_gi.csv\n",
      "    ├── [205K]  val_ui_te.csv\n",
      "    └── [300K]  val_ui_tr.csv\n",
      "\n",
      " 9.1M used in 1 directory, 10 files\n"
     ]
    }
   ],
   "source": [
    "!tree --du -h -C ./data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> References\n",
    "1. https://github.com/RecoHut-Stanzas/S168471/blob/main/reports/S168471_report.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-29 14:11:18\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.144+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "IPython: 5.5.0\n",
      "scipy  : 1.4.1\n",
      "recohut: 0.0.8\n",
      "torch  : 1.10.0+cu111\n",
      "numpy  : 1.19.5\n",
      "pandas : 1.1.5\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
