{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp models.deepfm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DeepFM\n",
    "> A pytorch implementation of DeepFM.\n",
    "\n",
    "DeepFM consists of an FM component and a deep component which are integrated in a parallel structure. The FM component is the same as the 2-way factorization machines which is used to model the low-order feature interactions. The deep component is a multi-layered perceptron that is used to capture high-order feature interactions and nonlinearities. These two components share the same inputs/embeddings and their outputs are summed up as the final prediction. It is worth pointing out that the spirit of DeepFM resembles that of the Wide & Deep architecture which can capture both memorization and generalization. The advantages of DeepFM over the Wide & Deep model is that it reduces the effort of hand-crafted feature engineering by identifying feature combinations automatically.\n",
    "\n",
    "![https://github.com/RecoHut-Stanzas/S021355/raw/main/images/img12.png](https://github.com/RecoHut-Stanzas/S021355/raw/main/images/img12.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *\n",
    "from fastcore.nb_imports import *\n",
    "from fastcore.test import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## v1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> **References:-**\n",
    "- H Guo, et al. DeepFM: A Factorization-Machine based Neural Network for CTR Prediction, 2017.\n",
    "- https://github.com/rixwew/pytorch-fm/blob/master/torchfm/model/dfm.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import torch\n",
    "\n",
    "from recohut.models.layers.common import FeaturesEmbedding, FeaturesLinear, MultiLayerPerceptron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class FactorizationMachine(torch.nn.Module):\n",
    "\n",
    "    def __init__(self, reduce_sum=True):\n",
    "        super().__init__()\n",
    "        self.reduce_sum = reduce_sum\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        :param x: Float tensor of size ``(batch_size, num_fields, embed_dim)``\n",
    "        \"\"\"\n",
    "        square_of_sum = torch.sum(x, dim=1) ** 2\n",
    "        sum_of_square = torch.sum(x ** 2, dim=1)\n",
    "        ix = square_of_sum - sum_of_square\n",
    "        if self.reduce_sum:\n",
    "            ix = torch.sum(ix, dim=1, keepdim=True)\n",
    "        return 0.5 * ix\n",
    "\n",
    "class DeepFM(torch.nn.Module):\n",
    "    \"\"\"\n",
    "    A pytorch implementation of DeepFM.\n",
    "    Reference:\n",
    "        H Guo, et al. DeepFM: A Factorization-Machine based Neural Network for CTR Prediction, 2017.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, field_dims, embed_dim, mlp_dims, dropout):\n",
    "        super().__init__()\n",
    "        self.linear = FeaturesLinear(field_dims)\n",
    "        self.fm = FactorizationMachine(reduce_sum=True)\n",
    "        self.embedding = FeaturesEmbedding(field_dims, embed_dim)\n",
    "        self.embed_output_dim = len(field_dims) * embed_dim\n",
    "        self.mlp = MultiLayerPerceptron(self.embed_output_dim, mlp_dims, dropout)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        :param x: Long tensor of size ``(batch_size, num_fields)``\n",
    "        \"\"\"\n",
    "        embed_x = self.embedding(x)\n",
    "        x = self.linear(x) + self.fm(embed_x) + self.mlp(embed_x.view(-1, self.embed_output_dim))\n",
    "        return torch.sigmoid(x.squeeze(1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## v2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> **References:-**\n",
    "- https://github.com/huangjunheng/recommendation_model/tree/master/deepFM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from collections import namedtuple, defaultdict\n",
    "\n",
    "import torch\n",
    "from torch import nn as nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#exporti\n",
    "class FM(nn.Module):\n",
    "    def __init__(self, p, k):\n",
    "        super(FM, self).__init__()\n",
    "        self.p = p\n",
    "        self.k = k\n",
    "        self.linear = nn.Linear(self.p, 1, bias=True)\n",
    "        self.v = nn.Parameter(torch.Tensor(self.p, self.k), requires_grad=True)\n",
    "        self.v.data.uniform_(-0.01, 0.01)\n",
    "        self.drop = nn.Dropout(0.3)\n",
    "\n",
    "    def forward(self, x):\n",
    "        linear_part = self.linear(x)\n",
    "        inter_part1 = torch.pow(torch.mm(x, self.v), 2)\n",
    "        inter_part2 = torch.mm(torch.pow(x, 2), torch.pow(self.v, 2))\n",
    "        pair_interactions = torch.sum(torch.sub(inter_part1, inter_part2), dim=1)\n",
    "        self.drop(pair_interactions)\n",
    "        output = linear_part.transpose(1, 0) + 0.5 * pair_interactions\n",
    "        return output.view(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class DeepFMv2(nn.Module):\n",
    "    def __init__(self, feat_sizes, sparse_feature_columns, dense_feature_columns,dnn_hidden_units=[400, 400,400], dnn_dropout=0.0, ebedding_size=4,\n",
    "                 l2_reg_linear=0.00001, l2_reg_embedding=0.00001, l2_reg_dnn=0, init_std=0.0001, seed=1024,\n",
    "                 device='cpu'):\n",
    "        super(DeepFMv2, self).__init__()\n",
    "        self.feat_sizes = feat_sizes\n",
    "        self.device = device\n",
    "        self.dense_feature_columns = dense_feature_columns\n",
    "        self.sparse_feature_columns = sparse_feature_columns\n",
    "        self.embedding_size = ebedding_size\n",
    "        self.l2_reg_linear = l2_reg_linear\n",
    "\n",
    "        self.bias = nn.Parameter(torch.zeros((1, )))\n",
    "        self.init_std = init_std\n",
    "        self.dnn_dropout = dnn_dropout\n",
    "\n",
    "        self.embedding_dic = nn.ModuleDict({feat:nn.Embedding(self.feat_sizes[feat], self.embedding_size, sparse=False)\n",
    "                                            for feat in self.sparse_feature_columns})\n",
    "        for tensor in self.embedding_dic.values():\n",
    "            nn.init.normal_(tensor.weight, mean=0, std=self.init_std)\n",
    "        self.embedding_dic.to(self.device)\n",
    "\n",
    "        self.feature_index = defaultdict(int)\n",
    "        start = 0\n",
    "        for feat in self.feat_sizes:\n",
    "            if feat in self.feature_index:\n",
    "                continue\n",
    "            self.feature_index[feat] = start\n",
    "            start += 1\n",
    "\n",
    "        self.input_size = self.embedding_size * len(self.sparse_feature_columns)+len(self.dense_feature_columns)\n",
    "        # fm\n",
    "        self.fm = FM(self.input_size, 10)\n",
    "\n",
    "        # DNN\n",
    "        self.dropout = nn.Dropout(self.dnn_dropout)\n",
    "        self.hidden_units = [self.input_size] + dnn_hidden_units\n",
    "        self.Linears = nn.ModuleList([nn.Linear(self.hidden_units[i], self.hidden_units[i+1]) for i in range(len(self.hidden_units)-1)])\n",
    "        self.relus = nn.ModuleList([nn.ReLU() for i in range(len(self.hidden_units)-1)])\n",
    "        for name, tensor in self.Linears.named_parameters():\n",
    "            if 'weight' in name:\n",
    "                nn.init.normal_(tensor, mean=0, std=self.init_std)\n",
    "        self.dnn_outlayer = nn.Linear(dnn_hidden_units[-1], 1, bias=False).to(self.device)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        # x shape 1024*39\n",
    "\n",
    "        sparse_embedding = [self.embedding_dic[feat](x[:, self.feature_index[feat]].long()) for feat in self.sparse_feature_columns]\n",
    "        sparse_embedding = torch.cat(sparse_embedding, dim=-1)\n",
    "        # print(sparse_embedding.shape)  # batch * 208\n",
    "\n",
    "        dense_value = [x[:, self.feature_index[feat]] for feat in\n",
    "                            self.dense_feature_columns]\n",
    "\n",
    "        dense_value = torch.cat(dense_value, dim=0)\n",
    "        dense_value = torch.reshape(dense_value, (len(self.dense_feature_columns), -1))\n",
    "        dense_value = dense_value.T\n",
    "        # print(dense_value.shape) # batch * 13\n",
    "\n",
    "        input_x = torch.cat((dense_value, sparse_embedding), dim=1)\n",
    "        # print(input_x.shape) # batch * 221\n",
    "\n",
    "        fm_logit = self.fm(input_x)\n",
    "\n",
    "        for i in range(len(self.Linears)):\n",
    "            fc = self.Linears[i](input_x)\n",
    "            fc = self.relus[i](fc)\n",
    "            fc = self.dropout(fc)\n",
    "            input_x = fc\n",
    "        dnn_logit = self.dnn_outlayer(input_x)\n",
    "\n",
    "        y_pre = torch.sigmoid(fm_logit+dnn_logit+self.bias)\n",
    "        return y_pre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch/epoches: 0/10, train loss: 0.570, test auc: 0.684\n",
      "epoch/epoches: 1/10, train loss: 0.534, test auc: 0.714\n",
      "epoch/epoches: 2/10, train loss: 0.511, test auc: 0.725\n",
      "epoch/epoches: 3/10, train loss: 0.486, test auc: 0.732\n",
      "epoch/epoches: 4/10, train loss: 0.459, test auc: 0.738\n",
      "epoch/epoches: 5/10, train loss: 0.431, test auc: 0.743\n",
      "epoch/epoches: 6/10, train loss: 0.401, test auc: 0.743\n",
      "epoch/epoches: 7/10, train loss: 0.368, test auc: 0.740\n",
      "epoch/epoches: 8/10, train loss: 0.337, test auc: 0.735\n",
      "epoch/epoches: 9/10, train loss: 0.312, test auc: 0.729\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import DataLoader\n",
    "from sklearn.metrics import log_loss, roc_auc_score\n",
    "\n",
    "from model import DeepFMv2 as DeepFM\n",
    "from recohut.datasets.criteo import CriteoSampleDataset\n",
    "\n",
    "\n",
    "def get_auc(loader, model):\n",
    "    pred, target = [], []\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for x, y in loader:\n",
    "            x, y = x.to(device).float(), y.to(device).float()\n",
    "            y_hat = model(x)\n",
    "            pred += list(y_hat.cpu().numpy())\n",
    "            target += list(y.cpu().numpy())\n",
    "    auc = roc_auc_score(target, pred)\n",
    "    return auc\n",
    "\n",
    "\n",
    "root = '/content/data'\n",
    "batch_size = 1024\n",
    "epochs = 10\n",
    "seed = 1024\n",
    "lr = 0.00005\n",
    "wd = 0.00001\n",
    "device = 'cpu'\n",
    "\n",
    "ds = CriteoSampleDataset(root=root)\n",
    "train_tensor_data, test_tensor_data = ds.load()\n",
    "train_loader = DataLoader(train_tensor_data, shuffle=True, batch_size=batch_size)\n",
    "test_loader = DataLoader(test_tensor_data, batch_size=batch_size)\n",
    "\n",
    "sparse_features = ['C' + str(i) for i in range(1, 27)]\n",
    "dense_features = ['I' + str(i) for i in range(1, 14)]\n",
    "\n",
    "# model = NFM(ds.feat_sizes, embedding_size, ds.linear_feature_columns, ds.dnn_feature_columns).to(device)\n",
    "model = DeepFMv2(ds.feat_sizes, sparse_feature_columns=sparse_features, dense_feature_columns=dense_features,\n",
    "                dnn_hidden_units=[1000, 500, 250], dnn_dropout=0.9, ebedding_size=16,\n",
    "                l2_reg_linear=1e-3, device=device)\n",
    "loss_func = nn.BCELoss(reduction='mean')\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=wd)\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    total_loss_epoch = 0.0\n",
    "    total_tmp = 0\n",
    "    model.train()\n",
    "    for index, (x, y) in enumerate(train_loader):\n",
    "        x, y = x.to(device).float(), y.to(device).float()\n",
    "        y_hat = model(x)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss = loss_func(y_hat, y)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        total_loss_epoch += loss.item()\n",
    "        total_tmp += 1\n",
    "    auc = get_auc(test_loader, model)\n",
    "    print('epoch/epoches: {}/{}, train loss: {:.3f}, test auc: {:.3f}'.format(epoch, epochs, total_loss_epoch / total_tmp, auc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class DeepFM(PointModel):\n",
    "\n",
    "#     def __init__(self, n_users, n_items, embedding_dim, batch_norm=True, dropout=0.1, num_layers=3, act_function='relu'):\n",
    "#         \"\"\"\n",
    "#         Args:\n",
    "#             n_users : int, the number of users\n",
    "#             n_items : int, the number of items\n",
    "#             embedding_dim : int, the number of latent factoact_function : str, activation function for hidden layer\n",
    "#             num_layers : int, number of hidden layers\n",
    "#             batch_norm : bool, whether to normalize a batch of data\n",
    "#             dropout : float, dropout rate\n",
    "#         \"\"\"\n",
    "#         super().__init__()\n",
    "\n",
    "#         self.num_layers = num_layers\n",
    "\n",
    "#         self.user_embedding = nn.Embedding(\n",
    "#             num_embeddings=n_users, embedding_dim=embedding_dim\n",
    "#         )\n",
    "#         self.item_embedding = nn.Embedding(\n",
    "#             num_embeddings=n_items, embedding_dim=embedding_dim\n",
    "#         )\n",
    "#         self.user_bias = nn.Embedding(n_users, 1)\n",
    "#         self.item_bias = nn.Embedding(n_items, 1)\n",
    "#         self.bias_ = nn.Parameter(torch.tensor([0.0]))\n",
    "\n",
    "#         fm_modules = []\n",
    "#         if batch_norm:\n",
    "#             fm_modules.append(nn.BatchNorm1d(embedding_dim))\n",
    "#         fm_modules.append(nn.Dropout(dropout))\n",
    "#         self.fm_layers = nn.Sequential(*fm_modules)\n",
    "\n",
    "#         deep_modules = []\n",
    "#         in_dim = embedding_dim * 2   # user & item\n",
    "#         for _ in range(num_layers):  # _ is dim if layers is list\n",
    "#             out_dim = in_dim\n",
    "#             deep_modules.append(nn.Linear(in_dim, out_dim))\n",
    "#             in_dim = out_dim\n",
    "#             if batch_norm:\n",
    "#                 deep_modules.append(nn.BatchNorm1d(out_dim))\n",
    "#             if act_function == 'relu':\n",
    "#                 deep_modules.append(nn.ReLU())\n",
    "#             elif act_function == 'sigmoid':\n",
    "#                 deep_modules.append(nn.Sigmoid())\n",
    "#             elif act_function == 'tanh':\n",
    "#                 deep_modules.append(nn.Tanh())\n",
    "#             deep_modules.append(nn.Dropout(dropout))\n",
    "\n",
    "#         self.deep_layers = nn.Sequential(*deep_modules)\n",
    "#         self.deep_out = nn.Linear(in_dim, 1, bias=False)\n",
    "\n",
    "#         self._init_weights()\n",
    "\n",
    "#     def _init_weights(self):\n",
    "#         nn.init.normal_(self.item_embedding.weight, std=0.01)\n",
    "#         nn.init.normal_(self.user_embedding.weight, std=0.01)\n",
    "#         nn.init.constant_(self.user_bias.weight, 0.0)\n",
    "#         nn.init.constant_(self.item_bias.weight, 0.0)\n",
    "\n",
    "#         # for deep layers\n",
    "#         for m in self.deep_layers:\n",
    "#             if isinstance(m, nn.Linear):\n",
    "#                 nn.init.xavier_normal_(m.weight)\n",
    "#         nn.init.xavier_normal_(self.deep_out.weight)\n",
    "\n",
    "#     def forward(self, users, items):\n",
    "#         embed_user = self.user_embedding(users)\n",
    "#         embed_item = self.item_embedding(items)\n",
    "\n",
    "#         fm = embed_user * embed_item\n",
    "#         fm = self.fm_layers(fm)\n",
    "#         y_fm = fm.sum(dim=-1)\n",
    "\n",
    "#         y_fm = y_fm + self.user_bias(users) + self.item_bias(items) + self.bias_\n",
    "\n",
    "#         if self.num_layers:\n",
    "#             fm = self.deep_layers(fm)\n",
    "\n",
    "#         y_deep = torch.cat((embed_user, embed_item), dim=-1)\n",
    "#         y_deep = self.deep_layers(y_deep)\n",
    "\n",
    "#         # since BCELoss will automatically transfer pred with sigmoid\n",
    "#         # there is no need to use extra nn.Sigmoid(pred)\n",
    "#         pred = y_fm + y_deep\n",
    "\n",
    "#         return pred.view(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2022-01-08 05:31:12\n",
      "\n",
      "recohut: 0.0.9\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.144+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "pandas    : 1.1.5\n",
      "numpy     : 1.19.5\n",
      "IPython   : 5.5.0\n",
      "PIL       : 7.1.2\n",
      "matplotlib: 3.2.2\n",
      "torch     : 1.10.0+cu111\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d -p recohut"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
