{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp evaluation.metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metrics\n",
    "> Metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *\n",
    "from fastcore.nb_imports import *\n",
    "from fastcore.test import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import numpy as np\n",
    "import torch\n",
    "from sklearn.metrics import precision_score, recall_score, ndcg_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def calculate_precision_recall(X, y_true, y_pred, N, threshold):\n",
    "    \"\"\"Calculate the precision and recall scores.\n",
    "\n",
    "    Args:\n",
    "        X\n",
    "        y_true\n",
    "        y_pred\n",
    "        N\n",
    "        threshold\n",
    "    \n",
    "    Returns:\n",
    "        precision_score (float)\n",
    "        recall_score (float)\n",
    "    \"\"\"\n",
    "    precision = 0\n",
    "    recall = 0\n",
    "    count = 0\n",
    "    \n",
    "    rec_true = np.array([1 if rating >= threshold else 0 for rating in y_true])\n",
    "    rec_pred = np.zeros(y_pred.size)\n",
    "    \n",
    "    for user_id in np.unique(X[:,0]):\n",
    "        indices = np.where(X[:,0] == user_id)[0]\n",
    "        \n",
    "        rec_true = np.array([1 if y_true[i] >= threshold else 0 for i in indices])\n",
    "\n",
    "        if (np.count_nonzero(rec_true) > 0): # ignore test users without relevant ratings\n",
    "        \n",
    "            user_pred = np.array([y_pred[i] for i in indices])\n",
    "            rec_pred = np.zeros(indices.size)\n",
    "\n",
    "            for pos in np.argsort(user_pred)[-N:]:\n",
    "                if user_pred[pos] >= threshold:\n",
    "                    rec_pred[pos] = 1\n",
    "            \n",
    "            precision += precision_score(rec_true, rec_pred, zero_division=0)\n",
    "            recall += recall_score(rec_true, rec_pred)\n",
    "            count += 1\n",
    "        \n",
    "    return precision/count, recall/count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def calculate_ndcg(X, y_true, y_pred, N):\n",
    "    \"\"\"Calculate the NDCG score.\n",
    "\n",
    "    Args:\n",
    "        X\n",
    "        y_true\n",
    "        y_pred\n",
    "        N\n",
    "    \n",
    "    Returns:\n",
    "        ndcg_score (float)\n",
    "    \"\"\"\n",
    "    ndcg = 0\n",
    "    count = 0\n",
    "    \n",
    "    for user_id in np.unique(X[:,0]):\n",
    "        indices = np.where(X[:,0] == user_id)[0]\n",
    "        \n",
    "        user_true = np.array([y_true[i] for i in indices])\n",
    "        user_pred = np.array([y_pred[i] for i in indices])  \n",
    "        \n",
    "        user_true = np.expand_dims(user_true, axis=0)\n",
    "        user_pred = np.expand_dims(user_pred, axis=0)\n",
    "                \n",
    "        if user_true.size > 1:\n",
    "            ndcg += ndcg_score(user_true, user_pred, k=N, ignore_ties=False)\n",
    "            count += 1\n",
    "    \n",
    "    return ndcg / count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def recall(scores, labels, k):\n",
    "    scores = scores.cpu()\n",
    "    labels = labels.cpu()\n",
    "    rank = (-scores).argsort(dim=1)\n",
    "    cut = rank[:, :k]\n",
    "    hit = labels.gather(1, cut)\n",
    "    return (hit.sum(1).float() / torch.min(torch.Tensor([k]).to(hit.device), labels.sum(1).float())).mean().cpu().item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def ndcg(scores, labels, k):\n",
    "    scores = scores.cpu()\n",
    "    labels = labels.cpu()\n",
    "    rank = (-scores).argsort(dim=1)\n",
    "    cut = rank[:, :k]\n",
    "    hits = labels.gather(1, cut)\n",
    "    position = torch.arange(2, 2+k)\n",
    "    weights = 1 / torch.log2(position.float())\n",
    "    dcg = (hits.float() * weights).sum(1)\n",
    "    idcg = torch.Tensor([weights[:min(int(n), k)].sum() for n in labels.sum(1)])\n",
    "    ndcg = dcg / idcg\n",
    "    return ndcg.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def recalls_and_ndcgs_for_ks(scores, labels, ks):\n",
    "    metrics = {}\n",
    "\n",
    "    scores = scores\n",
    "    labels = labels\n",
    "    answer_count = labels.sum(1)\n",
    "\n",
    "    labels_float = labels.float()\n",
    "    rank = (-scores).argsort(dim=1)\n",
    "    cut = rank\n",
    "    for k in sorted(ks, reverse=True):\n",
    "       cut = cut[:, :k]\n",
    "       hits = labels_float.gather(1, cut)\n",
    "       metrics['Recall@%d' % k] = \\\n",
    "           (hits.sum(1) / torch.min(torch.Tensor([k]).to(labels.device), labels.sum(1).float())).mean().cpu().item()\n",
    "\n",
    "       position = torch.arange(2, 2+k)\n",
    "       weights = 1 / torch.log2(position.float())\n",
    "       dcg = (hits * weights.to(hits.device)).sum(1)\n",
    "       idcg = torch.Tensor([weights[:min(int(n), k)].sum() for n in answer_count]).to(dcg.device)\n",
    "       ndcg = (dcg / idcg).mean()\n",
    "       metrics['NDCG@%d' % k] = ndcg.cpu().item()\n",
    "\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-58b30018-5fc4-4258-b49c-935742dcba6d\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>deepmf</th>\n",
       "      <th>item</th>\n",
       "      <th>ncf</th>\n",
       "      <th>user</th>\n",
       "      <th>vdeepmf</th>\n",
       "      <th>vncf</th>\n",
       "      <th>y_test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.726801</td>\n",
       "      <td>496</td>\n",
       "      <td>2.854854</td>\n",
       "      <td>68</td>\n",
       "      <td>2.472237</td>\n",
       "      <td>2.620151</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.349192</td>\n",
       "      <td>473</td>\n",
       "      <td>3.002311</td>\n",
       "      <td>633</td>\n",
       "      <td>2.847424</td>\n",
       "      <td>2.690057</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.726862</td>\n",
       "      <td>329</td>\n",
       "      <td>3.605561</td>\n",
       "      <td>1405</td>\n",
       "      <td>3.810497</td>\n",
       "      <td>3.466036</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.467009</td>\n",
       "      <td>328</td>\n",
       "      <td>3.389759</td>\n",
       "      <td>1240</td>\n",
       "      <td>3.639901</td>\n",
       "      <td>3.205043</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.140076</td>\n",
       "      <td>54</td>\n",
       "      <td>3.194410</td>\n",
       "      <td>841</td>\n",
       "      <td>2.887761</td>\n",
       "      <td>2.848487</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-58b30018-5fc4-4258-b49c-935742dcba6d')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-58b30018-5fc4-4258-b49c-935742dcba6d button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-58b30018-5fc4-4258-b49c-935742dcba6d');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "     deepmf  item       ncf  user   vdeepmf      vncf  y_test\n",
       "0  2.726801   496  2.854854    68  2.472237  2.620151     3.0\n",
       "1  3.349192   473  3.002311   633  2.847424  2.690057     3.5\n",
       "2  3.726862   329  3.605561  1405  3.810497  3.466036     4.0\n",
       "3  3.467009   328  3.389759  1240  3.639901  3.205043     0.5\n",
       "4  3.140076    54  3.194410   841  2.887761  2.848487     3.0"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "sample_data = pd.DataFrame.from_records(\n",
    "    [{'deepmf': 2.7268011569976807,\n",
    "  'item': 496,\n",
    "  'ncf': 2.854853630065918,\n",
    "  'user': 68,\n",
    "  'vdeepmf': 2.4722371101379395,\n",
    "  'vncf': 2.620150566101074,\n",
    "  'y_test': 3.0},\n",
    " {'deepmf': 3.3491923809051514,\n",
    "  'item': 473,\n",
    "  'ncf': 3.0023105144500732,\n",
    "  'user': 633,\n",
    "  'vdeepmf': 2.847424030303955,\n",
    "  'vncf': 2.6900570392608643,\n",
    "  'y_test': 3.5},\n",
    " {'deepmf': 3.7268624305725098,\n",
    "  'item': 329,\n",
    "  'ncf': 3.605560779571533,\n",
    "  'user': 1405,\n",
    "  'vdeepmf': 3.810497283935547,\n",
    "  'vncf': 3.466035842895508,\n",
    "  'y_test': 4.0},\n",
    " {'deepmf': 3.4670088291168213,\n",
    "  'item': 328,\n",
    "  'ncf': 3.389759063720703,\n",
    "  'user': 1240,\n",
    "  'vdeepmf': 3.6399013996124268,\n",
    "  'vncf': 3.205043315887451,\n",
    "  'y_test': 0.5},\n",
    " {'deepmf': 3.140076160430908,\n",
    "  'item': 54,\n",
    "  'ncf': 3.1944096088409424,\n",
    "  'user': 841,\n",
    "  'vdeepmf': 2.887760877609253,\n",
    "  'vncf': 2.848487138748169,\n",
    "  'y_test': 3.0}]\n",
    ")\n",
    "sample_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test Precision Recall\n",
    "num_recommendations = 2\n",
    "method = 'ncf'\n",
    "like_threshold = 3\n",
    "\n",
    "ids = sample_data[['user', 'item']].to_numpy()\n",
    "y_true = sample_data['y_test'].to_numpy()\n",
    "y_pred = sample_data[method].to_numpy()\n",
    "precision, recall = calculate_precision_recall(ids, y_true, y_pred, num_recommendations, like_threshold)\n",
    "\n",
    "test_eq(precision, 0.75)\n",
    "test_eq(recall, 0.75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample data of 5 users/items\n",
    "sample_data_2 = {\n",
    "    'y_true':np.array([1,2,3,4,5]),\n",
    "    'y_pred':np.array([1,3,3,2,4]),\n",
    "    'ids':np.array([[1,2],[1,3],[2,4],[2,5],[3,2]]),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test NDCG\n",
    "num_recommendations = 2\n",
    "ids = sample_data_2['ids']\n",
    "y_true = sample_data_2['y_true']\n",
    "y_pred = sample_data_2['y_pred']\n",
    "ndcg = calculate_ndcg(ids, y_true, y_pred, num_recommendations)\n",
    "test_close(ndcg, 0.9686, eps=1e-4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NDCG\n",
    "> Normalized Discounted Cumulative Gain.\n",
    "\n",
    "NDCG is a metric that evaluates how well the recommender performs in recommending ranked items to users. Therefore both hit of relevant items and correctness in ranking of these items matter to the NDCG evaluation. The total NDCG score is normalized by the total number of users.\n",
    "\n",
    "nDCG has three parts. First is ‘CG’ which stands for Cumulative Gains. It deals with the fact that most relevant items are more useful than somewhat relevant items that are more useful than irrelevant items. It sums the items based on its relevancy, hence, the term cumulative.\n",
    "\n",
    "But CG doesn’t account for the position of the items on the list. And hence, changing the item's position won’t change the CG. This is where the second part of nDCG comes in to play i.e. ‘D’. Discounted Cumulative Gain, DCG for short, penalized the items that appear lower in the list. A relevant item appearing at the end of the list is a result of a bad recommender system and hence that item should be discounted to indicate the bad performance of the model.\n",
    "\n",
    "nDCG normalized the DCG values of the different number of the items lists. To do so we sort the item list by relevancy and calculate the DCG for that list. This will be the perfect DCG score as items are sorted by their relevancy score.\n",
    "\n",
    "<img src='https://camo.githubusercontent.com/b8491f0e8346610f9ec4dd04753bfa14095e21cafd3251f3221b0500867f7648/68747470733a2f2f6769746875622e636f6d2f7265636f6875742f7265636f2d7374617469632f7261772f6d61737465722f6d656469612f696d616765732f6d6574726963732f6e6463672e706e67'>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def ndcg_at_k(y_true_list, y_reco_list, users=None, k=10, next_item=False,\n",
    "              all_item=False):\n",
    "    if next_item:\n",
    "        ndcg_all = []\n",
    "        y_true_list = y_true_list.tolist()\n",
    "        y_reco_list = y_reco_list.tolist()\n",
    "        for y_true, y_reco in zip(y_true_list, y_reco_list):\n",
    "            if y_true in y_reco:\n",
    "                index = y_reco.index(y_true)\n",
    "                ndcg = 1. / np.log2(index + 2)\n",
    "            else:\n",
    "                ndcg = 0.\n",
    "            ndcg_all.append(ndcg)\n",
    "        return np.mean(ndcg_all)\n",
    "\n",
    "    elif all_item:\n",
    "        ndcg_all = []\n",
    "        users = users.tolist()\n",
    "        y_reco_list = y_reco_list.tolist()\n",
    "        for i in range(len(y_reco_list)):\n",
    "            y_true = y_true_list[users[i]]\n",
    "            y_reco = y_reco_list[i]\n",
    "            ndcg_all.append(ndcg_one(y_true, y_reco, k))\n",
    "        return np.mean(ndcg_all)\n",
    "\n",
    "    else:\n",
    "        ndcg_all = list()\n",
    "        for u in users:\n",
    "            y_true = y_true_list[u]\n",
    "            y_reco = y_reco_list[u]\n",
    "            ndcg_all.append(ndcg_one(y_true, y_reco, k))\n",
    "        return np.mean(ndcg_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def ndcg_one(y_true, y_reco, k):\n",
    "    rank_list = np.zeros(k)\n",
    "    common_items, indices_in_true, indices_in_reco = np.intersect1d(\n",
    "        y_true, y_reco, assume_unique=False, return_indices=True)\n",
    "\n",
    "    if common_items.size > 0:\n",
    "        rank_list[indices_in_reco] = 1\n",
    "        ideal_list = np.sort(rank_list)[::-1]\n",
    "        #  np.sum(rank_list / np.log2(2, k+2))\n",
    "        dcg = np.sum(rank_list / np.log2(np.arange(2, k + 2)))\n",
    "        idcg = np.sum(ideal_list / np.log2(np.arange(2, k + 2)))\n",
    "        ndcg = dcg / idcg\n",
    "    else:\n",
    "        ndcg = 0.\n",
    "    return ndcg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def dcg_at_k(r, k, method=0):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "        r: Relevance scores (list or numpy) in rank order\n",
    "            (first element is the first item)\n",
    "        k: Number of results to consider\n",
    "        method: If 0 then weights are [1.0, 1.0, 0.6309, 0.5, 0.4307, ...]\n",
    "                If 1 then weights are [1.0, 0.6309, 0.5, 0.4307, ...]\n",
    "    Returns:\n",
    "        Discounted cumulative gain\n",
    "    \"\"\"\n",
    "    r = np.asfarray(r)[:k]\n",
    "    if r.size:\n",
    "        if method == 0:\n",
    "            return r[0] + np.sum(r[1:] / np.log2(np.arange(2, r.size + 1)))\n",
    "        elif method == 1:\n",
    "            return np.sum(r / np.log2(np.arange(2, r.size + 2)))\n",
    "        else:\n",
    "            raise ValueError('method must be 0 or 1.')\n",
    "    return 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = dcg_at_k([3, 2, 3, 0, 1, 2], 6)\n",
    "test_eq(np.round(result,4), 8.0972)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def ndcg_at_k_v2(r, k, method=0):\n",
    "    dcg_max = dcg_at_k(sorted(r, reverse=True), k, method)\n",
    "    if not dcg_max:\n",
    "        return 0.\n",
    "    return dcg_at_k(r, k, method) / dcg_max"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = ndcg_at_k_v2([3, 2, 3, 0, 1, 2], 6, method=1)\n",
    "test_eq(np.round(result,4), 0.9608)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install -q torchmetrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from abc import abstractmethod\n",
    "from typing import Dict, Optional, Union\n",
    "\n",
    "import torch\n",
    "import torchmetrics as tm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "FLOAT_TYPES = [\n",
    "    torch.float,\n",
    "    torch.double,\n",
    "    torch.half,\n",
    "    torch.float32,\n",
    "    torch.float64,\n",
    "    torch.float16,\n",
    "    torch.bfloat16,\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def _check_inputs(ks, scores, labels):\n",
    "    if len(ks.shape) > 1:\n",
    "        raise ValueError(\"ks should be a 1-dimensional tensor\")\n",
    "\n",
    "    if len(scores.shape) != 2:\n",
    "        raise ValueError(\"scores must be a 2-dimensional tensor\")\n",
    "\n",
    "    if len(labels.shape) != 2:\n",
    "        raise ValueError(\"labels must be a 2-dimensional tensor\")\n",
    "\n",
    "    if scores.shape != labels.shape:\n",
    "        raise ValueError(\"scores and labels must be the same shape\")\n",
    "\n",
    "    if ks.device != scores.device:\n",
    "        ks = ks.to(device=scores.device)\n",
    "\n",
    "    if labels.device != scores.device:\n",
    "        labels = labels.to(device=scores.device)\n",
    "\n",
    "    if scores.dtype not in FLOAT_TYPES:\n",
    "        scores = scores.to(dtype=torch.float)\n",
    "\n",
    "    if labels.dtype not in FLOAT_TYPES:\n",
    "        labels = labels.to(dtype=torch.float)\n",
    "\n",
    "    return (\n",
    "        ks,\n",
    "        scores,\n",
    "        labels,\n",
    "    )\n",
    "\n",
    "\n",
    "def _extract_topk(ks, scores, labels):\n",
    "    max_k = int(max(ks))\n",
    "    topk_scores, topk_indices = torch.topk(scores, max_k)\n",
    "    topk_labels = torch.gather(labels, 1, topk_indices)\n",
    "    return topk_scores, topk_indices, topk_labels\n",
    "\n",
    "\n",
    "def _create_output_placeholder(scores, ks):\n",
    "    return torch.zeros(\n",
    "        scores.shape[0], len(ks), dtype=scores.dtype, device=scores.device\n",
    "    )\n",
    "\n",
    "\n",
    "def _mask_with_nans(metrics, labels):\n",
    "    for row, values in enumerate(labels):\n",
    "        if values.sum() == 0.0:\n",
    "            metrics[row, :].fill_(float(\"NaN\"))\n",
    "\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def one_hot_1d(\n",
    "    labels: torch.Tensor,\n",
    "    num_classes: int,\n",
    "    device: Optional[torch.device] = None,\n",
    "    dtype: Optional[torch.dtype] = torch.float32,\n",
    ") -> torch.Tensor:\n",
    "    r\"\"\"Coverts a 1d label tensor to one-hot representation\n",
    "    Args:\n",
    "        labels (torch.Tensor) : tensor with labels of shape :math:`(N, H, W)`,\n",
    "                                where N is batch siz. Each value is an integer\n",
    "                                representing correct classification.\n",
    "        num_classes (int): number of classes in labels.\n",
    "        device (Optional[torch.device]): the desired device of returned tensor.\n",
    "         Default: if None, uses the current device for the default tensor type\n",
    "         (see torch.set_default_tensor_type()). device will be the CPU for CPU\n",
    "         tensor types and the current CUDA device for CUDA tensor types.\n",
    "        dtype (Optional[torch.dtype]): the desired data type of returned\n",
    "         tensor. Default: torch.float32\n",
    "    Returns:\n",
    "        torch.Tensor: the labels in one hot tensor.\n",
    "    Examples::\n",
    "        >>> labels = torch.LongTensor([0, 1, 2, 0])\n",
    "        >>> one_hot_1d(labels, num_classes=3)\n",
    "        tensor([[1., 0., 0.],\n",
    "                [0., 1., 0.],\n",
    "                [0., 0., 1.],\n",
    "                [1., 0., 0.],\n",
    "               ])\n",
    "    \"\"\"\n",
    "    if not torch.is_tensor(labels):\n",
    "        raise TypeError(\"Input labels type is not a torch.Tensor. Got {}\".format(type(labels)))\n",
    "    if not len(labels.shape) == 1:\n",
    "        raise ValueError(\"Expected tensor should have 1 dim. Got: {}\".format(labels.shape))\n",
    "    if not labels.dtype == torch.int64:\n",
    "        raise ValueError(\n",
    "            \"labels must be of the same dtype torch.int64. Got: {}\".format(labels.dtype)\n",
    "        )\n",
    "    if num_classes < 1:\n",
    "        raise ValueError(\n",
    "            \"The number of classes must be bigger than one.\" \" Got: {}\".format(num_classes)\n",
    "        )\n",
    "    if device is None:\n",
    "        device = labels.device\n",
    "    labels_size = labels.shape[0]\n",
    "    one_hot = torch.zeros(labels_size, num_classes, device=device, dtype=dtype)\n",
    "    return one_hot.scatter_(1, labels.unsqueeze(-1), 1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def _tranform_label_to_onehot(labels, vocab_size):\n",
    "    return one_hot_1d(labels.reshape(-1), vocab_size, dtype=torch.float32).detach()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class RankingMetric(tm.Metric):\n",
    "    \"\"\"\n",
    "    Metric wrapper for computing ranking metrics@K for session-based task.\n",
    "    Parameters\n",
    "    ----------\n",
    "    top_ks : list, default [2, 5])\n",
    "        list of cutoffs\n",
    "    labels_onehot : bool\n",
    "        Enable transform the labels to one-hot representation\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, top_ks=None, labels_onehot=False):\n",
    "        super(RankingMetric, self).__init__()\n",
    "        self.top_ks = top_ks or [2, 5]\n",
    "        self.labels_onehot = labels_onehot\n",
    "        # Store the mean of the batch metrics (for each cut-off at topk)\n",
    "        self.add_state(\"metric_mean\", default=[], dist_reduce_fx=\"cat\")\n",
    "\n",
    "    def update(self, preds: torch.Tensor, target: torch.Tensor, **kwargs):  # type: ignore\n",
    "        # Computing the metrics at different cut-offs\n",
    "        if self.labels_onehot:\n",
    "            target = _tranform_label_to_onehot(target, preds.size(-1))\n",
    "        metric = self._metric(torch.LongTensor(self.top_ks), preds.view(-1, preds.size(-1)), target)\n",
    "        self.metric_mean.append(metric)  # type: ignore\n",
    "\n",
    "    def compute(self):\n",
    "        # Computing the mean of the batch metrics (for each cut-off at topk)\n",
    "        return torch.cat(self.metric_mean, axis=0).mean(0)\n",
    "\n",
    "    @abstractmethod\n",
    "    def _metric(self, ks: torch.Tensor, preds: torch.Tensor, target: torch.Tensor) -> torch.Tensor:\n",
    "        \"\"\"\n",
    "        Compute a ranking metric over a predictions and one-hot targets.\n",
    "        This method should be overridden by subclasses.\n",
    "        \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class PrecisionAt(RankingMetric):\n",
    "    def __init__(self, top_ks=None, labels_onehot=False):\n",
    "        super(PrecisionAt, self).__init__(top_ks=top_ks, labels_onehot=labels_onehot)\n",
    "\n",
    "    def _metric(self, ks: torch.Tensor, scores: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:\n",
    "        \"\"\"Compute precision@K for each of the provided cutoffs\n",
    "        Parameters\n",
    "        ----------\n",
    "        ks : torch.Tensor or list\n",
    "            list of cutoffs\n",
    "        scores : torch.Tensor\n",
    "            predicted item scores\n",
    "        labels : torch.Tensor\n",
    "            true item labels\n",
    "        Returns\n",
    "        -------\n",
    "        torch.Tensor:\n",
    "            list of precisions at cutoffs\n",
    "        \"\"\"\n",
    "\n",
    "        ks, scores, labels = _check_inputs(ks, scores, labels)\n",
    "        _, _, topk_labels = _extract_topk(ks, scores, labels)\n",
    "        precisions = _create_output_placeholder(scores, ks)\n",
    "\n",
    "        for index, k in enumerate(ks):\n",
    "            precisions[:, index] = torch.sum(topk_labels[:, : int(k)], dim=1) / float(k)\n",
    "\n",
    "        return precisions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class RecallAt(RankingMetric):\n",
    "    def __init__(self, top_ks=None, labels_onehot=False):\n",
    "        super(RecallAt, self).__init__(top_ks=top_ks, labels_onehot=labels_onehot)\n",
    "\n",
    "    def _metric(self, ks: torch.Tensor, scores: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:\n",
    "        \"\"\"Compute recall@K for each of the provided cutoffs\n",
    "        Parameters\n",
    "        ----------\n",
    "        ks : torch.Tensor or list\n",
    "            list of cutoffs\n",
    "        scores : torch.Tensor\n",
    "            predicted item scores\n",
    "        labels : torch.Tensor\n",
    "            true item labels\n",
    "        Returns\n",
    "        -------\n",
    "            torch.Tensor: list of recalls at cutoffs\n",
    "        \"\"\"\n",
    "\n",
    "        ks, scores, labels = _check_inputs(ks, scores, labels)\n",
    "        _, _, topk_labels = _extract_topk(ks, scores, labels)\n",
    "        recalls = _create_output_placeholder(scores, ks)\n",
    "\n",
    "        # Compute recalls at K\n",
    "        num_relevant = torch.sum(labels, dim=-1)\n",
    "        rel_indices = (num_relevant != 0).nonzero()\n",
    "        rel_count = num_relevant[rel_indices].squeeze()\n",
    "\n",
    "        if rel_indices.shape[0] > 0:\n",
    "            for index, k in enumerate(ks):\n",
    "                rel_labels = topk_labels[rel_indices, : int(k)].squeeze()\n",
    "\n",
    "                recalls[rel_indices, index] = (\n",
    "                    torch.div(torch.sum(rel_labels, dim=-1), rel_count)\n",
    "                    .reshape(len(rel_indices), 1)\n",
    "                    .to(dtype=torch.float32)\n",
    "                )  # Ensuring type is double, because it can be float if --fp16\n",
    "\n",
    "        return recalls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class AvgPrecisionAt(RankingMetric):\n",
    "    def __init__(self, top_ks=None, labels_onehot=False):\n",
    "        super(AvgPrecisionAt, self).__init__(top_ks=top_ks, labels_onehot=labels_onehot)\n",
    "        self.precision_at = PrecisionAt(top_ks)._metric\n",
    "\n",
    "    def _metric(self, ks: torch.Tensor, scores: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:\n",
    "        \"\"\"Compute average precision at K for provided cutoffs\n",
    "        Parameters\n",
    "        ----------\n",
    "        ks : torch.Tensor or list\n",
    "            list of cutoffs\n",
    "        scores : torch.Tensor\n",
    "            2-dim tensor of predicted item scores\n",
    "        labels : torch.Tensor\n",
    "            2-dim tensor of true item labels\n",
    "        Returns\n",
    "        -------\n",
    "        torch.Tensor:\n",
    "            list of average precisions at cutoffs\n",
    "        \"\"\"\n",
    "        ks, scores, labels = _check_inputs(ks, scores, labels)\n",
    "        topk_scores, _, topk_labels = _extract_topk(ks, scores, labels)\n",
    "        avg_precisions = _create_output_placeholder(scores, ks)\n",
    "\n",
    "        # Compute average precisions at K\n",
    "        num_relevant = torch.sum(labels, dim=1)\n",
    "        max_k = ks.max().item()\n",
    "\n",
    "        precisions = self.precision_at(1 + torch.arange(max_k), topk_scores, topk_labels)\n",
    "        rel_precisions = precisions * topk_labels\n",
    "\n",
    "        for index, k in enumerate(ks):\n",
    "            total_prec = rel_precisions[:, : int(k)].sum(dim=1)\n",
    "            avg_precisions[:, index] = total_prec / num_relevant.clamp(min=1, max=k).to(\n",
    "                dtype=torch.float32, device=scores.device\n",
    "            )  # Ensuring type is double, because it can be float if --fp16\n",
    "\n",
    "        return avg_precisions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class MRRAt(RankingMetric):\n",
    "#     def __init__(self, top_ks=None, labels_onehot=False):\n",
    "#         super(MRRAt, self).__init__(top_ks=top_ks, labels_onehot=labels_onehot)\n",
    "\n",
    "#     def _metric(self, ks: torch.Tensor, scores: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:\n",
    "#         \"\"\"Compute mean reciprocal rank at K for provided cutoffs\n",
    "#         Parameters\n",
    "#         ----------\n",
    "#         ks : torch.Tensor or list\n",
    "#             list of cutoffs\n",
    "#         scores : torch.Tensor\n",
    "#             2-dim tensor of predicted item scores\n",
    "#         labels : torch.Tensor\n",
    "#             2-dim tensor of true item labels\n",
    "#         Returns\n",
    "#         -------\n",
    "#         torch.Tensor:\n",
    "#             list of mrr scores at cutoffs\n",
    "#         \"\"\"\n",
    "#         ks, scores, labels = _check_inputs(ks, scores, labels)\n",
    "#         topk_scores, _, topk_labels = _extract_topk(ks, scores, labels)\n",
    "#         mrrs = _create_output_placeholder(scores, ks)\n",
    "\n",
    "#         # Compute mrr scores at K\n",
    "#         # for index, k in enumerate(ks):\n",
    "#             # topk_labels = topk_labels[torch.argsort(topk_scores[:,:k], dim=-1, descending=True)]\n",
    "#             # position = torch.nonzero(topk_labels)\n",
    "#             # mrrs[:, index] = 1.0 / (position[0] + 1.0).to(\n",
    "#             #     dtype=torch.float32, device=scores.device\n",
    "#             # )\n",
    "\n",
    "#         return topk_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# num_items = 10000 # The number of candidate items to rank\n",
    "# batch_size = 10 # The number of ranked lists to process\n",
    "# cutoffs = [10, 20, 50, 100] # Cutoffs for the number of positions to include when computing metrics\n",
    "\n",
    "# # labels: A large tensor of binary relevance labels\n",
    "# probabilities = torch.empty(batch_size, num_items).fill_(0.5)\n",
    "# labels = torch.bernoulli(probabilities)\n",
    "\n",
    "# # scores: A large tensor of simulated relevance scores\n",
    "# scores = torch.empty(batch_size, num_items).uniform_(0, 1)\n",
    "\n",
    "# metric_at_k = MRRAt(cutoffs)\n",
    "\n",
    "# # metric has correct shape?\n",
    "# p_at_ks = metric_at_k(scores, labels)\n",
    "# # test_eq(len(p_at_ks.shape), 1)\n",
    "# # test_eq(len(p_at_ks), len(cutoffs))\n",
    "\n",
    "# # # metric is all zero when nothing is relevant\n",
    "# # p_at_ks = metric_at_k(scores, torch.zeros(batch_size, num_items))\n",
    "# # test_eq(int(torch.count_nonzero(p_at_ks)), 0)\n",
    "\n",
    "# # # metric is all one when everything is relevant\n",
    "# # p_at_ks = metric_at_k(scores, torch.ones(batch_size, num_items))\n",
    "# # test_eq(int(torch.count_nonzero(p_at_ks)), len(cutoffs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class DCGAt(RankingMetric):\n",
    "    def __init__(self, top_ks=None, labels_onehot=False):\n",
    "        super(DCGAt, self).__init__(top_ks=top_ks, labels_onehot=labels_onehot)\n",
    "\n",
    "    def _metric(\n",
    "        self, ks: torch.Tensor, scores: torch.Tensor, labels: torch.Tensor, log_base: int = 2\n",
    "    ) -> torch.Tensor:\n",
    "        \"\"\"Compute discounted cumulative gain at K for provided cutoffs (ignoring ties)\n",
    "        Parameters\n",
    "        ----------\n",
    "        ks : torch.Tensor or list\n",
    "            list of cutoffs\n",
    "        scores : torch.Tensor\n",
    "            predicted item scores\n",
    "        labels : torch.Tensor\n",
    "            true item labels\n",
    "        Returns\n",
    "        -------\n",
    "        torch.Tensor :\n",
    "            list of discounted cumulative gains at cutoffs\n",
    "        \"\"\"\n",
    "        ks, scores, labels = _check_inputs(ks, scores, labels)\n",
    "        topk_scores, topk_indices, topk_labels = _extract_topk(ks, scores, labels)\n",
    "        dcgs = _create_output_placeholder(scores, ks)\n",
    "\n",
    "        # Compute discounts\n",
    "        discount_positions = torch.arange(ks.max().item()).to(\n",
    "            device=scores.device, dtype=torch.float32\n",
    "        )\n",
    "\n",
    "        discount_log_base = torch.log(\n",
    "            torch.Tensor([log_base]).to(device=scores.device, dtype=torch.float32)\n",
    "        ).item()\n",
    "\n",
    "        discounts = 1 / (torch.log(discount_positions + 2) / discount_log_base)\n",
    "\n",
    "        # Compute DCGs at K\n",
    "        for index, k in enumerate(ks):\n",
    "            dcgs[:, index] = torch.sum(\n",
    "                (topk_labels[:, :k] * discounts[:k].repeat(topk_labels.shape[0], 1)), dim=1\n",
    "            ).to(\n",
    "                dtype=torch.float32, device=scores.device\n",
    "            )  # Ensuring type is double, because it can be float if --fp16\n",
    "\n",
    "        return dcgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class NDCGAt(RankingMetric):\n",
    "    def __init__(self, top_ks=None, labels_onehot=False):\n",
    "        super(NDCGAt, self).__init__(top_ks=top_ks, labels_onehot=labels_onehot)\n",
    "        self.dcg_at = DCGAt(top_ks)._metric\n",
    "\n",
    "    def _metric(\n",
    "        self, ks: torch.Tensor, scores: torch.Tensor, labels: torch.Tensor, log_base: int = 2\n",
    "    ) -> torch.Tensor:\n",
    "        \"\"\"Compute normalized discounted cumulative gain at K for provided cutoffs (ignoring ties)\n",
    "        Parameters\n",
    "        ----------\n",
    "        ks : torch.Tensor or list\n",
    "            list of cutoffs\n",
    "        scores : torch.Tensor\n",
    "            predicted item scores\n",
    "        labels : torch.Tensor\n",
    "            true item labels\n",
    "        Returns\n",
    "        -------\n",
    "        torch.Tensor :\n",
    "            list of discounted cumulative gains at cutoffs\n",
    "        \"\"\"\n",
    "        ks, scores, labels = _check_inputs(ks, scores, labels)\n",
    "        topk_scores, topk_indices, topk_labels = _extract_topk(ks, scores, labels)\n",
    "        # ndcgs = _create_output_placeholder(scores, ks) #TODO track if this line is needed\n",
    "\n",
    "        # Compute discounted cumulative gains\n",
    "        gains = self.dcg_at(ks, topk_scores, topk_labels)\n",
    "        normalizing_gains = self.dcg_at(ks, topk_labels, topk_labels)\n",
    "\n",
    "        # Prevent divisions by zero\n",
    "        relevant_pos = (normalizing_gains != 0).nonzero(as_tuple=True)\n",
    "        irrelevant_pos = (normalizing_gains == 0).nonzero(as_tuple=True)\n",
    "\n",
    "        gains[irrelevant_pos] = 0\n",
    "        gains[relevant_pos] /= normalizing_gains[relevant_pos]\n",
    "\n",
    "        return gains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_items = 10000 # The number of candidate items to rank\n",
    "batch_size = 10 # The number of ranked lists to process\n",
    "cutoffs = [10, 20, 50, 100] # Cutoffs for the number of positions to include when computing metrics\n",
    "\n",
    "# labels: A large tensor of binary relevance labels\n",
    "probabilities = torch.empty(batch_size, num_items).fill_(0.5)\n",
    "labels = torch.bernoulli(probabilities)\n",
    "\n",
    "# scores: A large tensor of simulated relevance scores\n",
    "scores = torch.empty(batch_size, num_items).uniform_(0, 1)\n",
    "\n",
    "ranking_metrics = [PrecisionAt, RecallAt, AvgPrecisionAt, DCGAt, NDCGAt]\n",
    "\n",
    "for metric in ranking_metrics:\n",
    "    metric_at_k = metric(cutoffs)\n",
    "\n",
    "    # metric has correct shape?\n",
    "    p_at_ks = metric_at_k(scores, labels)\n",
    "    test_eq(len(p_at_ks.shape), 1)\n",
    "    test_eq(len(p_at_ks), len(cutoffs))\n",
    "\n",
    "    # metric is all zero when nothing is relevant\n",
    "    p_at_ks = metric_at_k(scores, torch.zeros(batch_size, num_items))\n",
    "    test_eq(int(torch.count_nonzero(p_at_ks)), 0)\n",
    "\n",
    "    # metric is all one when everything is relevant\n",
    "    p_at_ks = metric_at_k(scores, torch.ones(batch_size, num_items))\n",
    "    test_eq(int(torch.count_nonzero(p_at_ks)), len(cutoffs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> **References:-**\n",
    "- https://github.com/massquantity/DBRL/blob/master/dbrl/evaluate/metrics.py\n",
    "- [https://github.com/NVIDIA-Merlin/Transformers4Rec/blob/main/transformers4rec/torch/ranking_metric.py](https://github.com/NVIDIA-Merlin/Transformers4Rec/blob/main/transformers4rec/torch/ranking_metric.py)\n",
    "- [https://github.com/karlhigley/ranking-metrics-torch](https://github.com/karlhigley/ranking-metrics-torch)\n",
    "- [https://github.com/mquad/sars_tutorial/blob/master/util/metrics.py](https://github.com/mquad/sars_tutorial/blob/master/util/metrics.py)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2022-01-06 09:02:26\n",
      "\n",
      "recohut: 0.0.9\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.144+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "torchmetrics: 0.6.2\n",
      "numpy       : 1.19.5\n",
      "torch       : 1.10.0+cu111\n",
      "PIL         : 7.1.2\n",
      "matplotlib  : 3.2.2\n",
      "IPython     : 5.5.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d -p recohut"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
