{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp evaluation.metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metrics\n",
    "> Metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *\n",
    "from fastcore.nb_imports import *\n",
    "from fastcore.test import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def NDCG(true, pred):\n",
    "    match = pred.eq(true).nonzero(as_tuple=True)[1]\n",
    "    ncdg = torch.log(torch.Tensor([2])).div(torch.log(match + 2))\n",
    "    ncdg = ncdg.sum().div(pred.shape[0]).item()\n",
    "    return ncdg\n",
    "\n",
    "\n",
    "def APAK(true, pred):\n",
    "    k = pred.shape[1]\n",
    "    apak = pred.eq(true).div(torch.arange(k) + 1)\n",
    "    apak = apak.sum().div(pred.shape[0]).item()\n",
    "    return apak\n",
    "\n",
    "\n",
    "def HR(true, pred):\n",
    "    hr = pred.eq(true).sum().div(pred.shape[0]).item()\n",
    "    return hr\n",
    "\n",
    "\n",
    "def get_eval_metrics(scores, true, k=10):\n",
    "    test_items = [torch.LongTensor(list(item_scores.keys())) for item_scores in scores]\n",
    "    test_scores = [torch.Tensor(list(item_scores.values())) for item_scores in scores]\n",
    "    topk_indices = [s.topk(k).indices for s in test_scores]\n",
    "    topk_items = [item[idx] for item, idx in zip(test_items, topk_indices)]\n",
    "    pred = torch.vstack(topk_items)\n",
    "    ncdg = NDCG(true, pred)\n",
    "    apak = APAK(true, pred)\n",
    "    hr = HR(true, pred)\n",
    "\n",
    "    return ncdg, apak, hr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.4261859357357025, 0.36666667461395264, 0.6000000238418579)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = [{1: 0.2, 2: 0.3, 3: 0.4, 4: 0.5, 9: 0.1},\n",
    "          {1: 0.2, 2: 0.3, 3: 0.4, 4: 0.5, 9: 0.1},\n",
    "          {1: 0.2, 2: 0.3, 3: 0.4, 4: 0.5, 9: 0.1},\n",
    "          {1: 0.2, 2: 0.3, 3: 0.4, 4: 0.5, 9: 0.1},\n",
    "          {1: 0.2, 2: 0.3, 3: 0.4, 4: 0.5, 9: 0.1}]\n",
    "\n",
    "true = torch.tensor([[1],[1],[2],[3],[4]])\n",
    "metric = get_eval_metrics(scores, true, k=3)\n",
    "metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.0, 1.0, 1.0)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# it should all 1, because all relevant items are in range k=3\n",
    "true = torch.tensor([[4],[4],[4],[4],[4]])\n",
    "metric = get_eval_metrics(scores, true, k=3)\n",
    "metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.0, 0.0, 0.0)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# it should all 0, because no relevant item is in range k=3\n",
    "true = torch.tensor([[9],[1],[9],[1],[1]])\n",
    "metric = get_eval_metrics(scores, true, k=3)\n",
    "metric"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> **References:-**\n",
    "- https://github.com/massquantity/DBRL/blob/master/dbrl/evaluate/metrics.py\n",
    "- [https://github.com/NVIDIA-Merlin/Transformers4Rec/blob/main/transformers4rec/torch/ranking_metric.py](https://github.com/NVIDIA-Merlin/Transformers4Rec/blob/main/transformers4rec/torch/ranking_metric.py)\n",
    "- [https://github.com/karlhigley/ranking-metrics-torch](https://github.com/karlhigley/ranking-metrics-torch)\n",
    "- [https://github.com/mquad/sars_tutorial/blob/master/util/metrics.py](https://github.com/mquad/sars_tutorial/blob/master/util/metrics.py)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2022-01-06 09:02:26\n",
      "\n",
      "recohut: 0.0.9\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.144+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "torchmetrics: 0.6.2\n",
      "numpy       : 1.19.5\n",
      "torch       : 1.10.0+cu111\n",
      "PIL         : 7.1.2\n",
      "matplotlib  : 3.2.2\n",
      "IPython     : 5.5.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d -p recohut"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
