{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
    "\n",
    "import pandas as pd\n",
    "from transformers import pipeline\n",
    "from transformers import BertTokenizer, BertForQuestionAnswering\n",
    "from transformers import AlbertTokenizer, AlbertForQuestionAnswering\n",
    "from transformers import AutoTokenizer, AutoModelForQuestionAnswering\n",
    "from transformers.data.processors.squad import (\n",
    "    SquadResult, \n",
    "    SquadV1Processor, \n",
    "    SquadV2Processor, \n",
    "    SquadExample, \n",
    "    SquadFeatures,\n",
    ")\n",
    "from transformers.data.processors.utils import DataProcessor, InputExample\n",
    "from transformers import (\n",
    "    AutoModelForQuestionAnswering,\n",
    "    AutoTokenizer,\n",
    "    squad_convert_examples_to_features,\n",
    ")\n",
    "from transformers.data.metrics.squad_metrics import (\n",
    "    compute_predictions_logits,\n",
    "    squad_evaluate,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# os.path.abspath(os.getcwd())\n",
    "data_dir = os.path.abspath('../../qary/data')\n",
    "path = os.path.abspath('../../qary/data/squad')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelnames = [\n",
    "    \"bert-large-uncased-whole-word-masking-finetuned-squad\",\n",
    "    \"ktrapeznikov/albert-xlarge-v2-squad-v2\",\n",
    "    \"mrm8488/bert-tiny-5-finetuned-squadv2\",\n",
    "    \"twmkn9/albert-base-v2-squad2\",\n",
    "    'distilbert-base-uncased',\n",
    "    \"distilbert-base-cased-distilled-squad\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad')\n",
    "model = AutoModelForQuestionAnswering.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForQuestionAnswering(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 1024, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 1024)\n",
       "      (token_type_embeddings): Embedding(2, 1024)\n",
       "      (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (12): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (13): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (14): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (15): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (16): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (17): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (18): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (19): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (20): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (21): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (22): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (23): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (qa_outputs): Linear(in_features=1024, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processor = SquadV2Processor()\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_list(tensor):\n",
    "    return tensor.detach().cpu().tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert SQuAD examples to features using processors:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 35/35 [00:12<00:00,  2.84it/s]\n"
     ]
    }
   ],
   "source": [
    "examples = processor.get_dev_examples(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "convert squad examples to features: 100%|████████████████████████████████████| 11873/11873 [01:40<00:00, 117.88it/s]\n",
      "add example index and unique id: 100%|████████████████████████████████████| 11873/11873 [00:00<00:00, 351633.02it/s]\n"
     ]
    }
   ],
   "source": [
    "features, dataset = squad_convert_examples_to_features(\n",
    "    examples=examples,\n",
    "    tokenizer=tokenizer,\n",
    "    max_seq_length=512,\n",
    "    doc_stride = 128,\n",
    "    max_query_length=256,\n",
    "    is_training=False,\n",
    "    return_dataset='pt',\n",
    "    threads=4,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_sampler = SequentialSampler(dataset)\n",
    "eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results = []\n",
    "\n",
    "for batch in tqdm(eval_dataloader):\n",
    "    model.eval()\n",
    "    batch = tuple(t.to(device) for t in batch)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        inputs = {\n",
    "            \"input_ids\": batch[0],\n",
    "            \"attention_mask\": batch[1],\n",
    "            \"token_type_ids\": batch[2],\n",
    "        }\n",
    "\n",
    "        example_indices = batch[3]\n",
    "\n",
    "        outputs = model(**inputs)\n",
    "\n",
    "        for i, example_index in enumerate(example_indices):\n",
    "            eval_feature = features[example_index.item()]\n",
    "            unique_id = int(eval_feature.unique_id)\n",
    "\n",
    "            output = [to_list(output[i]) for output in outputs]\n",
    "\n",
    "            start_logits, end_logits = output\n",
    "            result = SquadResult(unique_id, start_logits, end_logits)\n",
    "            all_results.append(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UsageError: Line magic function `%install_ext` not found.\n"
     ]
    }
   ],
   "source": [
    "%install_ext https://raw.github.com/cpcloud/ipython-autotime/master/autotime.py\n",
    "%load_ext autotime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_prediction_file = \"predictions.json\"\n",
    "output_nbest_file = \"nbest_predictions.json\"\n",
    "output_null_log_odds_file = \"null_predictions.json\"\n",
    "n_best_size = 1\n",
    "max_answer_length = 254\n",
    "do_lower_case = True\n",
    "null_score_diff_threshold = 0.5\n",
    "\n",
    "\n",
    "predictions = compute_predictions_logits(\n",
    "    examples,\n",
    "    features,\n",
    "    all_results,\n",
    "    n_best_size,\n",
    "    max_answer_length,\n",
    "    do_lower_case,\n",
    "    output_prediction_file,\n",
    "    output_nbest_file,\n",
    "    output_null_log_odds_file,\n",
    "    False,  # verbose_logging\n",
    "    True,  # version_2_with_negative\n",
    "    null_score_diff_threshold,\n",
    "    tokenizer,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "null_odds = json.load(open(\"null_predictions.json\", 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_f1_thresh = -21.030719757080078\n",
    "best_exact_thresh = -21.529870867729187"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = squad_evaluate(examples, \n",
    "                         predictions,\n",
    "                         no_answer_probs=null_odds, \n",
    "                         no_answer_probability_threshold=best_f1_thresh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_v1_path = os.path.abspath('../../qary/data/predictions-v1')\n",
    "preds_v2_path = os.path.abspath('../../qary/data/predictions-v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 48/48 [00:12<00:00,  3.93it/s]\n"
     ]
    }
   ],
   "source": [
    "processor_v1 = SquadV1Processor()\n",
    "examples_v1 = processor_v1.get_dev_examples(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 35/35 [00:11<00:00,  3.07it/s]\n"
     ]
    }
   ],
   "source": [
    "processor_v2 = SquadV2Processor()\n",
    "examples_v2 = processor_v2.get_dev_examples(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "reports_v1 = []\n",
    "for m in modelnames:\n",
    "    report = {}\n",
    "    modelname=m.replace('/', '-')\n",
    "    mpath = os.path.join(preds_v1_path, modelname)\n",
    "    mpreds_path = os.path.join(mpath, \"predictions.json\")\n",
    "    predictions = json.load(open(mpreds_path , 'rb'))\n",
    "    report = dict(squad_evaluate(examples_v1, \n",
    "                             predictions,\n",
    "                            ))\n",
    "    \n",
    "    report['MODEL'] = m\n",
    "    reports_v1.append(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "reports_v1_df = pd.DataFrame(reports_v1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-large-uncased-whole-word-masking-finetuned-squad\n",
      "ktrapeznikov-albert-xlarge-v2-squad-v2\n",
      "mrm8488-bert-tiny-5-finetuned-squadv2\n",
      "twmkn9-albert-base-v2-squad2\n",
      "distilbert-base-uncased\n",
      "distilbert-base-cased-distilled-squad\n"
     ]
    }
   ],
   "source": [
    "reports_v2 = []\n",
    "for m in modelnames:\n",
    "    report = {}\n",
    "    modelname=m.replace('/', '-')\n",
    "    print(modelname)\n",
    "    mpath = os.path.join(preds_v2_path, modelname)\n",
    "    mpreds_path = os.path.join(mpath, \"predictions.json\")\n",
    "    mnodds_path = os.path.join(mpath, \"null_predictions.json\")\n",
    "    predictions = json.load(open(mpreds_path , 'rb'))\n",
    "    null_odds = json.load(open(mnodds_path, 'rb'))\n",
    "    report = dict(squad_evaluate(examples_v2, \n",
    "                                 predictions, \n",
    "                                 no_answer_probs=null_odds, \n",
    "                                 no_answer_probability_threshold=0.5))\n",
    "    \n",
    "    report['MODEL'] = m\n",
    "    reports_v2.append(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "reports_v2_df = pd.DataFrame(reports_v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compact_dataframe(df):\n",
    "    columns_to_drop = [col for col \n",
    "                       in list(df.columns) \n",
    "                       if \"total\" in col]\n",
    "#     print(df.round(2).set_index('MODEL').drop(columns = columns_to_drop))\n",
    "    return df.round(2).set_index('MODEL').drop(columns = columns_to_drop).sort_values(by=['exact', 'f1'], ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "compact_v1_df = compact_dataframe(reports_v1_df)\n",
    "compact_v2_df = compact_dataframe(reports_v2_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>exact</th>\n",
       "      <th>f1</th>\n",
       "      <th>HasAns_exact</th>\n",
       "      <th>HasAns_f1</th>\n",
       "      <th>best_exact</th>\n",
       "      <th>best_exact_thresh</th>\n",
       "      <th>best_f1</th>\n",
       "      <th>best_f1_thresh</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MODEL</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>bert-large-uncased-whole-word-masking-finetuned-squad</th>\n",
       "      <td>86.29</td>\n",
       "      <td>92.47</td>\n",
       "      <td>86.29</td>\n",
       "      <td>92.47</td>\n",
       "      <td>86.29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>92.47</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ktrapeznikov/albert-xlarge-v2-squad-v2</th>\n",
       "      <td>81.49</td>\n",
       "      <td>87.14</td>\n",
       "      <td>81.49</td>\n",
       "      <td>87.14</td>\n",
       "      <td>81.49</td>\n",
       "      <td>0.0</td>\n",
       "      <td>87.14</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>distilbert-base-cased-distilled-squad</th>\n",
       "      <td>78.28</td>\n",
       "      <td>85.70</td>\n",
       "      <td>78.28</td>\n",
       "      <td>85.70</td>\n",
       "      <td>78.28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>85.70</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>twmkn9/albert-base-v2-squad2</th>\n",
       "      <td>74.04</td>\n",
       "      <td>80.59</td>\n",
       "      <td>74.04</td>\n",
       "      <td>80.59</td>\n",
       "      <td>74.04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.59</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mrm8488/bert-tiny-5-finetuned-squadv2</th>\n",
       "      <td>50.82</td>\n",
       "      <td>57.71</td>\n",
       "      <td>50.82</td>\n",
       "      <td>57.71</td>\n",
       "      <td>50.82</td>\n",
       "      <td>0.0</td>\n",
       "      <td>57.71</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>distilbert-base-uncased</th>\n",
       "      <td>0.03</td>\n",
       "      <td>1.16</td>\n",
       "      <td>0.03</td>\n",
       "      <td>1.16</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.16</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    exact     f1  \\\n",
       "MODEL                                                              \n",
       "bert-large-uncased-whole-word-masking-finetuned...  86.29  92.47   \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2              81.49  87.14   \n",
       "distilbert-base-cased-distilled-squad               78.28  85.70   \n",
       "twmkn9/albert-base-v2-squad2                        74.04  80.59   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2               50.82  57.71   \n",
       "distilbert-base-uncased                              0.03   1.16   \n",
       "\n",
       "                                                    HasAns_exact  HasAns_f1  \\\n",
       "MODEL                                                                         \n",
       "bert-large-uncased-whole-word-masking-finetuned...         86.29      92.47   \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                     81.49      87.14   \n",
       "distilbert-base-cased-distilled-squad                      78.28      85.70   \n",
       "twmkn9/albert-base-v2-squad2                               74.04      80.59   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                      50.82      57.71   \n",
       "distilbert-base-uncased                                     0.03       1.16   \n",
       "\n",
       "                                                    best_exact  \\\n",
       "MODEL                                                            \n",
       "bert-large-uncased-whole-word-masking-finetuned...       86.29   \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                   81.49   \n",
       "distilbert-base-cased-distilled-squad                    78.28   \n",
       "twmkn9/albert-base-v2-squad2                             74.04   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                    50.82   \n",
       "distilbert-base-uncased                                   0.03   \n",
       "\n",
       "                                                    best_exact_thresh  \\\n",
       "MODEL                                                                   \n",
       "bert-large-uncased-whole-word-masking-finetuned...                0.0   \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                            0.0   \n",
       "distilbert-base-cased-distilled-squad                             0.0   \n",
       "twmkn9/albert-base-v2-squad2                                      0.0   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                             0.0   \n",
       "distilbert-base-uncased                                           0.0   \n",
       "\n",
       "                                                    best_f1  best_f1_thresh  \n",
       "MODEL                                                                        \n",
       "bert-large-uncased-whole-word-masking-finetuned...    92.47             0.0  \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                87.14             0.0  \n",
       "distilbert-base-cased-distilled-squad                 85.70             0.0  \n",
       "twmkn9/albert-base-v2-squad2                          80.59             0.0  \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                 57.71             0.0  \n",
       "distilbert-base-uncased                                1.16             0.0  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compact_v1_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>exact</th>\n",
       "      <th>f1</th>\n",
       "      <th>HasAns_exact</th>\n",
       "      <th>HasAns_f1</th>\n",
       "      <th>NoAns_exact</th>\n",
       "      <th>NoAns_f1</th>\n",
       "      <th>best_exact</th>\n",
       "      <th>best_exact_thresh</th>\n",
       "      <th>best_f1</th>\n",
       "      <th>best_f1_thresh</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MODEL</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ktrapeznikov/albert-xlarge-v2-squad-v2</th>\n",
       "      <td>84.46</td>\n",
       "      <td>87.47</td>\n",
       "      <td>80.01</td>\n",
       "      <td>86.04</td>\n",
       "      <td>88.90</td>\n",
       "      <td>88.90</td>\n",
       "      <td>84.71</td>\n",
       "      <td>-1.31</td>\n",
       "      <td>87.68</td>\n",
       "      <td>-1.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>twmkn9/albert-base-v2-squad2</th>\n",
       "      <td>77.92</td>\n",
       "      <td>81.38</td>\n",
       "      <td>72.84</td>\n",
       "      <td>79.78</td>\n",
       "      <td>82.98</td>\n",
       "      <td>82.98</td>\n",
       "      <td>78.38</td>\n",
       "      <td>-3.34</td>\n",
       "      <td>81.73</td>\n",
       "      <td>-1.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mrm8488/bert-tiny-5-finetuned-squadv2</th>\n",
       "      <td>57.85</td>\n",
       "      <td>61.24</td>\n",
       "      <td>50.10</td>\n",
       "      <td>56.89</td>\n",
       "      <td>65.57</td>\n",
       "      <td>65.57</td>\n",
       "      <td>59.57</td>\n",
       "      <td>-2.07</td>\n",
       "      <td>62.20</td>\n",
       "      <td>-1.87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bert-large-uncased-whole-word-masking-finetuned-squad</th>\n",
       "      <td>42.63</td>\n",
       "      <td>45.93</td>\n",
       "      <td>85.24</td>\n",
       "      <td>91.85</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.13</td>\n",
       "      <td>65.83</td>\n",
       "      <td>-21.53</td>\n",
       "      <td>67.13</td>\n",
       "      <td>-21.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>distilbert-base-cased-distilled-squad</th>\n",
       "      <td>39.02</td>\n",
       "      <td>42.86</td>\n",
       "      <td>78.10</td>\n",
       "      <td>85.78</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>55.61</td>\n",
       "      <td>-22.67</td>\n",
       "      <td>56.85</td>\n",
       "      <td>-22.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>distilbert-base-uncased</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.23</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>50.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>50.07</td>\n",
       "      <td>-1.60</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    exact     f1  \\\n",
       "MODEL                                                              \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2              84.46  87.47   \n",
       "twmkn9/albert-base-v2-squad2                        77.92  81.38   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2               57.85  61.24   \n",
       "bert-large-uncased-whole-word-masking-finetuned...  42.63  45.93   \n",
       "distilbert-base-cased-distilled-squad               39.02  42.86   \n",
       "distilbert-base-uncased                              0.00   0.61   \n",
       "\n",
       "                                                    HasAns_exact  HasAns_f1  \\\n",
       "MODEL                                                                         \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                     80.01      86.04   \n",
       "twmkn9/albert-base-v2-squad2                               72.84      79.78   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                      50.10      56.89   \n",
       "bert-large-uncased-whole-word-masking-finetuned...         85.24      91.85   \n",
       "distilbert-base-cased-distilled-squad                      78.10      85.78   \n",
       "distilbert-base-uncased                                     0.00       1.23   \n",
       "\n",
       "                                                    NoAns_exact  NoAns_f1  \\\n",
       "MODEL                                                                       \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                    88.90     88.90   \n",
       "twmkn9/albert-base-v2-squad2                              82.98     82.98   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                     65.57     65.57   \n",
       "bert-large-uncased-whole-word-masking-finetuned...         0.13      0.13   \n",
       "distilbert-base-cased-distilled-squad                      0.05      0.05   \n",
       "distilbert-base-uncased                                    0.00      0.00   \n",
       "\n",
       "                                                    best_exact  \\\n",
       "MODEL                                                            \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                   84.71   \n",
       "twmkn9/albert-base-v2-squad2                             78.38   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                    59.57   \n",
       "bert-large-uncased-whole-word-masking-finetuned...       65.83   \n",
       "distilbert-base-cased-distilled-squad                    55.61   \n",
       "distilbert-base-uncased                                  50.07   \n",
       "\n",
       "                                                    best_exact_thresh  \\\n",
       "MODEL                                                                   \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                          -1.31   \n",
       "twmkn9/albert-base-v2-squad2                                    -3.34   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                           -2.07   \n",
       "bert-large-uncased-whole-word-masking-finetuned...             -21.53   \n",
       "distilbert-base-cased-distilled-squad                          -22.67   \n",
       "distilbert-base-uncased                                          0.00   \n",
       "\n",
       "                                                    best_f1  best_f1_thresh  \n",
       "MODEL                                                                        \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                87.68           -1.22  \n",
       "twmkn9/albert-base-v2-squad2                          81.73           -1.82  \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                 62.20           -1.87  \n",
       "bert-large-uncased-whole-word-masking-finetuned...    67.13          -21.03  \n",
       "distilbert-base-cased-distilled-squad                 56.85          -22.42  \n",
       "distilbert-base-uncased                               50.07           -1.60  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compact_v2_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-large-uncased-whole-word-masking-finetuned-squad\n",
      "ktrapeznikov-albert-xlarge-v2-squad-v2\n",
      "mrm8488-bert-tiny-5-finetuned-squadv2\n",
      "twmkn9-albert-base-v2-squad2\n",
      "distilbert-base-uncased\n",
      "distilbert-base-cased-distilled-squad\n"
     ]
    }
   ],
   "source": [
    "reports_v2_adjusted = []\n",
    "for m in modelnames:\n",
    "    report = {}\n",
    "    modelname=m.replace('/', '-')\n",
    "    print(modelname)\n",
    "    mpath = os.path.join(preds_v2_path, modelname)\n",
    "    mpreds_path = os.path.join(mpath, \"predictions.json\")\n",
    "    mnodds_path = os.path.join(mpath, \"null_predictions.json\")\n",
    "    predictions = json.load(open(mpreds_path , 'rb'))\n",
    "    null_odds = json.load(open(mnodds_path, 'rb'))\n",
    "    report_raw = dict(squad_evaluate(examples_v2, \n",
    "                                 predictions, \n",
    "                                 no_answer_probs=null_odds, \n",
    "                                 no_answer_probability_threshold=0.5))\n",
    "    \n",
    "    best_thresh = report_raw['best_exact_thresh']\n",
    "    \n",
    "    report = dict(squad_evaluate(examples_v2, \n",
    "                             predictions, \n",
    "                             no_answer_probs=null_odds, \n",
    "                             no_answer_probability_threshold=best_thresh ))\n",
    "    \n",
    "    report['MODEL'] = m\n",
    "    reports_v2_adjusted.append(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "reports_v2_adj_df = compact_dataframe(pd.DataFrame(reports_v2_adjusted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>exact</th>\n",
       "      <th>f1</th>\n",
       "      <th>HasAns_exact</th>\n",
       "      <th>HasAns_f1</th>\n",
       "      <th>NoAns_exact</th>\n",
       "      <th>NoAns_f1</th>\n",
       "      <th>best_exact</th>\n",
       "      <th>best_exact_thresh</th>\n",
       "      <th>best_f1</th>\n",
       "      <th>best_f1_thresh</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MODEL</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ktrapeznikov/albert-xlarge-v2-squad-v2</th>\n",
       "      <td>84.71</td>\n",
       "      <td>87.67</td>\n",
       "      <td>79.10</td>\n",
       "      <td>85.02</td>\n",
       "      <td>90.31</td>\n",
       "      <td>90.31</td>\n",
       "      <td>84.71</td>\n",
       "      <td>-1.31</td>\n",
       "      <td>87.68</td>\n",
       "      <td>-1.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>twmkn9/albert-base-v2-squad2</th>\n",
       "      <td>78.38</td>\n",
       "      <td>81.57</td>\n",
       "      <td>69.55</td>\n",
       "      <td>75.95</td>\n",
       "      <td>87.18</td>\n",
       "      <td>87.18</td>\n",
       "      <td>78.38</td>\n",
       "      <td>-3.34</td>\n",
       "      <td>81.73</td>\n",
       "      <td>-1.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bert-large-uncased-whole-word-masking-finetuned-squad</th>\n",
       "      <td>65.83</td>\n",
       "      <td>67.01</td>\n",
       "      <td>58.72</td>\n",
       "      <td>61.09</td>\n",
       "      <td>72.92</td>\n",
       "      <td>72.92</td>\n",
       "      <td>65.83</td>\n",
       "      <td>-21.53</td>\n",
       "      <td>67.13</td>\n",
       "      <td>-21.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mrm8488/bert-tiny-5-finetuned-squadv2</th>\n",
       "      <td>59.57</td>\n",
       "      <td>62.13</td>\n",
       "      <td>41.70</td>\n",
       "      <td>46.82</td>\n",
       "      <td>77.39</td>\n",
       "      <td>77.39</td>\n",
       "      <td>59.57</td>\n",
       "      <td>-2.07</td>\n",
       "      <td>62.20</td>\n",
       "      <td>-1.87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>distilbert-base-cased-distilled-squad</th>\n",
       "      <td>55.61</td>\n",
       "      <td>56.73</td>\n",
       "      <td>42.88</td>\n",
       "      <td>45.13</td>\n",
       "      <td>68.29</td>\n",
       "      <td>68.29</td>\n",
       "      <td>55.61</td>\n",
       "      <td>-22.67</td>\n",
       "      <td>56.85</td>\n",
       "      <td>-22.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>distilbert-base-uncased</th>\n",
       "      <td>6.23</td>\n",
       "      <td>6.85</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.23</td>\n",
       "      <td>12.45</td>\n",
       "      <td>12.45</td>\n",
       "      <td>50.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>50.07</td>\n",
       "      <td>-1.60</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    exact     f1  \\\n",
       "MODEL                                                              \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2              84.71  87.67   \n",
       "twmkn9/albert-base-v2-squad2                        78.38  81.57   \n",
       "bert-large-uncased-whole-word-masking-finetuned...  65.83  67.01   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2               59.57  62.13   \n",
       "distilbert-base-cased-distilled-squad               55.61  56.73   \n",
       "distilbert-base-uncased                              6.23   6.85   \n",
       "\n",
       "                                                    HasAns_exact  HasAns_f1  \\\n",
       "MODEL                                                                         \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                     79.10      85.02   \n",
       "twmkn9/albert-base-v2-squad2                               69.55      75.95   \n",
       "bert-large-uncased-whole-word-masking-finetuned...         58.72      61.09   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                      41.70      46.82   \n",
       "distilbert-base-cased-distilled-squad                      42.88      45.13   \n",
       "distilbert-base-uncased                                     0.00       1.23   \n",
       "\n",
       "                                                    NoAns_exact  NoAns_f1  \\\n",
       "MODEL                                                                       \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                    90.31     90.31   \n",
       "twmkn9/albert-base-v2-squad2                              87.18     87.18   \n",
       "bert-large-uncased-whole-word-masking-finetuned...        72.92     72.92   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                     77.39     77.39   \n",
       "distilbert-base-cased-distilled-squad                     68.29     68.29   \n",
       "distilbert-base-uncased                                   12.45     12.45   \n",
       "\n",
       "                                                    best_exact  \\\n",
       "MODEL                                                            \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                   84.71   \n",
       "twmkn9/albert-base-v2-squad2                             78.38   \n",
       "bert-large-uncased-whole-word-masking-finetuned...       65.83   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                    59.57   \n",
       "distilbert-base-cased-distilled-squad                    55.61   \n",
       "distilbert-base-uncased                                  50.07   \n",
       "\n",
       "                                                    best_exact_thresh  \\\n",
       "MODEL                                                                   \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                          -1.31   \n",
       "twmkn9/albert-base-v2-squad2                                    -3.34   \n",
       "bert-large-uncased-whole-word-masking-finetuned...             -21.53   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                           -2.07   \n",
       "distilbert-base-cased-distilled-squad                          -22.67   \n",
       "distilbert-base-uncased                                          0.00   \n",
       "\n",
       "                                                    best_f1  best_f1_thresh  \n",
       "MODEL                                                                        \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                87.68           -1.22  \n",
       "twmkn9/albert-base-v2-squad2                          81.73           -1.82  \n",
       "bert-large-uncased-whole-word-masking-finetuned...    67.13          -21.03  \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                 62.20           -1.87  \n",
       "distilbert-base-cased-distilled-squad                 56.85          -22.42  \n",
       "distilbert-base-uncased                               50.07           -1.60  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reports_v2_adj_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reports_v1_adjusted = []\n",
    "for m in modelnames:\n",
    "    report = {}\n",
    "    modelname=m.replace('/', '-')\n",
    "    print(modelname)\n",
    "    mpath = os.path.join(preds_v1_path, modelname)\n",
    "    mpreds_path = os.path.join(mpath, \"predictions.json\")\n",
    "    mnodds_path = os.path.join(mpath, \"null_predictions.json\")\n",
    "    predictions = json.load(open(mpreds_path , 'rb'))\n",
    "    null_odds = json.load(open(mnodds_path, 'rb'))\n",
    "    report_raw = dict(squad_evaluate(examples_v1, \n",
    "                                 predictions, \n",
    "                                 no_answer_probs=null_odds, \n",
    "                                 no_answer_probability_threshold=0.5))\n",
    "    \n",
    "    best_thresh = report_raw['best_exact_thresh']\n",
    "    \n",
    "#     report = dict(squad_evaluate(examples_v1, \n",
    "#                              predictions, \n",
    "#                              no_answer_probs=null_odds, \n",
    "#                              no_answer_probability_threshold=best_thresh ))\n",
    "    \n",
    "    report['MODEL'] = m\n",
    "    reports_v1_adjusted.append(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "keep_cols = ['exact', 'f1']\n",
    "# keep_cols = ['exact', 'f1', 'HasAns_exact', 'HasAns_f1']\n",
    "all_reports = compact_v1_df[keep_cols].merge(compact_v2_df[keep_cols], left_on=index, right_on=index, suffixes=('_v1', '_v2'))\n",
    "all_reports = all_reports.rename(columns = {'key_0':'MODEL'})\n",
    "# all_reports.set_index('MODEL')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MODEL</th>\n",
       "      <th>exact_v1</th>\n",
       "      <th>f1_v1</th>\n",
       "      <th>exact_v2</th>\n",
       "      <th>f1_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ktrapeznikov/albert-xlarge-v2-squad-v2</td>\n",
       "      <td>86.29</td>\n",
       "      <td>92.47</td>\n",
       "      <td>84.46</td>\n",
       "      <td>87.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>twmkn9/albert-base-v2-squad2</td>\n",
       "      <td>81.49</td>\n",
       "      <td>87.14</td>\n",
       "      <td>77.92</td>\n",
       "      <td>81.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bert-large-uncased-whole-word-masking-finetune...</td>\n",
       "      <td>78.28</td>\n",
       "      <td>85.70</td>\n",
       "      <td>57.85</td>\n",
       "      <td>61.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mrm8488/bert-tiny-5-finetuned-squadv2</td>\n",
       "      <td>74.04</td>\n",
       "      <td>80.59</td>\n",
       "      <td>42.63</td>\n",
       "      <td>45.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>distilbert-base-cased-distilled-squad</td>\n",
       "      <td>50.82</td>\n",
       "      <td>57.71</td>\n",
       "      <td>39.02</td>\n",
       "      <td>42.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>distilbert-base-uncased</td>\n",
       "      <td>0.03</td>\n",
       "      <td>1.16</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.61</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               MODEL  exact_v1  f1_v1  \\\n",
       "0             ktrapeznikov/albert-xlarge-v2-squad-v2     86.29  92.47   \n",
       "1                       twmkn9/albert-base-v2-squad2     81.49  87.14   \n",
       "2  bert-large-uncased-whole-word-masking-finetune...     78.28  85.70   \n",
       "3              mrm8488/bert-tiny-5-finetuned-squadv2     74.04  80.59   \n",
       "4              distilbert-base-cased-distilled-squad     50.82  57.71   \n",
       "5                            distilbert-base-uncased      0.03   1.16   \n",
       "\n",
       "   exact_v2  f1_v2  \n",
       "0     84.46  87.47  \n",
       "1     77.92  81.38  \n",
       "2     57.85  61.24  \n",
       "3     42.63  45.93  \n",
       "4     39.02  42.86  \n",
       "5      0.00   0.61  "
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "human = {'MODEL': 'human performance', 'exact_v1':86.831, 'f1_v1': 89.452}\n",
    "all_reports = all_reports.append(human, ignore_index=True).sort_values('exact_v1', ascending = False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MODEL</th>\n",
       "      <th>exact_v1</th>\n",
       "      <th>f1_v1</th>\n",
       "      <th>exact_v2</th>\n",
       "      <th>f1_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>human performance</td>\n",
       "      <td>86.831</td>\n",
       "      <td>89.452</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ktrapeznikov/albert-xlarge-v2-squad-v2</td>\n",
       "      <td>86.290</td>\n",
       "      <td>92.470</td>\n",
       "      <td>84.46</td>\n",
       "      <td>87.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>twmkn9/albert-base-v2-squad2</td>\n",
       "      <td>81.490</td>\n",
       "      <td>87.140</td>\n",
       "      <td>77.92</td>\n",
       "      <td>81.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bert-large-uncased-whole-word-masking-finetune...</td>\n",
       "      <td>78.280</td>\n",
       "      <td>85.700</td>\n",
       "      <td>57.85</td>\n",
       "      <td>61.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mrm8488/bert-tiny-5-finetuned-squadv2</td>\n",
       "      <td>74.040</td>\n",
       "      <td>80.590</td>\n",
       "      <td>42.63</td>\n",
       "      <td>45.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>distilbert-base-cased-distilled-squad</td>\n",
       "      <td>50.820</td>\n",
       "      <td>57.710</td>\n",
       "      <td>39.02</td>\n",
       "      <td>42.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>distilbert-base-uncased</td>\n",
       "      <td>0.030</td>\n",
       "      <td>1.160</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.61</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               MODEL  exact_v1   f1_v1  \\\n",
       "6                                  human performance    86.831  89.452   \n",
       "0             ktrapeznikov/albert-xlarge-v2-squad-v2    86.290  92.470   \n",
       "1                       twmkn9/albert-base-v2-squad2    81.490  87.140   \n",
       "2  bert-large-uncased-whole-word-masking-finetune...    78.280  85.700   \n",
       "3              mrm8488/bert-tiny-5-finetuned-squadv2    74.040  80.590   \n",
       "4              distilbert-base-cased-distilled-squad    50.820  57.710   \n",
       "5                            distilbert-base-uncased     0.030   1.160   \n",
       "\n",
       "   exact_v2  f1_v2  \n",
       "6       NaN    NaN  \n",
       "0     84.46  87.47  \n",
       "1     77.92  81.38  \n",
       "2     57.85  61.24  \n",
       "3     42.63  45.93  \n",
       "4     39.02  42.86  \n",
       "5      0.00   0.61  "
      ]
     },
     "execution_count": 233,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>exact</th>\n",
       "      <th>f1</th>\n",
       "      <th>HasAns_exact</th>\n",
       "      <th>HasAns_f1</th>\n",
       "      <th>NoAns_exact</th>\n",
       "      <th>NoAns_f1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MODEL</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ktrapeznikov/albert-xlarge-v2-squad-v2</th>\n",
       "      <td>84.46</td>\n",
       "      <td>87.47</td>\n",
       "      <td>80.01</td>\n",
       "      <td>86.04</td>\n",
       "      <td>88.90</td>\n",
       "      <td>88.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>twmkn9/albert-base-v2-squad2</th>\n",
       "      <td>77.92</td>\n",
       "      <td>81.38</td>\n",
       "      <td>72.84</td>\n",
       "      <td>79.78</td>\n",
       "      <td>82.98</td>\n",
       "      <td>82.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mrm8488/bert-tiny-5-finetuned-squadv2</th>\n",
       "      <td>57.85</td>\n",
       "      <td>61.24</td>\n",
       "      <td>50.10</td>\n",
       "      <td>56.89</td>\n",
       "      <td>65.57</td>\n",
       "      <td>65.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bert-large-uncased-whole-word-masking-finetuned-squad</th>\n",
       "      <td>42.63</td>\n",
       "      <td>45.93</td>\n",
       "      <td>85.24</td>\n",
       "      <td>91.85</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>distilbert-base-cased-distilled-squad</th>\n",
       "      <td>39.02</td>\n",
       "      <td>42.86</td>\n",
       "      <td>78.10</td>\n",
       "      <td>85.78</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>distilbert-base-uncased</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.23</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    exact     f1  \\\n",
       "MODEL                                                              \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2              84.46  87.47   \n",
       "twmkn9/albert-base-v2-squad2                        77.92  81.38   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2               57.85  61.24   \n",
       "bert-large-uncased-whole-word-masking-finetuned...  42.63  45.93   \n",
       "distilbert-base-cased-distilled-squad               39.02  42.86   \n",
       "distilbert-base-uncased                              0.00   0.61   \n",
       "\n",
       "                                                    HasAns_exact  HasAns_f1  \\\n",
       "MODEL                                                                         \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                     80.01      86.04   \n",
       "twmkn9/albert-base-v2-squad2                               72.84      79.78   \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                      50.10      56.89   \n",
       "bert-large-uncased-whole-word-masking-finetuned...         85.24      91.85   \n",
       "distilbert-base-cased-distilled-squad                      78.10      85.78   \n",
       "distilbert-base-uncased                                     0.00       1.23   \n",
       "\n",
       "                                                    NoAns_exact  NoAns_f1  \n",
       "MODEL                                                                      \n",
       "ktrapeznikov/albert-xlarge-v2-squad-v2                    88.90     88.90  \n",
       "twmkn9/albert-base-v2-squad2                              82.98     82.98  \n",
       "mrm8488/bert-tiny-5-finetuned-squadv2                     65.57     65.57  \n",
       "bert-large-uncased-whole-word-masking-finetuned...         0.13      0.13  \n",
       "distilbert-base-cased-distilled-squad                      0.05      0.05  \n",
       "distilbert-base-uncased                                    0.00      0.00  "
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compact_v2_df[['exact', 'f1', 'HasAns_exact', 'HasAns_f1', 'NoAns_exact', 'NoAns_f1']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MODEL</th>\n",
       "      <th>HasAns_exact</th>\n",
       "      <th>HasAns_f1</th>\n",
       "      <th>NoAns_exact</th>\n",
       "      <th>NoAns_f1</th>\n",
       "      <th>HasAns_exact_ADJ</th>\n",
       "      <th>HasAns_f1_ADJ</th>\n",
       "      <th>NoAns_exact_ADJ</th>\n",
       "      <th>NoAns_f1_ADJ</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ktrapeznikov/albert-xlarge-v2-squad-v2</td>\n",
       "      <td>80.01</td>\n",
       "      <td>86.04</td>\n",
       "      <td>88.90</td>\n",
       "      <td>88.90</td>\n",
       "      <td>79.10</td>\n",
       "      <td>85.02</td>\n",
       "      <td>90.31</td>\n",
       "      <td>90.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>twmkn9/albert-base-v2-squad2</td>\n",
       "      <td>72.84</td>\n",
       "      <td>79.78</td>\n",
       "      <td>82.98</td>\n",
       "      <td>82.98</td>\n",
       "      <td>69.55</td>\n",
       "      <td>75.95</td>\n",
       "      <td>87.18</td>\n",
       "      <td>87.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bert-large-uncased-whole-word-masking-finetune...</td>\n",
       "      <td>50.10</td>\n",
       "      <td>56.89</td>\n",
       "      <td>65.57</td>\n",
       "      <td>65.57</td>\n",
       "      <td>58.72</td>\n",
       "      <td>61.09</td>\n",
       "      <td>72.92</td>\n",
       "      <td>72.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mrm8488/bert-tiny-5-finetuned-squadv2</td>\n",
       "      <td>85.24</td>\n",
       "      <td>91.85</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.13</td>\n",
       "      <td>41.70</td>\n",
       "      <td>46.82</td>\n",
       "      <td>77.39</td>\n",
       "      <td>77.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>distilbert-base-cased-distilled-squad</td>\n",
       "      <td>78.10</td>\n",
       "      <td>85.78</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>42.88</td>\n",
       "      <td>45.13</td>\n",
       "      <td>68.29</td>\n",
       "      <td>68.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>distilbert-base-uncased</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.23</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.23</td>\n",
       "      <td>12.45</td>\n",
       "      <td>12.45</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               MODEL  HasAns_exact  HasAns_f1  \\\n",
       "0             ktrapeznikov/albert-xlarge-v2-squad-v2         80.01      86.04   \n",
       "1                       twmkn9/albert-base-v2-squad2         72.84      79.78   \n",
       "2  bert-large-uncased-whole-word-masking-finetune...         50.10      56.89   \n",
       "3              mrm8488/bert-tiny-5-finetuned-squadv2         85.24      91.85   \n",
       "4              distilbert-base-cased-distilled-squad         78.10      85.78   \n",
       "5                            distilbert-base-uncased          0.00       1.23   \n",
       "\n",
       "   NoAns_exact  NoAns_f1  HasAns_exact_ADJ  HasAns_f1_ADJ  NoAns_exact_ADJ  \\\n",
       "0        88.90     88.90             79.10          85.02            90.31   \n",
       "1        82.98     82.98             69.55          75.95            87.18   \n",
       "2        65.57     65.57             58.72          61.09            72.92   \n",
       "3         0.13      0.13             41.70          46.82            77.39   \n",
       "4         0.05      0.05             42.88          45.13            68.29   \n",
       "5         0.00      0.00              0.00           1.23            12.45   \n",
       "\n",
       "   NoAns_f1_ADJ  \n",
       "0         90.31  \n",
       "1         87.18  \n",
       "2         72.92  \n",
       "3         77.39  \n",
       "4         68.29  \n",
       "5         12.45  "
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keep_cols_v1 = ['HasAns_exact', 'HasAns_f1']\n",
    "keep_cols_v2 = ['HasAns_exact', 'HasAns_f1', 'NoAns_exact', 'NoAns_f1']\n",
    "reports_unpack = compact_v2_df[keep_cols_v2].merge(reports_v2_adj_df[keep_cols_v2], left_on=index, right_on=index, suffixes=('', '_ADJ'))\n",
    "reports_unpack = reports_unpack.rename(columns = {'key_0':'MODEL'}).reset_index(drop=True)\n",
    "reports_unpack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MODEL</th>\n",
       "      <th>HasAns_exact</th>\n",
       "      <th>HasAns_f1</th>\n",
       "      <th>NoAns_exact</th>\n",
       "      <th>NoAns_f1</th>\n",
       "      <th>HasAns_exact(ADJ)</th>\n",
       "      <th>HasAns_f1(ADJ)</th>\n",
       "      <th>NoAns_exact(ADJ)</th>\n",
       "      <th>NoAns_f1(ADJ)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mrm8488/bert-tiny-5-finetuned-squadv2</td>\n",
       "      <td>85.24</td>\n",
       "      <td>91.85</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.13</td>\n",
       "      <td>41.70</td>\n",
       "      <td>46.82</td>\n",
       "      <td>77.39</td>\n",
       "      <td>77.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ktrapeznikov/albert-xlarge-v2-squad-v2</td>\n",
       "      <td>80.01</td>\n",
       "      <td>86.04</td>\n",
       "      <td>88.90</td>\n",
       "      <td>88.90</td>\n",
       "      <td>79.10</td>\n",
       "      <td>85.02</td>\n",
       "      <td>90.31</td>\n",
       "      <td>90.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>distilbert-base-cased-distilled-squad</td>\n",
       "      <td>78.10</td>\n",
       "      <td>85.78</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>42.88</td>\n",
       "      <td>45.13</td>\n",
       "      <td>68.29</td>\n",
       "      <td>68.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>twmkn9/albert-base-v2-squad2</td>\n",
       "      <td>72.84</td>\n",
       "      <td>79.78</td>\n",
       "      <td>82.98</td>\n",
       "      <td>82.98</td>\n",
       "      <td>69.55</td>\n",
       "      <td>75.95</td>\n",
       "      <td>87.18</td>\n",
       "      <td>87.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bert-large-uncased-whole-word-masking-finetune...</td>\n",
       "      <td>50.10</td>\n",
       "      <td>56.89</td>\n",
       "      <td>65.57</td>\n",
       "      <td>65.57</td>\n",
       "      <td>58.72</td>\n",
       "      <td>61.09</td>\n",
       "      <td>72.92</td>\n",
       "      <td>72.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>distilbert-base-uncased</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.23</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.23</td>\n",
       "      <td>12.45</td>\n",
       "      <td>12.45</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               MODEL  HasAns_exact  HasAns_f1  \\\n",
       "3              mrm8488/bert-tiny-5-finetuned-squadv2         85.24      91.85   \n",
       "0             ktrapeznikov/albert-xlarge-v2-squad-v2         80.01      86.04   \n",
       "4              distilbert-base-cased-distilled-squad         78.10      85.78   \n",
       "1                       twmkn9/albert-base-v2-squad2         72.84      79.78   \n",
       "2  bert-large-uncased-whole-word-masking-finetune...         50.10      56.89   \n",
       "5                            distilbert-base-uncased          0.00       1.23   \n",
       "\n",
       "   NoAns_exact  NoAns_f1  HasAns_exact(ADJ)  HasAns_f1(ADJ)  NoAns_exact(ADJ)  \\\n",
       "3         0.13      0.13              41.70           46.82             77.39   \n",
       "0        88.90     88.90              79.10           85.02             90.31   \n",
       "4         0.05      0.05              42.88           45.13             68.29   \n",
       "1        82.98     82.98              69.55           75.95             87.18   \n",
       "2        65.57     65.57              58.72           61.09             72.92   \n",
       "5         0.00      0.00               0.00            1.23             12.45   \n",
       "\n",
       "   NoAns_f1(ADJ)  \n",
       "3          77.39  \n",
       "0          90.31  \n",
       "4          68.29  \n",
       "1          87.18  \n",
       "2          72.92  \n",
       "5          12.45  "
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keep_cols = ['HasAns_exact', 'HasAns_f1', 'NoAns_exact', 'NoAns_f1']\n",
    "v2_vs_adjusted = compact_v2_df[keep_cols].merge(reports_v2_adj_df[keep_cols], left_on=index, right_on=index, suffixes=('', '(ADJ)'))\n",
    "v2_vs_adjusted = v2_vs_adjusted.rename(columns = {'key_0':'MODEL'}).reset_index(drop=True)\n",
    "v2_vs_adjusted.sort_values(by=['HasAns_exact', 'HasAns_f1'], ascending = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Default model uses distilbert-base-cased-distilled-squad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "context = '''\n",
    "Computational linguistics is often grouped within the field of artificial intelligence \n",
    "but was present before the development of artificial intelligence. \n",
    "Computational linguistics originated with efforts in the United States in the 1950s to use computers to \n",
    "automatically translate texts from foreign languages, particularly Russian scientific journals, into English.[3]\n",
    "Since computers can make arithmetic (systematic) calculations much faster and more accurately than humans, \n",
    "it was thought to be only a short matter of time before they could also begin to process language.[4] \n",
    "Computational and quantitative methods are also used historically in the attempted reconstruction of earlier\n",
    "forms of modern languages and sub-grouping modern languages into language families. \n",
    "Earlier methods, such as lexicostatistics and glottochronology, have been proven to be premature and inaccurate. \n",
    "However, recent interdisciplinary studies that borrow concepts from biological studies, especially gene mapping, \n",
    "have proved to produce more sophisticated analytical tools and more reliable results.[5]\n",
    "'''\n",
    "\n",
    "questions=['When was computational linguistics invented?',\n",
    "          'Which problems computational linguistics is trying to solve?',\n",
    "          'What methods existed before the emergence of computational linguistics?',\n",
    "          'Who invented computational linguistics?',\n",
    "          'Who invented gene mapping?']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "265aa80d9c424defa4e0a0bcbc463a78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=230.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model: bert-large-uncased-whole-word-masking-finetuned-squad\n",
      "-----------------\n",
      "Question: When was computational linguistics invented?\n",
      "Answer: 1950s (confidence score 0.7105585285134239)\n",
      " \n",
      "Question: Which problems computational linguistics is trying to solve?\n",
      "Answer: earlier forms of modern languages and sub-grouping modern languages into language families. (confidence score 0.034796690637104444)\n",
      " \n",
      "Question: What methods existed before the emergence of computational linguistics?\n",
      "Answer: lexicostatistics and glottochronology, (confidence score 0.8949566496998465)\n",
      " \n",
      "Question: Who invented computational linguistics?\n",
      "Answer: United States (confidence score 0.5333964470000865)\n",
      " \n",
      "Question: Who invented gene mapping?\n",
      "Answer: biological studies, (confidence score 0.02638426599066701)\n",
      " \n",
      "Model: ktrapeznikov/albert-xlarge-v2-squad-v2\n",
      "-----------------\n",
      "Question: When was computational linguistics invented?\n",
      "Answer: 1950s (confidence score 0.6412413898187204)\n",
      " \n",
      "Question: Which problems computational linguistics is trying to solve?\n",
      "Answer: translate texts from foreign languages, (confidence score 0.1307672173261354)\n",
      " \n",
      "Question: What methods existed before the emergence of computational linguistics?\n",
      "Answer:  (confidence score 0.6308010582306451)\n",
      " \n",
      "Question: Who invented computational linguistics?\n",
      "Answer:  (confidence score 0.9748902345310917)\n",
      " \n",
      "Question: Who invented gene mapping?\n",
      "Answer:  (confidence score 0.9988990117797236)\n",
      " \n",
      "Model: mrm8488/bert-tiny-5-finetuned-squadv2\n",
      "-----------------\n",
      "Question: When was computational linguistics invented?\n",
      "Answer: 1950s (confidence score 0.5100432430158293)\n",
      " \n",
      "Question: Which problems computational linguistics is trying to solve?\n",
      "Answer: artificial intelligence. (confidence score 0.03275686739784334)\n",
      " \n",
      "Question: What methods existed before the emergence of computational linguistics?\n",
      "Answer:  (confidence score 0.06689302592967117)\n",
      " \n",
      "Question: Who invented computational linguistics?\n",
      "Answer:  (confidence score 0.05630986208743849)\n",
      " \n",
      "Question: Who invented gene mapping?\n",
      "Answer:  (confidence score 0.8440988190788303)\n",
      " \n",
      "Model: twmkn9/albert-base-v2-squad2\n",
      "-----------------\n",
      "Question: When was computational linguistics invented?\n",
      "Answer: 1950s (confidence score 0.630521506320747)\n",
      " \n",
      "Question: Which problems computational linguistics is trying to solve?\n",
      "Answer:  (confidence score 0.5901262729978356)\n",
      " \n",
      "Question: What methods existed before the emergence of computational linguistics?\n",
      "Answer:  (confidence score 0.2787252009804586)\n",
      " \n",
      "Question: Who invented computational linguistics?\n",
      "Answer:  (confidence score 0.9395531361082305)\n",
      " \n",
      "Question: Who invented gene mapping?\n",
      "Answer:  (confidence score 0.9998772777192002)\n",
      " \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02d2bab743de4800af0e4eb39e0d7425",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=230.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model: distilbert-base-uncased\n",
      "-----------------\n",
      "Question: When was computational linguistics invented?\n",
      "Answer: English.[3] Since computers can make arithmetic (systematic) (confidence score 1.582584644544934e-05)\n",
      " \n",
      "Question: Which problems computational linguistics is trying to solve?\n",
      "Answer: into language families. (confidence score 1.5566947074668424e-05)\n",
      " \n",
      "Question: What methods existed before the emergence of computational linguistics?\n",
      "Answer: into language families. (confidence score 1.5719257432964353e-05)\n",
      " \n",
      "Question: Who invented computational linguistics?\n",
      "Answer: English.[3] Since computers can make arithmetic (systematic) (confidence score 1.5842987051530207e-05)\n",
      " \n",
      "Question: Who invented gene mapping?\n",
      "Answer: English.[3] Since computers can make arithmetic (systematic) (confidence score 1.5883810795251857e-05)\n",
      " \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9916a14168748779384e75c85ced39d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=230.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model: distilbert-base-cased-distilled-squad\n",
      "-----------------\n",
      "Question: When was computational linguistics invented?\n",
      "Answer: 1950s (confidence score 0.7759537003546768)\n",
      " \n",
      "Question: Which problems computational linguistics is trying to solve?\n",
      "Answer: gene mapping, (confidence score 0.4235580072416312)\n",
      " \n",
      "Question: What methods existed before the emergence of computational linguistics?\n",
      "Answer: lexicostatistics and glottochronology, (confidence score 0.8573431178602817)\n",
      " \n",
      "Question: Who invented computational linguistics?\n",
      "Answer: computers (confidence score 0.7313878935375229)\n",
      " \n",
      "Question: Who invented gene mapping?\n",
      "Answer: biological studies, (confidence score 0.4788379586462099)\n",
      " \n"
     ]
    }
   ],
   "source": [
    "for model in modelnames:\n",
    "    # Sentiment analysis pipeline\n",
    "    nlp = pipeline('question-answering', model=model, tokenizer=model)\n",
    "    print(f'Model: {model}')\n",
    "    print('-----------------')\n",
    "    for q in questions:\n",
    "        print(f'Question: {q}')\n",
    "        result = nlp(context=context, question=q, handle_impossible_answer=True)\n",
    "        answer = result['answer']\n",
    "        conf = result['score']\n",
    "        print(f'Answer: {answer} (confidence score {conf})')\n",
    "        print(' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b3b4be15537e4e02b87234c3d1d8f83f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=230.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model: bert-large-uncased-whole-word-masking-finetuned-squad\n",
      "-----------------\n",
      "Question: When was computational linguistics invented?\n",
      "Answer: 1950s (confidence score 0.7105585285134239)\n",
      " \n",
      "Question: Which problems computational linguistics is trying to solve?\n",
      "Answer: earlier forms of modern languages and sub-grouping modern languages into language families. (confidence score 0.034796690637104444)\n",
      " \n",
      "Question: What methods existed before the emergence of computational linguistics?\n",
      "Answer: lexicostatistics and glottochronology, (confidence score 0.8949566496998465)\n",
      " \n",
      "Question: Who invented computational linguistics?\n",
      "Answer: United States (confidence score 0.5333964470000865)\n",
      " \n",
      "Question: Who invented gene mapping?\n",
      "Answer: biological studies, (confidence score 0.02638426599066701)\n",
      " \n",
      "Model: ktrapeznikov/albert-xlarge-v2-squad-v2\n",
      "-----------------\n",
      "Question: When was computational linguistics invented?\n",
      "Answer: 1950s (confidence score 0.6412413898187204)\n",
      " \n",
      "Question: Which problems computational linguistics is trying to solve?\n",
      "Answer: translate texts from foreign languages, (confidence score 0.1307672173261354)\n",
      " \n",
      "Question: What methods existed before the emergence of computational linguistics?\n",
      "Answer: lexicostatistics and glottochronology, (confidence score 0.03728622768787493)\n",
      " \n",
      "Question: Who invented computational linguistics?\n",
      "Answer: efforts in the United States in the 1950s to use computers (confidence score 1.0536395728961978e-05)\n",
      " \n",
      "Question: Who invented gene mapping?\n",
      "Answer: gene mapping, (confidence score 3.271786086923688e-10)\n",
      " \n",
      "Model: mrm8488/bert-tiny-5-finetuned-squadv2\n",
      "-----------------\n",
      "Question: When was computational linguistics invented?\n",
      "Answer: 1950s (confidence score 0.5100432430158293)\n",
      " \n",
      "Question: Which problems computational linguistics is trying to solve?\n",
      "Answer: artificial intelligence. (confidence score 0.03275686739784334)\n",
      " \n",
      "Question: What methods existed before the emergence of computational linguistics?\n",
      "Answer: artificial intelligence. (confidence score 0.048131708241787585)\n",
      " \n",
      "Question: Who invented computational linguistics?\n",
      "Answer: artificial intelligence. Computational linguistics originated with efforts in the United States (confidence score 0.02255180868260709)\n",
      " \n",
      "Question: Who invented gene mapping?\n",
      "Answer: Computational linguistics (confidence score 0.0005385152777170749)\n",
      " \n",
      "Model: twmkn9/albert-base-v2-squad2\n",
      "-----------------\n",
      "Question: When was computational linguistics invented?\n",
      "Answer: 1950s (confidence score 0.630521506320747)\n",
      " \n",
      "Question: Which problems computational linguistics is trying to solve?\n",
      "Answer: earlier forms of modern languages and sub-grouping modern languages into language families. (confidence score 0.0030569392577274868)\n",
      " \n",
      "Question: What methods existed before the emergence of computational linguistics?\n",
      "Answer: lexicostatistics and glottochronology, (confidence score 0.1910335840485171)\n",
      " \n",
      "Question: Who invented computational linguistics?\n",
      "Answer: the United States (confidence score 0.00043000163404216876)\n",
      " \n",
      "Question: Who invented gene mapping?\n",
      "Answer: gene mapping, (confidence score 4.254114755646597e-12)\n",
      " \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58f15b61a3d04bcda3a0799a2a7e32bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=230.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model: distilbert-base-uncased\n",
      "-----------------\n",
      "Question: When was computational linguistics invented?\n",
      "Answer: languages and sub-grouping modern languages into language families. (confidence score 1.6736800661229642e-05)\n",
      " \n",
      "Question: Which problems computational linguistics is trying to solve?\n",
      "Answer: languages and sub-grouping modern languages into language families. (confidence score 1.6955155309750225e-05)\n",
      " \n",
      "Question: What methods existed before the emergence of computational linguistics?\n",
      "Answer: languages and sub-grouping modern languages into language families. (confidence score 1.6916083581960213e-05)\n",
      " \n",
      "Question: Who invented computational linguistics?\n",
      "Answer: languages and sub-grouping modern languages into language families. (confidence score 1.6536293692165794e-05)\n",
      " \n",
      "Question: Who invented gene mapping?\n",
      "Answer: languages and sub-grouping modern languages into language families. (confidence score 1.6527565861201984e-05)\n",
      " \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "589fb2ecdff24932a2eed80289965840",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=230.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model: distilbert-base-cased-distilled-squad\n",
      "-----------------\n",
      "Question: When was computational linguistics invented?\n",
      "Answer: 1950s (confidence score 0.7759537003546768)\n",
      " \n",
      "Question: Which problems computational linguistics is trying to solve?\n",
      "Answer: gene mapping, (confidence score 0.4235580072416312)\n",
      " \n",
      "Question: What methods existed before the emergence of computational linguistics?\n",
      "Answer: lexicostatistics and glottochronology, (confidence score 0.8573431178602817)\n",
      " \n",
      "Question: Who invented computational linguistics?\n",
      "Answer: computers (confidence score 0.7313878935375229)\n",
      " \n",
      "Question: Who invented gene mapping?\n",
      "Answer: biological studies, (confidence score 0.4788379586462099)\n",
      " \n"
     ]
    }
   ],
   "source": [
    "for model in modelnames:\n",
    "    # Sentiment analysis pipeline\n",
    "    nlp = pipeline('question-answering', model=model, tokenizer=model)\n",
    "    print(f'Model: {model}')\n",
    "    print('-----------------')\n",
    "    for q in questions:\n",
    "        print(f'Question: {q}')\n",
    "        result = nlp(context=context, question=q, handle_impossible_answer=False)\n",
    "        answer = result['answer']\n",
    "        conf = result['score']\n",
    "        print(f'Answer: {answer} (confidence score {conf})')\n",
    "        print(' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b67857bd672c4727ad3f8bc21d0f506f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=230.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "context = '''\n",
    "Stan Lee[1] (born Stanley Martin Lieber /ˈliːbər/; December 28, 1922 – November 12, 2018) was an American \n",
    "comic book writer, editor, publisher, and producer. He rose through the ranks of a family-run business to become \n",
    "Marvel Comics' primary creative leader for two decades, leading its expansion from a small division of a publishing \n",
    "house to a multimedia corporation that dominated the comics industry. \n",
    "'''\n",
    "nlp = pipeline('question-answering')\n",
    "result = nlp(context=context, question=\"Who is Stan Lee?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 0.2854291316652837,\n",
       " 'start': 95,\n",
       " 'end': 159,\n",
       " 'answer': 'an American comic book writer, editor, publisher, and producer.'}"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (qaryenv)",
   "language": "python",
   "name": "qaryenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
