{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "from sklearn.metrics import (accuracy_score, dcg_score, roc_auc_score, \n",
    "                             precision_score, recall_score)\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import Adam\n",
    "\n",
    "from htools import assert_raises, save, load\n",
    "from incendio.core import *\n",
    "from incendio.callbacks import *\n",
    "from incendio.metrics import *\n",
    "from incendio.optimizers import *\n",
    "from incendio.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reproducible testing.\n",
    "np.random.seed(0)\n",
    "torch.manual_seed(0)\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Data(Dataset):\n",
    "    \n",
    "    def __init__(self, n=64, dim=2):\n",
    "        self.x = torch.rand(n, dim).float()\n",
    "        self.y = torch.clamp(\n",
    "            (self.x[:, 0]*.75 + self.x[:, 1]*.25).round(), 0, 1\n",
    "        ).abs().unsqueeze(-1)\n",
    "        \n",
    "    def __getitem__(self, i):\n",
    "        return self.x[i], self.y[i]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MulticlassData(Dataset):\n",
    "    \n",
    "    def __init__(self, n=64, dim=2):\n",
    "        self.x = torch.rand(n, dim).float()\n",
    "        # Integer labels between 0 and 4, inclusive.\n",
    "        self.y = torch.clamp(\n",
    "            torch.round(\n",
    "                torch.randint(6, (n, 1)).float() \n",
    "                * (self.x[:, 0]*.75 + self.x[:, 1]*.25).unsqueeze(-1)\n",
    "            ), 0, 4).long().flatten()\n",
    "        \n",
    "    def __getitem__(self, i):\n",
    "        return self.x[i], self.y[i]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[0.4963, 0.7682],\n",
       "         [0.0885, 0.1320],\n",
       "         [0.3074, 0.6341],\n",
       "         [0.4901, 0.8964],\n",
       "         [0.4556, 0.6323]]), tensor([0, 1, 1, 0, 1]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ypred_ = torch.rand(5, 2)\n",
    "y_ = torch.tensor([0, 1, 1, 0, 1])\n",
    "ypred_, y_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleModel(BaseModel):\n",
    "    \n",
    "    def __init__(self, dim):\n",
    "        super().__init__()  \n",
    "        self.fc1 = nn.Linear(dim, 2)\n",
    "        self.fc2 = nn.Linear(2, 1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.leaky_relu(self.fc1(x))\n",
    "        return self.fc2(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GroupedModel(BaseModel):\n",
    "    \n",
    "    def __init__(self, dim):\n",
    "        super().__init__()  \n",
    "        g1 = nn.Sequential(\n",
    "            nn.Linear(dim, 8),\n",
    "            nn.LeakyReLU(),\n",
    "            nn.Linear(8, 4),\n",
    "            nn.LeakyReLU()\n",
    "        )\n",
    "        g2 = nn.Linear(4, 1)\n",
    "        self.groups = nn.ModuleList([g1, g2])\n",
    "        \n",
    "    def forward(self, x):\n",
    "        for group in self.groups:\n",
    "            x = group(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIM = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SimpleModel(\n",
       "  (fc1): Linear(in_features=2, out_features=2, bias=True)\n",
       "  (fc2): Linear(in_features=2, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snet = SimpleModel(DIM)\n",
    "snet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Adam (\n",
       "Parameter Group 0\n",
       "    amsgrad: False\n",
       "    betas: (0.9, 0.999)\n",
       "    eps: 0.001\n",
       "    lr: 0.002\n",
       "    weight_decay: 0\n",
       ")"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optim = variable_lr_optimizer(snet, 2e-3)\n",
    "optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As expected, got ValueError(Received more learning rates than layer groups.).\n"
     ]
    }
   ],
   "source": [
    "with assert_raises(ValueError) as ar:\n",
    "    optim = variable_lr_optimizer(snet, [3e-3, 1e-1])\n",
    "    optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Adam (\n",
       "Parameter Group 0\n",
       "    amsgrad: False\n",
       "    betas: (0.9, 0.999)\n",
       "    eps: 0.001\n",
       "    lr: 0.001\n",
       "    weight_decay: 0\n",
       ")"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "update_optimizer(optim, 1e-3, 0.5)\n",
    "optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 [((2, 2), False), ((2,), False), ((1, 2), False), ((1,), False)]\n",
      "1 [((2, 2), False), ((2,), False), ((1, 2), False), ((1,), True)]\n",
      "2 [((2, 2), False), ((2,), False), ((1, 2), True), ((1,), True)]\n",
      "3 [((2, 2), False), ((2,), True), ((1, 2), True), ((1,), True)]\n",
      "4 [((2, 2), True), ((2,), True), ((1, 2), True), ((1,), True)]\n"
     ]
    }
   ],
   "source": [
    "snet.freeze()\n",
    "for n in range(5):\n",
    "    snet.unfreeze(n_layers=n)\n",
    "    print(n, snet.trainable())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As expected, got AttributeError('SimpleModel' object has no attribute 'groups').\n"
     ]
    }
   ],
   "source": [
    "snet.freeze()\n",
    "with assert_raises(AttributeError) as ar:\n",
    "    for n in range(3):\n",
    "        snet.unfreeze(n_groups=n)\n",
    "        print(n, snet.trainable())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIM = 2\n",
    "metrics = [accuracy_score, \n",
    "           precision_score, \n",
    "           recall_score, \n",
    "           percent_positive,\n",
    "           mean_soft_prediction\n",
    "          ]\n",
    "\n",
    "# Model starts out unfrozen and freezes last group startingn with epoch 3.\n",
    "# This is not useful but is done here for testing purposes.\n",
    "callbacks = [EarlyStopper('max', 'accuracy', patience=3),\n",
    "             PerformanceThreshold('recall', 'max', 0.25, skip_epochs=5),\n",
    "             MetricHistory(),\n",
    "             ModelUnfreezer({3:1}, 'groups'),\n",
    "             ModelCheckpoint(),\n",
    "             CosineLRScheduler(),\n",
    "             S3Uploader('gg-datascience', 'hmamin/incendio/v1')\n",
    "            ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = Data(n=300, dim=DIM)\n",
    "val = Data(n=30, dim=DIM)\n",
    "\n",
    "dl_train = DataLoader(train, batch_size=8, shuffle=True)\n",
    "dl_val = DataLoader(val, batch_size=8, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Trainer(criterion='binary_cross_entropy_with_logits', out_dir='../data/v1')\n",
       "\n",
       "Datasets: 300 train rows, 30 val rows\n",
       "\n",
       "Optimizer: None\n",
       "\n",
       "GroupedModel(\n",
       "  (groups): ModuleList(\n",
       "    (0): Sequential(\n",
       "      (0): Linear(in_features=2, out_features=8, bias=True)\n",
       "      (1): LeakyReLU(negative_slope=0.01)\n",
       "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
       "      (3): LeakyReLU(negative_slope=0.01)\n",
       "    )\n",
       "    (1): Linear(in_features=4, out_features=1, bias=True)\n",
       "  )\n",
       "))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gnet = GroupedModel(DIM)\n",
    "t = Trainer(gnet, train, val, dl_train, dl_val, F.binary_cross_entropy_with_logits, \n",
    "            'binary', '../data/v1', torch.optim.RMSprop, \n",
    "            last_act=torch.sigmoid, metrics=metrics, callbacks=callbacks)\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data written to ../data/v1/tmp.pkl.\n"
     ]
    }
   ],
   "source": [
    "t.save('tmp.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Object loaded from ../data/v1/tmp.pkl.\n",
      "\n",
      "2020-02-18 17:06:42,841\n",
      " Could not load optimizer.  Loading model weights only.\n"
     ]
    }
   ],
   "source": [
    "t.load('tmp.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Object loaded from ../data/v1/tmp.pkl.\n"
     ]
    }
   ],
   "source": [
    "d = load('../data/v1/tmp.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-18 17:05:55,471 [INFO]: Removing files from output directory.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19d79ef9c4e34303a4d1c1dcdcd0c087",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=38), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-18 17:05:55,642\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 0\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.7622 |       0.7841 |\n",
      "| accuracy             |  0.5200 |       0.5667 |\n",
      "| precision            |  0.2786 |       0.5667 |\n",
      "| recall               |  0.4689 |       1.0000 |\n",
      "| percent_positive     |  0.4500 |       1.0000 |\n",
      "| mean_soft_prediction |  0.4903 |       0.7714 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-18 17:05:55,643\n",
      " Saving model. Loss improved from inf to 0.7841.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/harrisonmamin/.pyenv/versions/3.7.4/envs/main/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9071320cd1d3481c82aa1f541015c77c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=38), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-18 17:05:55,786\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 1\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.7072 |       0.7335 |\n",
      "| accuracy             |  0.5200 |       0.4333 |\n",
      "| precision            |  0.3200 |       0.0000 |\n",
      "| recall               |  0.5778 |       0.0000 |\n",
      "| percent_positive     |  0.5767 |       0.0000 |\n",
      "| mean_soft_prediction |  0.5299 |       0.3847 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-18 17:05:55,787\n",
      " Saving model. Loss improved from 0.7841 to 0.7335.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/harrisonmamin/.pyenv/versions/3.7.4/envs/main/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 due to no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b18d75a59cf40afba71962862f61b10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=38), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-18 17:05:55,920\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 2\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.5200 |       0.3595 |\n",
      "| accuracy             |  0.7800 |       0.8333 |\n",
      "| precision            |  0.6821 |       0.7711 |\n",
      "| recall               |  0.7364 |       1.0000 |\n",
      "| percent_positive     |  0.4900 |       0.7333 |\n",
      "| mean_soft_prediction |  0.5552 |       0.7156 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-18 17:05:55,921\n",
      " Saving model. Loss improved from 0.7335 to 0.3595.\n",
      "Data written to ../data/v1/trainer.pkl.\n",
      "\n",
      "2020-02-18 17:05:55,922\n",
      " Training complete. Model in eval mode.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File: ../data/v1/train.log\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v1/train.log\n",
      "File: ../data/v1/history.csv\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v1/history.csv\n",
      "File: ../data/v1/trainer.pkl\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v1/trainer.pkl\n",
      "File: ../data/v1/lrs.png\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v1/lrs.png\n",
      "File: ../data/v1/history.png\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v1/history.png\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 112544 characters]\n",
      "text/plain": [
       "<Figure size 864x864 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "t.fit(3, 3e-1, 0.5, clean=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data written to ../data/v1/trainer.zip.\n"
     ]
    }
   ],
   "source": [
    "t.save('trainer.zip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Object loaded from ../data/v1/trainer.zip.\n",
      "Trainer(criterion='binary_cross_entropy_with_logits', out_dir='../data/v1')\n",
      "\n",
      "Datasets: 300 train rows, 30 val rows\n",
      "\n",
      "Optimizer: RMSprop (\n",
      "Parameter Group 0\n",
      "    alpha: 0.99\n",
      "    centered: False\n",
      "    eps: 0.001\n",
      "    lr: 0.015052040053751204\n",
      "    momentum: 0\n",
      "    weight_decay: 0\n",
      "\n",
      "Parameter Group 1\n",
      "    alpha: 0.99\n",
      "    centered: False\n",
      "    eps: 0.001\n",
      "    lr: 0.030104080107502408\n",
      "    momentum: 0\n",
      "    weight_decay: 0\n",
      ")\n",
      "\n",
      "GroupedModel(\n",
      "  (groups): ModuleList(\n",
      "    (0): Sequential(\n",
      "      (0): Linear(in_features=2, out_features=8, bias=True)\n",
      "      (1): LeakyReLU(negative_slope=0.01)\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): LeakyReLU(negative_slope=0.01)\n",
      "    )\n",
      "    (1): Linear(in_features=4, out_features=1, bias=True)\n",
      "  )\n",
      "))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "19995"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2 = Trainer.from_file(os.path.join('..', 'data', 'v1', 'trainer.zip'))\n",
    "print(t2)\n",
    "del t2; gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multiclass classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleMulticlassModel(BaseModel):\n",
    "    \n",
    "    def __init__(self, dim, classes):\n",
    "        super().__init__()  \n",
    "        self.fc1 = nn.Linear(dim, 10)\n",
    "        self.fc2 = nn.Linear(10, classes)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.leaky_relu(self.fc1(x))\n",
    "        return self.fc2(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIM = 2\n",
    "metrics = [accuracy_score,\n",
    "           mean_soft_prediction\n",
    "          ]\n",
    "\n",
    "# Model starts out unfrozen and freezes last group startingn with epoch 3.\n",
    "# This is not useful but is done here for testing purposes.\n",
    "callbacks = [EarlyStopper('max', 'accuracy', patience=3),\n",
    "             PerformanceThreshold('loss', 'min', 2, skip_epochs=5),\n",
    "             MetricHistory(),\n",
    "             ModelUnfreezer({3:2}, 'layers'),\n",
    "             ModelCheckpoint(),\n",
    "             MetricHistory(),\n",
    "             CosineLRScheduler(),\n",
    "             S3Uploader('gg-datascience', 'hmamin/incendio/v2')\n",
    "            ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = MulticlassData(n=34, dim=DIM)\n",
    "val = MulticlassData(n=30, dim=DIM)\n",
    "\n",
    "dl_train = DataLoader(train, batch_size=8, shuffle=True)\n",
    "dl_val = DataLoader(val, batch_size=8, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor([[0.2199, 0.8899],\n",
       "         [0.5928, 0.9209],\n",
       "         [0.1033, 0.7044],\n",
       "         [0.7221, 0.8767],\n",
       "         [0.9785, 0.8563],\n",
       "         [0.1482, 0.4460],\n",
       "         [0.1631, 0.2388],\n",
       "         [0.3072, 0.0535]]), tensor([1, 1, 1, 3, 4, 1, 1, 1])]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(iter(dl_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Trainer(criterion='cross_entropy', out_dir='../data/v1')\n",
       "\n",
       "Datasets: 34 train rows, 30 val rows\n",
       "\n",
       "Optimizer: None\n",
       "\n",
       "SimpleMulticlassModel(\n",
       "  (fc1): Linear(in_features=2, out_features=10, bias=True)\n",
       "  (fc2): Linear(in_features=10, out_features=5, bias=True)\n",
       "))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smnet = SimpleMulticlassModel(DIM, 5)\n",
    "t = Trainer(smnet, train, val, dl_train, dl_val, \n",
    "            F.cross_entropy, 'multiclass', '../data/v1', torch.optim.RMSprop, \n",
    "            last_act=F.softmax, metrics=metrics, callbacks=callbacks)\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9766fc8e8a34de2bf2dd3c4c0bb2462",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-18 15:57:20,486\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 0\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.5430 |       1.7918 |\n",
      "| accuracy             |  0.2647 |       0.3667 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-18 15:57:20,487\n",
      " Saving model. Loss improved from inf to 1.7918.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4792d3cf4d7f46fabbd191fa5735f8d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-18 15:57:20,525\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 1\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.3794 |       1.4423 |\n",
      "| accuracy             |  0.5000 |       0.3667 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-18 15:57:20,526\n",
      " Saving model. Loss improved from 1.7918 to 1.4423.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb9b4002d9ee4c21b58980bfde36c2c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-18 15:57:20,561\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 2\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.2873 |       1.4404 |\n",
      "| accuracy             |  0.4706 |       0.3333 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-18 15:57:20,562\n",
      " Saving model. Loss improved from 1.4423 to 1.4404.\n",
      "Data written to ../data/v1/trainer.pkl.\n",
      "\n",
      "2020-02-18 15:57:20,564\n",
      " Epoch 3: Unfreezing last 2 layers.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8481a7516727408d8995cc3b08710489",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-18 15:57:20,596\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 3\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.2710 |       1.4384 |\n",
      "| accuracy             |  0.4706 |       0.3333 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-18 15:57:20,597\n",
      " Saving model. Loss improved from 1.4404 to 1.4384.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36bdf825faa54e40a7ee543288668190",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-18 15:57:20,628\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 4\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.2655 |       1.4421 |\n",
      "| accuracy             |  0.4706 |       0.3667 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-18 15:57:20,629\n",
      " EarlyStopper halting training: validation accuracy has not improved enough in 4 epochs.\n",
      "\n",
      "2020-02-18 15:57:20,630\n",
      " Training complete. Model in eval mode.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/6 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File: ../data/v1/train.log\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v2/train.log\n",
      "File: ../data/v1/history.csv\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v2/history.csv\n",
      "File: ../data/v1/trainer.pkl\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v2/trainer.pkl\n",
      "File: ../data/v1/trainer.zip\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v2/trainer.zip\n",
      "File: ../data/v1/lrs.png\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v2/lrs.png\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/6 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File: ../data/v1/history.png\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v2/history.png\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 35236 characters]\n",
      "text/plain": [
       "<Figure size 864x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "t.fit(10, 3e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "*x, y = next(iter(dl_train))\n",
    "x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat = smnet(*x)\n",
    "yhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y.shape, yhat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "F.softmax(yhat, dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "F.cross_entropy(yhat, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Trainer(smnet, train, val, dl_train, dl_val, F.cross_entropy, 'multiclass',\n",
    "        '../data/v2', 'datascience-delphi-dev', last_act=partial(F.softmax, dim=-1),\n",
    "            metrics=metrics, callbacks=callbacks)\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from functools import wraps\n",
    "\n",
    "def catch_keyboard_interrupt(func):\n",
    "    @wraps(func)\n",
    "    def wrapper(*args, **kwargs):\n",
    "        try:\n",
    "            func(*args, **kwargs)\n",
    "        except KeyboardInterrupt:\n",
    "            print('interrupt', dir(func))\n",
    "        return\n",
    "    return wrapper\n",
    "\n",
    "class Foo:\n",
    "    @catch_keyboard_interrupt\n",
    "    def bar(a, b=3):\n",
    "        for i in range(b):\n",
    "            time.sleep(1)\n",
    "            print(a)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
