{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:12.634404Z",
     "start_time": "2020-06-02T18:51:12.241902Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:14.601601Z",
     "start_time": "2020-06-02T18:51:12.636529Z"
    }
   },
   "outputs": [],
   "source": [
    "from comet_ml import Experiment\n",
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "from sklearn.metrics import (accuracy_score, dcg_score, roc_auc_score, \n",
    "                             precision_score, recall_score)\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import Adam\n",
    "\n",
    "from htools import assert_raises, save, load, hdir\n",
    "from incendio.core import *\n",
    "from incendio.callbacks import *\n",
    "from incendio.metrics import *\n",
    "from incendio.optimizers import *\n",
    "from incendio.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:14.633414Z",
     "start_time": "2020-06-02T18:51:14.603595Z"
    }
   },
   "outputs": [],
   "source": [
    "# Reproducible testing.\n",
    "np.random.seed(0)\n",
    "torch.manual_seed(0)\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:14.666781Z",
     "start_time": "2020-06-02T18:51:14.635338Z"
    }
   },
   "outputs": [],
   "source": [
    "class Data(Dataset):\n",
    "    \n",
    "    def __init__(self, n=64, dim=2):\n",
    "        self.x = torch.rand(n, dim).float()\n",
    "        self.y = torch.clamp(\n",
    "            (self.x[:, 0]*.75 + self.x[:, 1]*.25).round(), 0, 1\n",
    "        ).abs().unsqueeze(-1)\n",
    "        \n",
    "    def __getitem__(self, i):\n",
    "        return self.x[i], self.y[i]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:14.698203Z",
     "start_time": "2020-06-02T18:51:14.668300Z"
    }
   },
   "outputs": [],
   "source": [
    "class EasyDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, n=10, scalar=8):\n",
    "        \"\"\"Larger scalar makes data easier to classify.\"\"\"\n",
    "        self.x = torch.rand(n).float().unsqueeze(-1)\n",
    "        self.y = (self.x + torch.randn_like(self.x)/scalar > 0.5).float()\n",
    "        \n",
    "    def __getitem__(self, i):\n",
    "        return self.x[i], self.y[i]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:14.729333Z",
     "start_time": "2020-06-02T18:51:14.699675Z"
    }
   },
   "outputs": [],
   "source": [
    "class MulticlassData(Dataset):\n",
    "    \n",
    "    def __init__(self, n=64, dim=2):\n",
    "        self.x = torch.rand(n, dim).float()\n",
    "        # Integer labels between 0 and 4, inclusive.\n",
    "        self.y = torch.clamp(\n",
    "            torch.round(\n",
    "                torch.randint(6, (n, 1)).float() \n",
    "                * (self.x[:, 0]*.75 + self.x[:, 1]*.25).unsqueeze(-1)\n",
    "            ), 0, 4).long().flatten()\n",
    "        \n",
    "    def __getitem__(self, i):\n",
    "        return self.x[i], self.y[i]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:14.765057Z",
     "start_time": "2020-06-02T18:51:14.731021Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[0.4963, 0.7682],\n",
       "         [0.0885, 0.1320],\n",
       "         [0.3074, 0.6341],\n",
       "         [0.4901, 0.8964],\n",
       "         [0.4556, 0.6323]]), tensor([0, 1, 1, 0, 1]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ypred_ = torch.rand(5, 2)\n",
    "y_ = torch.tensor([0, 1, 1, 0, 1])\n",
    "ypred_, y_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:14.795451Z",
     "start_time": "2020-06-02T18:51:14.767622Z"
    }
   },
   "outputs": [],
   "source": [
    "class SimpleModel(BaseModel):\n",
    "    \n",
    "    def __init__(self, dim):\n",
    "        super().__init__()  \n",
    "        self.fc1 = nn.Linear(dim, 2)\n",
    "        self.fc2 = nn.Linear(2, 1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.leaky_relu(self.fc1(x))\n",
    "        return self.fc2(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:14.825940Z",
     "start_time": "2020-06-02T18:51:14.797562Z"
    }
   },
   "outputs": [],
   "source": [
    "class GroupedModel(BaseModel):\n",
    "    \n",
    "    def __init__(self, dim):\n",
    "        super().__init__()  \n",
    "        g1 = nn.Sequential(\n",
    "            nn.Linear(dim, 8),\n",
    "            nn.LeakyReLU(),\n",
    "            nn.Linear(8, 4),\n",
    "            nn.LeakyReLU()\n",
    "        )\n",
    "        g2 = nn.Linear(4, 1)\n",
    "        self.groups = nn.ModuleList([g1, g2])\n",
    "        \n",
    "    def forward(self, x):\n",
    "        for group in self.groups:\n",
    "            x = group(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:14.856799Z",
     "start_time": "2020-06-02T18:51:14.827609Z"
    }
   },
   "outputs": [],
   "source": [
    "DIM = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:14.890637Z",
     "start_time": "2020-06-02T18:51:14.859234Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SimpleModel(\n",
       "  (fc1): Linear(in_features=2, out_features=2, bias=True)\n",
       "  (fc2): Linear(in_features=2, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snet = SimpleModel(DIM)\n",
    "snet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:14.924398Z",
     "start_time": "2020-06-02T18:51:14.892455Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Adam (\n",
       "Parameter Group 0\n",
       "    amsgrad: False\n",
       "    betas: (0.9, 0.999)\n",
       "    eps: 0.001\n",
       "    lr: 0.002\n",
       "    weight_decay: 0\n",
       ")"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optim = variable_lr_optimizer(snet, 2e-3)\n",
    "optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:14.954753Z",
     "start_time": "2020-06-02T18:51:14.926767Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As expected, got ValueError(Received more learning rates than layer groups.).\n"
     ]
    }
   ],
   "source": [
    "with assert_raises(ValueError) as ar:\n",
    "    optim = variable_lr_optimizer(snet, [3e-3, 1e-1])\n",
    "    optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:14.983845Z",
     "start_time": "2020-06-02T18:51:14.956478Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Adam (\n",
       "Parameter Group 0\n",
       "    amsgrad: False\n",
       "    betas: (0.9, 0.999)\n",
       "    eps: 0.001\n",
       "    lr: 0.001\n",
       "    weight_decay: 0\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "update_optimizer(optim, 1e-3, 0.5)\n",
    "optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:15.017293Z",
     "start_time": "2020-06-02T18:51:14.985551Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 [((2, 2), False), ((2,), False), ((1, 2), False), ((1,), False)]\n",
      "1 [((2, 2), False), ((2,), False), ((1, 2), False), ((1,), True)]\n",
      "2 [((2, 2), False), ((2,), False), ((1, 2), True), ((1,), True)]\n",
      "3 [((2, 2), False), ((2,), True), ((1, 2), True), ((1,), True)]\n",
      "4 [((2, 2), True), ((2,), True), ((1, 2), True), ((1,), True)]\n"
     ]
    }
   ],
   "source": [
    "snet.freeze()\n",
    "for n in range(5):\n",
    "    snet.unfreeze(n_layers=n)\n",
    "    print(n, snet.trainable())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:15.048038Z",
     "start_time": "2020-06-02T18:51:15.019101Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As expected, got AttributeError('SimpleModel' object has no attribute 'groups').\n"
     ]
    }
   ],
   "source": [
    "snet.freeze()\n",
    "with assert_raises(AttributeError) as ar:\n",
    "    for n in range(3):\n",
    "        snet.unfreeze(n_groups=n)\n",
    "        print(n, snet.trainable())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:15.077868Z",
     "start_time": "2020-06-02T18:51:15.049401Z"
    }
   },
   "outputs": [],
   "source": [
    "from collections.abc import Iterable, Mapping\n",
    "from functools import partial\n",
    "\n",
    "from htools import flatten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:52:38.163867Z",
     "start_time": "2020-06-02T18:52:38.135482Z"
    }
   },
   "outputs": [],
   "source": [
    "def is_builtin(x, drop_callables=True):\n",
    "    def _builtin(x, drop_callables):\n",
    "        if callable(x) and drop_callables:\n",
    "            return False\n",
    "        return x.__class__.__module__ == 'builtins'\n",
    "    \n",
    "    builtin = partial(_builtin, drop_callables=drop_callables)\n",
    "    # Check mapping before iterable because mappings are iterable.\n",
    "    if isinstance(x, Mapping):\n",
    "        return builtin(x) and all(builtin(o) for o in flatten(x.items()))\n",
    "    elif isinstance(x, Iterable):\n",
    "        return builtin(x) and all(builtin(o) for o in flatten(x))\n",
    "    return builtin(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TODO:\n",
    "-maybe add val batch metrics\n",
    "\n",
    "-maybe add train metrics (batch and/or epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-03T17:48:28.845852Z",
     "start_time": "2020-06-03T17:48:28.770961Z"
    }
   },
   "outputs": [],
   "source": [
    "class CometCallback(TorchCallback):\n",
    "    \n",
    "    def __init__(self, project_name, order=100):\n",
    "        # High order so this comes after MetricHandler.\n",
    "        self.project_name = project_name\n",
    "        self.order = order\n",
    "        # Define when training begins so reported time is more meaningful.\n",
    "        self.exp = None\n",
    "    \n",
    "    def on_train_begin(self, trainer, epochs, lrs, lr_mult, **kwargs):\n",
    "        self.exp = Experiment(project_name=self.project_name)\n",
    "        params = {k: v for k, v in vars(trainer).items() if is_builtin(v)}\n",
    "        self.exp.log_parameters(\n",
    "            dict(**params, **kwargs, epochs=epochs, lrs=lrs, lr_mult=lr_mult)\n",
    "        )\n",
    "\n",
    "    def on_train_end(self, trainer, epoch, val_stats):\n",
    "        self.exp.end()\n",
    "\n",
    "    def on_epoch_end(self, trainer, epoch, val_stats):\n",
    "        self.exp.log_metrics(trainer.stats, prefix='train', epoch=epoch)\n",
    "        self.exp.log_metrics(val_stats, prefix='val', epoch=epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-03T17:48:39.257773Z",
     "start_time": "2020-06-03T17:48:39.227016Z"
    }
   },
   "outputs": [],
   "source": [
    "DIM = 2\n",
    "metrics = [accuracy_score, \n",
    "           precision_score, \n",
    "           recall_score, \n",
    "           percent_positive,\n",
    "           mean_soft_prediction\n",
    "          ]\n",
    "\n",
    "# Model starts out unfrozen and freezes last group startingn with epoch 3.\n",
    "# This is not useful but is done here for testing purposes.\n",
    "callbacks = [EarlyStopper('accuracy', 'max', patience=3),\n",
    "             PerformanceThreshold('recall', 'max', 0.25, skip_epochs=5),\n",
    "             MetricHistory(),\n",
    "             ModelUnfreezer({3:1}, 'groups'),\n",
    "             ModelCheckpoint(),\n",
    "             CosineLRScheduler(),\n",
    "             S3Uploader('gg-datascience', 'hmamin/incendio/v1'),\n",
    "#              CometCallback('incendio-test')\n",
    "            ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-03T17:48:39.947372Z",
     "start_time": "2020-06-03T17:48:39.916468Z"
    }
   },
   "outputs": [],
   "source": [
    "train = Data(n=10_000, dim=DIM)\n",
    "val = Data(n=30, dim=DIM)\n",
    "\n",
    "dl_train = DataLoader(train, batch_size=8, shuffle=True)\n",
    "dl_val = DataLoader(val, batch_size=8, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-03T17:48:40.250836Z",
     "start_time": "2020-06-03T17:48:40.213929Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Trainer(criterion='binary_cross_entropy_with_logits', out_dir='../data/v5')\n",
       "\n",
       "Datasets: 10000 train rows, 30 val rows\n",
       "\n",
       "Optimizer: None\n",
       "\n",
       "GroupedModel(\n",
       "  (groups): ModuleList(\n",
       "    (0): Sequential(\n",
       "      (0): Linear(in_features=2, out_features=8, bias=True)\n",
       "      (1): LeakyReLU(negative_slope=0.01)\n",
       "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
       "      (3): LeakyReLU(negative_slope=0.01)\n",
       "    )\n",
       "    (1): Linear(in_features=4, out_features=1, bias=True)\n",
       "  )\n",
       "))"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gnet = GroupedModel(DIM)\n",
    "t = Trainer(gnet, train, val, dl_train, dl_val, F.binary_cross_entropy_with_logits, \n",
    "            'binary', '../data/v5', optim_type=torch.optim.RMSprop, \n",
    "            last_act=torch.sigmoid, metrics=metrics, callbacks=callbacks)\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-03T17:48:41.061865Z",
     "start_time": "2020-06-03T17:48:41.031969Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'BasicConfig': BasicConfig(order=0),\n",
       " 'StatsHandler': StatsHandler(order=5),\n",
       " 'MetricPrinter': MetricPrinter(pbar_metric='loss', order=10, batch_freq=1),\n",
       " 'CosineLRScheduler': CosineLRScheduler(warm=0.3, cycle_len=5, cycle_decay=0.0, restarts=False, verbose=False, min_lr=0.03, order=10),\n",
       " 'EarlyStopper': EarlyStopper(order=15, metric='accuracy', min_improvement=0.0, patience=3),\n",
       " 'PerformanceThreshold': PerformanceThreshold(order=15, metric='recall', threshold=0.25, skip_epochs=5, split='val'),\n",
       " 'ModelUnfreezer': ModelUnfreezer(order=25, mode='epoch'),\n",
       " 'ModelCheckpoint': ModelCheckpoint(order=25, metric='loss'),\n",
       " 'MetricHistory': MetricHistory(fname='history.csv', plot_fname='history.png', order=90),\n",
       " 'S3Uploader': S3Uploader(bucket='gg-datascience', prefix='hmamin/incendio/v1', order=95)}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-03T17:48:42.281755Z",
     "start_time": "2020-06-03T17:48:42.253812Z"
    }
   },
   "outputs": [],
   "source": [
    "# t.save('tmp.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-03T17:48:42.818902Z",
     "start_time": "2020-06-03T17:48:42.791610Z"
    }
   },
   "outputs": [],
   "source": [
    "# !ls ../data/v5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-03T17:48:43.365106Z",
     "start_time": "2020-06-03T17:48:43.338211Z"
    }
   },
   "outputs": [],
   "source": [
    "# t.load('tmp.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-03T17:48:45.383185Z",
     "start_time": "2020-06-03T17:48:45.356339Z"
    }
   },
   "outputs": [],
   "source": [
    "# d = load('../data/v1/trainer.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-03T17:50:11.390214Z",
     "start_time": "2020-06-03T17:50:11.363030Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-08-26 21:21:54,114 [INFO]: RMSprop (\n",
      "Parameter Group 0\n",
      "    alpha: 0.99\n",
      "    centered: False\n",
      "    eps: 0.001\n",
      "    lr: 0.03\n",
      "    momentum: 0\n",
      "    weight_decay: 0\n",
      "\n",
      "Parameter Group 1\n",
      "    alpha: 0.99\n",
      "    centered: False\n",
      "    eps: 0.001\n",
      "    lr: 0.3\n",
      "    momentum: 0\n",
      "    weight_decay: 0\n",
      ")\n",
      "2020-08-26 21:21:54,114 [INFO]: Removing files from output directory.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2c66a9c754b4839bf6653bcdb69a46e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e2f76caf1a14bb182fc5a6eff3cfce9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=4.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=====\n",
      "\n",
      "Epoch 0\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.1223 |       0.0309 |\n",
      "| accuracy             |  0.9531 |       1.0000 |\n",
      "| precision            |  0.9487 |       1.0000 |\n",
      "| recall               |  0.9534 |       1.0000 |\n",
      "| percent_positive     |  0.4957 |       0.4333 |\n",
      "| mean_soft_prediction |  0.4933 |       0.4532 |\n",
      "\n",
      "\n",
      "Saving model. Loss improved from inf to 0.0309.\n",
      "Writing data to ../data/v5/best_val_metrics.json.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2fd698acfab140eea961efcdd8d41ae6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a33d5ff3548e4b6bbe6a937ddfad095f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=4.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=====\n",
      "\n",
      "Epoch 1\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.0396 |       0.0307 |\n",
      "| accuracy             |  0.9823 |       0.9667 |\n",
      "| precision            |  0.9791 |       1.0000 |\n",
      "| recall               |  0.9766 |       0.9111 |\n",
      "| percent_positive     |  0.4921 |       0.4000 |\n",
      "| mean_soft_prediction |  0.4914 |       0.4134 |\n",
      "\n",
      "\n",
      "Saving model. Loss improved from 0.0309 to 0.0307.\n",
      "Writing data to ../data/v5/best_val_metrics.json.\n",
      "\n",
      "Training complete. Model in eval mode.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/6 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File: ../data/v5/history.csv\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v1/history.csv\n",
      "File: ../data/v5/train.log\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v1/train.log\n",
      "File: ../data/v5/best_val_metrics.json\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v1/best_val_metrics.json\n",
      "File: ../data/v5/trainer.pkl\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v1/trainer.pkl\n",
      "File: ../data/v5/lrs.png\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v1/lrs.png\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/6 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File: ../data/v5/history.png\n",
      "Bucket: gg-datascience\n",
      "Key: hmamin/incendio/v1/history.png\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 72372 characters]\n",
      "text/plain": [
       "<Figure size 864x648 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "t.fit(2, 3e-1, .1, clean=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_val_metrics.json history.png           train.log\n",
      "history.csv           lrs.png               trainer.pkl\n"
     ]
    }
   ],
   "source": [
    "!ls ../data/v5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t.load('trainer.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Object loaded from ../data/v1/trainer.zip.\n",
      "Trainer(criterion='binary_cross_entropy_with_logits', out_dir='../data/v1')\n",
      "\n",
      "Datasets: 300 train rows, 30 val rows\n",
      "\n",
      "Optimizer: RMSprop (\n",
      "Parameter Group 0\n",
      "    alpha: 0.99\n",
      "    centered: False\n",
      "    eps: 0.001\n",
      "    lr: 0.015052040053751204\n",
      "    momentum: 0\n",
      "    weight_decay: 0\n",
      "\n",
      "Parameter Group 1\n",
      "    alpha: 0.99\n",
      "    centered: False\n",
      "    eps: 0.001\n",
      "    lr: 0.030104080107502408\n",
      "    momentum: 0\n",
      "    weight_decay: 0\n",
      ")\n",
      "\n",
      "GroupedModel(\n",
      "  (groups): ModuleList(\n",
      "    (0): Sequential(\n",
      "      (0): Linear(in_features=2, out_features=8, bias=True)\n",
      "      (1): LeakyReLU(negative_slope=0.01)\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): LeakyReLU(negative_slope=0.01)\n",
      "    )\n",
      "    (1): Linear(in_features=4, out_features=1, bias=True)\n",
      "  )\n",
      "))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "19995"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2 = Trainer.from_file(os.path.join('..', 'data', 'v1', 'trainer.zip'))\n",
    "print(t2)\n",
    "del t2; gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "Interrupt\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    for i in range(10):\n",
    "        time.sleep(1)\n",
    "        print(i)\n",
    "except KeyboardInterrupt:\n",
    "    print('Interrupt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test is_builtin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:55.101327Z",
     "start_time": "2020-06-02T18:51:55.066249Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_builtin([{'3': t}])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:15.408724Z",
     "start_time": "2020-06-02T18:51:12.309Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_builtin([2,4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(True, False)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# classes have class `type` so are always considered builtin (if we specify\n",
    "# drop_callable=False). Can't remember what my original use case was so not\n",
    "# sure if this is desired.\n",
    "is_builtin(F.relu, False), is_builtin(nn.ReLU(), False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:15.409781Z",
     "start_time": "2020-06-02T18:51:12.311Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.optim.rmsprop.RMSprop'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(False, True)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(t.optim_type)\n",
    "is_builtin(t.optim_type), is_builtin(t.optim_type, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:15.410563Z",
     "start_time": "2020-06-02T18:51:12.313Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(False, True)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_builtin(torch.sigmoid), is_builtin(torch.sigmoid, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:15.411727Z",
     "start_time": "2020-06-02T18:51:12.315Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_builtin([1,2,3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:15.412496Z",
     "start_time": "2020-06-02T18:51:12.317Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(False, True)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_builtin([TorchCallback, Trainer]), is_builtin([TorchCallback, Trainer], False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:15.413432Z",
     "start_time": "2020-06-02T18:51:12.319Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_builtin({'a': 1, 'c': 33, 'd': [2, 4]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:15.414794Z",
     "start_time": "2020-06-02T18:51:12.321Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(False, True)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_builtin({'a': 1, 'c': 33, 't': t.optim_type}), is_builtin({'a': 1, 'c': 33, 't': t.optim_type}, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:15.415726Z",
     "start_time": "2020-06-02T18:51:12.323Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'optim': None,\n",
       " 'eps': 0.001,\n",
       " 'mode': 'binary',\n",
       " 'thresh': 0.5,\n",
       " '_stop_training': False,\n",
       " 'out_dir': '../data/v5'}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = {k: v for k, v in vars(t).items() if is_builtin(v)}\n",
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:15.416671Z",
     "start_time": "2020-06-02T18:51:12.325Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'optim': None,\n",
       " 'eps': 0.001,\n",
       " 'mode': 'binary',\n",
       " 'thresh': 0.5,\n",
       " '_stop_training': False,\n",
       " 'out_dir': '../data/v5',\n",
       " 'c': 3333,\n",
       " 'z': [44, 55],\n",
       " 'x': True,\n",
       " 'e': 44}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d2 = dict(c=3333, z=[44, 55], x=True)\n",
    "{**d, **d2, 'e': 44}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-02T18:51:15.417739Z",
     "start_time": "2020-06-02T18:51:12.327Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'optim': None,\n",
       " 'eps': 0.001,\n",
       " 'mode': 'binary',\n",
       " 'thresh': 0.5,\n",
       " '_stop_training': False,\n",
       " 'out_dir': '../data/v5',\n",
       " 'c': 3333,\n",
       " 'z': [44, 55],\n",
       " 'x': True,\n",
       " 'abc': 33,\n",
       " 'xyz': 'abc'}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(d, **d2, abc=33, xyz='abc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sawtooth Scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIM = 2\n",
    "metrics = [accuracy_score, \n",
    "           precision_score, \n",
    "           recall_score]\n",
    "\n",
    "callbacks = [MetricHistory(),\n",
    "             AdaptiveSawtoothScheduler(1e-2, .8)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = EasyDataset(400, 100)\n",
    "val = EasyDataset(40, 100)\n",
    "\n",
    "dl_train = DataLoader(train, batch_size=8, shuffle=True)\n",
    "dl_val = DataLoader(val, batch_size=8, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Trainer(criterion='binary_cross_entropy_with_logits', out_dir='../data/v1')\n",
       "\n",
       "Datasets: 400 train rows, 40 val rows\n",
       "\n",
       "Optimizer: None\n",
       "\n",
       "SimpleModel(\n",
       "  (fc1): Linear(in_features=1, out_features=2, bias=True)\n",
       "  (fc2): Linear(in_features=2, out_features=1, bias=True)\n",
       "))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snet = SimpleModel(1)\n",
    "t = Trainer(snet, train, val, dl_train, dl_val, \n",
    "            F.binary_cross_entropy_with_logits, 'multiclass', '../data/v1', \n",
    "            metrics=metrics, callbacks=callbacks)\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-25 21:17:05,968 [INFO]: Adam (\n",
      "Parameter Group 0\n",
      "    amsgrad: False\n",
      "    betas: (0.9, 0.999)\n",
      "    eps: 0.001\n",
      "    lr: 0.1\n",
      "    weight_decay: 0\n",
      ")\n",
      "2020-03-25 21:17:05,968 [INFO]: Removing files from output directory.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "877fa9bde1e140d497acbce20d3e8be4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-03-25 21:17:06,163\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 0\n",
      "\n",
      "| Metric    |   Train |   Validation |\n",
      "|-----------|---------|--------------|\n",
      "| loss      |  0.3081 |       0.0764 |\n",
      "| accuracy  |  0.5150 |       0.4500 |\n",
      "| precision |  0.0000 |       0.0000 |\n",
      "| recall    |  0.0000 |       0.0000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13a41a12560f4082b4888511fc5eee64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-03-25 21:17:06,340\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 1\n",
      "\n",
      "| Metric    |   Train |   Validation |\n",
      "|-----------|---------|--------------|\n",
      "| loss      |  0.0940 |       0.0453 |\n",
      "| accuracy  |  0.5150 |       0.4500 |\n",
      "| precision |  0.0000 |       0.0000 |\n",
      "| recall    |  0.0000 |       0.0000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed046828c27044bb97146f06c49e2c90",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-03-25 21:17:06,508\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 2\n",
      "\n",
      "| Metric    |   Train |   Validation |\n",
      "|-----------|---------|--------------|\n",
      "| loss      |  0.0738 |       0.0420 |\n",
      "| accuracy  |  0.5150 |       0.4500 |\n",
      "| precision |  0.0000 |       0.0000 |\n",
      "| recall    |  0.0000 |       0.0000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24ce478c2ab44219ba05bef640a22508",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-03-25 21:17:06,658\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 3\n",
      "\n",
      "| Metric    |   Train |   Validation |\n",
      "|-----------|---------|--------------|\n",
      "| loss      |  0.0699 |       0.0422 |\n",
      "| accuracy  |  0.5150 |       0.4500 |\n",
      "| precision |  0.0000 |       0.0000 |\n",
      "| recall    |  0.0000 |       0.0000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9108ac388b8c4b35905e56360423ef3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-03-25 21:17:06,817\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 4\n",
      "\n",
      "| Metric    |   Train |   Validation |\n",
      "|-----------|---------|--------------|\n",
      "| loss      |  0.0695 |       0.0422 |\n",
      "| accuracy  |  0.5150 |       0.4500 |\n",
      "| precision |  0.0000 |       0.0000 |\n",
      "| recall    |  0.0000 |       0.0000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-03-25 21:17:06,818\n",
      " Training complete. Model in eval mode.\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 32848 characters]\n",
      "text/plain": [
       "<Figure size 864x576 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "t.fit(5, 1e-1, clean=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t.callbacks['AdaptiveSawtoothScheduler'].plot_lrs('adaptive_sawtooth_lrs.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multiclass classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleMulticlassModel(BaseModel):\n",
    "    \n",
    "    def __init__(self, dim, classes):\n",
    "        super().__init__()  \n",
    "        self.fc1 = nn.Linear(dim, 10)\n",
    "        self.fc2 = nn.Linear(10, classes)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.leaky_relu(self.fc1(x))\n",
    "        return self.fc2(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIM = 2\n",
    "metrics = [accuracy_score,\n",
    "           mean_soft_prediction\n",
    "          ]\n",
    "\n",
    "callbacks = [\n",
    "#              EarlyStopper('accuracy', 'max', patience=10),\n",
    "             PerformanceThreshold('loss', 'min', 2, skip_epochs=5),\n",
    "             MetricHistory(),\n",
    "             ModelCheckpoint(),\n",
    "             MetricHistory(),\n",
    "             CosineLRScheduler(),\n",
    "             S3Uploader('gg-datascience', 'hmamin/incendio/v2')\n",
    "            ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = MulticlassData(n=88, dim=DIM)\n",
    "val = MulticlassData(n=40, dim=DIM)\n",
    "\n",
    "dl_train = DataLoader(train, batch_size=8, shuffle=True)\n",
    "dl_val = DataLoader(val, batch_size=8, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Trainer(criterion='cross_entropy', out_dir='../data/v1')\n",
       "\n",
       "Datasets: 88 train rows, 40 val rows\n",
       "\n",
       "Optimizer: None\n",
       "\n",
       "SimpleMulticlassModel(\n",
       "  (fc1): Linear(in_features=2, out_features=10, bias=True)\n",
       "  (fc2): Linear(in_features=10, out_features=5, bias=True)\n",
       "))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smnet = SimpleMulticlassModel(DIM, 5)\n",
    "t = Trainer(smnet, train, val, dl_train, dl_val, \n",
    "            F.cross_entropy, 'multiclass', '../data/v1',\n",
    "            last_act=F.softmax, metrics=metrics, callbacks=callbacks)\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adam (\n",
      "Parameter Group 0\n",
      "    amsgrad: False\n",
      "    betas: (0.9, 0.999)\n",
      "    eps: 0.001\n",
      "    lr: 0.3\n",
      "    weight_decay: 0\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c0a41ffbad040a8a0b6670175a2804e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,246\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 0\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.5746 |       1.5252 |\n",
      "| accuracy             |  0.2273 |       0.3000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:50:12,247\n",
      " Saving model. Loss improved from inf to 1.5252.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "231379ff01a447e28e52286d9309a46e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,286\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 1\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.4841 |       1.5531 |\n",
      "| accuracy             |  0.3636 |       0.3000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dad830bd3e094a74a62c85bd0d97c487",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,445\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 2\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.4658 |       1.5053 |\n",
      "| accuracy             |  0.2841 |       0.2750 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:50:12,446\n",
      " Saving model. Loss improved from 1.5252 to 1.5053.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8f801661ced478a8772e55865f8d82c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,486\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 3\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.4311 |       1.4825 |\n",
      "| accuracy             |  0.3523 |       0.3000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:50:12,487\n",
      " Saving model. Loss improved from 1.5053 to 1.4825.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07b0c95e1b5f471b9c04a2281bbc5abd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,528\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 4\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.3938 |       1.4890 |\n",
      "| accuracy             |  0.3182 |       0.3000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71876c30c4224f6b88abdbb4c488e2f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,567\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 5\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.3632 |       1.4671 |\n",
      "| accuracy             |  0.3295 |       0.3750 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:50:12,568\n",
      " Saving model. Loss improved from 1.4825 to 1.4671.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43a4390427224d658181068ecc5bf565",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,607\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 6\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.3797 |       1.5319 |\n",
      "| accuracy             |  0.3750 |       0.4000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cc710da780e4451ac2f92202ebf8fde",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,645\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 7\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.4439 |       1.4302 |\n",
      "| accuracy             |  0.2500 |       0.3750 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:50:12,646\n",
      " Saving model. Loss improved from 1.4671 to 1.4302.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bef989343a1d4dbb84ab65cda4907cdc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,683\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 8\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.4423 |       1.4877 |\n",
      "| accuracy             |  0.3750 |       0.2750 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2a93e0eff124bccac616873dfeb66fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,720\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 9\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.4118 |       1.4260 |\n",
      "| accuracy             |  0.2614 |       0.3750 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:50:12,721\n",
      " Saving model. Loss improved from 1.4302 to 1.4260.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7f5d548fbdf4c66bd5bdd1fb66faba4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,762\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 10\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.3246 |       1.6506 |\n",
      "| accuracy             |  0.3409 |       0.3000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e422fcfd19bb446e82ac2654faa4a1df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,800\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 11\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.4122 |       1.4588 |\n",
      "| accuracy             |  0.3636 |       0.2250 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b8c9015644f4af58ee9e2e54a4e40e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,838\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 12\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.3587 |       1.5368 |\n",
      "| accuracy             |  0.3750 |       0.3000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57dc879feea046ff97f40608bc7fbd1e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,879\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 13\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.3318 |       1.4431 |\n",
      "| accuracy             |  0.3523 |       0.4000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b20a7a78de140cdb1cb126b66ac60fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,918\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 14\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.3460 |       1.4037 |\n",
      "| accuracy             |  0.3636 |       0.4000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:50:12,919\n",
      " Saving model. Loss improved from 1.4260 to 1.4037.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8531f2f032f4d9bbe0432aea0e9121b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:12,959\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 15\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.3139 |       1.4650 |\n",
      "| accuracy             |  0.3864 |       0.4000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0514564cdcf24977995404055e90b3e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:13,000\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 16\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.3258 |       1.4417 |\n",
      "| accuracy             |  0.3864 |       0.4000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cf3aefc29ce46fe8fbb39c2731b5f99",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:13,039\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 17\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.2968 |       1.4104 |\n",
      "| accuracy             |  0.3864 |       0.4000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f218d5b5367140cea3c8fee4c26bf1cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:13,079\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 18\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.2847 |       1.4272 |\n",
      "| accuracy             |  0.3977 |       0.4000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "82bb05be1dd44910a3e1a35e3e14e586",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=11), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:50:13,117\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 19\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  1.2826 |       1.4363 |\n",
      "| accuracy             |  0.3864 |       0.4000 |\n",
      "| mean_soft_prediction |  0.2000 |       0.2000 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:50:13,118\n",
      " Training complete. Model in eval mode.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2020-02-19 16:50:13,808\n",
      " Failed to upload ../data/v1/train.log to gg-datascience/hmamin/incendio/v2/train.log: An error occurred (ExpiredToken) when calling the PutObject operation: The provided token has expired.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 73428 characters]\n",
      "text/plain": [
       "<Figure size 864x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "t.fit(20, .3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Easy binary classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIM = 2\n",
    "metrics = [accuracy_score,\n",
    "           mean_soft_prediction\n",
    "          ]\n",
    "\n",
    "# Model starts out unfrozen and freezes last group startingn with epoch 3.\n",
    "# This is not useful but is done here for testing purposes.\n",
    "callbacks = [\n",
    "    MetricHistory(),\n",
    "    ModelCheckpoint(),\n",
    "    MetricHistory(),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = EasyDataset(400, 100)\n",
    "val = EasyDataset(40, 100)\n",
    "\n",
    "dl_train = DataLoader(train, batch_size=8, shuffle=True)\n",
    "dl_val = DataLoader(val, batch_size=8, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Trainer(criterion='binary_cross_entropy_with_logits', out_dir='../data/v1')\n",
       "\n",
       "Datasets: 400 train rows, 40 val rows\n",
       "\n",
       "Optimizer: None\n",
       "\n",
       "SimpleModel(\n",
       "  (fc1): Linear(in_features=1, out_features=2, bias=True)\n",
       "  (fc2): Linear(in_features=2, out_features=1, bias=True)\n",
       "))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snet = SimpleModel(1)\n",
    "t = Trainer(snet, train, val, dl_train, dl_val, \n",
    "            F.binary_cross_entropy_with_logits, 'multiclass', '../data/v1', \n",
    "            last_act=torch.sigmoid, metrics=metrics, callbacks=callbacks)\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adam (\n",
      "Parameter Group 0\n",
      "    amsgrad: False\n",
      "    betas: (0.9, 0.999)\n",
      "    eps: 0.001\n",
      "    lr: 0.1\n",
      "    weight_decay: 0\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fbb8ee730fb3475680062d6412f9a84b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:53:31,492\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 0\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.6242 |       0.4281 |\n",
      "| accuracy             |  0.5125 |       0.4250 |\n",
      "| mean_soft_prediction |  0.4767 |       0.4384 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:53:31,492\n",
      " Saving model. Loss improved from inf to 0.4281.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "741ce96186cc43b996200aaeabdc04d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:53:31,576\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 1\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.2702 |       0.1758 |\n",
      "| accuracy             |  0.5125 |       0.4250 |\n",
      "| mean_soft_prediction |  0.4262 |       0.5940 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:53:31,577\n",
      " Saving model. Loss improved from 0.4281 to 0.1758.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af1187fbe1bf4209b58febcf1b75b7c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:53:31,653\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 2\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.1424 |       0.1053 |\n",
      "| accuracy             |  0.5125 |       0.4250 |\n",
      "| mean_soft_prediction |  0.4689 |       0.5977 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:53:31,654\n",
      " Saving model. Loss improved from 0.1758 to 0.1053.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0e312e2f7274da89121339f9a910f1a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:53:31,734\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 3\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.0963 |       0.0617 |\n",
      "| accuracy             |  0.5125 |       0.4250 |\n",
      "| mean_soft_prediction |  0.4778 |       0.5728 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:53:31,734\n",
      " Saving model. Loss improved from 0.1053 to 0.0617.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12d92fc9b8424a408c2ce09f42e49c74",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:53:31,812\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 4\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.0894 |       0.0497 |\n",
      "| accuracy             |  0.5125 |       0.4250 |\n",
      "| mean_soft_prediction |  0.4825 |       0.5574 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:53:31,812\n",
      " Saving model. Loss improved from 0.0617 to 0.0497.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbfc7b7147574eca961b904d898fc5b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:53:31,890\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 5\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.0752 |       0.0413 |\n",
      "| accuracy             |  0.5125 |       0.4250 |\n",
      "| mean_soft_prediction |  0.4840 |       0.5723 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:53:31,891\n",
      " Saving model. Loss improved from 0.0497 to 0.0413.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93c19aa3c388407b9d6c2097cb8bd4ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:53:31,970\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 6\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.0694 |       0.0355 |\n",
      "| accuracy             |  0.5125 |       0.4250 |\n",
      "| mean_soft_prediction |  0.4844 |       0.5674 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:53:31,972\n",
      " Saving model. Loss improved from 0.0413 to 0.0355.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "513f4362d5c24cd0bcd8bdcafea342a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:53:32,051\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 7\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.0642 |       0.0316 |\n",
      "| accuracy             |  0.5125 |       0.4250 |\n",
      "| mean_soft_prediction |  0.4819 |       0.5716 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:53:32,051\n",
      " Saving model. Loss improved from 0.0355 to 0.0316.\n",
      "Data written to ../data/v1/trainer.pkl.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "078e31ce9983474eade38411300f1fb5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:53:32,128\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 8\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.0623 |       0.0731 |\n",
      "| accuracy             |  0.5125 |       0.4250 |\n",
      "| mean_soft_prediction |  0.4835 |       0.6086 |\n",
      "\n",
      "=====\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "899f415695fc456081dd1d327edef288",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "2020-02-19 16:53:32,208\n",
      " \n",
      "=====\n",
      "\n",
      "Epoch 9\n",
      "\n",
      "| Metric               |   Train |   Validation |\n",
      "|----------------------|---------|--------------|\n",
      "| loss                 |  0.0514 |       0.0296 |\n",
      "| accuracy             |  0.5125 |       0.4250 |\n",
      "| mean_soft_prediction |  0.4870 |       0.5620 |\n",
      "\n",
      "=====\n",
      "\n",
      "2020-02-19 16:53:32,209\n",
      " Saving model. Loss improved from 0.0316 to 0.0296.\n",
      "Data written to ../data/v1/trainer.pkl.\n",
      "\n",
      "2020-02-19 16:53:32,211\n",
      " Training complete. Model in eval mode.\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 28508 characters]\n",
      "text/plain": [
       "<Figure size 864x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "t.fit(10, .1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import inspect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[SimpleModel(\n",
       "   (fc1): Linear(in_features=1, out_features=2, bias=True)\n",
       "   (fc2): Linear(in_features=2, out_features=1, bias=True)\n",
       " ),\n",
       " <__main__.EasyDataset at 0x1444f6a90>,\n",
       " <__main__.EasyDataset at 0x1444f6d10>,\n",
       " <torch.utils.data.dataloader.DataLoader at 0x1444f65d0>,\n",
       " <torch.utils.data.dataloader.DataLoader at 0x144506210>,\n",
       " <function torch.nn.functional.binary_cross_entropy_with_logits(input, target, weight=None, size_average=None, reduce=None, reduction='mean', pos_weight=None)>,\n",
       " 'multiclass',\n",
       " '../data/v1',\n",
       " torch.optim.adam.Adam,\n",
       " 0.001,\n",
       " <function _VariableFunctions.sigmoid>,\n",
       " None,\n",
       " [<function incendio.metrics.batch_size(y_true, y_pred)>,\n",
       "  <function sklearn.metrics._classification.accuracy_score(y_true, y_pred, normalize=True, sample_weight=None)>,\n",
       "  <function incendio.metrics.mean_soft_prediction(y_true, y_score)>],\n",
       " {'BasicConfig': BasicConfig(priority=0),\n",
       "  'StatsHandler': StatsHandler(priority=5),\n",
       "  'MetricPrinter': MetricPrinter(priority=10),\n",
       "  'ModelCheckpoint': ModelCheckpoint(priority=25, metric='loss'),\n",
       "  'MetricHistory': MetricHistory(fname='history.csv', plot_fname='history.png', priority=90)},\n",
       " device(type='cpu')]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[getattr(t, k, None) for k in inspect.signature(Trainer).parameters]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from spellotape.dl import inverse_sigmoid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5108256237659907"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inverse_sigmoid(.625)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "*x, y = next(iter(dl_train))\n",
    "x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat = smnet(*x)\n",
    "yhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y.shape, yhat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "F.softmax(yhat, dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "F.cross_entropy(yhat, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Trainer(smnet, train, val, dl_train, dl_val, F.cross_entropy, 'multiclass',\n",
    "        '../data/v2', 'datascience-delphi-dev', last_act=partial(F.softmax, dim=-1),\n",
    "            metrics=metrics, callbacks=callbacks)\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<function Fizz.train at 0x14264a0e0>\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from functools import wraps\n",
    "\n",
    "def catch_keyboard_interrupt(func):\n",
    "    @wraps(func)\n",
    "    def wrapper(*args, **kwargs):\n",
    "        try:\n",
    "            func(*args, **kwargs)\n",
    "        except KeyboardInterrupt:\n",
    "            print('interrupt', dir(func))\n",
    "            # This sets a class variable, not an instance var. Works but not ideal.\n",
    "            setattr(eval(Foo.bar.__qualname__.split('.')[0]), 'stop', True)\n",
    "        return\n",
    "    return wrapper\n",
    "\n",
    "\n",
    "def catch_method_interrupt(meth_name):\n",
    "    def decorator(cls):\n",
    "        func = getattr(cls, meth_name)\n",
    "        print(func)\n",
    "        @wraps(func)\n",
    "        def wrapper(*args, **kwargs):\n",
    "            print(args, kwargs)\n",
    "            try:\n",
    "                func(*args, **kwargs)\n",
    "            except KeyboardInterrupt:\n",
    "                print('interrupt', dir(func))\n",
    "                # This sets a class variable, not an instance var. Works but not ideal.\n",
    "                setattr(cls, 'stop', True)\n",
    "            return\n",
    "        return wrapper\n",
    "    return decorator\n",
    "\n",
    "\n",
    "class Foo:\n",
    "    \n",
    "    def __init__(self, a):\n",
    "        self.a = a\n",
    "        \n",
    "    @catch_keyboard_interrupt\n",
    "    def bar(self, b=3):\n",
    "        for i in range(b):\n",
    "            time.sleep(1)\n",
    "            print(self.a)\n",
    "            \n",
    "\n",
    "@catch_method_interrupt('train')\n",
    "class Fizz:\n",
    "\n",
    "        def __init__(self, a):\n",
    "            self.a = a\n",
    "            \n",
    "        def walk(self, t=5):\n",
    "            for i in range(t):\n",
    "                time.sleep(1)\n",
    "                print(i)\n",
    "                \n",
    "        def train(self, epochs):\n",
    "            for e in epochs:\n",
    "                print(e)\n",
    "                time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'a': 5}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f = Foo(5)\n",
    "f.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n",
      "interrupt ['__annotations__', '__call__', '__class__', '__closure__', '__code__', '__defaults__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__get__', '__getattribute__', '__globals__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__kwdefaults__', '__le__', '__lt__', '__module__', '__name__', '__ne__', '__new__', '__qualname__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__']\n"
     ]
    }
   ],
   "source": [
    "f.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'a': 5}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Foo.stop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.stop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6,) {}\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "train() missing 1 required positional argument: 'epochs'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-413-ea99ea3a43b1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mfizz\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFizz\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m6\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mfizz\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwalk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-407-f8b7b7d6cfd8>\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     23\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m                 \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'interrupt'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: train() missing 1 required positional argument: 'epochs'"
     ]
    }
   ],
   "source": [
    "fizz = Fizz(6)\n",
    "fizz.walk()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
