{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T22:41:07.606600Z",
     "start_time": "2020-07-03T22:41:07.566244Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:52:59.582567Z",
     "start_time": "2020-07-03T23:52:59.541895Z"
    }
   },
   "outputs": [],
   "source": [
    "from fastai2.vision.all import (create_body, create_head, create_cnn_model, \n",
    "                                AdaptiveConcatPool2d)\n",
    "from fastai2.vision import models\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from htools import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:05:19.742773Z",
     "start_time": "2020-07-03T23:05:19.700327Z"
    }
   },
   "outputs": [],
   "source": [
    "class ConvBlock(nn.Module):\n",
    "    \n",
    "    def __init__(self, c_in, c_out, kernel=3, **kwargs):\n",
    "        super().__init__()\n",
    "        self.layers = nn.Sequential(nn.Conv2d(c_in, c_out, kernel, **kwargs),\n",
    "                                    nn.BatchNorm2d(c_out),\n",
    "                                    nn.LeakyReLU())\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.layers(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:31:34.514525Z",
     "start_time": "2020-07-03T23:31:34.467275Z"
    }
   },
   "outputs": [],
   "source": [
    "class ConvNet(nn.Module):\n",
    "    \n",
    "    def __init__(self, c_dims, kernel=3, **kwargs):\n",
    "        super().__init__()\n",
    "        self.layers = nn.Sequential(*flatten(\n",
    "            [ConvBlock(c_in, c_out).layers \n",
    "             for c_in, c_out in zip(dims, dims[1:])]\n",
    "        ))\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.layers(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:39:17.198707Z",
     "start_time": "2020-07-03T23:39:17.155729Z"
    }
   },
   "outputs": [],
   "source": [
    "x = torch.randn(2, 3, 128, 128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:40:01.853958Z",
     "start_time": "2020-07-03T23:40:01.799901Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ConvNet(\n",
       "  (layers): Sequential(\n",
       "    (0): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): LeakyReLU(negative_slope=0.01)\n",
       "    (3): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (4): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): LeakyReLU(negative_slope=0.01)\n",
       "    (6): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (8): LeakyReLU(negative_slope=0.01)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dims = [3, 16, 32, 64]\n",
    "net = ConvNet(dims)\n",
    "net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:40:04.075670Z",
     "start_time": "2020-07-03T23:40:03.956198Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 64, 122, 122])"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net(x).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:40:15.877862Z",
     "start_time": "2020-07-03T23:40:15.832212Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 3, 1, 1])"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mp = nn.AdaptiveMaxPool2d(1)\n",
    "mp(x).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:40:16.585670Z",
     "start_time": "2020-07-03T23:40:16.546482Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 3, 1, 1])"
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ap = nn.AdaptiveAvgPool2d(1)\n",
    "ap(x).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:40:19.428462Z",
     "start_time": "2020-07-03T23:40:19.361134Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 6])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 3.6667e+00,  4.1879e+00,  3.5557e+00, -7.1334e-03, -2.3195e-03,\n",
       "         -2.9794e-03],\n",
       "        [ 3.8067e+00,  3.8550e+00,  4.0427e+00, -1.7173e-03, -6.3261e-03,\n",
       "          1.0651e-02]])"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat = torch.cat([mp(x), ap(x)], dim=1).squeeze()\n",
    "print(cat.shape)\n",
    "cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:40:26.777677Z",
     "start_time": "2020-07-03T23:40:26.733632Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 6])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[0.2736, 0.4607, 0.2448, 0.0069, 0.0070, 0.0070],\n",
       "        [0.2957, 0.3103, 0.3743, 0.0066, 0.0065, 0.0066]])"
      ]
     },
     "execution_count": 240,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w = F.softmax(cat, dim=1)\n",
    "print(w.shape)\n",
    "w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:40:28.348144Z",
     "start_time": "2020-07-03T23:40:28.305514Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.0032e+00,  1.9294e+00,  8.7052e-01, -4.9527e-05, -1.6182e-05,\n",
       "         -2.0772e-05],\n",
       "        [ 1.1255e+00,  1.1961e+00,  1.5133e+00, -1.1263e-05, -4.1299e-05,\n",
       "          7.0727e-05]])"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w*cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:40:29.331562Z",
     "start_time": "2020-07-03T23:40:29.237864Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 64, 122, 122])"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x2 = net(x)\n",
    "x2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:56:03.949750Z",
     "start_time": "2020-07-03T23:56:03.899658Z"
    }
   },
   "outputs": [],
   "source": [
    "class AttentiveConcatPool2d(nn.Module):\n",
    "    \n",
    "    def __init__(self, c_in, size=1, soften=True):\n",
    "        super().__init__()\n",
    "        # TODO: logic currently only sound when size > 1\n",
    "        self.adapt = AdaptiveConcatPool2d(size)\n",
    "#         self.temp = temp\n",
    "        self.c_in = c_in\n",
    "        self.soften = soften\n",
    "        self.scalar = self._compute_scalar()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        res = self.adapt(x).squeeze()\n",
    "        weights = res.div(self.scalar).softmax(dim=1)\n",
    "#         return res * weights, weights\n",
    "        return res * weights # Fix later, avoid returning tuple for now for fastai2 compatibility\n",
    "    \n",
    "    def _compute_scalar(self):\n",
    "        # Larger denominator smoothes out softmax. The 2 inside the sqrt\n",
    "        # accounts for the fact that the concat pooling doubles the channel\n",
    "        # dimension (using bert's method of sqrt(n_features) as a reference).\n",
    "        # The one outside the sqrt is just an idea, w/ images we often have a\n",
    "        # number of channels by the time we're ready for the final concat\n",
    "        # pool, whereas bert was operating on vectors of fixed length 64.\n",
    "        # Worried we'd smooth the softmax too much and just get a uniform \n",
    "        # dist.\n",
    "        return np.sqrt(self.c_in*2) / 2 if self.soften else 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:44:57.922828Z",
     "start_time": "2020-07-03T23:44:57.879651Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AttentiveConcatPool2d(\n",
       "  (adapt): AdaptiveConcatPool2d(\n",
       "    (ap): AdaptiveAvgPool2d(output_size=1)\n",
       "    (mp): AdaptiveMaxPool2d(output_size=1)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 253,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attn = AttentiveConcatPool2d(c_in=64)\n",
    "attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:44:58.157729Z",
     "start_time": "2020-07-03T23:44:58.106331Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.656854249492381"
      ]
     },
     "execution_count": 254,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attn.scalar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:44:58.714910Z",
     "start_time": "2020-07-03T23:44:58.655201Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 128])"
      ]
     },
     "execution_count": 255,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y, a = attn(x2)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:44:58.901472Z",
     "start_time": "2020-07-03T23:44:58.841065Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0449, 0.0464, 0.0472, 0.0361, 0.0448, 0.0392, 0.0462, 0.0359, 0.0433,\n",
       "         0.0396, 0.0390, 0.0417, 0.0398, 0.0339, 0.0428, 0.0350, 0.0426, 0.0340,\n",
       "         0.0491, 0.0299, 0.0365, 0.0342, 0.0417, 0.0426, 0.0423, 0.0409, 0.0551,\n",
       "         0.0435, 0.0341, 0.0444, 0.0413, 0.0450, 0.0502, 0.0461, 0.0334, 0.0364,\n",
       "         0.0375, 0.0604, 0.0419, 0.0509, 0.0467, 0.0482, 0.0425, 0.0392, 0.0395,\n",
       "         0.0353, 0.0425, 0.0336, 0.0433, 0.0411, 0.0387, 0.0374, 0.0402, 0.0389,\n",
       "         0.0411, 0.0409, 0.0426, 0.0359, 0.0505, 0.0487, 0.0347, 0.0360, 0.0411,\n",
       "         0.0338, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021,\n",
       "         0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021,\n",
       "         0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021,\n",
       "         0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021,\n",
       "         0.0022, 0.0021, 0.0021, 0.0021, 0.0022, 0.0021, 0.0021, 0.0021, 0.0021,\n",
       "         0.0021, 0.0022, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0020,\n",
       "         0.0020, 0.0022, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021,\n",
       "         0.0020, 0.0021],\n",
       "        [0.0367, 0.0441, 0.0452, 0.0421, 0.0413, 0.0422, 0.0357, 0.0410, 0.0356,\n",
       "         0.0353, 0.0508, 0.0333, 0.0488, 0.0364, 0.0409, 0.0430, 0.0364, 0.0341,\n",
       "         0.0404, 0.0415, 0.0414, 0.0427, 0.0411, 0.0410, 0.0361, 0.0523, 0.0462,\n",
       "         0.0379, 0.0274, 0.0363, 0.0545, 0.0348, 0.0347, 0.0424, 0.0385, 0.0279,\n",
       "         0.0359, 0.0385, 0.0459, 0.0362, 0.0392, 0.0493, 0.0395, 0.0456, 0.0461,\n",
       "         0.0383, 0.0593, 0.0418, 0.0478, 0.0423, 0.0605, 0.0534, 0.0489, 0.0486,\n",
       "         0.0387, 0.0376, 0.0474, 0.0430, 0.0496, 0.0410, 0.0430, 0.0379, 0.0543,\n",
       "         0.0320, 0.0021, 0.0021, 0.0021, 0.0021, 0.0022, 0.0021, 0.0021, 0.0021,\n",
       "         0.0021, 0.0022, 0.0021, 0.0021, 0.0021, 0.0022, 0.0021, 0.0021, 0.0022,\n",
       "         0.0022, 0.0022, 0.0021, 0.0022, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021,\n",
       "         0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0022, 0.0021, 0.0021, 0.0021,\n",
       "         0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021,\n",
       "         0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0021, 0.0022,\n",
       "         0.0022, 0.0021, 0.0022, 0.0021, 0.0021, 0.0021, 0.0021, 0.0022, 0.0021,\n",
       "         0.0022, 0.0021]], grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 256,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:40:54.939466Z",
     "start_time": "2020-07-03T23:40:54.873002Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 128])"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:40:55.809291Z",
     "start_time": "2020-07-03T23:40:55.727600Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0106, 0.0108, 0.0108, 0.0097, 0.0106, 0.0100, 0.0108, 0.0097, 0.0105,\n",
       "         0.0101, 0.0100, 0.0103, 0.0101, 0.0095, 0.0104, 0.0096, 0.0104, 0.0095,\n",
       "         0.0110, 0.0090, 0.0098, 0.0095, 0.0103, 0.0104, 0.0104, 0.0102, 0.0116,\n",
       "         0.0105, 0.0095, 0.0106, 0.0102, 0.0106, 0.0111, 0.0107, 0.0094, 0.0097,\n",
       "         0.0099, 0.0121, 0.0103, 0.0112, 0.0108, 0.0109, 0.0104, 0.0100, 0.0101,\n",
       "         0.0096, 0.0104, 0.0094, 0.0105, 0.0102, 0.0100, 0.0098, 0.0101, 0.0100,\n",
       "         0.0102, 0.0102, 0.0104, 0.0097, 0.0112, 0.0110, 0.0096, 0.0097, 0.0102,\n",
       "         0.0095, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0054],\n",
       "        [0.0097, 0.0105, 0.0106, 0.0103, 0.0102, 0.0103, 0.0096, 0.0102, 0.0096,\n",
       "         0.0096, 0.0112, 0.0094, 0.0110, 0.0097, 0.0102, 0.0104, 0.0097, 0.0095,\n",
       "         0.0101, 0.0102, 0.0102, 0.0104, 0.0102, 0.0102, 0.0097, 0.0113, 0.0107,\n",
       "         0.0099, 0.0087, 0.0097, 0.0115, 0.0095, 0.0095, 0.0103, 0.0099, 0.0088,\n",
       "         0.0097, 0.0099, 0.0107, 0.0097, 0.0100, 0.0110, 0.0100, 0.0106, 0.0107,\n",
       "         0.0099, 0.0120, 0.0103, 0.0109, 0.0103, 0.0121, 0.0114, 0.0110, 0.0110,\n",
       "         0.0099, 0.0098, 0.0108, 0.0104, 0.0111, 0.0102, 0.0104, 0.0099, 0.0115,\n",
       "         0.0092, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0053, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054, 0.0054,\n",
       "         0.0054, 0.0054]], grad_fn=<SoftmaxBackward>)"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:41:06.166353Z",
     "start_time": "2020-07-03T23:41:06.113851Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0180, 0.0194, 0.0202, 0.0108, 0.0179, 0.0130, 0.0193, 0.0107, 0.0164,\n",
       "         0.0133, 0.0129, 0.0150, 0.0135, 0.0093, 0.0159, 0.0100, 0.0158, 0.0094,\n",
       "         0.0223, 0.0072, 0.0111, 0.0096, 0.0151, 0.0158, 0.0156, 0.0143, 0.0299,\n",
       "         0.0166, 0.0095, 0.0174, 0.0146, 0.0181, 0.0236, 0.0191, 0.0091, 0.0110,\n",
       "         0.0117, 0.0380, 0.0152, 0.0245, 0.0197, 0.0213, 0.0157, 0.0130, 0.0132,\n",
       "         0.0102, 0.0157, 0.0092, 0.0164, 0.0145, 0.0126, 0.0117, 0.0138, 0.0128,\n",
       "         0.0145, 0.0143, 0.0158, 0.0106, 0.0240, 0.0219, 0.0099, 0.0107, 0.0145,\n",
       "         0.0093, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004],\n",
       "        [0.0107, 0.0163, 0.0174, 0.0146, 0.0140, 0.0148, 0.0100, 0.0137, 0.0099,\n",
       "         0.0098, 0.0232, 0.0086, 0.0209, 0.0105, 0.0137, 0.0154, 0.0105, 0.0091,\n",
       "         0.0133, 0.0142, 0.0141, 0.0152, 0.0138, 0.0137, 0.0102, 0.0250, 0.0183,\n",
       "         0.0114, 0.0057, 0.0104, 0.0277, 0.0095, 0.0094, 0.0149, 0.0119, 0.0059,\n",
       "         0.0101, 0.0119, 0.0180, 0.0104, 0.0124, 0.0215, 0.0126, 0.0177, 0.0182,\n",
       "         0.0117, 0.0345, 0.0144, 0.0199, 0.0148, 0.0365, 0.0264, 0.0210, 0.0208,\n",
       "         0.0120, 0.0112, 0.0195, 0.0154, 0.0218, 0.0138, 0.0154, 0.0115, 0.0274,\n",
       "         0.0079, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004,\n",
       "         0.0004, 0.0004]], grad_fn=<SoftmaxBackward>)"
      ]
     },
     "execution_count": 251,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attn.adapt(x2).squeeze().softmax(dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TODO\n",
    "\n",
    "- find error w/ new head() func\n",
    "- experiment w/ conv1d -> split (see how this creates Q, K, V)\n",
    "    - try incorporating this into concat pool, either for Mike's text version or this img version\n",
    "- looks like gpt2 uses dropout on attn weights. Look into this and consider if it would be useful here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:59:01.103346Z",
     "start_time": "2020-07-03T23:59:01.043971Z"
    }
   },
   "outputs": [],
   "source": [
    "# 99% FASTAI2 CODE\n",
    "from fastai2.vision.all import Flatten, SigmoidRange, LinBnDrop, L\n",
    "\n",
    "\n",
    "def new_create_head(nf, n_out, lin_ftrs=None, ps=0.5, bn_final=False, lin_first=False, y_range=None):\n",
    "    \"Model head that takes `nf` features, runs through `lin_ftrs`, and out `n_out` classes.\"\n",
    "    lin_ftrs = [nf, 512, n_out] if lin_ftrs is None else [nf] + lin_ftrs + [n_out]\n",
    "    ps = L(ps)\n",
    "    if len(ps) == 1: ps = [ps[0]/2] * (len(lin_ftrs)-2) + ps\n",
    "    actns = [nn.ReLU(inplace=True)] * (len(lin_ftrs)-2) + [None]\n",
    "    layers = [AttentiveConcatPool2d(lin_ftrs[1]), Flatten()]\n",
    "    if lin_first: layers.append(nn.Dropout(ps.pop(0)))\n",
    "    for ni,no,p,actn in zip(lin_ftrs[:-1], lin_ftrs[1:], ps, actns):\n",
    "        layers += LinBnDrop(ni, no, bn=True, p=p, act=actn, lin_first=lin_first)\n",
    "    if lin_first: layers.append(nn.Linear(lin_ftrs[-2], n_out))\n",
    "    if bn_final: layers.append(nn.BatchNorm1d(lin_ftrs[-1], momentum=0.01))\n",
    "    if y_range is not None: layers.append(SigmoidRange(*y_range))\n",
    "    return nn.Sequential(*layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:59:01.326746Z",
     "start_time": "2020-07-03T23:59:01.272851Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): AttentiveConcatPool2d(\n",
       "    (adapt): AdaptiveConcatPool2d(\n",
       "      (ap): AdaptiveAvgPool2d(output_size=1)\n",
       "      (mp): AdaptiveMaxPool2d(output_size=1)\n",
       "    )\n",
       "  )\n",
       "  (1): Flatten(full=False)\n",
       "  (2): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (3): Dropout(p=0.25, inplace=False)\n",
       "  (4): Linear(in_features=64, out_features=512, bias=False)\n",
       "  (5): ReLU(inplace=True)\n",
       "  (6): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (7): Dropout(p=0.5, inplace=False)\n",
       "  (8): Linear(in_features=512, out_features=3, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 289,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "head = new_create_head(nf=64, n_out=3)\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T23:59:54.006505Z",
     "start_time": "2020-07-03T23:59:53.938899Z"
    }
   },
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "running_mean should contain 128 elements not 64",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-292-2b782ad7b767>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhead\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    548\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    551\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     98\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    101\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    548\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    551\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/batchnorm.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    104\u001b[0m             \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrunning_mean\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrunning_var\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrack_running_stats\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m             exponential_average_factor, self.eps)\n\u001b[0m\u001b[1;32m    107\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mbatch_norm\u001b[0;34m(input, running_mean, running_var, weight, bias, training, momentum, eps)\u001b[0m\n\u001b[1;32m   1921\u001b[0m     return torch.batch_norm(\n\u001b[1;32m   1922\u001b[0m         \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrunning_mean\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrunning_var\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1923\u001b[0;31m         \u001b[0mtraining\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmomentum\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackends\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcudnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menabled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1924\u001b[0m     )\n\u001b[1;32m   1925\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: running_mean should contain 128 elements not 64"
     ]
    }
   ],
   "source": [
    "head[:3](x2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-04T06:29:36.933140Z",
     "start_time": "2020-07-04T06:29:36.886699Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 0.2957,  0.8758, -0.0259, -0.8718,  0.4397,  1.3841,  1.6302,  0.4875,\n",
       "        -1.0211, -0.5959,  1.2964, -0.5931,  0.7207, -0.5879, -0.3265, -0.1064,\n",
       "        -0.6025, -1.1680,  0.8615, -0.8901,  0.2129,  0.8318,  0.7593, -0.1956,\n",
       "        -0.2041,  0.0039,  0.0162, -0.6255, -0.4588, -0.5954,  1.1492,  0.9183,\n",
       "         0.3214,  1.3216, -0.5161,  0.0196, -1.0844, -0.4268, -0.0475, -0.3155,\n",
       "         0.4768,  1.8904, -1.1151, -0.0113, -0.2272, -0.0434,  1.6390,  0.7456,\n",
       "         0.5126, -1.2891, -0.2839, -1.0576,  0.2108,  1.4733, -1.2572,  0.6419,\n",
       "         0.6894, -1.5202,  2.2025, -0.1163, -1.6674,  0.2190, -0.8412,  2.6658])"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z = torch.randn(64)\n",
    "z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-04T06:31:10.343615Z",
     "start_time": "2020-07-04T06:31:10.302546Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-1.6674, -1.5202, -1.2891, -1.2572, -1.1680, -1.1151, -1.0844, -1.0576,\n",
       "        -1.0211, -0.8901, -0.8718, -0.8412, -0.6255, -0.6025, -0.5959, -0.5954,\n",
       "        -0.5931, -0.5879, -0.5161, -0.4588, -0.4268, -0.3265, -0.3155, -0.2839,\n",
       "        -0.2272, -0.2041, -0.1956, -0.1163, -0.1064, -0.0475, -0.0434, -0.0259,\n",
       "        -0.0113,  0.0039,  0.0162,  0.0196,  0.2108,  0.2129,  0.2190,  0.2957,\n",
       "         0.3214,  0.4397,  0.4768,  0.4875,  0.5126,  0.6419,  0.6894,  0.7207,\n",
       "         0.7456,  0.7593,  0.8318,  0.8615,  0.8758,  0.9183,  1.1492,  1.2964,\n",
       "         1.3216,  1.3841,  1.4733,  1.6302,  1.6390,  1.8904,  2.2025,  2.6658])"
      ]
     },
     "execution_count": 312,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z.sort()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-04T06:51:36.346789Z",
     "start_time": "2020-07-04T06:51:36.306873Z"
    }
   },
   "outputs": [],
   "source": [
    "def plot_bar(data, act_fn=identity, title=None, **kwargs):\n",
    "    data = act_fn(data).sort()[0]\n",
    "    fig, ax = plt.subplots(**kwargs)\n",
    "    plt.bar(range(data.shape[0]), data)\n",
    "    if title: plt.title(title)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-04T06:49:12.008066Z",
     "start_time": "2020-07-04T06:49:11.964149Z"
    }
   },
   "outputs": [],
   "source": [
    "def plot_scatter(data, act_fn=identity, title=None, **kwargs):\n",
    "    fig, ax = plt.subplots(**kwargs)\n",
    "    plt.scatter(data, act_fn(data), s=1)\n",
    "    plt.xlabel('Original')\n",
    "    plt.ylabel('Transformed')\n",
    "    if title: plt.title(title)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-04T06:39:04.464271Z",
     "start_time": "2020-07-04T06:39:04.209685Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD4CAYAAADxeG0DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAALLElEQVR4nO3dX4yld1kH8O9jgaiAAeyqhBYXE6I2RgpO+BOMwYpawGg0mtBEwoWmN5CUhMS0IdF4xxXRGGKysZULCVzwRxogYkUM4QbZhaLbLpWqbWha7RJjMF4YK48XcxqHcTY7O+ftnPPMfD7JZOb8dvu+32lnvvP0d973THV3AJjruzYdAID1KHKA4RQ5wHCKHGA4RQ4w3LM2cdLrr7++z549u4lTA4x14cKFb3b3mf3rGynys2fP5vz585s4NcBYVfXoQeu2VgCGU+QAwylygOEUOcBwihxgOEUOMJwiBxhOkQMMt5EbggBOi7N3fuo7Hj/y3rcsfg4TOcBwihxgOEUOMJwiBxhOkQMMp8gBhlPkAMMpcoDhFDnAcIocYDhFDjCcIgcYTpEDDKfIAYZT5ADDKXKA4RQ5wHCKHGA4RQ4wnCIHGM4vXwZYyHH8ouWDmMgBhlPkAMMpcoDhFDnAcIocYLi1i7yqbqyqz1XVpap6oKruWCIYAIezxOWHTyV5d3d/uaqen+RCVd3X3Q8ucGwArmLtiby7n+juL68+/o8kl5K8ZN3jAnA4i+6RV9XZJK9M8sUljwvAlS1W5FX1vCQfTfKu7v7WAX9+e1Wdr6rzly9fXuq0AKfeIkVeVc/Obol/sLs/dtDf6e5z3b3T3TtnzpxZ4rQAZJmrVirJ3Ukudff71o8EwLVYYiJ/fZK3Jbmlqu5fvb15geMCcAhrX37Y3V9IUgtkAeAIvIwtwBFs6iVrD+IWfYDhFDnAcIocYDhFDjCcIgcYTpEDDKfIAYZT5ADDKXKA4RQ5wHBu0QfYZ+/t94+89y1bdTv+QUzkAMMpcoDhFDnAcIocYDhFDjCcIgcYTpEDDOc6cuBU2/ZrxA/DRA4wnCIHGE6RAwxnjxw4NU7CfvhBTOQAwylygOFsrQAn0kndRjmIIgdOhP2vIX6a2FoBGE6RAwynyAGGU+QAwylygOFctQKMc5ouLTwMEznAcIocYDhFDjDcIkVeVfdU1ZNVdXGJ4wFweEtN5B9IcutCxwLgGixS5N39+ST/tsSxALg2x7ZHXlW3V9X5qjp/+fLl4zotwIl3bEXe3ee6e6e7d86cOXNcpwU48Vy1AjCcIgcYbpFb9KvqQ0nekOT6qnosye91991LHBs43dyOf3WLFHl337bEcQC4dl40C9gapu+jUeTARijt5Shy4FjsL26W46oVgOEUOcBwtlaAxdn/Pl6KHFiL0t48RQ4cmtLeToocuKK9xa20t5cnOwGGU+QAw9laAZLY/57MRA4wnCIHGE6RAwynyAGG82QnnEKe2DxZTOQAw5nI4YTZfzem6fvkM5EDDGcih8FM2yQmcoDxFDnAcIocYDhFDjCcJzthCE9sciWKHLaQ0uZa2FoBGE6RAwynyAGGs0cOz6CD9rq9FgpLM5EDDKfIAYZT5ADDKXKA4RQ5wHCKHGC4RYq8qm6tqoeq6uGqunOJYwJwOGsXeVVdl+T9Sd6U5KYkt1XVTeseF4DDWeKGoFcnebi7/ylJqurDSX4lyYMLHBu20tVu9Hl6DY5Ddfd6B6j69SS3dvdvrx6/Lclruvud+/7e7UluT5KXvvSlP/Xoo48e6XyH/Qbav7bfYf+5w6xt67EOsq1Zl/681zk+bKuqutDdO/vXl5jI64C1//fTobvPJTmXJDs7O+v99ICFKG5OgiWK/LEkN+55fEOSxxc4Lqxlf0krbU6qJa5a+VKSl1fVy6rqOUnemuTeBY4LwCGsPZF391NV9c4kn0lyXZJ7uvuBtZMBcCiLvIxtd386yaeXOBZczUFbJLZNOM3c2QkwnCIHGE6RAwznV72xNex9w9EocjZGScMybK0ADKfIAYaztcKxsI0CzxxFzuKUNhwvRc5alDZsnj1ygOEUOcBwihxgOHvkXBN74rB9TOQAw5nIuSLTN8xgIgcYzkROEtM3TGYiBxhOkQMMp8gBhrNHPsRhf3vOYfa67YfDyaLIt5CiBa6FrRWA4RQ5wHCKHGA4RQ4wnCIHGE6RAwynyAGGU+QAwylygOHc2bkF3MkJrMNEDjCcifwZdNQXtQK4FiZygOHWKvKq+o2qeqCqvl1VO0uFAuDw1p3ILyb5tSSfXyALAEew1h55d19KkqpaJg0A1+zY9sir6vaqOl9V5y9fvnxcpwU48a46kVfVXyX5oQP+6D3d/YnDnqi7zyU5lyQ7Ozt96IRDuBoF2JSrFnl3v/E4ggBwNK4jPyITOLAt1iryqvrVJH+U5EyST1XV/d39i4sk2yJKG9hm61618vEkH18oCwBHYGtlH9M3MI1b9AGGO9UTuekbOAlM5ADDKXKA4RQ5wHCnZo/cfjhwUpnIAYY7sRO5CRw4LUzkAMMpcoDhFDnAcIocYDhFDjCcIgcYTpEDDKfIAYZT5ADDnYg7O93FCZxmJnKA4RQ5wHCKHGA4RQ4wnCIHGE6RAww37vJDlxoCfCcTOcBwihxgOEUOMJwiBxhOkQMMp8gBhlPkAMMpcoDhFDnAcNXdx3/SqstJHl3zMNcn+eYCcTZlcv7J2RP5N21y/k1n/+HuPrN/cSNFvoSqOt/dO5vOcVST80/Onsi/aZPzb2t2WysAwylygOEmF/m5TQdY0+T8k7Mn8m/a5PxbmX3sHjkAuyZP5ABEkQOMN7LIq+rWqnqoqh6uqjs3nedqquqeqnqyqi7uWXtRVd1XVV9fvX/hJjNeSVXdWFWfq6pLVfVAVd2xWp+S/7ur6m+r6qur/L+/Wh+RP0mq6rqq+kpVfXL1eFL2R6rq76vq/qo6v1qblP8FVfWRqvra6nvgdduYf1yRV9V1Sd6f5E1JbkpyW1XdtNlUV/WBJLfuW7szyWe7++VJPrt6vI2eSvLu7v7xJK9N8o7Vv+8p+f8ryS3d/YokNye5tapemzn5k+SOJJf2PJ6UPUl+trtv3nP99aT8f5jkL7r7x5K8Irv/HbYvf3ePekvyuiSf2fP4riR3bTrXIXKfTXJxz+OHkrx49fGLkzy06YyH/Dw+keTnJ+ZP8r1JvpzkNVPyJ7khu2VxS5JPTvvaSfJIkuv3rY3In+T7kvxzVheFbHP+cRN5kpck+caex4+t1qb5we5+IklW739gw3muqqrOJnllki9mUP7V1sT9SZ5Mcl93T8r/B0l+J8m396xNyZ4kneQvq+pCVd2+WpuS/0eSXE7yp6utrT+pqudmC/NPLPI6YM01lM+wqnpeko8meVd3f2vTea5Fd/9Pd9+c3en21VX1E5vOdBhV9UtJnuzuC5vOsobXd/ersrsV+o6q+plNB7oGz0ryqiR/3N2vTPKf2YZtlANMLPLHkty45/ENSR7fUJZ1/GtVvThJVu+f3HCeK6qqZ2e3xD/Y3R9bLY/J/7Tu/vckf5Pd5ysm5H99kl+uqkeSfDjJLVX1Z5mRPUnS3Y+v3j+Z5ONJXp05+R9L8tjq/+CS5CPZLfatyz+xyL+U5OVV9bKqek6Stya5d8OZjuLeJG9fffz27O49b52qqiR3J7nU3e/b80dT8p+pqhesPv6eJG9M8rUMyN/dd3X3Dd19Nrtf53/d3b+ZAdmTpKqeW1XPf/rjJL+Q5GKG5O/uf0nyjar60dXSzyV5MNuYf9Ob9Ed8EuLNSf4hyT8mec+m8xwi74eSPJHkv7P7U/63knx/dp/E+vrq/Ys2nfMK2X86u1tXf5fk/tXbmwfl/8kkX1nlv5jkd1frI/Lv+TzekP97snNE9uzuMX919fbA09+rU/Kvst6c5Pzq6+fPk7xwG/O7RR9guIlbKwDsocgBhlPkAMMpcoDhFDnAcIocYDhFDjDc/wIPmppVixMzjAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_bar(z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-05T00:29:32.253443Z",
     "start_time": "2020-07-05T00:29:31.364627Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 8488 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_bar(z, nn.Softmax(dim=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-04T06:37:21.603920Z",
     "start_time": "2020-07-04T06:37:21.560637Z"
    }
   },
   "outputs": [],
   "source": [
    "class SmoothSoftmax(nn.Module):\n",
    "    \n",
    "    def __init__(self, scale):\n",
    "        super().__init__()\n",
    "        self.scale = scale\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return x.div(self.scale).softmax(dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-05T00:55:23.006232Z",
     "start_time": "2020-07-05T00:55:22.946334Z"
    }
   },
   "outputs": [],
   "source": [
    "z = torch.randn(512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-05T00:55:58.090384Z",
     "start_time": "2020-07-05T00:55:56.850510Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 5768 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_bar(z, SmoothSoftmax(z.shape[-1]**.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-04T06:53:47.517317Z",
     "start_time": "2020-07-04T06:53:31.093601Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 4688 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 8852 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5400 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 9652 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5800 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 10340 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5704 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 9760 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 6204 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 9608 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 6124 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 9728 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5684 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 10516 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 6072 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 9260 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5620 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 9236 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5568 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 8912 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5600 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 10756 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5500 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 10696 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5132 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 10716 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5120 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 10624 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5104 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 10512 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5032 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 10468 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for s in np.r_[.1:1:.2, 1:12]:\n",
    "    plot_bar(z, SmoothSoftmax(s), f'Scale={s:.1f}', figsize=(8, 2))\n",
    "    plot_scatter(z, SmoothSoftmax(s), f'Scale={s:.1f}', figsize=(8, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-05T00:29:21.200612Z",
     "start_time": "2020-07-05T00:29:20.933488Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 5336 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = torch.randn(64)\n",
    "plot_bar(x, nn.Softmax(dim=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-05T00:26:38.525115Z",
     "start_time": "2020-07-05T00:26:36.241601Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAACcCAYAAACwYehSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAALKklEQVR4nO3dcahe913H8fdnNwsbdaNq7+ZIWm8LYTOI7UpIK5WxTbel7TD+M+hgbg4kFFrdwCFRBqI47B8iTigNpdZt6CxjrhpsWFeqYxOc5karbdcWQ4j2km5pGW7qpDXu6x/nXH28uTf3JLn3Pr/7nPcLLs9zfuf8nuf3Jbn55Pc75zlPqgpJktSmV017AJIkaW0GtSRJDTOoJUlqmEEtSVLDDGpJkhpmUEuS1DCDWpKkhhnU0sgluTvJYpKXk3xq2uOR9P/tmPYAJE3dGeA3gfcAr53yWCStYFBLI1dVXwBIsg/YPeXhSFrBpW9JkhpmUEuS1DCDWpKkhhnUkiQ1zIvJpJFLsoPu34I5YC7Ja4BzVXVuuiOTBM6oJcHHgf8EDgMf6J9/fKojkvS/UlXTHoMkSVqDM2pJkhpmUEuS1DCDWpKkhhnUkiQ1zKCWJKlhTX6O+qqrrqqFhYVpD0OSpC1x4sSJl6pqfrV9TQb1wsICi4uL0x6GJElbIsk/r7XPpW9JkhpmUEuS1DCDWpKkhjV5jlqSpFYsHH7kvLbT99y+Ze/vjFqSpIYZ1JIkNcygliSpYQa1JEkNM6glSWqYQS1JUsMMakmSGmZQS5LUMINakqSGGdSSJDXMoJYkqWEGtSRJDTOoJUlqmEEtSVLDDGpJkhpmUEuS1LBBQZ3kQJLnkpxMcniV/W9J8tdJXk7ysYvpK0mS1rZuUCeZA+4FbgX2Au9PsnfFYd8CfhH47UvoK0mS1jBkRr0fOFlVp6rqFeAh4ODkAVV1tqqOA/91sX0lSdLahgT1LuD5ie2lvm2Iy+krSdLoDQnqrNJWA19/cN8kh5IsJll88cUXB768JEmzbUhQLwFXT2zvBs4MfP3Bfavq/qraV1X75ufnB768JEmzbUhQHwf2JLk2yU7gDuDowNe/nL6SJI3ejvUOqKpzSe4GHgXmgAer6ukkd/b7jyT5IWAReD3wvSQfBfZW1XdW67tZxUiSNGvWDWqAqjoGHFvRdmTi+TfolrUH9ZUkScN4ZzJJkhpmUEuS1DCDWpKkhhnUkiQ1zKCWJKlhBrUkSQ0zqCVJaphBLUlSwwxqSZIaZlBLktQwg1qSpIYNute3JEmzbOHwI+e1nb7n9imM5HzOqCVJaphBLUlSwwxqSZIaZlBLktQwg1qSpIYZ1JIkNcygliSpYQa1JEkNM6glSWqYQS1JUsMMakmSGmZQS5LUMINakqSGGdSSJDXMoJYkqWEGtSRJDTOoJUlqmEEtSVLDDGpJkhpmUEuS1LBBQZ3kQJLnkpxMcniV/Unye/3+f0xy48S+00meTPJEksWNHLwkSbNux3oHJJkD7gXeBSwBx5McraqvTxx2K7Cn/7kJuK9/XPaOqnppw0YtSdIlWDj8yHltp++5fQojGW7doAb2Ayer6hRAkoeAg8BkUB8EPlNVBXwtyZVJ3lRVL2z4iCVJWsd2DOS1DFn63gU8P7G91LcNPaaALyU5keTQpQ5UkqQxGjKjziptdRHH3FJVZ5K8AXgsybNV9ZXz3qQL8UMA11xzzYBhSZI0+4bMqJeAqye2dwNnhh5TVcuPZ4GH6ZbSz1NV91fVvqraNz8/P2z0kiTNuCFBfRzYk+TaJDuBO4CjK445Cnywv/r7ZuDbVfVCkiuSvA4gyRXAu4GnNnD8kiTNtHWXvqvqXJK7gUeBOeDBqno6yZ39/iPAMeA24CTwXeDDffc3Ag8nWX6vz1bVFze8CkmSZtSQc9RU1TG6MJ5sOzLxvIC7Vul3Crj+MscoSdJ5ZunK7gvxzmSSJDXMoJYkqWGDlr4lSZqWsSxxr8UZtSRJDTOoJUlqmEEtSVLDPEctSdoya51vXq19ed/YGdSSpA039gvANpJL35IkNcygliSpYS59S5IuicvbW8MZtSRJDTOoJUlqmEEtSVLDPEctSbogz0VPlzNqSZIaZlBLktQwg1qSpIYZ1JIkNcygliSpYQa1JEkN8+NZkrSNXczXRl6o/UKvpelyRi1JUsOcUUtS45zpjpszakmSGuaMWpIuw0acI16vj8bNoJak3qVcgCVtNoNa0rbg1c0aK4Na0qa4lCVeA1Q6n0EtjdBGzkINV2lzGdRSY1zilTTJoJY2wEaGqyRNMqjVnK1YlvX8qaTtYlBQJzkAfBKYAx6oqntW7E+//zbgu8DPVdXfDemr7cVlWUnaWusGdZI54F7gXcAScDzJ0ar6+sRhtwJ7+p+bgPuAmwb21UVw5ihJ4zJkRr0fOFlVpwCSPAQcBCbD9iDwmaoq4GtJrkzyJmBhQN8mbcXM0aCUJK1nSFDvAp6f2F6imzWvd8yugX2nxkCUJLUu3ST4Agck7wPeU1U/32//LLC/qn5h4phHgN+qqr/qtx8Hfhm4br2+E69xCDjUb74ZeO4ya1vNVcBLm/C624X1j7f+MdcO465/zLXD9qn/h6tqfrUdQ2bUS8DVE9u7gTMDj9k5oC8AVXU/cP+A8VyyJItVtW8z36Nl1j/e+sdcO4y7/jHXDrNR/5CvuTwO7ElybZKdwB3A0RXHHAU+mM7NwLer6oWBfSVJ0hrWnVFX1bkkdwOP0n3E6sGqejrJnf3+I8Axuo9mnaT7eNaHL9R3UyqRJGkGDfocdVUdowvjybYjE88LuGto3yna1KX1bcD6x2vMtcO46x9z7TAD9a97MZkkSZqeIeeoJUnSlIwmqJMcSPJckpNJDk97PJstyYNJziZ5aqLtB5I8luSf+sfvn+YYN0uSq5P8ZZJnkjyd5CN9+1jqf02Sv03yD339v963j6J+6O6omOTvk/x5vz2m2k8neTLJE0kW+7ZR1N/fbOvzSZ7tf/9/fBZqH0VQT9zK9FZgL/D+JHunO6pN9yngwIq2w8DjVbUHeLzfnkXngF+qqh8Bbgbu6v+8x1L/y8A7q+p64AbgQP9pjLHUD/AR4JmJ7THVDvCOqrph4mNJY6n/k8AXq+otwPV0fwe2fe2jCGomboNaVa8Ay7cynVlV9RXgWyuaDwKf7p9/GviZLR3UFqmqF5a/FKaq/o3ul3UX46m/qurf+81X9z/FSOpPshu4HXhgonkUtV/AzNef5PXA24DfB6iqV6rqX5mB2scS1Gvd4nRs3th/vp3+8Q1THs+mS7IAvBX4G0ZUf7/0+wRwFnisqsZU/+/S3RnxexNtY6kduv+UfSnJif6OjzCO+q8DXgT+oD/t8UCSK5iB2scS1FmlzcvdZ1yS7wP+BPhoVX1n2uPZSlX131V1A93dAPcn+dFpj2krJHkvcLaqTkx7LFN0S1XdSHeq764kb5v2gLbIDuBG4L6qeivwH2zDZe7VjCWoh9wGdQy+2X+rGf3j2SmPZ9MkeTVdSP9RVX2hbx5N/cv6pb8v012vMIb6bwF+OslpulNc70zyh4yjdgCq6kz/eBZ4mO7U3xjqXwKW+tUjgM/TBfe2r30sQe2tTDtHgQ/1zz8E/NkUx7JpkoTuPNUzVfU7E7vGUv98kiv7568Ffgp4lhHUX1W/UlW7q2qB7vf8L6rqA4ygdoAkVyR53fJz4N3AU4yg/qr6BvB8kjf3TT9J95XK27720dzwJMltdOeulm9l+okpD2lTJflj4O103xzzTeDXgD8FPgdcA/wL8L6qWnnB2baX5CeArwJP8n/nKX+V7jz1GOr/MbqLZubo/jP+uar6jSQ/yAjqX5bk7cDHquq9Y6k9yXV0s2joloI/W1WfGFH9N9BdRLgTOEV3O+tXsc1rH01QS5K0HY1l6VuSpG3JoJYkqWEGtSRJDTOoJUlqmEEtSVLDDGpJkhpmUEuS1DCDWpKkhv0PPI4hyL7avTYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 4208 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 4620 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 4568 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 4108 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 4152 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5236 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5332 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 5316 characters]\n",
      "text/plain": [
       "<Figure size 576x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(1, 10):\n",
    "    plot_bar(x, SmoothSoftmax(i), title=str(i), figsize=(8, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-05T00:41:02.992341Z",
     "start_time": "2020-07-05T00:41:02.805268Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 6128 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_bar(torch.randn(4), SmoothSoftmax(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-05T00:41:06.416634Z",
     "start_time": "2020-07-05T00:41:06.235780Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 5672 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_bar(torch.randn(4), SmoothSoftmax(8))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
