{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Abstract Attributes\n",
    "\n",
    "Not working yet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-23T05:51:35.156336Z",
     "start_time": "2020-11-23T05:51:35.150651Z"
    }
   },
   "outputs": [],
   "source": [
    "def abstractattrs(*attrs):\n",
    "    def wrapper(cls):\n",
    "        \n",
    "        def method(cls, *args, **kwargs):\n",
    "            super(cls, cls).__init_subclass__(*args, **kwargs)\n",
    "            for attr in attrs:\n",
    "                if not hasattr(cls, attr):\n",
    "                    raise RuntimeError(f'Class must have attribute {attr}.') \n",
    "                    \n",
    "        setattr(cls, '__init_subclass__', classmethod(method))\n",
    "        return cls\n",
    "    return wrapper              "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-23T05:51:35.759875Z",
     "start_time": "2020-11-23T05:51:35.756318Z"
    }
   },
   "outputs": [],
   "source": [
    "@abstractattrs('name', 'age')\n",
    "class Base:\n",
    "    def __init__(self, age=7):\n",
    "        self.age = age "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-23T05:51:40.646466Z",
     "start_time": "2020-11-23T05:51:36.471154Z"
    }
   },
   "outputs": [
    {
     "ename": "RecursionError",
     "evalue": "maximum recursion depth exceeded while calling a Python object",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRecursionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-122-75cc12bc3db6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mChild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBase\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'a'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-120-cedaaefac923>\u001b[0m in \u001b[0;36mmethod\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m             \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init_subclass__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mattr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "... last 1 frames repeated, from the frame below ...\n",
      "\u001b[0;32m<ipython-input-120-cedaaefac923>\u001b[0m in \u001b[0;36mmethod\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m             \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init_subclass__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mattr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRecursionError\u001b[0m: maximum recursion depth exceeded while calling a Python object"
     ]
    }
   ],
   "source": [
    "class Child(Base):\n",
    "    a = 'a'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-23T05:50:14.670781Z",
     "start_time": "2020-11-23T05:50:14.664637Z"
    }
   },
   "outputs": [],
   "source": [
    "class AbstractAttrs:\n",
    "    \n",
    "    def __init__(self, *args):\n",
    "        print(args)\n",
    "        self.args = list(args)\n",
    "    \n",
    "    @classmethod\n",
    "    def __init_subclass__(cls, **kwargs):\n",
    "        super().__init_subclass__(**kwargs)\n",
    "#         if not hasattr(cls, f'_{cls.__name__}__attrs'):\n",
    "#             print('no attrs')\n",
    "        print('init subclass', AbstractAttrs._args())\n",
    "        for arg in AbstractAttrs.args:\n",
    "            assert hasattr(cls, arg), \\\n",
    "                f'{cls.__name__} must have class variable {arg}.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-23T05:50:15.093761Z",
     "start_time": "2020-11-23T05:50:15.075674Z"
    }
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "_args() missing 1 required positional argument: 'self'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-119-8be9ad7b1fab>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mCallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mAbstractAttrs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-118-9f0e0d6150a2>\u001b[0m in \u001b[0;36m__init_subclass__\u001b[0;34m(cls, **kwargs)\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;31m#         if not hasattr(cls, f'_{cls.__name__}__attrs'):\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;31m#             print('no attrs')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'init subclass'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAbstractAttrs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mAbstractAttrs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: _args() missing 1 required positional argument: 'self'"
     ]
    }
   ],
   "source": [
    "class Callback(AbstractAttrs):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-23T05:46:47.154418Z",
     "start_time": "2020-11-23T05:46:47.149946Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('order', 'is_val')\n"
     ]
    }
   ],
   "source": [
    "c = Callback('order', 'is_val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-23T05:46:54.144913Z",
     "start_time": "2020-11-23T05:46:54.137769Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre super\n",
      "NewCallback\n",
      "no attrs\n"
     ]
    }
   ],
   "source": [
    "class NewCallback(Callback):\n",
    "    \n",
    "    def __init__(self, a, order=3):\n",
    "        self.a = a\n",
    "        self.order = order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-23T05:45:42.173623Z",
     "start_time": "2020-11-23T05:45:42.167310Z"
    }
   },
   "outputs": [],
   "source": [
    "nc = NewCallback('a', 'my_order')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-23T05:45:42.762174Z",
     "start_time": "2020-11-23T05:45:42.755871Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<__main__.NewCallback at 0x1a1353b358>"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-23T05:45:45.676917Z",
     "start_time": "2020-11-23T05:45:45.670265Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['order', 'val_loop']"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nc._attrs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-23T23:56:26.366857Z",
     "start_time": "2020-11-23T23:56:26.363937Z"
    }
   },
   "outputs": [],
   "source": [
    "from abc import ABC, abstractmethod, abstractproperty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-23T23:57:25.624700Z",
     "start_time": "2020-11-23T23:57:25.619574Z"
    }
   },
   "outputs": [],
   "source": [
    "class Abstract(ABC):\n",
    "    def __init__(self, a):\n",
    "        self.a = a\n",
    "    @abstractmethod\n",
    "    def walk(self):\n",
    "        pass\n",
    "    \n",
    "    @abstractproperty\n",
    "    def b(self):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T00:10:56.285468Z",
     "start_time": "2020-11-24T00:10:56.280697Z"
    }
   },
   "outputs": [],
   "source": [
    "class Child(Abstract):\n",
    "    b = 3\n",
    "        \n",
    "    def walk(self):\n",
    "        return self.a\n",
    "    \n",
    "#     def b(self):\n",
    "#         print('in b, method not prop')\n",
    "#         return self.a*2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T00:10:56.577462Z",
     "start_time": "2020-11-24T00:10:56.574092Z"
    }
   },
   "outputs": [],
   "source": [
    "c = Child(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T00:11:02.636602Z",
     "start_time": "2020-11-24T00:11:02.630073Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Metaclass approach\n",
    "\n",
    "This seems more promising."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T21:31:34.256384Z",
     "start_time": "2020-11-25T21:31:34.252111Z"
    }
   },
   "outputs": [],
   "source": [
    "from inspect import ismethod, getattr_static"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T21:48:48.547821Z",
     "start_time": "2020-11-25T21:48:48.544203Z"
    }
   },
   "outputs": [],
   "source": [
    "def hasstatic(cls, meth_name):\n",
    "    \"\"\"Check if a class possesses a staticmethod of a given name. Similar to\n",
    "    hasattr\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    cls: Type or any\n",
    "        A class or an instance (seems to work on both, though more extensive\n",
    "        testing may be needed for more complex scenarios).\n",
    "    meth_name: str\n",
    "        Name of method to check. If the class/instance does not contain any\n",
    "        attribute with this name, function returns False.\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    bool: True if `cls` has a staticmethod with name `meth_name`.\n",
    "    \"\"\"\n",
    "    return isinstance(getattr_static(cls, meth_name, None), staticmethod)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T21:48:49.198568Z",
     "start_time": "2020-11-25T21:48:49.195347Z"
    }
   },
   "outputs": [],
   "source": [
    "class Foo:\n",
    "    def __init__(self, a):\n",
    "        self.a = a\n",
    "    @staticmethod\n",
    "    def bar(self):\n",
    "        return 'bar'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T21:48:49.420954Z",
     "start_time": "2020-11-25T21:48:49.414795Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hasstatic(Foo, 'bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T21:48:49.846406Z",
     "start_time": "2020-11-25T21:48:49.835371Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hasstatic(Foo, '__init__')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T21:48:49.846406Z",
     "start_time": "2020-11-25T21:48:49.835371Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hasstatic(Foo(3), 'bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T21:48:49.985580Z",
     "start_time": "2020-11-25T21:48:49.978211Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hasstatic(Foo(3), '__init__')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T21:48:50.172309Z",
     "start_time": "2020-11-25T21:48:50.152465Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hasstatic(Foo(3), 'fake')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:54.259916Z",
     "start_time": "2020-11-25T22:18:54.242940Z"
    }
   },
   "outputs": [],
   "source": [
    "class AbstractAttrs(type):\n",
    "    \"\"\"Basically the attribute equivalent of abc.abstractmethod: this allows \n",
    "    us to define an abstract parent class that requires its children to \n",
    "    possess certain class and/or instance attributes. This differs from\n",
    "    abc.abstractproperty in a few ways:\n",
    "    \n",
    "    1. abstractproperty ignores instance attributes. AbstractAttrs lets us\n",
    "    specify required instance attributes and/or class attributes and\n",
    "    distinguish between the two.\n",
    "    2. abstractproperty considers the requirement fulfilled by methods,\n",
    "    properties, and class attributes. AbstractAttrs does not allow methods \n",
    "    (including classmethods and staticmethods) to fulfill either requirement,\n",
    "    though properties can fulfill either.\n",
    "    \n",
    "    Examples\n",
    "    --------\n",
    "    This class defines required instance attributes and class attributes, \n",
    "    but you can also specify one or the other. If you don't care whether an\n",
    "    attribute is at the class or instance level, you can simply use\n",
    "    @abc.abstractproperty.\n",
    "    \n",
    "    class Parent(metaclass=AbstractAttrs, inst_attrs=['name'], \n",
    "                 class_attrs=['order', 'is_val']):\n",
    "        pass\n",
    "    \"\"\"\n",
    "\n",
    "    def __new__(cls, name, bases, methods, **meta_kwargs):\n",
    "        \"\"\"This provides user-defined parent classes with an \n",
    "        `__init_subclass__` method that checks for class attributes. Errors\n",
    "        will occur when the parent class is defined, not when instances of it\n",
    "        are constructed.\n",
    "        \"\"\"\n",
    "        class_ = type.__new__(cls, name, bases, methods)\n",
    "        class_attrs = meta_kwargs.get('class_attrs', [])\n",
    "        inst_attrs = meta_kwargs.get('inst_attrs', [])\n",
    "        \n",
    "        def __init_subclass__(cls, **kwargs):\n",
    "            super().__init_subclass__(**kwargs)\n",
    "            for attr in class_attrs:\n",
    "                # TypeError maintains consistency with abstractmethod.\n",
    "                # Remaining checks occur at instantiation.\n",
    "                if not hasattr(cls, attr):\n",
    "                    raise TypeError(f'{cls} must have class attribute '\n",
    "                                    f'`{attr}`.')\n",
    "        \n",
    "        # Make sure we distinguish between the abstract parent class that \n",
    "        # defines an interface and the child classes that implement it. The\n",
    "        # abstract parent should not define the required attributes: it merely\n",
    "        # enforces the requirement that its children do. We want the children\n",
    "        # to inherit class_attrs and inst_attrs without overwriting them when\n",
    "        # their own __new__ is called, so that AbstractAttrs.__call__ can\n",
    "        # use them for validation. Only change this if you're very confident\n",
    "        # you understand the repercussions.\n",
    "        if class_attrs or inst_attrs:\n",
    "            class_.__init_subclass__ = classmethod(__init_subclass__)\n",
    "            class_._is_parent = True\n",
    "            class_.class_attrs = class_attrs\n",
    "            class_.inst_attrs = inst_attrs\n",
    "        else:\n",
    "            class_._is_parent = False\n",
    "        return class_\n",
    "\n",
    "    def __call__(cls, *args, **kwargs):\n",
    "        \"\"\"This is called when we create instances of our classes. Parents are\n",
    "        initialized normally, while children undergo a series of checks for\n",
    "        each of our required attributes.\n",
    "        \"\"\"\n",
    "        inst = cls.__new__(cls, *args, **kwargs)\n",
    "        if not isinstance(inst, cls): return inst\n",
    "        \n",
    "        inst.__init__(*args, **kwargs)\n",
    "        if cls._is_parent: return inst\n",
    "        \n",
    "        # Validate children.\n",
    "        for attr in inst.inst_attrs:\n",
    "            # TypeError maintains consistency with abstractmethod.\n",
    "            if not hasattr(inst, attr):\n",
    "                raise TypeError(f'Instances of {type(inst)} must '\n",
    "                                f'have instance attribute `{attr}`.')\n",
    "            elif ismethod(getattr(inst, attr)):\n",
    "                raise TypeError(f'`{attr}` must be an instance attribute, '\n",
    "                                'not a method.')\n",
    "                \n",
    "        # In AbstractAttrs.__new__, methods are still unbound so we couldn't\n",
    "        # easily check this until now.\n",
    "        for attr in inst.class_attrs:\n",
    "            # `ismethod` must check inst, not cls (cls.method is a function \n",
    "            # while inst.method is a method). staticmethod can be retrieved \n",
    "            # from either.\n",
    "            if ismethod(getattr(inst, attr)) or hasstatic(inst, attr):\n",
    "                raise TypeError(f'`{attr}` must be a class attribute, not a '\n",
    "                                'method.')\n",
    "            elif isinstance(getattr(cls, attr), property):\n",
    "                raise TypeError(\n",
    "                    f'`{attr}` must be a class attribute, not a property. '\n",
    "                    'Properties fulfill instance attribute requirements but '\n",
    "                    'not class attribute requirements.'\n",
    "                )\n",
    "        return inst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:17:49.099243Z",
     "start_time": "2020-11-25T22:17:49.095397Z"
    }
   },
   "outputs": [],
   "source": [
    "class Parent(metaclass=AbstractAttrs, inst_attrs=['name'],\n",
    "             class_attrs=['order', 'is_val']):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:17:49.658307Z",
     "start_time": "2020-11-25T22:17:49.654188Z"
    }
   },
   "outputs": [],
   "source": [
    "p = Parent()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:17:50.211541Z",
     "start_time": "2020-11-25T22:17:50.207089Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['order', 'is_val'], ['name'], True)"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p.class_attrs, p.inst_attrs, p._is_parent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:17:51.358384Z",
     "start_time": "2020-11-25T22:17:51.353558Z"
    }
   },
   "outputs": [],
   "source": [
    "# Implements required class vars but not required inst vars. Error will be\n",
    "# thrown on object instantiation, not class definition.\n",
    "class Child(Parent):\n",
    "    \n",
    "    order = 33\n",
    "    is_val = True\n",
    "    \n",
    "    def __init__(self, a):\n",
    "        self.a = a\n",
    "        print('child init')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:17:52.029726Z",
     "start_time": "2020-11-25T22:17:52.014379Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "child init\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "Instances of <class '__main__.Child'> must have instance attribute `name`.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-179-5745deb256bc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mChild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-174-0a2284c79e4b>\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m     76\u001b[0m             \u001b[0;31m# TypeError maintains consistency with abstractmethod.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m                 raise TypeError(f'Instances of {type(inst)} must '\n\u001b[0m\u001b[1;32m     79\u001b[0m                                 f'have instance attribute `{attr}`.')\n\u001b[1;32m     80\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mismethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Instances of <class '__main__.Child'> must have instance attribute `name`."
     ]
    }
   ],
   "source": [
    "c = Child(4)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:01.109074Z",
     "start_time": "2020-11-25T22:18:01.071752Z"
    }
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "<class '__main__.BadChild'> must have class attribute `is_val`.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-180-6198368fede5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# instance var. Throws error at class definition. If we want to accept either,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# we could use abc.abstractmethod or simply define our instance variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mBadChild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mParent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0morder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-174-0a2284c79e4b>\u001b[0m in \u001b[0;36m__new__\u001b[0;34m(cls, name, bases, methods, **meta_kwargs)\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0mare\u001b[0m \u001b[0mconstructed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m         \"\"\"\n\u001b[0;32m---> 33\u001b[0;31m         \u001b[0mclass_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__new__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbases\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethods\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m         \u001b[0mclass_attrs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmeta_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'class_attrs'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m         \u001b[0minst_attrs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmeta_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'inst_attrs'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-174-0a2284c79e4b>\u001b[0m in \u001b[0;36m__init_subclass__\u001b[0;34m(cls, **kwargs)\u001b[0m\n\u001b[1;32m     41\u001b[0m                 \u001b[0;31m# Remaining checks occur at instantiation.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 43\u001b[0;31m                     raise TypeError(f'{cls} must have class attribute '\n\u001b[0m\u001b[1;32m     44\u001b[0m                                     f'`{attr}`.')\n\u001b[1;32m     45\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: <class '__main__.BadChild'> must have class attribute `is_val`."
     ]
    }
   ],
   "source": [
    "# Possesses one of the required class vars but the other is defined as an\n",
    "# instance var. Throws error at class definition. If we want to accept either,\n",
    "# we could use abc.abstractmethod or simply define our instance variables.\n",
    "class BadChild(Parent):\n",
    "    \n",
    "    order = 1\n",
    "    \n",
    "    def __init__(self, b, is_val):\n",
    "        self.b = b\n",
    "        self.is_val = is_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:03.688107Z",
     "start_time": "2020-11-25T22:18:03.683509Z"
    }
   },
   "outputs": [],
   "source": [
    "# Trying to sneakily use a method to fulfill an instance var requirement.\n",
    "class TrickyChild(Parent):\n",
    "    \n",
    "    order = 33\n",
    "    is_val = True\n",
    "    \n",
    "    def __init__(self, a):\n",
    "        self.a = a\n",
    "        print('child init')\n",
    "        \n",
    "    def name(self):\n",
    "        print('in name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:06.134108Z",
     "start_time": "2020-11-25T22:18:06.119762Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "child init\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "`name` must be an instance attribute, not a method.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-182-06a057fee448>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrickyChild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m44\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-174-0a2284c79e4b>\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m     79\u001b[0m                                 f'have instance attribute `{attr}`.')\n\u001b[1;32m     80\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mismethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m                 raise TypeError(f'`{attr}` must be an instance attribute, '\n\u001b[0m\u001b[1;32m     82\u001b[0m                                 'not a method.')\n\u001b[1;32m     83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: `name` must be an instance attribute, not a method."
     ]
    }
   ],
   "source": [
    "tc = TrickyChild(44)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:07.210242Z",
     "start_time": "2020-11-25T22:18:07.204535Z"
    }
   },
   "outputs": [],
   "source": [
    "# Trying to use a property to fulfill an instance var requirement \n",
    "# (acceptable).\n",
    "class TrickyChild2(Parent):\n",
    "    \n",
    "    order = 33\n",
    "    is_val = True\n",
    "    \n",
    "    def __init__(self, a):\n",
    "        self.a = a\n",
    "        print('child init')\n",
    "        \n",
    "    @property\n",
    "    def name(self):\n",
    "        return 'name'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:08.025231Z",
     "start_time": "2020-11-25T22:18:08.021405Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "child init\n"
     ]
    }
   ],
   "source": [
    "tc = TrickyChild2(44)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:08.719931Z",
     "start_time": "2020-11-25T22:18:08.714440Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'name'"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tc.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:09.927525Z",
     "start_time": "2020-11-25T22:18:09.912604Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "child init\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'name'"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TrickyChild2(1).name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:10.636386Z",
     "start_time": "2020-11-25T22:18:10.629715Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<property at 0x10b0b84a8>"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TrickyChild2.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:23.450601Z",
     "start_time": "2020-11-25T22:18:23.446435Z"
    }
   },
   "outputs": [],
   "source": [
    "# Trying to use a property to fulfill a class var requirement. UNacceptable.\n",
    "class TrickyChild3(Parent):\n",
    "    \n",
    "    order = 33\n",
    "    \n",
    "    def __init__(self, a, name):\n",
    "        self.a = a\n",
    "        self.name = name\n",
    "        print('child init')\n",
    "        \n",
    "    @property\n",
    "    def is_val(self):\n",
    "        return 'is val'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:24.789177Z",
     "start_time": "2020-11-25T22:18:24.775257Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "child init\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "`is_val` must be a class attribute, not a property. Properties fulfill instance attribute requirements but not class attribute requirements.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-189-7748292bef5a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrickyChild3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m44\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'sam'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-174-0a2284c79e4b>\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m     93\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproperty\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m                 raise TypeError(\n\u001b[0;32m---> 95\u001b[0;31m                     \u001b[0;34mf'`{attr}` must be a class attribute, not a property. '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     96\u001b[0m                     \u001b[0;34m'Properties fulfill instance attribute requirements but '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m                     \u001b[0;34m'not class attribute requirements.'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: `is_val` must be a class attribute, not a property. Properties fulfill instance attribute requirements but not class attribute requirements."
     ]
    }
   ],
   "source": [
    "tc = TrickyChild3(44, 'sam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:31.613743Z",
     "start_time": "2020-11-25T22:18:31.607170Z"
    }
   },
   "outputs": [],
   "source": [
    "# Trying to use a method to fulfill a class var req. UNacceptable.\n",
    "class TrickyChild4(Parent):\n",
    "    \n",
    "    order = 33\n",
    "    \n",
    "    def __init__(self, a, name):\n",
    "        self.a = a\n",
    "        self.name = name\n",
    "        print('child init')\n",
    "        \n",
    "    def is_val(self):\n",
    "        print(self, type(self))\n",
    "        return self.a, 'is_val'\n",
    "        \n",
    "    def testing(self):\n",
    "        return self.a*2, 'is_val'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:32.238383Z",
     "start_time": "2020-11-25T22:18:32.223166Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "child init\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "`is_val` must be a class attribute, not a method.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-191-4bd69b22702b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrickyChild4\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m44\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'sam'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-174-0a2284c79e4b>\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0;31m# cls, not inst. staticmethod can be retrieved from either.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mismethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mhasstatic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m                 raise TypeError(f'`{attr}` must be a class attribute, not a '\n\u001b[0m\u001b[1;32m     92\u001b[0m                                 'method.')\n\u001b[1;32m     93\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproperty\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: `is_val` must be a class attribute, not a method."
     ]
    }
   ],
   "source": [
    "tc = TrickyChild4(44, 'sam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:35.949307Z",
     "start_time": "2020-11-25T22:18:35.944716Z"
    }
   },
   "outputs": [],
   "source": [
    "# Trying to use a classmethod to fulfill a class var req. UNacceptable.\n",
    "class TrickyChild5(Parent):\n",
    "    \n",
    "    order = 33\n",
    "    \n",
    "    def __init__(self, a, name):\n",
    "        self.a = a\n",
    "        self.name = name\n",
    "        print('child init')\n",
    "        \n",
    "    @classmethod\n",
    "    def is_val(cls):\n",
    "        return 'testing classmethod'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:37.311386Z",
     "start_time": "2020-11-25T22:18:37.282636Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "child init\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "`is_val` must be a class attribute, not a method.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-193-91962abed3e3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrickyChild5\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m44\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'sam'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-174-0a2284c79e4b>\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0;31m# cls, not inst. staticmethod can be retrieved from either.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mismethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mhasstatic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m                 raise TypeError(f'`{attr}` must be a class attribute, not a '\n\u001b[0m\u001b[1;32m     92\u001b[0m                                 'method.')\n\u001b[1;32m     93\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproperty\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: `is_val` must be a class attribute, not a method."
     ]
    }
   ],
   "source": [
    "tc = TrickyChild5(44, 'sam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:35.949307Z",
     "start_time": "2020-11-25T22:18:35.944716Z"
    }
   },
   "outputs": [],
   "source": [
    "# Trying to use a staticmethod to fulfill a class var req. UNacceptable.\n",
    "class TrickyChild6(Parent):\n",
    "    \n",
    "    order = 33\n",
    "    \n",
    "    def __init__(self, a, name):\n",
    "        self.a = a\n",
    "        self.name = name\n",
    "        print('child init')\n",
    "        \n",
    "    @staticmethod\n",
    "    def is_val(a):\n",
    "        return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:37.311386Z",
     "start_time": "2020-11-25T22:18:37.282636Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "child init\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "`is_val` must be a class attribute, not a method.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-195-9c6101162906>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrickyChild6\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m44\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'sam'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-174-0a2284c79e4b>\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0;31m# cls, not inst. staticmethod can be retrieved from either.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mismethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mhasstatic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m                 raise TypeError(f'`{attr}` must be a class attribute, not a '\n\u001b[0m\u001b[1;32m     92\u001b[0m                                 'method.')\n\u001b[1;32m     93\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproperty\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: `is_val` must be a class attribute, not a method."
     ]
    }
   ],
   "source": [
    "tc = TrickyChild6(44, 'sam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:37.984014Z",
     "start_time": "2020-11-25T22:18:37.976428Z"
    }
   },
   "outputs": [],
   "source": [
    "# Implements desired interface.\n",
    "class GoodChild(Parent):\n",
    "    \n",
    "    order = 100\n",
    "    is_val = False\n",
    "    \n",
    "    def __init__(self, name):\n",
    "        self.name = name\n",
    "        \n",
    "    def walk(self):\n",
    "        return self.order, self.is_val, self.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:38.748960Z",
     "start_time": "2020-11-25T22:18:38.746440Z"
    }
   },
   "outputs": [],
   "source": [
    "ian = GoodChild('Ian')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-25T22:18:39.076294Z",
     "start_time": "2020-11-25T22:18:39.070290Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, False, 'Ian')"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ian.walk()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data\n",
    "\n",
    "Get top 1000 sites for lsh dict testing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-28T06:46:42.309177Z",
     "start_time": "2020-11-28T06:46:41.374372Z"
    }
   },
   "outputs": [],
   "source": [
    "r = requests.get('https://dataforseo.com/top-1000-websites')\n",
    "soup = bs(r.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-28T06:54:20.042797Z",
     "start_time": "2020-11-28T06:54:18.802141Z"
    }
   },
   "outputs": [],
   "source": [
    "s3_path = ('s3://commoncrawl/projects/hyperlinkgraph/cc-main-2018-aug-sep-oct'\n",
    "           '/domain/cc-main-2018-aug-sep-oct-domain-ranks.txt.gz')\n",
    "df = pd.read_csv(s3_path, nrows=10_000, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 421,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-28T06:56:08.715036Z",
     "start_time": "2020-11-28T06:56:08.709814Z"
    }
   },
   "outputs": [],
   "source": [
    "def _process_hostname(host):\n",
    "    parts = host.split('.')\n",
    "    return '.'.join(parts[1:]) + '.' + parts[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 423,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-28T06:56:20.406386Z",
     "start_time": "2020-11-28T06:56:20.394649Z"
    }
   },
   "outputs": [],
   "source": [
    "df['host'] = df['#host_rev'].apply(_process_hostname)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:52.475838Z",
     "start_time": "2020-11-30T21:15:52.468064Z"
    }
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as bs\n",
    "from datasketch import lsh, MinHashLSH, MinHash, MinHashLSHForest\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import requests\n",
    "from tldextract import extract\n",
    "\n",
    "# from htools import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:54.002167Z",
     "start_time": "2020-11-30T21:15:53.953159Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['m2', 'm3']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set1 = set(['minhash', 'is', 'a', 'probabilistic', 'data', 'structure', 'for',\n",
    "            'estimating', 'the', 'similarity', 'between', 'datasets'])\n",
    "set2 = set(['minhash', 'is', 'a', 'probability', 'data', 'structure', 'for',\n",
    "            'estimating', 'the', 'similarity', 'between', 'documents'])\n",
    "set3 = set(['minhash', 'is', 'probability', 'data', 'structure', 'for',\n",
    "            'estimating', 'the', 'similarity', 'between', 'documents'])\n",
    "\n",
    "m1 = MinHash(num_perm=128)\n",
    "m2 = MinHash(num_perm=128)\n",
    "m3 = MinHash(num_perm=128)\n",
    "for d in set1:\n",
    "    m1.update(d.encode('utf8'))\n",
    "for d in set2:\n",
    "    m2.update(d.encode('utf8'))\n",
    "for d in set3:\n",
    "    m3.update(d.encode('utf8'))\n",
    "\n",
    "# Create LSH index\n",
    "lsh = MinHashLSH(threshold=0.5, num_perm=128)\n",
    "lsh.insert(\"m2\", m2)\n",
    "lsh.insert(\"m3\", m3)\n",
    "result = lsh.query(m1)\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:15.987298Z",
     "start_time": "2020-11-30T21:15:15.976742Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hmamin/anaconda3/lib/python3.7/site-packages/ipykernel/ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "def ngrams(word, n=3, step=1, drop_last=False):\n",
    "    \"\"\"To get non-overlapping sequences, pass in same value for `step` as `n`.\"\"\"\n",
    "    stop = max(1, step+len(word)-n)\n",
    "    ngrams_ = []\n",
    "    for i in range(0, stop, step):\n",
    "        ngrams_.append(word[i:i+n])\n",
    "    if drop_last and len(ngrams_[-1]) < n: ngrams_ = ngrams_[:-1]\n",
    "    return ngrams_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:17.124629Z",
     "start_time": "2020-11-30T21:15:17.121460Z"
    }
   },
   "outputs": [],
   "source": [
    "w = 'dinosaur'\n",
    "tw = 'I'\n",
    "ws = ['dino', 'rhino', 'dog', 'cat', 'lion', 'tyrannosaurus rex',\n",
    "      'tiger', 'owl', 'rhinocerous', 'kino']\n",
    "s = 'The mailman was running late today, possibly due to the heavy '\\\n",
    "    'snowfall from the previous night.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:17.792499Z",
     "start_time": "2020-11-30T21:15:17.782719Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['din', 'ino', 'nos', 'osa', 'sau', 'aur']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ngrams(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:18.577320Z",
     "start_time": "2020-11-30T21:15:18.572066Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['di', 'in', 'no', 'os', 'sa', 'au', 'ur']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ngrams(w, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:19.296712Z",
     "start_time": "2020-11-30T21:15:19.289859Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['din', 'osa', 'ur']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ngrams(w, 3, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:20.133759Z",
     "start_time": "2020-11-30T21:15:20.127959Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['din', 'osa']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ngrams(w, 3, 3, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:20.810743Z",
     "start_time": "2020-11-30T21:15:20.799733Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The',\n",
       " 'he ',\n",
       " 'e m',\n",
       " ' ma',\n",
       " 'mai',\n",
       " 'ail',\n",
       " 'ilm',\n",
       " 'lma',\n",
       " 'man',\n",
       " 'an ',\n",
       " 'n w',\n",
       " ' wa',\n",
       " 'was',\n",
       " 'as ',\n",
       " 's r',\n",
       " ' ru',\n",
       " 'run',\n",
       " 'unn',\n",
       " 'nni',\n",
       " 'nin',\n",
       " 'ing',\n",
       " 'ng ',\n",
       " 'g l',\n",
       " ' la',\n",
       " 'lat',\n",
       " 'ate',\n",
       " 'te ',\n",
       " 'e t',\n",
       " ' to',\n",
       " 'tod',\n",
       " 'oda',\n",
       " 'day',\n",
       " 'ay,',\n",
       " 'y, ',\n",
       " ', p',\n",
       " ' po',\n",
       " 'pos',\n",
       " 'oss',\n",
       " 'ssi',\n",
       " 'sib',\n",
       " 'ibl',\n",
       " 'bly',\n",
       " 'ly ',\n",
       " 'y d',\n",
       " ' du',\n",
       " 'due',\n",
       " 'ue ',\n",
       " 'e t',\n",
       " ' to',\n",
       " 'to ',\n",
       " 'o t',\n",
       " ' th',\n",
       " 'the',\n",
       " 'he ',\n",
       " 'e h',\n",
       " ' he',\n",
       " 'hea',\n",
       " 'eav',\n",
       " 'avy',\n",
       " 'vy ',\n",
       " 'y s',\n",
       " ' sn',\n",
       " 'sno',\n",
       " 'now',\n",
       " 'owf',\n",
       " 'wfa',\n",
       " 'fal',\n",
       " 'all',\n",
       " 'll ',\n",
       " 'l f',\n",
       " ' fr',\n",
       " 'fro',\n",
       " 'rom',\n",
       " 'om ',\n",
       " 'm t',\n",
       " ' th',\n",
       " 'the',\n",
       " 'he ',\n",
       " 'e p',\n",
       " ' pr',\n",
       " 'pre',\n",
       " 'rev',\n",
       " 'evi',\n",
       " 'vio',\n",
       " 'iou',\n",
       " 'ous',\n",
       " 'us ',\n",
       " 's n',\n",
       " ' ni',\n",
       " 'nig',\n",
       " 'igh',\n",
       " 'ght',\n",
       " 'ht.']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ngrams(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:21.584523Z",
     "start_time": "2020-11-30T21:15:21.581094Z"
    }
   },
   "outputs": [],
   "source": [
    "def lsh_hash_word(word, num_perm=128, **ngram_kwargs):\n",
    "    mhash = MinHash(num_perm=num_perm)\n",
    "    for ng in ngrams(word, **ngram_kwargs):\n",
    "        mhash.update(ng.encode('utf8'))\n",
    "    return mhash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:22.121850Z",
     "start_time": "2020-11-30T21:15:22.116831Z"
    }
   },
   "outputs": [],
   "source": [
    "m = lsh_hash_word(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:22.512486Z",
     "start_time": "2020-11-30T21:15:22.492641Z"
    }
   },
   "outputs": [],
   "source": [
    "forest = MinHashLSHForest(num_perm=128)\n",
    "\n",
    "for i, word in enumerate(ws):\n",
    "    forest.add(i, lsh_hash_word(word))\n",
    "forest.index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:23.445536Z",
     "start_time": "2020-11-30T21:15:23.440200Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['dino',\n",
       " 'rhino',\n",
       " 'dog',\n",
       " 'cat',\n",
       " 'lion',\n",
       " 'tyrannosaurus rex',\n",
       " 'tiger',\n",
       " 'owl',\n",
       " 'rhinocerous',\n",
       " 'kino']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:15:24.236814Z",
     "start_time": "2020-11-30T21:15:24.224462Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'select' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-c9d084a38355>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0midx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mforest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlsh_hash_word\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mws\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'select' is not defined"
     ]
    }
   ],
   "source": [
    "idx = forest.query(lsh_hash_word(w), 3)\n",
    "idx, select(ws, idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:16:09.848117Z",
     "start_time": "2020-11-30T21:16:09.844898Z"
    }
   },
   "outputs": [],
   "source": [
    "from htools.core import select"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:16:10.786864Z",
     "start_time": "2020-11-30T21:16:10.776535Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([4], ['lion'])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = forest.query(lsh_hash_word('ionjwl'), 3)\n",
    "idx, select(ws, idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:16:11.881760Z",
     "start_time": "2020-11-30T21:16:11.863846Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([4, 7], ['lion', 'owl'])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = forest.query(lsh_hash_word('ionowl'), 7)\n",
    "idx, select(ws, idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:16:12.217613Z",
     "start_time": "2020-11-30T21:16:12.209209Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = forest.query(lsh_hash_word('xxx'), 7)\n",
    "idx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data\n",
    "\n",
    "Get top 1000 sites for lsh dict testing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:16:13.999958Z",
     "start_time": "2020-11-30T21:16:13.996749Z"
    }
   },
   "outputs": [],
   "source": [
    "def _process_hostname(host):\n",
    "    parts = host.split('.')\n",
    "    return '.'.join(parts[1:]) + '.' + parts[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:16:21.747265Z",
     "start_time": "2020-11-30T21:16:18.137815Z"
    }
   },
   "outputs": [],
   "source": [
    "s3_path = ('s3://commoncrawl/projects/hyperlinkgraph/cc-main-2018-aug-sep-oct'\n",
    "           '/domain/cc-main-2018-aug-sep-oct-domain-ranks.txt.gz')\n",
    "df = pd.read_csv(s3_path, nrows=100_000, sep='\\t', \n",
    "                 usecols=['#pr_pos', '#host_rev'])\n",
    "df['host'] = df['#host_rev'].apply(_process_hostname)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSHDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:16:40.245708Z",
     "start_time": "2020-11-30T21:16:40.242030Z"
    }
   },
   "outputs": [],
   "source": [
    "from abc import ABC, abstractmethod\n",
    "from fuzzywuzzy import fuzz, process\n",
    "import matplotlib.pyplot as plt\n",
    "import inspect\n",
    "\n",
    "from htools.meta import add_docstring"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:16:40.723824Z",
     "start_time": "2020-11-30T21:16:40.605228Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x151900bb70>]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 10760 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualizing how n_candidates changes as a function of vocab size.\n",
    "x = np.arange(0, 1_000_000, 1_000)\n",
    "y = np.clip(x // 1000, 10, 500)\n",
    "plt.plot(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:19:58.662462Z",
     "start_time": "2020-11-30T21:19:58.647918Z"
    }
   },
   "outputs": [],
   "source": [
    "# abstractmethod doesn't work because we're inheriting from dict which has a\n",
    "# metaclass that's not ABCMeta.\n",
    "class _FuzzyDictBase(dict):\n",
    "    \n",
    "    def __init_subclass__(cls, **kwargs):\n",
    "        super().__init_subclass__(**kwargs)\n",
    "        if not callable(getattr(cls, 'similar', None)):\n",
    "            raise TypeError('Children of _FuzzyDictBase must define method '\n",
    "                            '`similar`.')\n",
    "    \n",
    "    def __getitem__(self, key):\n",
    "        try:\n",
    "            return super().__getitem__(key)\n",
    "        except KeyError:\n",
    "            return self.similar(key, mode='values', n_keys=1)[0]\n",
    "        \n",
    "    def _filter_similarity_pairs(self, pairs, mode='keys_values'):\n",
    "        \"\"\"\n",
    "        mode: str\n",
    "            Determines what information to return. In each case, items are\n",
    "            sorted in descending order of similarity.\n",
    "            - \"keys_values\": list of (key, value) tuples.\n",
    "            - \"keys_similarities\": list of (key, similarity) tuples.\n",
    "            - \"keys_values_similarities\": list of (key, value, similarity) \n",
    "                tuples.\n",
    "            - \"keys\": list of keys (strings).\n",
    "            - \"values\": list of values corresponding to the nearest keys (type\n",
    "                is dependent on what your dict values are).\n",
    "        \"\"\"\n",
    "        if mode == 'values':\n",
    "            return [self[p[0]] for p in pairs]\n",
    "        elif mode == 'keys_values':\n",
    "            return [(p[0], self[p[0]]) for p in pairs]\n",
    "        elif mode == 'keys_similarities':\n",
    "            return pairs\n",
    "        elif mode == 'keys':\n",
    "            return [p[0] for p in pairs]\n",
    "        elif mode == 'keys_values_similarities':\n",
    "            return [(p[0], self[p[0]], p[1]) for p in pairs]\n",
    "        else:\n",
    "            raise ValueError(\n",
    "                'Unrecognized value for `mode`. Should be one of (\"values\", '\n",
    "                '\"keys\", keys_values\", \"keys_similarities\", or '\n",
    "                '\"keys_values_similarities\").'\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:19:59.675875Z",
     "start_time": "2020-11-30T21:19:59.666092Z"
    }
   },
   "outputs": [],
   "source": [
    "class LSHDict(_FuzzyDictBase):\n",
    "    \"\"\"Dictionary that returns the value corresponding to a key's nearest\n",
    "    neighbor if the key isn't present in the dict. This is intended for use\n",
    "    as a word2index dict when using embeddings in deep learning: e.g. if we\n",
    "    have domain embeddings for the top 100k websites, some of our options for \n",
    "    dealing with unknown domains are:\n",
    "    \n",
    "    1. Encode all of them as <UNK>. This loses a lot of information.\n",
    "    2. Create a FuzzyKeyDict which will search for similar keys using variants\n",
    "    of Levenshtein distance. Lookup is O(N) and for 100k domains, that comes\n",
    "    out to 0.6 seconds per item. We might have thousands or millions of \n",
    "    lookups over the course of training so this can be a significant cost.\n",
    "    3. Create an LSHDict (lookups are O(1)). Indexing into the dict as usual \n",
    "    (e.g. my_lsh_dict[key]) will provide the key's index if present and the\n",
    "    (approximate) nearest neighbor's index otherwise. Either way, the result\n",
    "    can be used to index into your embedding layer. \n",
    "    4. Create an LSHDict and use the `similar_values` method to return n>1\n",
    "    neighbors. Then pass their indices to an Embedding layer and \n",
    "    compute the sum/average/weighted average of the results. This may be \n",
    "    preferable to #3 cases such as web domain lookup, where similar URLs are\n",
    "    not guaranteed to represent similar sites. (This is basically\n",
    "    equivalent to an EmbeddingBag layer, but in torch that doesn't store\n",
    "    intermediate representations so we wouldn't be able to use our pretrained\n",
    "    embeddings.)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, data, n_candidates=None, n_keys=3, ngram_size=3,\n",
    "                 scorer=fuzz.ratio):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        data: dict or list[tuple]\n",
    "            The base dictionary. Unlike FuzzyKeyDict, we require this since\n",
    "            adding items one by one is computationally infeasible for large\n",
    "            datasets. Just build up your dictionary first.\n",
    "        n_candidates: int or None\n",
    "            Number of reasonably similar keys to retrieve when trying to index\n",
    "            in with a key that's missing (or when using the `similar` method).\n",
    "            You can override this in `similar` but not when using \n",
    "            __getitem__'s square bracket syntax. If not specified, this will\n",
    "            be auto initialized to vocab size/1,000, clipped to lie in \n",
    "            [20, 500]. See `similar` docstring for more on this.\n",
    "        n_keys: int\n",
    "            Default number of similar keys to retrieve in `similar`.\n",
    "        scorer: function\n",
    "            Default scoring function to use to narrow `n_candidates` keys down\n",
    "            to `n_keys`. Should be a fuzzywuzzy function where scores lie in \n",
    "            [0, 100] and higher values indicate high similarity.\n",
    "        \"\"\"\n",
    "        if len(data) < 10_000 and len(next(iter(data))) < 100:\n",
    "            warnings.warn(\n",
    "                'It looks like you\\'re working with a relatively small '\n",
    "                'amount of data. FuzzyKeyDict may be fast enough for your '\n",
    "                'use case and would provide the set of strictly most similar '\n",
    "                'keys rather than an approximation of that set.'\n",
    "            )\n",
    "            \n",
    "        super().__init__(data)\n",
    "        self.scorer = scorer\n",
    "        self.hash_word = partial(lsh_hash_word, n=ngram_size)\n",
    "        self.forest = MinHashLSHForest(num_perm=128)\n",
    "        self._initialize_forest()\n",
    "        \n",
    "        # Datasketch's LSH implementation usually gives pretty decent results\n",
    "        # even with numbers as low as 5-10, but increasing that by a factor of\n",
    "        # 10 comes with minimal time cost: Fuzzywuzzy matching doesn't get \n",
    "        # particularly slow until we get into the thousands. The fact that \n",
    "        # we cap this at 500 makes this lookup asymptotically O(1) while\n",
    "        # FuzzyKeyDict's is O(N).\n",
    "        self.n_candidates = n_candidates or np.clip(len(self) // 1_000,\n",
    "                                                    20, 500)\n",
    "        self.n_keys = n_keys\n",
    "        \n",
    "    def __setitem__(self, key, val):\n",
    "        \"\"\"Try to add keys all at once in the constructor because adding new\n",
    "        keys can be extremely slow.\n",
    "        \"\"\"\n",
    "        super().__setitem__(key, val)\n",
    "        self._update_forest(key, val)\n",
    "        \n",
    "    def _update_forest(self, key, val, index=True):\n",
    "        \"\"\"Used in __setitem__ to update our LSH Forest. Forest's index method\n",
    "        seems to recompute everything so adding items to a large LSHDict will\n",
    "        be incredibly slow. Luckily, our deep learning use case rarely/never\n",
    "        requires us to update object2index dicts after instantiation so that's\n",
    "        not as troubling as it might seem.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        key: str\n",
    "        val: any\n",
    "        index: bool\n",
    "            If True, reindex the forest (essentially making the key \n",
    "            queryable). This should be False when initializing the forest so\n",
    "            we just index once after everything's been added.\n",
    "        \"\"\"\n",
    "        self.forest.add(key, self.hash_word(key))\n",
    "        if index: self.forest.index()\n",
    "        \n",
    "    def _initialize_forest(self):\n",
    "        \"\"\"Called once in __init__ to add all items to LSH Forest. This is\n",
    "        necessary because dict specifically calls its own __setitem__, not\n",
    "        its children's.\n",
    "        \"\"\"\n",
    "        for k, v in self.items():\n",
    "            self._update_forest(k, v, False)\n",
    "        self.forest.index()\n",
    "    \n",
    "    @add_docstring(_FuzzyDictBase._filter_similarity_pairs)\n",
    "    def similar(self, key, mode='keys_values', n_candidates=None, \n",
    "                n_keys=None, scorer=None):\n",
    "        \"\"\"Find a list of similar keys. This is used in __getitem__ but can\n",
    "        also be useful as a user-facing method if you want to get more than\n",
    "        1 neighbor or you want to get similarity scores as well.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        key: str\n",
    "            Word/URL/etc. to find similar keys to.\n",
    "        mode: str\n",
    "            See section below `Returns`.\n",
    "        n_candidates: int or None\n",
    "            Number of similar candidats to retrieve. This uses Jaccard\n",
    "            Similarity which isn't always a great metric for string \n",
    "            similarity. This is also where the LSH comes in so they're not\n",
    "            strictly the n best candidates, but rather a close approximation\n",
    "            of that set. If None, this will fall back to self.n_candidates.\n",
    "            Keep in mind this determines how many keys to \n",
    "        n_keys: int or None\n",
    "            Number of similar keys to return. If None, this will fall back to\n",
    "            self.n_keys. \n",
    "        scorer: function or None\n",
    "            Fuzzywuzzy scoring function, e.g. fuzz.ratio or \n",
    "            fuzz.partial_ratio, which will be used to score each candidate and\n",
    "            select which to return. Higher scores indicate higher levels of\n",
    "            similarity. If None, this will fall back to self.scorer.\n",
    "            \n",
    "        Returns\n",
    "        -------\n",
    "        list: List if `mode` is \"keys\" or \"values\". List of tuples otherwise.\n",
    "        \"\"\"\n",
    "        candidates = self.forest.query(self.hash_word(key), \n",
    "                                       n_candidates or self.n_candidates)\n",
    "        if not candidates: raise KeyError('No similar keys found.')\n",
    "            \n",
    "        # List of (key, score) where higher means more similar.\n",
    "        pairs = process.extract(key, candidates, \n",
    "                                limit=n_keys or self.n_keys,\n",
    "                                scorer=scorer or self.scorer)\n",
    "        return self._filter_similarity_pairs(pairs, mode=mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T21:19:38.837702Z",
     "start_time": "2020-11-30T21:19:38.826990Z"
    }
   },
   "outputs": [],
   "source": [
    "class FuzzyKeyDict(_FuzzyDictBase):\n",
    "    \"\"\"Dictionary that will try to find the most similar key if a key is\n",
    "    missing and return its corresponding value. This could be useful when\n",
    "    working with embeddings, where we could map missing items to the indices\n",
    "    of one or more existing embeddings.\n",
    "\n",
    "    Examples\n",
    "    --------\n",
    "    d = FuzzyKeyDict(limit=3)\n",
    "    d['dog'] = 0\n",
    "    d['cat'] = 1\n",
    "    d['alley cat'] = 2\n",
    "    d['pig'] = 3\n",
    "    d['cow'] = 4\n",
    "    d['cowbell'] = 5\n",
    "    d['baby cow'] = 6\n",
    "\n",
    "    # Keys and similarity scores for the most similar keys.\n",
    "    >>> d.similar('house cat', mode='keys_similarities')\n",
    "    [('alley cat', 56), ('cat', 50), ('cowbell', 25)]\n",
    "\n",
    "    # \"house cat\" not in dict so we get the value for the most similar key.\n",
    "    >>> d['house cat']\n",
    "    2\n",
    "\n",
    "    # \"cat\" is in dict so __getitem__ functions like a regular dict.\n",
    "    >>> d['cat']\n",
    "    1\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, data=(), n_keys=3, scorer=fuzz.ratio):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        data: Iterable (optional)\n",
    "            Sequence of pairs, such as a dictionary or a list of tuples. If\n",
    "            provided, this will be used to populate the FuzzyKeyDict.\n",
    "        n_keys: int\n",
    "            Default number of similar keys to retrieve when using the \n",
    "            `similar` method.\n",
    "        scorer: function\n",
    "            Default scoring function to use to select the most similar keys in\n",
    "            `similar`. Should be a fuzzywuzzy function where scores lie in \n",
    "            [0, 100] and higher values indicate high similarity.\n",
    "        \"\"\"\n",
    "        super().__init__(data)\n",
    "        self.n_keys = n_keys\n",
    "        self.scorer = scorer\n",
    "\n",
    "    @add_docstring(_FuzzyDictBase._filter_similarity_pairs)\n",
    "    def similar(self, key, mode='keys_values', n_keys=None, scorer=None):\n",
    "        \"\"\"Find the keys in the dictionary that are most similar to the given\n",
    "        key and return some relevant information (the keys, their similarity\n",
    "        scores, their corresponding values, etc.) depending on what the user\n",
    "        specifies.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        key: str\n",
    "            This can be present or missing from the dictionary, though in\n",
    "            practice it's often more useful when it's missing. We'll search\n",
    "            the existing keys and find the strings that are most similar.\n",
    "        mode: str\n",
    "            See section below `Returns`.\n",
    "        n_keys: int or None\n",
    "            Number of similar keys to return. If None, this will fall back to\n",
    "            self.n_keys. \n",
    "        scorer: function or None\n",
    "            Fuzzywuzzy scoring function, e.g. fuzz.ratio or \n",
    "            fuzz.partial_ratio, which will be used to score each candidate and\n",
    "            select which to return. Higher scores indicate higher levels of\n",
    "            similarity. If None, this will fall back to self.scorer.\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        list: List if `mode` is \"keys\" or \"values\". List of tuples otherwise.\n",
    "        \"\"\"\n",
    "        pairs = process.extract(key, \n",
    "                                self.keys(), \n",
    "                                limit=n_keys or self.n_keys,\n",
    "                                scorer=scorer or fuzz.ratio)\n",
    "        return self._filter_similarity_pairs(pairs, mode=mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 897,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T01:25:50.234669Z",
     "start_time": "2020-11-30T01:25:50.232108Z"
    }
   },
   "outputs": [],
   "source": [
    "# df = pd.read_csv('../data/top500Domains.csv', usecols=['Root Domain', 'Rank'])\n",
    "# df = df.rename(columns={'Root Domain': 'domain', 'Rank': 'rank_'})\n",
    "# d2i = dict(zip(df.domain, df.rank_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 891,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T01:24:56.980792Z",
     "start_time": "2020-11-30T01:24:56.968169Z"
    }
   },
   "outputs": [],
   "source": [
    "d2i = dict(df.head(1_000).set_index('host')['#pr_pos'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 892,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T01:25:01.692241Z",
     "start_time": "2020-11-30T01:24:57.658868Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hmamin/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:209: UserWarning: It looks like you're working with a relatively small amount of data. FuzzyKeyDict may be fast enough for your use case and would provide the set of strictly most similar keys rather than an approximation of that set.\n"
     ]
    }
   ],
   "source": [
    "ld = LSHDict(d2i, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 893,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T01:25:09.758669Z",
     "start_time": "2020-11-30T01:25:09.750099Z"
    }
   },
   "outputs": [],
   "source": [
    "fd = FuzzyKeyDict(d2i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 863,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:54:08.742000Z",
     "start_time": "2020-11-30T00:54:08.722930Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 863,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld['youtube.com']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 864,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:54:08.765978Z",
     "start_time": "2020-11-30T00:54:08.744808Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "149"
      ]
     },
     "execution_count": 864,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld['goguardian.com']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 865,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:54:08.780164Z",
     "start_time": "2020-11-30T00:54:08.770841Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 865,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'goguardian.com' in ld"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 835,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:48:04.767521Z",
     "start_time": "2020-11-30T00:48:04.757720Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('theguardian.com', 149),\n",
       " ('lavanguardia.com', 3401),\n",
       " ('gigapan.com', 42313),\n",
       " ('co.guardian.uk', 972)]"
      ]
     },
     "execution_count": 835,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld.similar('goguardian.com', n_candidates=50, n_keys=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 837,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:48:25.652782Z",
     "start_time": "2020-11-30T00:48:25.642930Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('theguardian.com', 149),\n",
       " ('lavanguardia.com', 3401),\n",
       " ('gigapan.com', 42313),\n",
       " ('co.guardian.uk', 972)]"
      ]
     },
     "execution_count": 837,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld.similar('goguardian.com', 'keys_values', n_candidates=50, n_keys=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 838,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:48:26.675824Z",
     "start_time": "2020-11-30T00:48:26.664697Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('theguardian.com', 83),\n",
       " ('lavanguardia.com', 73),\n",
       " ('gigapan.com', 72),\n",
       " ('co.guardian.uk', 71)]"
      ]
     },
     "execution_count": 838,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld.similar('goguardian.com', 'keys_similarities', n_candidates=50, n_keys=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 868,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:55:23.130940Z",
     "start_time": "2020-11-30T00:55:23.120867Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('theguardian.com', 149, 83),\n",
       " ('lavanguardia.com', 3401, 73),\n",
       " ('gigapan.com', 42313, 72),\n",
       " ('co.guardian.uk', 972, 71)]"
      ]
     },
     "execution_count": 868,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld.similar('goguardian.com', 'keys_values_similarities', n_candidates=50, n_keys=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 839,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:48:49.165435Z",
     "start_time": "2020-11-30T00:48:49.155038Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['theguardian.com', 'lavanguardia.com', 'gigapan.com', 'co.guardian.uk']"
      ]
     },
     "execution_count": 839,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld.similar('goguardian.com', 'keys', n_candidates=50, n_keys=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 840,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:48:49.165435Z",
     "start_time": "2020-11-30T00:48:49.155038Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[149, 3401, 42313, 972]"
      ]
     },
     "execution_count": 840,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld.similar('goguardian.com', 'values', n_candidates=50, n_keys=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 841,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:48:51.539953Z",
     "start_time": "2020-11-30T00:48:51.480063Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "149"
      ]
     },
     "execution_count": 841,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fd['goguardian.com']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 842,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:48:59.478804Z",
     "start_time": "2020-11-30T00:48:59.341297Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('theguardian.com', 149), ('garmin.com', 2440), ('youcaring.com', 4403)]"
      ]
     },
     "execution_count": 842,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fd.similar('goguardian.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 844,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:49:09.611673Z",
     "start_time": "2020-11-30T00:49:09.549848Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('theguardian.com', 83),\n",
       " ('garmin.com', 75),\n",
       " ('youcaring.com', 74),\n",
       " ('lavanguardia.com', 73),\n",
       " ('gogetfunding.com', 73)]"
      ]
     },
     "execution_count": 844,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fd.similar('goguardian.com', 'keys_similarities', n_keys=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 845,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:49:13.452275Z",
     "start_time": "2020-11-30T00:49:13.442846Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7594"
      ]
     },
     "execution_count": 845,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld['miniclip.com']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 846,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:49:13.651390Z",
     "start_time": "2020-11-30T00:49:13.627046Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('blogblog.com', 253), ('bloomberg.com', 269), ('mxtoolbox.com', 5351)]"
      ]
     },
     "execution_count": 846,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld.similar('bloodyelbow.com', n_candidates=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 847,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:49:14.297657Z",
     "start_time": "2020-11-30T00:49:14.238255Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('oneyellow.com', 71), ('bebo.com', 70), ('yelloyello.com', 69)]"
      ]
     },
     "execution_count": 847,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fd.similar('bloodyelbow.com', 'keys_similarities')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 848,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:49:14.757114Z",
     "start_time": "2020-11-30T00:49:14.745743Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27962"
      ]
     },
     "execution_count": 848,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld['pausd.edu']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 849,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:49:16.130016Z",
     "start_time": "2020-11-30T00:49:16.067266Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('usd.edu', 27962), ('ucsd.edu', 1401), ('sfusd.edu', 5552)]"
      ]
     },
     "execution_count": 849,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fd.similar('pausd.edu', 'keys_values')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 850,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:49:16.897863Z",
     "start_time": "2020-11-30T00:49:16.888489Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2109"
      ]
     },
     "execution_count": 850,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld['addictinggames.com']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 851,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:49:17.690034Z",
     "start_time": "2020-11-30T00:49:17.680178Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('epicgames.com', 5300), ('aeriagames.com', 28344), ('chinatimes.com', 10750)]"
      ]
     },
     "execution_count": 851,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld.similar('addictinggames.com', n_candidates=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 852,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T00:49:18.792121Z",
     "start_time": "2020-11-30T00:49:18.724960Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('epicgames.com', 5300),\n",
       " ('oxforddictionaries.com', 2800),\n",
       " ('latimes.com', 352)]"
      ]
     },
     "execution_count": 852,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fd.similar('addictinggames.com')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Toy Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 609,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-28T18:09:00.015859Z",
     "start_time": "2020-11-28T18:09:00.007413Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dino': 2,\n",
       " 'rhino': 1,\n",
       " 'dog': 7,\n",
       " 'cat': 3,\n",
       " 'lion': 8,\n",
       " 'tyrannosaurus rex': 0,\n",
       " 'tiger': 5,\n",
       " 'owl': 9,\n",
       " 'rhinocerous': 4,\n",
       " 'kino': 6}"
      ]
     },
     "execution_count": 609,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2i = dict(zip(ws, np.random.permutation(len(ws))))\n",
    "w2i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-28T06:29:01.669268Z",
     "start_time": "2020-11-28T06:29:01.628097Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dino': 6,\n",
       " 'rhino': 3,\n",
       " 'dog': 5,\n",
       " 'cat': 1,\n",
       " 'lion': 0,\n",
       " 'tyrannosaurus rex': 8,\n",
       " 'tiger': 4,\n",
       " 'owl': 9,\n",
       " 'rhinocerous': 2,\n",
       " 'kino': 7}"
      ]
     },
     "execution_count": 331,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld = LSHDict(w2i)\n",
    "ld"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-28T06:29:19.673560Z",
     "start_time": "2020-11-28T06:29:19.657424Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['lion']\n",
      "['kino']\n",
      "[]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(5, 0, 7, -1)"
      ]
     },
     "execution_count": 333,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld['dog'], ld['sea lion'], ld['munchkin'], ld['kitten']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-28T06:27:19.872397Z",
     "start_time": "2020-11-28T06:27:19.864529Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['lion']"
      ]
     },
     "execution_count": 321,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld.forest.query(lsh_hash_word('estion'), 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-28T06:27:36.461703Z",
     "start_time": "2020-11-28T06:27:36.451918Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['tyrannosaurus rex']"
      ]
     },
     "execution_count": 323,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld.forest.query(lsh_hash_word('brontosaur'), 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-28T06:27:23.287019Z",
     "start_time": "2020-11-28T06:27:23.280216Z"
    }
   },
   "outputs": [],
   "source": [
    "ld['dragon'] = 6"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
