{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "08b0264c",
   "metadata": {},
   "source": [
    "# Utility Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "93dd45bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp util"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32551def",
   "metadata": {},
   "source": [
    "## SHA-1 Digest"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b529f373",
   "metadata": {},
   "source": [
    "Calculating the base 32 encoded SHA-1 digest that is commonly used in WARC files and CDX indexes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "57e2518b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from hashlib import sha1\n",
    "from base64 import b32encode\n",
    "\n",
    "def sha1_digest(content: bytes) -> str:\n",
    "    return b32encode(sha1(content).digest()).decode('ascii')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8c7b6bfb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sha1_digest(b'12345')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "454e9abc",
   "metadata": {},
   "source": [
    "## Making URLs Pretty\n",
    "\n",
    "Sometimes I want to return something that looks like a URL in Jupyter, but works in other environments. Adapted from [here](https://github.com/jupyterlab/jupyterlab/issues/7393)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6e8cf0ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from dataclasses import dataclass\n",
    "\n",
    "@dataclass(frozen=True)\n",
    "class URL:\n",
    "    \"\"\"Wrapper around a URL string to provide nice display in IPython environments.\"\"\"\n",
    "\n",
    "    url: str\n",
    "\n",
    "    def _repr_html_(self):\n",
    "        \"\"\"HTML link to this URL.\"\"\"\n",
    "        return f'<a href=\"{self.url}\">{self.url}</a>'\n",
    "\n",
    "    def __str__(self):\n",
    "        \"\"\"Return the underlying string.\"\"\"\n",
    "        return self.url"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "462d0944",
   "metadata": {},
   "source": [
    "It displays nicely"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2a6254b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href=\"https://commoncrawl.org/\">https://commoncrawl.org/</a>"
      ],
      "text/plain": [
       "URL(url='https://commoncrawl.org/')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = URL('https://commoncrawl.org/')\n",
    "url"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "250af5ef",
   "metadata": {},
   "source": [
    "The repr is usable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a97f123f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"URL(url='https://commoncrawl.org/')\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repr(url)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b4ddaef",
   "metadata": {},
   "source": [
    "The string form is what we need"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "56101c2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://commoncrawl.org/'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(url)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1506b57e",
   "metadata": {},
   "source": [
    "Or we can extract it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "39b1e3b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://commoncrawl.org/'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url.url"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "860566e1",
   "metadata": {},
   "source": [
    "# Session Helpers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "054a2f2a",
   "metadata": {},
   "source": [
    "Make a session that can run multiple concurrent requests and retry for intermittent failures."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "af0195a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import requests\n",
    "from requests.adapters import HTTPAdapter\n",
    "from requests.packages.urllib3.util.retry import Retry\n",
    "\n",
    "def make_session(pool_maxsize):\n",
    "    retry_strategy =  Retry(total=5, backoff_factor=1, status_forcelist=set([504, 500]))\n",
    "    adapter = HTTPAdapter(max_retries=retry_strategy, pool_maxsize=pool_maxsize, pool_block=True)\n",
    "    session = requests.Session()\n",
    "    session.mount('http://', adapter)\n",
    "    session.mount('https://', adapter)\n",
    "    return session"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b5f80c3",
   "metadata": {},
   "source": [
    "## Joblib Helpers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ee7e5ef",
   "metadata": {},
   "source": [
    "Forcing a function with joblib.Memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5712be80",
   "metadata": {},
   "outputs": [],
   "source": [
    "def _forced(f, force):\n",
    "    \"\"\"Forced version of memoized function with Memory\"\"\"\n",
    "    assert hasattr(f, 'call')\n",
    "    if not force:\n",
    "        return f\n",
    "    def result(*args, **kwargs):\n",
    "        # Force returns a tuple of result,metadata\n",
    "        return f.call(*args, **kwargs)[0]\n",
    "    return result"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
