{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8f0bc55e",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a8d4bc5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp query"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "811ebd94",
   "metadata": {},
   "source": [
    "# Query"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc0a04bf",
   "metadata": {},
   "source": [
    "This is an interface to *querying* resources that can be fetched.\n",
    "\n",
    "It covers local files, the [Common Crawl Index](http://commoncrawl.org/2015/04/announcing-the-common-crawl-index/) and the [Internet Archive's CDX Server](https://github.com/internetarchive/wayback/tree/master/wayback-cdx-server) for finding archived resources based on URL patterns.\n",
    "The latter two have  a lot of overlap with [cdx_toolkit](https://github.com/cocrawler/cdx_toolkit) but this is more optimised for speed and ease of use."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a830ace2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "# Typing\n",
    "from __future__ import annotations # For Python <3.9\n",
    "from typing import Any, Callable, Generator, Optional, Union\n",
    "from collections.abc import Iterable\n",
    "from pathlib import Path\n",
    "from dataclasses import dataclass\n",
    "\n",
    "from datetime import datetime\n",
    "\n",
    "import requests\n",
    "from requests.sessions import Session\n",
    "import json\n",
    "\n",
    "from warcio.recordloader import ArcWarcRecord\n",
    "import warcio\n",
    "\n",
    "from webrefine.util import sha1_digest, URL, make_session"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26350a57",
   "metadata": {},
   "source": [
    "# Local WARC Files"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b441b5b6",
   "metadata": {},
   "source": [
    "## Object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "045bc2b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "\n",
    "@dataclass(frozen=True)\n",
    "class WarcFileRecord:\n",
    "    url: str\n",
    "    timestamp: datetime\n",
    "    mime: str\n",
    "    status: int\n",
    "    path: Path\n",
    "    offset: int\n",
    "    digest: str\n",
    "        \n",
    "    def get_content(self):\n",
    "        with open(self.path, 'rb') as f:\n",
    "            f.seek(self.offset)\n",
    "            record = next(warcio.ArchiveIterator(f))\n",
    "            return record.content_stream().read() \n",
    "        \n",
    "    def preview(self, filename):\n",
    "        with open(filename, 'wb') as f:\n",
    "            f.write(self.content)\n",
    "        return FileLink(filename) \n",
    "    \n",
    "    @property\n",
    "    def content(self):\n",
    "        return self.get_content()\n",
    "    \n",
    "def get_warc_url(record: ArcWarcRecord) -> str:\n",
    "    return record.rec_headers.get_header('WARC-Target-URI')\n",
    "\n",
    "_WARC_TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%SZ'\n",
    "def get_warc_timestamp(record: ArcWarcRecord) -> datetime:\n",
    "    return datetime.strptime(record.rec_headers.get_header('WARC-Date'), _WARC_TIMESTAMP_FORMAT)\n",
    "\n",
    "def get_warc_mime(record: ArcWarcRecord) -> str:\n",
    "    return record.http_headers.get_header('Content-Type').split(';')[0]\n",
    "\n",
    "def get_warc_status(record: ArcWarcRecord) -> int:\n",
    "    return int(record.http_headers.get_statuscode())\n",
    "\n",
    "def get_warc_digest(record: ArcWarcRecord) -> str:\n",
    "    digest = record.rec_headers.get_header('WARC-Payload-Digest')\n",
    "    prefix = 'sha1:'\n",
    "    if not digest.startswith(prefix):\n",
    "        raise ValueError('Expected %s to start with %s', digest, prefix)\n",
    "    return digest[len(prefix):]\n",
    "\n",
    "class WarcFileQuery:\n",
    "    def __init__(self, path: Union[str, Path]) -> None:\n",
    "        self.path = Path(path)\n",
    "        \n",
    "    def query(self) -> Generator[WarcRecord, None, None]:\n",
    "        results = []\n",
    "        with open(self.path, 'rb') as f:\n",
    "            archive = warcio.ArchiveIterator(f)\n",
    "            for record in archive:\n",
    "                if record.rec_type != 'response':\n",
    "                    continue\n",
    "                warc_record = WarcFileRecord(url=get_warc_url(record),\n",
    "                                         timestamp=get_warc_timestamp(record),\n",
    "                                         mime = get_warc_mime(record),\n",
    "                                         status = get_warc_status(record),\n",
    "                                         digest = get_warc_digest(record),\n",
    "                                         offset = archive.get_record_offset(),\n",
    "                                         path = self.path)\n",
    "                                         \n",
    "                                         \n",
    "                results.append(warc_record)\n",
    "        return results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7bd0444",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67271d3c",
   "metadata": {},
   "source": [
    "Generated some test data with:\n",
    "\n",
    "```\n",
    "wget -r -Q1M --domains skeptric.com --warc-file=skeptric --delete-after --no-directories https://skeptric.com/pagination-wayback-cdx/\n",
    "```\n",
    "\n",
    "See [warcio library](https://github.com/webrecorder/warcio#writing-warc-records) for how to do this in Python."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8bbe86c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = '../resources/test/skeptric.warc.gz'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "01c1f785",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[WarcFileRecord(url='https://skeptric.com/pagination-wayback-cdx/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 34), mime='text/html', status='200', path=PosixPath('../resources/test/skeptric.warc.gz'), offset=889, digest='Q6HKH563Z7HF2333QILSSSHY2K3B6NOK'),\n",
       " WarcFileRecord(url='https://skeptric.com/robots.txt', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 34), mime='text/html', status='404', path=PosixPath('../resources/test/skeptric.warc.gz'), offset=5804, digest='QRNGXIUXE4LAI3XR5RVATIUX5GTB33HX'),\n",
       " WarcFileRecord(url='https://skeptric.com/style.main.min.5ea2f07be7e07e221a7112a3095b89d049b96c48b831f16f1015bf2d95d914e5.css', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 35), mime='text/css', status='200', path=PosixPath('../resources/test/skeptric.warc.gz'), offset=7197, digest='LINCDTSPQGAQGZZ6LY2XFXZHG2X476H6'),\n",
       " WarcFileRecord(url='https://skeptric.com/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 36), mime='text/html', status='200', path=PosixPath('../resources/test/skeptric.warc.gz'), offset=17122, digest='JJVB3MQERHRZJCHOJNKS5VDOODXPZAV2'),\n",
       " WarcFileRecord(url='https://skeptric.com/about/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 37), mime='text/html', status='200', path=PosixPath('../resources/test/skeptric.warc.gz'), offset=125261, digest='Z5NRUTRW3XTKZDCJFDKGPJ5BWIBNQCG7'),\n",
       " WarcFileRecord(url='https://skeptric.com/tags/data', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 37), mime='text/html', status='302', path=PosixPath('../resources/test/skeptric.warc.gz'), offset=129093, digest='ZZZXDZTTV2KTABRO64ESHVWFPNKB4I5H'),\n",
       " WarcFileRecord(url='https://skeptric.com/tags/data/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 38), mime='text/html', status='200', path=PosixPath('../resources/test/skeptric.warc.gz'), offset=130269, digest='R7CLAACFU5L7T5LKI5G53RZSMCNUNV6F'),\n",
       " WarcFileRecord(url='https://skeptric.com/images/wayback_empty_returns.png', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 38), mime='image/png', status='200', path=PosixPath('../resources/test/skeptric.warc.gz'), offset=160971, digest='SU7JRTHNW6KFCJQFL5PMMKV33U2VLV7T'),\n",
       " WarcFileRecord(url='https://skeptric.com/searching-100b-pages-cdx', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 39), mime='text/html', status='302', path=PosixPath('../resources/test/skeptric.warc.gz'), offset=173368, digest='AYVHQLVFIVGZGUYPEHX46CHMZ5NUDDBF'),\n",
       " WarcFileRecord(url='https://skeptric.com/searching-100b-pages-cdx/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 39), mime='text/html', status='200', path=PosixPath('../resources/test/skeptric.warc.gz'), offset=174558, digest='CMT3ZNELTRC7H7ICVCGAYYS6GQ2NSZGP'),\n",
       " WarcFileRecord(url='https://skeptric.com/fast-web-data-workflow/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 39), mime='text/html', status='200', path=PosixPath('../resources/test/skeptric.warc.gz'), offset=188608, digest='UMQCERJOQ3AGE3Z576ABKKKNDGSNBQAX'),\n",
       " WarcFileRecord(url='https://skeptric.com/key-web-captures/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 40), mime='text/html', status='200', path=PosixPath('../resources/test/skeptric.warc.gz'), offset=195651, digest='5GLUI5DL5PFUM6QY53NWUCMJBC2CTXKS'),\n",
       " WarcFileRecord(url='https://skeptric.com/emacs-tempfile-hugo/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 40), mime='text/html', status='200', path=PosixPath('../resources/test/skeptric.warc.gz'), offset=201243, digest='JY2UE6SKCYPHXSUN5IR6SQVMZRRAGU64')]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "warc = WarcFileQuery(test_data)\n",
    "results = warc.query()\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a61cee8",
   "metadata": {},
   "source": [
    "Try fetching a record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9024343f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 16196 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_record = [r for r in results if r.mime == 'image/png'][0]\n",
    "\n",
    "content = image_record.content\n",
    "from IPython.display import Image\n",
    "Image(content)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "683e3510",
   "metadata": {},
   "source": [
    "Check the digests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a876fd34",
   "metadata": {},
   "outputs": [],
   "source": [
    "for result in results:\n",
    "    assert result.digest == sha1_digest(result.get_content())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a941c9d9",
   "metadata": {},
   "source": [
    "# Internet Archive\n",
    "\n",
    "\n",
    "## Querying CDX\n",
    "\n",
    "The Internet Archive runs it's own [Java CDX Server](https://github.com/internetarchive/wayback/tree/master/wayback-cdx-server). [Hakernoon has an article](https://hackernoon.com/guide-to-handling-internet-archives-cdx-server-api-response-c469df5b81f4) that gives a good overview of querying it.\n",
    "\n",
    "See the documentation for all the paramemters; here are the ones that I find most interesting\n",
    "\n",
    "* url: The URL to query, a wildcard `*` can implicitly define it (which means we don't need `matchType`)\n",
    "* output: \"json\" returns JSON instead of space separated\n",
    "* fl: Comma separated list of fields to return\n",
    "* from, to: Date filtering 1-14 digits yyyyMMddhhmmss, inclusive (this seems to do the right thing for truncation).\n",
    "* filter: `[!]field:regex` filter a returned field with [Java Regex](https://docs.oracle.com/javase/6/docs/api/java/util/regex/Pattern.html), inverting with `!`\n",
    "* limit/offset: For getting a small number of results (sampling). Internal limit is 150000 results.\n",
    "\n",
    "Default fields returned: [\"urlkey\",\"timestamp\",\"original\",\"mimetype\",\"statuscode\",\"digest\",\"length\"]\n",
    "\n",
    "Pagination isn't really useful because it's applied *before* filtering (including date filtering) so most pages are empty with a date filter.\n",
    "Unpaginated requests are fast enough anyway."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0d7e70b",
   "metadata": {},
   "source": [
    "### Exporting the data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d59a6309",
   "metadata": {},
   "source": [
    "The data is a JSON array or arrays; the first line contains the headers and the subsequent the data.\n",
    "Let's transform it into a list of dictionaries of keys to values which make it a bit easier to work with (although less efficient in memory).\n",
    "\n",
    "An alternative would be to directly use something like Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7a1e0ab4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def header_and_rows_to_dict(rows: Iterable[list[Any]]) -> list[dict[Any, Any]]:\n",
    "    header = None\n",
    "    data = []\n",
    "    for row in rows:\n",
    "        if header is None:\n",
    "            header = row\n",
    "        else:\n",
    "            assert len(row) == len(header), \"Row should be same length as header\"\n",
    "            data.append(dict(zip(header, row)))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4400afe",
   "metadata": {},
   "source": [
    "Check it on some data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c67767e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert header_and_rows_to_dict([['col_1', 'col_2'], [1, 'a'], [2, 'b']]) == [\n",
    "    {'col_1': 1, 'col_2': 'a'},\n",
    "    {'col_1': 2, 'col_2': 'b'}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "68604f65",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert header_and_rows_to_dict([['col_1', 'col_2']]) == []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "83e9c33d",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert header_and_rows_to_dict([]) == []"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21e3d242",
   "metadata": {},
   "source": [
    "### Making a query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dcf57eba",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "IA_CDX_URL = 'http://web.archive.org/cdx/search/cdx'\n",
    "\n",
    "# This could be more precise\n",
    "CaptureIndexRecord = dict\n",
    "\n",
    "def mimetypes_to_regex(mime: list[str], prefix='mimetype:') -> str:\n",
    "    return prefix + '|'.join('(' + s.replace('*', '.*') + ')' for s in mime)\n",
    "\n",
    "def query_wayback_cdx(url: str, start: Optional[str], end: Optional[str],\n",
    "                      status_ok: bool = True, \n",
    "                      mime: Optional[Union[str, Iterable[str]]] = None,\n",
    "                      limit: Optional[int] = None, offset: Optional[int] = None,\n",
    "                      session: Optional[Session] = None) -> list[CaptureIndexRecord]:\n",
    "    \"\"\"Get references to Wayback Machine Captures for url.\n",
    "    \n",
    "    Queries the Internet Archive Capture Index (CDX) for url.\n",
    "    \n",
    "    Arguments:\n",
    "      * start: Minimum date in format YYYYmmddHHMMSS (or any substring) inclusive\n",
    "      * end: Maximum date in format YYYYmmddHHMMSS (or any substring) inclusive\n",
    "      * status_ok: Only return those with a HTTP status 200\n",
    "      * mime: Filter on mimetypes, '*' is a wildcard (e.g. 'image/*')\n",
    "      * limit: Only return first limit records\n",
    "      * offset: Skip the first offset records, combine with limit\n",
    "      * session: Session to use when making requests\n",
    "    Filters results between start and end inclusive, in format YYYYmmddHHMMSS or any substring\n",
    "    (e.g. start=\"202001\", end=\"202001\" will get all captures in January 2020)\n",
    "    \"\"\"\n",
    "    if session is None:\n",
    "        session = requests\n",
    "        \n",
    "    params = {'url': url,\n",
    "              'output': 'json',\n",
    "              'from': start,\n",
    "              'to': end,\n",
    "              'limit': limit,\n",
    "              'offset': offset}\n",
    "    \n",
    "    filter = []\n",
    "    if status_ok:\n",
    "        filter.append('statuscode:200')\n",
    "    if mime:\n",
    "        # Turn our list of mimetypes into a regex\n",
    "        if isinstance(mime, str):\n",
    "            mime = [mime]\n",
    "        filter.append(mimetypes_to_regex(mime))\n",
    "    params['filter'] = filter\n",
    "    \n",
    "    params = {k:v for k,v in params.items() if v}\n",
    "    response = session.get(IA_CDX_URL, params=params)\n",
    "    response.raise_for_status()\n",
    "    return header_and_rows_to_dict(response.json())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08cada58",
   "metadata": {},
   "source": [
    "#### Test querying"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "eda75893",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 22.6 ms, sys: 3.83 ms, total: 26.5 ms\n",
      "Wall time: 1.15 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "full_sample = query_wayback_cdx('skeptric.com/*', start=None, end=None, status_ok=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6aea9c2b",
   "metadata": {},
   "source": [
    "#### Test statuscode filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1b5f2f0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16.5 ms, sys: 596 µs, total: 17.1 ms\n",
      "Wall time: 808 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "ok_sample = query_wayback_cdx('skeptric.com/*', start=None, end=None) #status_ok=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a6b0a931",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert ok_sample == [x for x in full_sample if x['statuscode'] == '200']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab5499ad",
   "metadata": {},
   "source": [
    "#### Test mimetypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "36d8aed9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.2 ms, sys: 183 µs, total: 10.4 ms\n",
      "Wall time: 611 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "html_sample = query_wayback_cdx('skeptric.com/*', start=None, end=None, status_ok=False, mime='text/html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c69fafd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert html_sample == [x for x in full_sample if x['mimetype'] == 'text/html']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c88a6921",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.91 ms, sys: 2.61 ms, total: 8.52 ms\n",
      "Wall time: 698 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "image_sample = query_wayback_cdx('skeptric.com/*', start=None, end=None, status_ok=False, mime='image/*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b61d2097",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert image_sample == [x for x in full_sample if x['mimetype'].startswith('image/')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "64391fa9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.76 ms, sys: 454 µs, total: 6.21 ms\n",
      "Wall time: 446 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "prog_sample = query_wayback_cdx('skeptric.com/*', start=None, end=None, status_ok=False, \n",
    "                                 mime=['text/css', 'application/javascript'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "262a5fc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert prog_sample == [x for x in full_sample if x['mimetype'] in ['text/css', 'application/javascript']]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f77f2442",
   "metadata": {},
   "source": [
    "#### Test date filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1ae552eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.68 ms, sys: 6.3 ms, total: 11 ms\n",
      "Wall time: 444 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "106"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "sample_2020 = query_wayback_cdx('skeptric.com/*', start='2020', end='2020', status_ok=False)\n",
    "len(sample_2020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "89ed2b51",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert sample_2020 == [x for x in full_sample if '2020' <= x['timestamp'] < '2021']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "fb54dc8d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11.5 ms, sys: 343 µs, total: 11.8 ms\n",
      "Wall time: 441 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "sample_to_2020 = query_wayback_cdx('skeptric.com/*', start=None, end='2020', status_ok=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e40cebc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert sample_to_2020 == [x for x in full_sample if x['timestamp'] < '2021']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "aa2e9d04",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16.6 ms, sys: 684 µs, total: 17.3 ms\n",
      "Wall time: 975 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "sample_from_2020 = query_wayback_cdx('skeptric.com/*', start='2020', end=None, status_ok=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5aa5a888",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert sample_from_2020 == [x for x in full_sample if x['timestamp'] >= '2020']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a01655d",
   "metadata": {},
   "source": [
    "#### Test limits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "87dbb1fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8.4 ms, sys: 382 µs, total: 8.78 ms\n",
      "Wall time: 415 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "sample_10 = query_wayback_cdx('skeptric.com/*', start=None, end=None, status_ok=False, limit=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "066f8b55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_10 == full_sample[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "112d8e95",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8.54 ms, sys: 366 µs, total: 8.91 ms\n",
      "Wall time: 711 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "sample_10_offset_20 = query_wayback_cdx('skeptric.com/*', start=None, end=None, status_ok=False, limit=10, offset=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "77d5c06a",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert sample_10_offset_20  == full_sample[20:20+10]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8928fec4",
   "metadata": {},
   "source": [
    "## Fetching Content"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68cd777d",
   "metadata": {},
   "source": [
    "The original content is at `http://web.archive.org/web/{timestamp}_id/{url}`, and the Wayback Machine version is the same excluding `_id`.\n",
    "The Wayback Machine version [makes some changes](https://skeptric.com/restoring-wayback-html/) that make it better for interactive viewing, but the content is different."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "e2d19a75",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def wayback_url(timestamp: str, url: str, wayback: bool = False) -> str:\n",
    "    postfix = '' if wayback else 'id_'\n",
    "    return f'http://web.archive.org/web/{timestamp}{postfix}/{url}'\n",
    "\n",
    "def fetch_wayback_content(timestamp: str, url: str,\n",
    "                          session: Optional[Session] = None) -> Optional[bytes]:\n",
    "    if session is None:\n",
    "        session = requests\n",
    "        \n",
    "    url = wayback_url(timestamp, url)\n",
    "    response = session.get(url)\n",
    "    # Sometimes Internet Archive deletes records\n",
    "    if response.status_code == 404:\n",
    "        logging.warning(f'Missing {url}')\n",
    "        return None\n",
    "    response.raise_for_status()\n",
    "    return response.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "252a7b86",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'urlkey': 'com,skeptric)/favicon.ico',\n",
       " 'timestamp': '20180819051126',\n",
       " 'original': 'http://skeptric.com/favicon.ico',\n",
       " 'mimetype': 'image/vnd.microsoft.icon',\n",
       " 'statuscode': '200',\n",
       " 'digest': 'R6YE2GPPT4BM4IAMHGUJPDJF6BGKKHDA',\n",
       " 'length': '851'}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "record = image_sample[0]\n",
    "record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "d6282e7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "content = fetch_wayback_content(record['timestamp'], record['original'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "ff18f8d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADgAAACYAAAAmwAAAIwAAABjAAAAFgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHoAAAD/AAAAQQAAAAEAAAABAAAANAAAAOoAAAAnAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACwAAAD/AAAAewAAAAAAAAAAAAAAAAAAAAAAAAB7AAAA6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACoAAAA9gAAAJQAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAP8AAABPAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAzQAAAPwAAAD/AAAAAAAAAAAAAAAAAAAAAAAAACUAAAD7AAAAkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKoAAAD/AAAA/wAAAAAAAABUAAAAAAAAAAAAAAAmAAAA/QAAAJwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAuAAAA/wAAAP8AAAACAAAA0gAAAAAAAAAAAAAAYgAAAP8AAABlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGEAAAD/AAAAkgAAAPwAAAAAAAAAAgAAAPkAAAD8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAXwAAAPEAAAD/AAAACgAAAMwAAAD/AAAASgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPAAAAqwAAAOwAAAD/AAAAYQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQwAAAPoAAADqAAAAPAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYwAAAP8AAADEAAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGAAAAPIAAAC5AAAAAQAAAAAAAAADAAAAHgAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC8AAADxAAAAAAAAAAAAAAABAAAArgAAAPgAAABgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaAAAA5gAAACAAAAAbAAAAfQAAAO4AAACbAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACUAAAB7AAAAegAAAE8AAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/H8AAPvfAAD37wAA4+8AAOPnAADj5wAA8u8AAPjPAAD8nwAA/j8AAP5/AAD8/wAA+f8AAPufAAD7nwAA//8AAA==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import Image\n",
    "Image(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "f41a00f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert record['digest'] == sha1_digest(content)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7391fff7",
   "metadata": {},
   "source": [
    "And fetching a HTML webpage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "48737310",
   "metadata": {},
   "outputs": [],
   "source": [
    "record = ok_sample[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e0f12231",
   "metadata": {},
   "outputs": [],
   "source": [
    "content = fetch_wayback_content(record['timestamp'], record['original'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "af8afb94",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert record['digest'] == sha1_digest(content)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1602812c",
   "metadata": {},
   "source": [
    "Get the Wayback URL for preview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "915526c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href=\"http://web.archive.org/web/20180805132731/http://skeptric.com/\">http://web.archive.org/web/20180805132731/http://skeptric.com/</a>"
      ],
      "text/plain": [
       "URL(url='http://web.archive.org/web/20180805132731/http://skeptric.com/')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "URL(wayback_url(record['timestamp'], record['original'], wayback=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "196b798d",
   "metadata": {},
   "source": [
    "## Wayback Record Object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "07f77f96",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "\n",
    "@dataclass(frozen=True)\n",
    "class WaybackRecord:\n",
    "    url: str\n",
    "    timestamp: datetime\n",
    "    mime: str\n",
    "    status: Optional[int]\n",
    "    digest: str\n",
    "        \n",
    "    def preview(self) -> URL:\n",
    "        return URL(wayback_url(self.timestamp_str, self.url, wayback=True))\n",
    "\n",
    "    @property\n",
    "    def timestamp_str(self) -> str:\n",
    "        return self.timestamp.strftime(_WAYBACK_TIMESTAMP_FORMAT)\n",
    "        \n",
    "    def get_content(self, session=None) -> Optional[bytes]:\n",
    "        return fetch_wayback_content(self.timestamp_str, self.url, session=None)\n",
    "    \n",
    "    @property\n",
    "    def content(self):\n",
    "        return self.get_content()\n",
    "    \n",
    "    @classmethod\n",
    "    def from_dict(cls, record: dict):\n",
    "        return _wayback_cdx_to_record(record)\n",
    "        \n",
    "_WAYBACK_TIMESTAMP_FORMAT = '%Y%m%d%H%M%S'\n",
    "def _wayback_cdx_to_record(record: dict) -> WaybackRecord:\n",
    "    return WaybackRecord(url = record['original'],\n",
    "                         timestamp = datetime.strptime(record['timestamp'], _WAYBACK_TIMESTAMP_FORMAT),\n",
    "                         mime = record['mimetype'],\n",
    "                         status = None if record['statuscode'] == '-' else int(record['statuscode']),\n",
    "                         digest = record['digest'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "51e71a51",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WaybackRecord(url='http://skeptric.com/', timestamp=datetime.datetime(2018, 8, 5, 13, 27, 31), mime='text/html', status=200, digest='2N5QQYZFAM36CSESTGSDKRTRV7I5HEXJ')"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "record = WaybackRecord.from_dict(full_sample[0])\n",
    "record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "067547db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href=\"http://web.archive.org/web/20180805132731/http://skeptric.com/\">http://web.archive.org/web/20180805132731/http://skeptric.com/</a>"
      ],
      "text/plain": [
       "URL(url='http://web.archive.org/web/20180805132731/http://skeptric.com/')"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "record.preview()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "4fee3266",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8.38 ms, sys: 3.03 ms, total: 11.4 ms\n",
      "Wall time: 718 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "b'<!DOCTYPE html>\\n<html lang=\"en\">\\n  <head>\\n    <meta charset=\"utf-8\" />\\n    <title>Edward Ross - An \\xc9'"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "content = record.content\n",
    "content[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "bc8b014c",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert sha1_digest(content) == record.digest"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a72a835",
   "metadata": {},
   "source": [
    "## Wayback Query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "6fa3b1ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@dataclass\n",
    "class WaybackQuery:\n",
    "    url: str\n",
    "    start: Optional[str]\n",
    "    end: Optional[str]\n",
    "    status_ok: bool = True\n",
    "    mime: Optional[Union[str, Iterable[str]]] = None\n",
    "        \n",
    "    def query(self,\n",
    "              limit: Optional[int] = None,\n",
    "              session: Optional[Session] = None) -> Generator[WaybackRecord, None, None]:\n",
    "        for r in query_wayback_cdx(self.url, self.start, self.end, self.status_ok, self.mime, limit, session=session):\n",
    "            yield _wayback_cdx_to_record(r)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bee966fc",
   "metadata": {},
   "source": [
    "#### Test fetching one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "217129ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "wb = WaybackQuery('skeptric.com/*', start=None, end=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "66ed8ea8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 53.9 ms, sys: 3.47 ms, total: 57.3 ms\n",
      "Wall time: 1.15 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "items = list(wb.query())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "2df52f00",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(items) > 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "84519530",
   "metadata": {},
   "outputs": [],
   "source": [
    "content = items[0].content\n",
    "assert sha1_digest(content) == items[0].digest"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "404f57f8",
   "metadata": {},
   "source": [
    "#### Test fetching many"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1b9c28e",
   "metadata": {},
   "source": [
    "I have no idea whether Session is [actually thread-safe](https://github.com/urllib3/urllib3/issues/1252); if it's not we should have 1 session per thread. As far as I can tell the issue occurs when you have lots of hosts, so in this case it should be ok.\n",
    "\n",
    "I guess we'll try it and see. Maybe long term we're better going with asyncio.\n",
    "Using a Session makes things slightly faster; we can always turn it off by passing session=False."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "818ee0e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import delayed, Parallel\n",
    "\n",
    "def wayback_fetch_parallel(items, threads=8, session=None):\n",
    "    if session is None:\n",
    "        session = make_session(threads)\n",
    "    return Parallel(n_jobs=threads, prefer='threads')(delayed(item.get_content)(session=session) for item in items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "ef3a2781",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 165 ms, sys: 16.7 ms, total: 181 ms\n",
      "Wall time: 846 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "contents_16 = wayback_fetch_parallel(items[:16], threads=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "1be81e33",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert contents_16[0] == content"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88a3a25b",
   "metadata": {},
   "source": [
    "# Common Crawl"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7e7523f",
   "metadata": {},
   "source": [
    "Common Crawl is similar, but slightly different, because it is split over many indexes, roughly monthly covering 2 weeks."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57c71a3d",
   "metadata": {},
   "source": [
    "## Indexes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c29750f7",
   "metadata": {},
   "source": [
    "Cache within a session, but not between sessions since over months the indexes change."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "2a2b002c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from functools import lru_cache\n",
    "@lru_cache(maxsize=None)\n",
    "def get_cc_indexes() -> List[Dict[str, str]]:\n",
    "    response = requests.get(\"https://index.commoncrawl.org/collinfo.json\")\n",
    "    response.raise_for_status()\n",
    "    return response.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "b804e130",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': 'CC-MAIN-2021-43',\n",
       "  'name': 'October 2021 Index',\n",
       "  'timegate': 'https://index.commoncrawl.org/CC-MAIN-2021-43/',\n",
       "  'cdx-api': 'https://index.commoncrawl.org/CC-MAIN-2021-43-index'},\n",
       " {'id': 'CC-MAIN-2021-39',\n",
       "  'name': 'September 2021 Index',\n",
       "  'timegate': 'https://index.commoncrawl.org/CC-MAIN-2021-39/',\n",
       "  'cdx-api': 'https://index.commoncrawl.org/CC-MAIN-2021-39-index'},\n",
       " {'id': 'CC-MAIN-2021-31',\n",
       "  'name': 'July 2021 Index',\n",
       "  'timegate': 'https://index.commoncrawl.org/CC-MAIN-2021-31/',\n",
       "  'cdx-api': 'https://index.commoncrawl.org/CC-MAIN-2021-31-index'}]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indexes = get_cc_indexes()\n",
    "indexes[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49240fd7",
   "metadata": {},
   "source": [
    "Notice the oldest ones have a different format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "5c1ed6fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': 'CC-MAIN-2013-20',\n",
       "  'name': 'Summer 2013 Index',\n",
       "  'timegate': 'https://index.commoncrawl.org/CC-MAIN-2013-20/',\n",
       "  'cdx-api': 'https://index.commoncrawl.org/CC-MAIN-2013-20-index'},\n",
       " {'id': 'CC-MAIN-2012',\n",
       "  'name': 'Index of 2012 ARC files',\n",
       "  'timegate': 'https://index.commoncrawl.org/CC-MAIN-2012/',\n",
       "  'cdx-api': 'https://index.commoncrawl.org/CC-MAIN-2012-index'},\n",
       " {'id': 'CC-MAIN-2009-2010',\n",
       "  'name': 'Index of 2009 - 2010 ARC files',\n",
       "  'timegate': 'https://index.commoncrawl.org/CC-MAIN-2009-2010/',\n",
       "  'cdx-api': 'https://index.commoncrawl.org/CC-MAIN-2009-2010-index'},\n",
       " {'id': 'CC-MAIN-2008-2009',\n",
       "  'name': 'Index of 2008 - 2009 ARC files',\n",
       "  'timegate': 'https://index.commoncrawl.org/CC-MAIN-2008-2009/',\n",
       "  'cdx-api': 'https://index.commoncrawl.org/CC-MAIN-2008-2009-index'}]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indexes[-4:]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b7a4d71",
   "metadata": {},
   "source": [
    "### Parsing the times\n",
    "\n",
    "We can parse approximate periods for the crawls using the same logic as [cdx_toolkit](https://github.com/cocrawler/cdx_toolkit/blob/0.9.33/cdx_toolkit/timeutils.py#L68).\n",
    "\n",
    "However in reality [it's complicated](https://skeptric.com/common-crawl-time-ranges/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "e9098401",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import re\n",
    "def parse_cc_crawl_date(crawl_id: str) -> datetime:\n",
    "    year_week = re.match(r'^CC-MAIN-(\\d{4}-\\d{2})$', crawl_id)\n",
    "    year = re.match(r'^CC-MAIN-(?:\\d{4}-)?(\\d{4})', crawl_id)\n",
    "    if year_week:\n",
    "        return datetime.strptime(year_week.group(1)  + '-0', '%Y-%W-%w')\n",
    "    elif year:\n",
    "        return datetime.strptime(year.group(1)  + '-12-01', '%Y-%m-%d')\n",
    "    else:\n",
    "        raise ValueError(f'Unexpected id: {crawl_id}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "b0113b9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "dates = [parse_cc_crawl_date(i['id']) for i in indexes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "481f8e52",
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def cc_index_by_time(start:Optional[datetime]=None, end:Optional[datetime]=None, indexes:Optional[list[str]]=None) -> list[str]:\n",
    "    \"\"\"Gets all indexes that may contain entries between start and end\n",
    "    \n",
    "    Generally errs on the side of giving an additional index\"\"\"\n",
    "    if indexes is None:\n",
    "        indexes = [i['id'] for i in get_cc_indexes()]\n",
    "        \n",
    "    if start and end and end < start:\n",
    "        raise ValueError(f\"Expect start >= end: start={start}, end={end}\")\n",
    "    \n",
    "        \n",
    "    dates = [parse_cc_crawl_date(i) for i in indexes]\n",
    "    if start:\n",
    "        previous_date = max((ts for ts in dates if ts < start), default=min(dates))\n",
    "    else:\n",
    "        previous_date = min(dates)\n",
    "    \n",
    "    if end:\n",
    "        next_date = min((ts for ts in dates if ts > end), default=max(dates))\n",
    "    else:\n",
    "        next_date = max(dates)\n",
    "\n",
    "    return [i for i in indexes if previous_date <= parse_cc_crawl_date(i) <= next_date]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "a8d85ba7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CC-MAIN-2021-43',\n",
       " 'CC-MAIN-2021-39',\n",
       " 'CC-MAIN-2021-31',\n",
       " 'CC-MAIN-2021-25',\n",
       " 'CC-MAIN-2021-21',\n",
       " 'CC-MAIN-2021-17',\n",
       " 'CC-MAIN-2021-10']"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cc_index_by_time(start=datetime(2021,5,1), end=datetime(2021,12,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "9b23cf9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CC-MAIN-2009-2010', 'CC-MAIN-2008-2009']"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cc_index_by_time(end=datetime(2010,1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "4dcff1cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CC-MAIN-2021-43',\n",
       " 'CC-MAIN-2021-39',\n",
       " 'CC-MAIN-2021-31',\n",
       " 'CC-MAIN-2021-25',\n",
       " 'CC-MAIN-2021-21',\n",
       " 'CC-MAIN-2021-17',\n",
       " 'CC-MAIN-2021-10',\n",
       " 'CC-MAIN-2021-04',\n",
       " 'CC-MAIN-2020-50']"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cc_index_by_time(start=datetime(2021,1,1))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b8cc1ec",
   "metadata": {},
   "source": [
    "## Querying CDX"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e65eeb1",
   "metadata": {},
   "source": [
    "https://pywb.readthedocs.io/en/latest/manual/cdxserver_api.html\n",
    "\n",
    "http://commoncrawl.org/2015/04/announcing-the-common-crawl-index/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "bdac8493",
   "metadata": {},
   "outputs": [],
   "source": [
    "cc_sample_api = 'https://index.commoncrawl.org/CC-MAIN-2021-43-index'\n",
    "assert cc_sample_api in [i['cdx-api'] for i in indexes]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1c7da86",
   "metadata": {},
   "source": [
    "The Common Crawl CDX is *much slower* than the Wayback Machines.\n",
    "It also uses the zipped CDX by default which automatically paginates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "3de00256",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import json\n",
    "\n",
    "def jsonl_loads(jsonl):\n",
    "    return [json.loads(line) for line in jsonl.splitlines()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "7585e1b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_jline = b'{\"status\": \"200\", \"mime\": \"text/html\"}\\n{\"status\": \"301\", \"mime\": \"-\"}\\n'\n",
    "test_jdict = [{'status': '200', 'mime': 'text/html'}, {'status': '301', 'mime': '-'}]\n",
    "assert jsonl_loads(test_jline) == test_jdict\n",
    "assert jsonl_loads(test_jline.rstrip()) == test_jdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "2379b35f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "\n",
    "# Default page size for queries (the underlying default is 5)\n",
    "CC_PAGE_SIZE = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "357bc201",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "\n",
    "def query_cc_cdx_num_pages(api: str,  url: str, page_size: int = CC_PAGE_SIZE,\n",
    "                           session: Optional[Session] = None) -> int:\n",
    "    if session is None:\n",
    "        session = requests\n",
    "        \n",
    "    response = session.get(api, params=dict(url=url, output='json',\n",
    "                                            showNumPages=True, pageSize=page_size))\n",
    "                                            \n",
    "    response.raise_for_status()\n",
    "    data = response.json()\n",
    "    return data[\"pages\"]\n",
    "\n",
    "def query_cc_cdx_page(\n",
    "                 api: str, url: str, page: int, \n",
    "                 start: Optional[str] = None, end: Optional[str] = None,\n",
    "                 status_ok: bool = True, mime: Optional[Union[str, Iterable[str]]] = None,\n",
    "                 limit: Optional[int] = None, offset: Optional[int] = None,\n",
    "                 page_size: int = CC_PAGE_SIZE,\n",
    "                 session: Optional[Session] = None) -> List[CaptureIndexRecord]:\n",
    "    \"\"\"Get references to Common Crawl Captures for url.\n",
    "    \n",
    "    Queries the Common Crawl Capture Index (CDX) for url.\n",
    "    \n",
    "    Filters:\n",
    "      * api: API endpoint to use (e.g. 'https://index.commoncrawl.org/CC-MAIN-2021-43-index')\n",
    "      * start: Minimum date in format YYYYmmddHHMMSS (or any substring) inclusive\n",
    "      * end: Maximum date in format YYYYmmddHHMMSS (or any substring) inclusive\n",
    "      * status_ok: Only return those with a HTTP status 200\n",
    "      * mime: Filter on mimetypes, '*' is a wildcard (e.g. 'image/*')\n",
    "      * limit: Only return first limit records\n",
    "      * offset: Skip the first offset records, combine with limit\n",
    "      * session: Session to use when making requests\n",
    "    Filters results between start and end inclusive, in format YYYYmmddHHMMSS or any substring\n",
    "    (e.g. start=\"202001\", end=\"202001\" will get all captures in January 2020)\n",
    "    \"\"\"\n",
    "    if session is None:\n",
    "        session = requests\n",
    "        \n",
    "    params = {'url': url,\n",
    "              'page': page,\n",
    "              'output': 'json',\n",
    "              'page': page,\n",
    "              'from': start,\n",
    "              'to': end,\n",
    "              'pageSize': page_size,\n",
    "              'limit': limit,\n",
    "              'offset': offset}\n",
    "    \n",
    "    filter = []\n",
    "    if status_ok:\n",
    "        # N.B. Different to IA\n",
    "        filter.append('=status:200')\n",
    "    if mime:\n",
    "        if isinstance(mime, str):\n",
    "            mime = [mime]\n",
    "        # N.B. Different to IA\n",
    "        filter.append(mimetypes_to_regex(mime, prefix='~mime:'))\n",
    "    params['filter'] = filter\n",
    "    \n",
    "    params = {k:v for k,v in params.items() if v}\n",
    "    response = session.get(api, params=params)\n",
    "    response.raise_for_status()\n",
    "    return jsonl_loads(response.content)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6ef7337",
   "metadata": {},
   "source": [
    "### Testing pagination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "46a17d2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_url = 'rmy.wikipedia.org/*'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "ea2c822e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#slow\n",
    "pages = query_cc_cdx_num_pages(cc_sample_api, test_url, page_size=1)\n",
    "pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "e3c41a3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "576"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#slow\n",
    "records = []\n",
    "for page in range(pages):\n",
    "    records += query_cc_cdx_page(cc_sample_api, test_url, page=page, mime='text/html', page_size=1)\n",
    "len(records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "dd46cb21",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('20211017144648', '20211026191502')"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#slow\n",
    "timestamps = [record['timestamp'] for record in records]\n",
    "min(timestamps), max(timestamps)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4591a96f",
   "metadata": {},
   "source": [
    "We should get an error querying past the last page (400: Invalid Request)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "d492ea9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#slow\n",
    "from requests.exceptions import HTTPError\n",
    "\n",
    "try:\n",
    "    query_cc_cdx_page(cc_sample_api, test_url, page=pages, mime='text/html', page_size=1)\n",
    "    raise AssertionError('Expected Failure')\n",
    "except HTTPError as e:\n",
    "    status = e.response.status_code\n",
    "    if status != 400:\n",
    "        raise AssertionError(f'Expected 400, got {status}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d4a06ca",
   "metadata": {},
   "source": [
    "### Error for missing fields"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e29d182",
   "metadata": {},
   "source": [
    "For CC-MAIN-2015-11 and CC-MAIN-2015-06 both the status and mime are missing from the index which leads to *problems*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "8a4a0c2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "api_2015_11 = next(idx['cdx-api'] for idx in get_cc_indexes() if idx['id'] == 'CC-MAIN-2015-11')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee19d50f",
   "metadata": {},
   "source": [
    "This turns up in this particular case as a 404"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "41c647a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#slow\n",
    "try:\n",
    "    results = query_cc_cdx_page(api_2015_11, test_url, page=0, page_size=1)\n",
    "    raise ValueError(\"Has it been fixed? Remove the blacklist\")\n",
    "except requests.exceptions.HTTPError as e:\n",
    "    if e.response.status_code != 404:\n",
    "        raise"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5561bf5",
   "metadata": {},
   "source": [
    "Which doesn't happen when we remove the filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "b57b30a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#slow\n",
    "results = query_cc_cdx_page(api_2015_11, test_url, page=0, status_ok=False, page_size=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "8e4c100b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "CC_API_FILTER_BLACKLIST = ['CC-MAIN-2015-11', 'CC-MAIN-2015-06']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "497b1f7b",
   "metadata": {},
   "source": [
    "## Downloading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "3850b705",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from warcio import ArchiveIterator\n",
    "from io import BytesIO\n",
    "\n",
    "CC_DATA_URL = \"https://commoncrawl.s3.amazonaws.com/\"\n",
    "def fetch_cc(filename: str, offset: int, length: int, session: Optional[Session] = None) -> bytes:\n",
    "    if session is None:\n",
    "        session = requests\n",
    "    data_url = CC_DATA_URL + filename\n",
    "    start_byte = int(offset)\n",
    "    end_byte = start_byte + int(length)\n",
    "    headers = {\"Range\": f\"bytes={start_byte}-{end_byte}\"}\n",
    "    r = session.get(data_url, headers=headers)\n",
    "    r.raise_for_status()\n",
    "    \n",
    "    # Decord WARC\n",
    "    response_content = r.content\n",
    "    archive = ArchiveIterator(BytesIO(response_content))\n",
    "    record = next(archive)\n",
    "    content = record.content_stream().read()\n",
    "    \n",
    "    # Archive should have just 1 record\n",
    "    assert not any(True for _ in archive), \"Expected 1 result in archive\"\n",
    "    \n",
    "    return content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "6b27219f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'urlkey': 'org,wikipedia,rmy)/wiki/%c3%93lvega',\n",
       " 'timestamp': '20211019031358',\n",
       " 'url': 'https://rmy.wikipedia.org/wiki/%C3%93lvega',\n",
       " 'mime': 'text/html',\n",
       " 'mime-detected': 'text/html',\n",
       " 'status': '200',\n",
       " 'digest': 'QZSQHAQSLMBHHCJSADEUS3I4U3EJALH5',\n",
       " 'length': '11604',\n",
       " 'offset': '641473227',\n",
       " 'filename': 'crawl-data/CC-MAIN-2021-43/segments/1634323585231.62/warc/CC-MAIN-20211019012407-20211019042407-00526.warc.gz',\n",
       " 'languages': 'ron,nno,vie',\n",
       " 'encoding': 'UTF-8'}"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#slow\n",
    "record = records[0]\n",
    "record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "a92567c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "record = {'urlkey': 'org,wikipedia,en)/?curid=3516101',\n",
    " 'timestamp': '20211024051554',\n",
    " 'url': 'https://en.wikipedia.org/?curid=3516101',\n",
    " 'mime': 'text/html',\n",
    " 'mime-detected': 'text/html',\n",
    " 'status': '200',\n",
    " 'digest': 'TFZFWXWKS3NFJLLLXLYU6JTNZ77D3IVD',\n",
    " 'length': '8922',\n",
    " 'offset': '332329771',\n",
    " 'filename': 'crawl-data/CC-MAIN-2021-43/segments/1634323585911.17/warc/CC-MAIN-20211024050128-20211024080128-00689.warc.gz',\n",
    " 'languages': 'eng',\n",
    " 'encoding': 'UTF-8'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "c1c6c8b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 25.2 ms, sys: 56 µs, total: 25.3 ms\n",
      "Wall time: 1.17 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "content = fetch_cc(record['filename'], record['offset'], record['length'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "bc5c239d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'<!DOCTYPE html>\\n<html class=\"client-nojs\" lang=\"en\" dir=\"ltr\">\\n<head>\\n<meta charset=\"UTF-8\"/>\\n<title>Category:Tad Morose albums - Wikipedia</title>\\n<script>document.documentElement.className=\"client-js\";RLCONF={\"wgBreakFrames\":!1,\"wgSeparatorTransformTable\":[\"\",\"\"],\"wgDigitTransformTable\":[\"\",\"\"],\"wgDefaultDateFormat\":\"dmy\",\"wgMonthNames\":[\"\",\"January\",\"February\",\"March\",\"April\",\"May\",\"June\",\"July\",\"August\",\"September\",\"October\",\"November\",\"December\"],\"wgRequestId\":\"362a73a3-175d-4af3-92bb-4707797d6067\",\"wgCSPNonce\":!1,\"wgCanonicalNamespace\":\"Category\",\"wgCanonicalSpecialPageName\":!1,\"wgNamespaceNumber\":14,\"wgPageName\":\"Category:Tad_Morose_albums\",\"wgTitle\":\"Tad Morose albums\",\"wgCurRevisionId\":906041584,\"wgRevisionId\":906041584,\"wgArticleId\":3516101,\"wgIsArticle\":!0,\"wgIsRedirect\":!1,\"wgAction\":\"view\",\"wgUserName\":null,\"wgUserGroups\":[\"*\"],\"wgCategories\":[\"Set categories\",\"Albums by artist\",\"Power metal albums by Swedish artists\"],\"wgPageContentLanguage\":\"en\",\"wgPageContentModel\":\"wik'"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "content[:1000]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bae2349",
   "metadata": {},
   "source": [
    "Check the digest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "7bfc0d96",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert sha1_digest(content) == record['digest']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89e8f533",
   "metadata": {},
   "source": [
    "## Put it into an Object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "432d8648",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "_CC_TIMESTAMP_FORMAT = '%Y%m%d%H%M%S'\n",
    "from IPython.display import FileLink\n",
    "\n",
    "@dataclass(frozen=True)\n",
    "class CommonCrawlRecord:\n",
    "    url: str\n",
    "    timestamp: datetime\n",
    "    filename: str\n",
    "    offset: int\n",
    "    length: int\n",
    "    mime: Optional[str]\n",
    "    status: Optional[int]\n",
    "    digest: Optional[str]\n",
    "        \n",
    "    def preview(self, filename):\n",
    "        with open(filename, 'wb') as f:\n",
    "            f.write(self.content)\n",
    "        return FileLink(filename)                    \n",
    "        \n",
    "    @property\n",
    "    def timestamp_str(self) -> str:\n",
    "        return self.timestamp.strftime(_CC_TIMESTAMP_FORMAT)\n",
    "          \n",
    "    def get_content(self, session=None) -> Optional[bytes]:\n",
    "        return fetch_cc(self.filename, self.offset, self.length, session=session)\n",
    "    \n",
    "    @property\n",
    "    def content(self):\n",
    "        return self.get_content()\n",
    "    \n",
    "    @classmethod\n",
    "    def from_dict(cls, record: dict):\n",
    "        return _cc_cdx_to_record(record)\n",
    "        \n",
    "_WAYBACK_TIMESTAMP_FORMAT = '%Y%m%d%H%M%S'\n",
    "def _cc_cdx_to_record(record: dict) -> WaybackRecord:\n",
    "    return CommonCrawlRecord(url = record['url'],\n",
    "                         timestamp = datetime.strptime(record['timestamp'], _WAYBACK_TIMESTAMP_FORMAT),\n",
    "                             filename=record['filename'],\n",
    "                             offset=record['offset'],\n",
    "                             length=record['length'],\n",
    "                         mime = record.get('mime'),\n",
    "                         status = None if record.get('status', '-') == '-' else int(record['status']),\n",
    "                         digest = record.get('digest'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "9588ccc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'urlkey': 'org,wikipedia,en)/?curid=3516101',\n",
       " 'timestamp': '20211024051554',\n",
       " 'url': 'https://en.wikipedia.org/?curid=3516101',\n",
       " 'mime': 'text/html',\n",
       " 'mime-detected': 'text/html',\n",
       " 'status': '200',\n",
       " 'digest': 'TFZFWXWKS3NFJLLLXLYU6JTNZ77D3IVD',\n",
       " 'length': '8922',\n",
       " 'offset': '332329771',\n",
       " 'filename': 'crawl-data/CC-MAIN-2021-43/segments/1634323585911.17/warc/CC-MAIN-20211024050128-20211024080128-00689.warc.gz',\n",
       " 'languages': 'eng',\n",
       " 'encoding': 'UTF-8'}"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "a9f1ae31",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CommonCrawlRecord(url='https://en.wikipedia.org/?curid=3516101', timestamp=datetime.datetime(2021, 10, 24, 5, 15, 54), filename='crawl-data/CC-MAIN-2021-43/segments/1634323585911.17/warc/CC-MAIN-20211024050128-20211024080128-00689.warc.gz', offset='332329771', length='8922', mime='text/html', status=200, digest='TFZFWXWKS3NFJLLLXLYU6JTNZ77D3IVD')"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = _cc_cdx_to_record(record)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "ab5c26d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'<!DOCTYPE html>\\n<html class=\"client-nojs\" lang=\"en\" dir=\"ltr\">\\n<head>\\n<meta charset=\"UTF-8\"/>\\n<title'"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.content[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "0d327e17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='tmpn7cny9ks.html' target='_blank'>tmpn7cny9ks.html</a><br>"
      ],
      "text/plain": [
       "/home/edward/src/webrefine/nbs/tmpn7cny9ks.html"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tempfile import NamedTemporaryFile\n",
    "f = NamedTemporaryFile('wb', dir='.', suffix='.html')\n",
    "name = Path(f.name).relative_to(Path('.').absolute())\n",
    "x.preview(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "c7aac64e",
   "metadata": {},
   "outputs": [],
   "source": [
    "f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fbdcf13",
   "metadata": {},
   "source": [
    "## Common Crawl Query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "347b2dd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import logging\n",
    "\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class CommonCrawlQuery:\n",
    "    url: str\n",
    "    start: Optional[str] = None\n",
    "    end: Optional[str] = None\n",
    "    apis: Optional[list[str]] = None\n",
    "    status_ok: bool = True\n",
    "    mime: Optional[Union[str, Iterable[str]]] = None\n",
    "    \n",
    "    @property\n",
    "    def cdx_apis(self) -> Dict[str, str]:\n",
    "        all_apis = get_cc_indexes()\n",
    "        if self.apis is None:\n",
    "            apis = cc_index_by_time(self.start, self.end)\n",
    "        else:\n",
    "            apis = self.apis\n",
    "            \n",
    "        return {x['id']: x['cdx-api'] for x in all_apis if x['id'] in apis}\n",
    "    \n",
    "    def query(self, page_size=CC_PAGE_SIZE, session=None) -> Generator[CommonCrawlRecord, None, None]:\n",
    "        for api_id, api in self.cdx_apis.items():\n",
    "            num_pages = query_cc_cdx_num_pages(api, self.url, page_size=page_size, session=session)\n",
    "            \n",
    "            for page in range(num_pages):\n",
    "                if api_id not in CC_API_FILTER_BLACKLIST:\n",
    "                    results_page = query_cc_cdx_page(api, self.url, page, page_size=page_size, status_ok=self.status_ok, mime=self.mime, session=session)\n",
    "                else:\n",
    "                    # Deal with missing Status OK and Mime\n",
    "                    results_page = query_cc_cdx_page(api, self.url, page, page_size=page_size, status_ok=False, mime=None, session=session)\n",
    "\n",
    "                for result in results_page:\n",
    "                    yield _cc_cdx_to_record(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "705642b8",
   "metadata": {},
   "source": [
    "### Test querying on blacklist APIs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "8bb8faac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CC-MAIN-2015-11', 'CC-MAIN-2015-06']"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CC_API_FILTER_BLACKLIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "0ce6dd00",
   "metadata": {},
   "outputs": [],
   "source": [
    "query_cc_all = CommonCrawlQuery('www.commoncrawl.org/*',\n",
    "                           start=datetime(2015,3,1), end=datetime(2015,3,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "4df03121",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.73 ms, sys: 307 µs, total: 5.04 ms\n",
      "Wall time: 5.02 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'CC-MAIN-2015-11': 'https://index.commoncrawl.org/CC-MAIN-2015-11-index',\n",
       " 'CC-MAIN-2015-06': 'https://index.commoncrawl.org/CC-MAIN-2015-06-index'}"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "query_cc_all.cdx_apis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "fc412129",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# slow\n",
    "x = list(query_cc_all.query())\n",
    "len(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "906b1853",
   "metadata": {},
   "source": [
    "Notice there is no Mime or Status. We *could* fill it in from the archive record."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "d8f36cb5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CommonCrawlRecord(url='http://commoncrawl.org/', timestamp=datetime.datetime(2015, 3, 2, 3, 27, 5), filename='crawl-data/CC-MAIN-2015-11/segments/1424936462700.28/warc/CC-MAIN-20150226074102-00159-ip-10-28-5-156.ec2.internal.warc.gz', offset='53235662', length='2526', mime=None, status=None, digest='QE4UUUWUJWEZBBK6PUG3CHFAGEKDMDBZ')"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#slow\n",
    "x[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "82fcdf7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'\\n<!doctype html>\\n<html class=\"no-js\" lang=\"en\">\\n<head>\\n<meta charset=\"utf-8\"/>\\n<meta name=\"viewport\"'"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#slow\n",
    "x[0].content[:100]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "846c4625",
   "metadata": {},
   "source": [
    "### Test querying on happy case data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "f8d9c165",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_url = 'upload.wikimedia.org/wikipedia/commons/*'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "e739a26f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({('application/pdf', 200): 561,\n",
       "         ('text/html', 404): 181,\n",
       "         ('image/vnd.djvu', 200): 125,\n",
       "         ('image/jpeg', 200): 65,\n",
       "         ('image/png', 200): 20,\n",
       "         ('unk', 301): 31,\n",
       "         ('application/ogg', 200): 2,\n",
       "         ('application/sla', 200): 2,\n",
       "         ('image/svg+xml', 200): 4,\n",
       "         ('image/gif', 200): 2,\n",
       "         ('image/x-xcf', 200): 2,\n",
       "         ('text/html', 412): 2,\n",
       "         ('warc/revisit', 304): 1,\n",
       "         ('text/html', 400): 154})"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#slow\n",
    "cc_archive_all = CommonCrawlQuery(test_url,\n",
    "                                  apis=['CC-MAIN-2021-43'],\n",
    "                                  status_ok=False)\n",
    "\n",
    "results = list(cc_archive_all.query(page_size=5))\n",
    "\n",
    "from collections import Counter\n",
    "Counter((r.mime, r.status) for r in results)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd8fcc16",
   "metadata": {},
   "source": [
    "### Test status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "02377e6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#slow\n",
    "results_ok = list(CommonCrawlQuery(test_url,\n",
    "                                  apis=['CC-MAIN-2021-43']).query(page_size=5))\n",
    "\n",
    "assert results_ok == [r for r in results if r.status == 200]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ae169bb",
   "metadata": {},
   "source": [
    "### Test mimetypes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ac64d02",
   "metadata": {},
   "source": [
    "These would fail in the BLACKLIST apis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "b5cc6e93",
   "metadata": {},
   "outputs": [],
   "source": [
    "#slow\n",
    "results_html = list(CommonCrawlQuery(test_url,\n",
    "                                  apis=['CC-MAIN-2021-43'],\n",
    "                                  status_ok=False,\n",
    "                                  mime='text/html').query(page_size=5))\n",
    "\n",
    "assert results_html == [r for r in results if r.mime == 'text/html']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "647308d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#slow\n",
    "mimes = ['application/pdf', 'image/vnd.djvu', 'application/ogg']\n",
    "results_mime = list(CommonCrawlQuery(test_url,\n",
    "                                  apis=['CC-MAIN-2021-43'],\n",
    "                                  status_ok=False,\n",
    "                                  mime=mimes).query(page_size=5))\n",
    "\n",
    "assert results_mime == [r for r in results if r.mime in mimes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "b9d1fe10",
   "metadata": {},
   "outputs": [],
   "source": [
    "#slow\n",
    "results_image = list(CommonCrawlQuery(test_url,\n",
    "                                  apis=['CC-MAIN-2021-43'],\n",
    "                                  status_ok=False,\n",
    "                                  mime='image/*').query(page_size=5))\n",
    "\n",
    "assert results_image == [r for r in results if r.mime.startswith('image/')]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cad8a7bc",
   "metadata": {},
   "source": [
    "### Check fetching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "f9e5aff7",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = _cc_cdx_to_record({'urlkey': 'com,skeptric)/',\n",
    " 'timestamp': '20211028110756',\n",
    " 'url': 'https://skeptric.com/',\n",
    " 'mime': 'text/html',\n",
    " 'mime-detected': 'text/html',\n",
    " 'status': '200',\n",
    " 'digest': '7RBLUZ55MD4FUPDRPVVJVTM7YDYQXRS3',\n",
    " 'length': '107238',\n",
    " 'offset': '650789450',\n",
    " 'filename': 'crawl-data/CC-MAIN-2021-43/segments/1634323588284.71/warc/CC-MAIN-20211028100619-20211028130619-00465.warc.gz',\n",
    " 'languages': 'eng',\n",
    " 'encoding': 'UTF-8'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "9f4ff344",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert sha1_digest(result.content) == result.digest"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be4cac42",
   "metadata": {},
   "source": [
    "### Check fetching in parallel\n",
    "\n",
    "Since they're hosted on AWS S3 we can go wild with parallelism.\n",
    "This code is actually identical to the wayback version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "8df2f11f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import delayed, Parallel\n",
    "\n",
    "\n",
    "def cc_fetch_parallel(items, threads=64, session=None):\n",
    "    if session is None:\n",
    "        session = make_session(threads)\n",
    "    return Parallel(n_jobs=threads, prefer='threads')(delayed(item.get_content)(session=session) for item in items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "d3e41bcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "#slow\n",
    "many_content = cc_fetch_parallel(results_html[:100])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "480px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
