{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ed0d3b64",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7a2487a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp runners"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94a27398",
   "metadata": {},
   "source": [
    "# Process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "67ccd133",
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "from __future__ import annotations\n",
    "from dataclasses import dataclass\n",
    "from typing import Callable\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ef7b38b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "import logging\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "@dataclass\n",
    "class ProcessMemory:\n",
    "    queries: list[Callable]\n",
    "    steps: list[Callable]\n",
    "    filter: Callable\n",
    "    progress_bar: bool = True\n",
    "\n",
    "    def query(self):\n",
    "        for query in tqdm(self.queries, desc='query', disable=not self.progress_bar):\n",
    "            for record in query.query():\n",
    "                yield record\n",
    "\n",
    "    def prepare(self, records):\n",
    "        return self.filter(tqdm(records, desc='filter', disable=not self.progress_bar))\n",
    "\n",
    "    def fetch(self, records):\n",
    "        for record in tqdm(records, desc='fetch', disable=not self.progress_bar):\n",
    "            yield (record.content, record)\n",
    "\n",
    "    def transform(self, content_records):\n",
    "        for content, record in tqdm(content_records, desc='transform', disable=not self.progress_bar):\n",
    "            error = False\n",
    "            for step in self.steps:\n",
    "                try:\n",
    "                    content = step(content, record)\n",
    "                except Exception as e:\n",
    "                    error = True\n",
    "                    logging.error('Error processing %s at step %s: %s' % (record, step.__name__, e))\n",
    "                    break\n",
    "            if not error:\n",
    "                yield content\n",
    "\n",
    "    def run(self):\n",
    "        records = self.prepare(self.query())\n",
    "        content_records = self.fetch(records)\n",
    "        return self.transform(content_records) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d7cff28",
   "metadata": {},
   "source": [
    "## Simple test process"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7f48e96",
   "metadata": {},
   "source": [
    "In practice we'd use a something like parsel, beautifulsoup or selectolax.\n",
    "However for a simple demo this has no external dependencies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "14be7a72",
   "metadata": {},
   "outputs": [],
   "source": [
    "from html.parser import HTMLParser\n",
    "\n",
    "class SkeptricHTMLParser(HTMLParser):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.extract = {}\n",
    "        self.field = None\n",
    "        \n",
    "    def handle_starttag(self, tag, attrs):\n",
    "        if dict(attrs).get('class') == 'post-full-title':\n",
    "            self.field = 'title'\n",
    "        if dict(attrs).get('class') == 'byline-meta-date':\n",
    "            self.field = 'date'\n",
    "\n",
    "    def handle_endtag(self, tag):\n",
    "        self.field = None\n",
    "\n",
    "    def handle_data(self, data):\n",
    "        if self.field is not None:\n",
    "            self.extract[self.field] = data\n",
    "\n",
    "def skeptric_filter(records):\n",
    "    return [r for r in records if r.mime == 'text/html' and r.status == 200]\n",
    "            \n",
    "def skeptric_extract(content, metadata):\n",
    "    parser = SkeptricHTMLParser()\n",
    "    html = content.decode('utf-8')\n",
    "    parser.feed(html)\n",
    "    data = parser.extract\n",
    "    data['url'] = metadata.url\n",
    "    data['timestamp'] = metadata.timestamp\n",
    "    return data\n",
    "\n",
    "def skeptric_verify_extract(content, metadata):\n",
    "    if not content.get('title'):\n",
    "        raise ValueError('Missing title')\n",
    "    if not content.get('date'):\n",
    "        raise ValueError('Missing date')\n",
    "    return content\n",
    "\n",
    "from datetime import datetime\n",
    "def skeptric_normalise(content, metadata):\n",
    "    content = content.copy()\n",
    "    content['date'] = datetime.strptime(content['date'], '%d %B %Y')\n",
    "    return content\n",
    "\n",
    "from webrefine.query import WarcFileQuery\n",
    "test_data = '../resources/test/skeptric.warc.gz'\n",
    "\n",
    "skeptric_query = WarcFileQuery(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7bf1a9dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "skeptric_process = ProcessMemory(queries=[skeptric_query],\n",
    "                     filter=skeptric_filter,\n",
    "                     steps=[skeptric_extract, skeptric_verify_extract, skeptric_normalise])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ccba7eeb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "782199b1f5824558a733e4e9f1d2c19f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "filter: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffcad02e58f748c9b17830b2912145ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "query:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6c4ad55a1d84e619409b61e147db066",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "transform: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ddeff3ded29a42c9a08edcf591f35f34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "fetch:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:Error processing WarcFileRecord(url='https://skeptric.com/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 36), mime='text/html', status=200, path=PosixPath('../resources/test/skeptric.warc.gz'), offset=17122, digest='JJVB3MQERHRZJCHOJNKS5VDOODXPZAV2') at step skeptric_verify_extract: Missing title\n",
      "ERROR:root:Error processing WarcFileRecord(url='https://skeptric.com/tags/data/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 38), mime='text/html', status=200, path=PosixPath('../resources/test/skeptric.warc.gz'), offset=130269, digest='R7CLAACFU5L7T5LKI5G53RZSMCNUNV6F') at step skeptric_verify_extract: Missing title\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'title': \"Pagination in Internet Archive's Wayback Machine with CDX\",\n",
       "  'date': datetime.datetime(2021, 11, 23, 0, 0),\n",
       "  'url': 'https://skeptric.com/pagination-wayback-cdx/',\n",
       "  'timestamp': datetime.datetime(2021, 11, 26, 11, 28, 34)},\n",
       " {'title': 'About Skeptric',\n",
       "  'date': datetime.datetime(2021, 10, 18, 0, 0),\n",
       "  'url': 'https://skeptric.com/about/',\n",
       "  'timestamp': datetime.datetime(2021, 11, 26, 11, 28, 37)},\n",
       " {'title': 'Searching 100 Billion Webpages Pages With Capture Index',\n",
       "  'date': datetime.datetime(2020, 6, 11, 0, 0),\n",
       "  'url': 'https://skeptric.com/searching-100b-pages-cdx/',\n",
       "  'timestamp': datetime.datetime(2021, 11, 26, 11, 28, 39)},\n",
       " {'title': 'Fast Web Dataset Extraction Worfklow',\n",
       "  'date': datetime.datetime(2021, 11, 21, 0, 0),\n",
       "  'url': 'https://skeptric.com/fast-web-data-workflow/',\n",
       "  'timestamp': datetime.datetime(2021, 11, 26, 11, 28, 39)},\n",
       " {'title': 'Unique Key for Web Captures',\n",
       "  'date': datetime.datetime(2021, 11, 19, 0, 0),\n",
       "  'url': 'https://skeptric.com/key-web-captures/',\n",
       "  'timestamp': datetime.datetime(2021, 11, 26, 11, 28, 40)},\n",
       " {'title': 'Hugo Readdir Error with Emacs',\n",
       "  'date': datetime.datetime(2021, 11, 22, 0, 0),\n",
       "  'url': 'https://skeptric.com/emacs-tempfile-hugo/',\n",
       "  'timestamp': datetime.datetime(2021, 11, 26, 11, 28, 40)}]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(skeptric_process.run())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "24ba6f36",
   "metadata": {},
   "outputs": [],
   "source": [
    "skeptric_process.progress_bar = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "fbb993aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:Error processing WarcFileRecord(url='https://skeptric.com/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 36), mime='text/html', status=200, path=PosixPath('../resources/test/skeptric.warc.gz'), offset=17122, digest='JJVB3MQERHRZJCHOJNKS5VDOODXPZAV2') at step skeptric_verify_extract: Missing title\n",
      "ERROR:root:Error processing WarcFileRecord(url='https://skeptric.com/tags/data/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 38), mime='text/html', status=200, path=PosixPath('../resources/test/skeptric.warc.gz'), offset=130269, digest='R7CLAACFU5L7T5LKI5G53RZSMCNUNV6F') at step skeptric_verify_extract: Missing title\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'title': \"Pagination in Internet Archive's Wayback Machine with CDX\",\n",
       "  'date': datetime.datetime(2021, 11, 23, 0, 0),\n",
       "  'url': 'https://skeptric.com/pagination-wayback-cdx/',\n",
       "  'timestamp': datetime.datetime(2021, 11, 26, 11, 28, 34)},\n",
       " {'title': 'About Skeptric',\n",
       "  'date': datetime.datetime(2021, 10, 18, 0, 0),\n",
       "  'url': 'https://skeptric.com/about/',\n",
       "  'timestamp': datetime.datetime(2021, 11, 26, 11, 28, 37)},\n",
       " {'title': 'Searching 100 Billion Webpages Pages With Capture Index',\n",
       "  'date': datetime.datetime(2020, 6, 11, 0, 0),\n",
       "  'url': 'https://skeptric.com/searching-100b-pages-cdx/',\n",
       "  'timestamp': datetime.datetime(2021, 11, 26, 11, 28, 39)},\n",
       " {'title': 'Fast Web Dataset Extraction Worfklow',\n",
       "  'date': datetime.datetime(2021, 11, 21, 0, 0),\n",
       "  'url': 'https://skeptric.com/fast-web-data-workflow/',\n",
       "  'timestamp': datetime.datetime(2021, 11, 26, 11, 28, 39)},\n",
       " {'title': 'Unique Key for Web Captures',\n",
       "  'date': datetime.datetime(2021, 11, 19, 0, 0),\n",
       "  'url': 'https://skeptric.com/key-web-captures/',\n",
       "  'timestamp': datetime.datetime(2021, 11, 26, 11, 28, 40)},\n",
       " {'title': 'Hugo Readdir Error with Emacs',\n",
       "  'date': datetime.datetime(2021, 11, 22, 0, 0),\n",
       "  'url': 'https://skeptric.com/emacs-tempfile-hugo/',\n",
       "  'timestamp': datetime.datetime(2021, 11, 26, 11, 28, 40)}]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(skeptric_process.run())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74cc04ed",
   "metadata": {},
   "source": [
    "We can always look up an error\n",
    "\n",
    "Would be nicer if everything was a string so we didn't have to handle the imports..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "bfc4cd45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WarcFileRecord(url='https://skeptric.com/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 36), mime='text/html', status=200, path=PosixPath('../resources/test/skeptric.warc.gz'), offset=17122, digest='JJVB3MQERHRZJCHOJNKS5VDOODXPZAV2')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from webrefine.query import WarcFileRecord\n",
    "import datetime\n",
    "from pathlib import PosixPath\n",
    "WarcFileRecord(url='https://skeptric.com/', timestamp=datetime.datetime(2021, 11, 26, 11, 28, 36), mime='text/html', status=200, path=PosixPath('../resources/test/skeptric.warc.gz'), offset=17122, digest='JJVB3MQERHRZJCHOJNKS5VDOODXPZAV2')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
