{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bf9f48f-cbf9-42fe-8d86-ef5389971dee",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from df_cereal.df_widget import DFWidget, BaseWidget\n",
    "import pandas as pd\n",
    "import pyarrow as pa\n",
    "import numpy as np\n",
    "import itertools\n",
    "import pyarrow.feather as feather\n",
    "import base64\n",
    "import io\n",
    "from ipywidgets import HTML\n",
    "\n",
    "def split_string(text, stride):\n",
    "    return [text[i:i+stride] for i in range(0, len(text), stride)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "663b81ed-27bc-4278-9275-93a91c0056f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from traitlets import Unicode, List, Dict, observe, Any, Bytes, observe\n",
    "\n",
    "class Base64Widget(BaseWidget):\n",
    "    _view_name = Unicode('Base64WidgetView').tag(sync=True)\n",
    "    df_base64 = Any(\"\").tag(sync=True)\n",
    "\n",
    "class BytesWidget(BaseWidget):\n",
    "    _view_name = Unicode('BytesWidgetView').tag(sync=True)\n",
    "    df_arrow_bytes = Bytes().tag(sync=True)\n",
    "\n",
    "class BytesBenchmarkWidget(BaseWidget):\n",
    "    _view_name = Unicode('BytesBenchmarkWidgetView').tag(sync=True)\n",
    "    df_arrow_bytes = Bytes().tag(sync=True)\n",
    "    timing_info = Dict({}).tag(sync=True)\n",
    "    \n",
    "    @observe('timing_info')\n",
    "    def _timing_info_change(self, change):\n",
    "        display(self.timing_info)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "453baded-4dc7-4104-8600-2e8543bc52a2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def format_td(td):\n",
    "    return \"{td.seconds}.{td.microseconds}\".format(td=td)\n",
    "\n",
    "class BytesBenchmarkWidget(BaseWidget):\n",
    "    _view_name = Unicode('BytesBenchmarkWidgetView').tag(sync=True)\n",
    "    df_arrow_bytes = Bytes().tag(sync=True)\n",
    "    timing_info = Dict({}).tag(sync=True)\n",
    "\n",
    "    def __init__(self, html_output, serial_t1, serial_t2, **kwargs):\n",
    "        self.serial_t1, self.serial_t2 = serial_t1, serial_t2\n",
    "        self.html_output = html_output\n",
    "        super().__init__(**kwargs)\n",
    "    \n",
    "    @observe('timing_info')\n",
    "    def _timing_info_change(self, change):\n",
    "        self.js_t1 = pd.Timestamp(self.timing_info['t1'])\n",
    "        self.js_t2 = pd.Timestamp(self.timing_info['t2'])\n",
    "        self.js_t3 = pd.Timestamp(self.timing_info['t3'])\n",
    "        self.t4 = pd.Timestamp.utcnow()\n",
    "        \n",
    "        self.timedeltas = dict(\n",
    "            py_serialize = self.serial_t2 - self.serial_t1,\n",
    "            widget_xfer = self.js_t1 - self.serial_t2,\n",
    "            js_deserialize = self.js_t2 - self.js_t1,\n",
    "            js_iterate = self.js_t3 - self.js_t2,\n",
    "            py_notify = self.t4 - self.js_t3,\n",
    "            roundtrip = self.t4 - self.serial_t1\n",
    "        )\n",
    "        template = \"\"\"\n",
    "        <table border=\"1\">\n",
    "            <thead>\n",
    "                <th>python serialize</th>\n",
    "                <th>widget transmit</th>\n",
    "                <th>js deserialize</th>\n",
    "                <th>js iterate</th>\n",
    "                <th>python notify</th>\n",
    "                <th>roundtrip</th>\n",
    "            </thead>\n",
    "            <tbody >\n",
    "                <tr>\n",
    "                    <td> {py_serialize} </td>\n",
    "                    <td> {widget_xfer} </td>\n",
    "                    <td> {js_deserialize} </td>\n",
    "                    <td> {js_iterate} </td>\n",
    "                    <td> {py_notify} </td>\n",
    "                    <td> {roundtrip} </td>\n",
    "                </tr>\n",
    "            </tbody>\n",
    "        </table>\n",
    "        \"\"\"\n",
    "        self.html_output.value = template.format(\n",
    "            **{k: format_td(v) for k,v in self.timedeltas.items()})\n",
    "\n",
    "def benchmark(df):\n",
    "    output_area = HTML(\n",
    "        value=\"Hello <b>World</b>\",\n",
    "        placeholder='Some HTML2',\n",
    "        description='Some HTML3')\n",
    "    display(output_area)\n",
    "    t1 = pd.Timestamp.utcnow()\n",
    "    table = pa.Table.from_pandas(df)\n",
    "    fake_file = io.BytesIO()\n",
    "    feather.write_feather(table, fake_file, compression='uncompressed')\n",
    "    fake_file.seek(0)\n",
    "    full_bytes = fake_file.read()\n",
    "    t2 = pd.Timestamp.utcnow()\n",
    "    bbw = BytesBenchmarkWidget(output_area, df_arrow_bytes=full_bytes, serial_t1=t1, serial_t2=t2)\n",
    "    return bbw\n",
    "\n",
    "\n",
    "df = pd.DataFrame({'a':[99,27], 'b': ['paddy', 'margaret']})\n",
    "bnch= benchmark(df)\n",
    "bnch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c9cd480-3661-43f1-ae48-19f880f6a9a5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "N = 10_000_000\n",
    "df = pd.DataFrame({\n",
    "    'one': [1]*N,\n",
    "    'str': [\"foo\", \"barr\", \"bazz\", \"bofffff\"] * (N//4),\n",
    "    'log_normal2': np.random.lognormal(25, .3, N),\n",
    "    'log_normal': np.random.lognormal(25, .3, N)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3272d369-088a-4398-9295-e8f46bd60c16",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "bbw = benchmark(df)\n",
    "bbw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ba85edf-d999-4d36-beed-d6d35f5e72e4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ab = bbw.td2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdebba6a-6f1b-430f-b260-b4f27ee57c00",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ab.seconds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ed9ac05-7a91-4932-bebb-2b6ab078294e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ab.microseconds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6139dcd1-824a-422f-927d-8c497a168410",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "bench"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78f30918-9aa9-4e2f-9357-bd68d374203d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ab = benchmark(df)\n",
    "ab.timing_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ddd8826-6e1f-40d7-aee3-98f3cfd262c4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ab.timing_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c82b101-51f6-406c-bed3-e82f429e5bc3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "Base64Widget(df_base64=base64table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4899000-9eae-4bb3-93eb-5e8e5e2e8f2a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1eefb23-4875-4e1d-88a8-df8048a13282",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'a':[99,27], 'b': ['paddy', 'margaret']})\n",
    "#df = pd.DataFrame({'a':[10,530], 'c': [-3.5, 9.7], 'd': [1,2]})\n",
    "\n",
    "table = pa.Table.from_pandas(df)\n",
    "feather.write_feather(table, \"simple_df.feath\", compression='uncompressed')\n",
    "feath_text = open(\"simple_df.feath\", \"rb\").read()\n",
    "usable_feath_text = base64.b64encode(feath_text).decode('utf8')\n",
    "\n",
    "print(len(feath_text))\n",
    "feath_text[:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12237758-ed22-4726-ab24-1988011c71c7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import io\n",
    "table = pa.Table.from_pandas(df)\n",
    "fake_file = io.BytesIO()\n",
    "feather.write_feather(table, fake_file, compression='uncompressed')\n",
    "#feath_text = open(\"simple_df.feath\", \"rb\").read()\n",
    "fake_file.seek(0)\n",
    "BytesWidget(df_arrow_bytes=fake_file.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd51ae9f-8692-469e-80c9-f89ca24e939b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BytesWidget(df_arrow_bytes=feath_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "978aa7be-b0df-4a3c-a12e-fc88c1910963",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "usable_feath_text = base64.b64encode(feath_text).decode('utf8')\n",
    "Base64Widget(df_base64=usable_feath_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb65ad8e-f441-4d40-8a6d-8e3980bf3e02",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def B64Viewer(df):\n",
    "    table = pa.Table.from_pandas(df)\n",
    "    feather.write_feather(table, \"simple_df.feath\", compression='uncompressed')\n",
    "    feath_text = open(\"simple_df.feath\", \"rb\").read()\n",
    "    usable_feath_text = base64.b64encode(feath_text).decode('utf8')\n",
    "    usable_feath_text = base64.b64encode(feath_text).decode('utf8')\n",
    "    return [Base64Widget(df_base64=usable_feath_text), usable_feath_text]\n",
    "ab = B64Viewer(pd.DataFrame({'a': [1, None, 2, 3,-3]}))\n",
    "ab[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc5551aa-a7a8-4c4a-b538-5e72bb736428",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "split_string(ab[1], 70)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b50bcf3-57d5-4b89-bdbf-0aee65240e05",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0853c3d6-1050-4ffc-bc31-ce7d5322dd67",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ab = B64Viewer(pd.DataFrame({'a': [1, None, np.nan, np.inf, np.inf * -1]}))\n",
    "ab[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86814090-a61f-48c7-bc0d-d053b5ba3fa2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "split_string(ab[1], 70)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fb0b4bd-a7d2-4c38-957a-30056d870551",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "split_string(usable_feath_text, 70)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "218fab29-0234-429c-9b66-a42864cc32b4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "b64_t2 = base64.b64encode(feath_text)\n",
    "print(b64_t2[:30])\n",
    "print(\"  %s\" % b64_t2.decode('utf8')[:30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb1d196e-ccef-4f9d-9a24-184dd6df62b1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "b64_t2.decode('utf8')[::70]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d2de3ba-f687-4352-a3f5-32bd0fb50037",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def split_string(text, stride):\n",
    "    return [text[i:i+stride] for i in range(0, len(text), stride)]\n",
    "#split_string(\"asdfasdfasdf\", 3)\n",
    "split_string(b64_t2.decode('utf8'), 70)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fda37d01-bc1f-4ae9-9fe7-c7b0c6f76edc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2cf879df-a49f-40f0-85a1-d48be86427a5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import base64\n",
    "b64_t2 = base64.b64encode(feath_text)\n",
    "b64_t2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f8784f6-9b87-4d8e-847b-20cb7c8e3ed6",
   "metadata": {},
   "outputs": [],
   "source": [
    "pa.fea"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d02a6ae-63e5-4f9b-9214-45424239623a",
   "metadata": {},
   "outputs": [],
   "source": [
    "sink = pa.BufferOutputStream()\n",
    "buf = sink.getvalue()\n",
    "with pa.ipc.open_stream(buf) as reader:\n",
    "      schema = reader.schema\n",
    "      batches = [b for b in reader]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d75243c9-7391-4324-a391-de820a6f90ac",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BytesWidget(df_arrow_bytes=pabuffer.to_pybytes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7039598-0e02-4397-8261-719932a2393b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import base64\n",
    "b64_t2 = base64.b64encode(pabuffer.to_pybytes())\n",
    "#Base64Widget(df_base64=b64_t2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27c071d5-18c4-4e6f-8ab6-751c15c11e20",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "b64_t2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e857785d-eec0-43cc-b4d4-2c427aa6968b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "b64_t3 = '/////8gAAAAUAAAAAAAAAAwAFgAGAAUACAAMAAwAAAAAAwQAGAAAADAAAAAAAAAAAAAKABgADAAEAAgACgAAAGwAAAAQAAAAAgAAAAAAAAAAAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAADAAAAAAAAAAgAAAAAAAAAA0AAAAAAAAAAAAAAAIAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAoAAAAAAAAAMgAAAAAAAAAAAAAABQAAAA0AAAAAAAAAcGFkZHltYXJnYXJldAAAAA=='"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5a1e18f-d9e5-48b9-8dd9-45a5e64573c0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "Base64Widget(df_base64=b64_t3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "581fae01-4499-4b88-a6cf-92b9202f32be",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "len(b64_t2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e301da0-8e3f-4b0c-abf7-7b351a2db6eb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pabuffer.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7dc27d26-405b-495e-bfbe-fd1eadb741af",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BytesWidget(df_arrow_bytes=pabuffer.to_pybytes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "928dcb27-ee47-403d-8437-fd11757f4d88",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BytesWidget(df_arrow_bytes=pabuffer.to_pybytes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21eb1c01-4521-4a48-a87c-a3b3c87177a3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
