{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vector stores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['vs_687b5ab75e9c8191ac0eba7e8ec6998e', 'vs_bvh1XzBEmCrH0crjtrzS5QyF']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from oa.stores import OaStores\n",
    "\n",
    "oa_stores = OaStores()  # specify api key if needed, but taken from config (e.g. env var) by default\n",
    "vector_stores = oa_stores.vector_stores\n",
    "list(vector_stores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['my_legal_documents', None]\n"
     ]
    }
   ],
   "source": [
    "names = [vs.name for vs in vector_stores.values()]\n",
    "print(names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VectorStore(id='vs_687b5ab75e9c8191ac0eba7e8ec6998e', created_at=1752914615, file_counts=FileCounts(cancelled=0, completed=400, failed=0, in_progress=0, total=400), last_active_at=1752920433, metadata={}, name='my_legal_documents', object='vector_store', status='completed', usage_bytes=7604632, expires_after=None, expires_at=None)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vs = vector_stores['vs_687b5ab75e9c8191ac0eba7e8ec6998e']\n",
    "vs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VectorStore(id='vs_687b5ab75e9c8191ac0eba7e8ec6998e', created_at=1752914615, file_counts=FileCounts(cancelled=0, completed=400, failed=0, in_progress=0, total=400), last_active_at=1752920433, metadata={}, name='my_legal_documents', object='vector_store', status='completed', usage_bytes=7604632, expires_after=None, expires_at=None)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oa_stores.client.vector_stores.retrieve(vector_store_id='vs_687b5ab75e9c8191ac0eba7e8ec6998e')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'vs_687b5ab75e9c8191ac0eba7e8ec6998e',\n",
       " 'created_at': 1752914615,\n",
       " 'file_counts': FileCounts(cancelled=0, completed=400, failed=0, in_progress=0, total=400),\n",
       " 'last_active_at': 1752920433,\n",
       " 'metadata': {},\n",
       " 'name': 'my_legal_documents',\n",
       " 'object': 'vector_store',\n",
       " 'status': 'completed',\n",
       " 'usage_bytes': 7604632,\n",
       " 'expires_after': None,\n",
       " 'expires_at': None}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(vs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "from oa.vector_stores import mk_search_func_for_oa_vector_store\n",
    "\n",
    "search = mk_search_func_for_oa_vector_store('vs_687b5ab75e9c8191ac0eba7e8ec6998e')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Sig (query: 'Union[str, List[str]]', *, filters: 'vector_store_search_params.Filters | NotGiven' = NOT_GIVEN, max_num_results: 'int | NotGiven' = NOT_GIVEN, ranking_options: 'vector_store_search_params.RankingOptions | NotGiven' = NOT_GIVEN, rewrite_query: 'bool | NotGiven' = NOT_GIVEN, extra_headers: 'Headers | None' = None, extra_query: 'Query | None' = None, extra_body: 'Body | None' = None, timeout: 'float | httpx.Timeout | None | NotGiven' = NOT_GIVEN)>"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from i2 import Sig \n",
    "\n",
    "Sig(search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "sync_page = search('finances')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'attributes': {},\n",
       " 'content': [Content(text='# Jurisprudence: Justice Administrative\\n            \\n                ## REFERENCE: Ordonnance du Tribunal administratif n° 2500284 du 26 juin 2025\\n    \\n                ## URL: [https://lexpol.cloud.pf/LexpolAfficheTexte.php?texte=1025811](entry[\"decision_links-href\"])\\n    \\n                ### CONTENU: Vu la procédure suivante :\\nPar une requête, enregistrée le 25 juin 2025, Mme C B épouse A demande au juge des référés :\\n1°) d\\'ordonner, sur le fondement des dispositions de l\\'article L. 521-1 du code de justice administrative, la suspension de la décision de la direction régionale des finances publiques en date du 18 juin 2025, portant rejet de la demande de paiement n°500004954 relative à la prise en charge des frais de changement de résidence après congé administratif.\\n2°) de mettre à la charge de l\\'Etat la somme de 120.000 francs CFP au titre des frais irrépétibles ;\\nElle soutient que :\\n- la condition d\\'urgence est remplie ;  elle doit rejoindre son nouveau poste dans un délai contraint, son entrée en fonctions à Mayotte est prévue le 14 août 2025 ; la charge financière à supporter, un total de 1.113.120,00 XPF, est  insupportable à court terme, incompatible avec ses ressources personnelles immédiates ; l\\'impossibilité matérielle de rejoindre son poste mettrait en péril la continuité du service à Mayotte ; son intégration dans la juridiction et, potentiellement, sa situation statutaire ; il résulte de cette situation une détresse financière et émotionnelle ; le service administratif régional (SAR) l\\'a informée que le bon de commande initialement émis pour couvrir le coût de ses billets d\\'avion serait annulé à compter du jeudi 26 juin 2025 ;\\n- la condition de doute sérieux sur la légalité est satisfaite :  les articles 38 et 41 du décret n°98-844 du 22 octobre 1998 sont méconnus ; l\\'administration a commis une erreur manifeste d\\'appréciation en confondant le trajet pris en charge ; le refus de prise en charge du trajet aller-retour est illégal au regard de l\\'article 41 du décret n°98-844 du 22 octobre 1998 ; il est porté atteinte au congé administratif prévu à l\\'article 41 ; la charge financière imposée à l\\'agent pour rejoindre sa nouvelle résidence administrative est illégale ; cette situation induit une rupture d\\'égalité entre les agents selon le lieu de leur congé administratif et de leur nouvelle affectation ; les articles 8 et premier du protocole n°1 de la convention européenne de sauvegarde des droits de l\\'homme et libertés fondamentaux sont méconnus, de même que le principe de proportionnalité ; à titre subsidiaire devrait être admis le cumul partiel des droits issus des articles 38 et 41 du décret du 22 octobre 1998.\\nVu :\\n- les autres pièces du dossier ;\\n- la requête enregistrée le 25 juin 2025 sous le numéro 2500283 par laquelle Mme C B épouse A demande l\\'annulation de la décision attaquée.', type='text')],\n",
       " 'file_id': 'file-RnSgUgeLjTbuYExuqstVGv',\n",
       " 'filename': 'entry_389:Ordonnance_du_Tribunal_administratif_#_2500284_du_26_juin_2025.md',\n",
       " 'score': 0.720569209482211}"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(sync_page.data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parse openai pricing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TODO: Need to finish the html page parser. Not getting all the tables!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "112759"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from oa._resources import Resources\n",
    "\n",
    "r = Resources()\n",
    "\n",
    "html = r.pricing_page_html\n",
    "len(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Latest models - Text tokens',\n",
       " 'Latest models - Audio tokens',\n",
       " 'Fine tuning',\n",
       " 'Built-in tools',\n",
       " 'Web search',\n",
       " 'Transcription and speech generation',\n",
       " 'Image generation',\n",
       " 'Embeddings',\n",
       " 'Moderation',\n",
       " 'Other models']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(r.pricing_info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'category': 'text',\n",
       "  'Model': 'gpt-4.5-preview',\n",
       "  'Alternate_Model': 'gpt-4.5-preview-2025-02-27',\n",
       "  'Input': '$75.00',\n",
       "  'Cached input': '$37.50',\n",
       "  'Output': '$150.00'},\n",
       " {'category': 'text',\n",
       "  'Model': 'gpt-4o',\n",
       "  'Alternate_Model': 'gpt-4o-2024-08-06',\n",
       "  'Input': '$2.50',\n",
       "  'Cached input': '$1.25',\n",
       "  'Output': '$10.00'},\n",
       " {'category': 'text',\n",
       "  'Model': 'gpt-4o-audio-preview',\n",
       "  'Alternate_Model': 'gpt-4o-audio-preview-2024-12-17',\n",
       "  'Input': '$2.50',\n",
       "  'Cached input': '-',\n",
       "  'Output': '$10.00'}]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from oa.util import pricing_info\n",
    "\n",
    "list(pricing_info())[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': 'Latest models - Text tokens',\n",
       " 'audio': 'Latest models - Audio tokens',\n",
       " 'finetune': 'Fine tuning',\n",
       " 'tools': 'Built-in tools',\n",
       " 'search': 'Web search',\n",
       " 'speech': 'Transcription and speech generation',\n",
       " 'images': 'Image generation',\n",
       " 'embeddings': 'Embeddings',\n",
       " 'moderation': 'Moderation',\n",
       " 'other': 'Other models'}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pricing_info.category_aliases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Alternate_Model</th>\n",
       "      <th>Input</th>\n",
       "      <th>Cached input</th>\n",
       "      <th>Output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gpt-4o-audio-preview</td>\n",
       "      <td>gpt-4o-audio-preview-2024-12-17</td>\n",
       "      <td>$40.00</td>\n",
       "      <td>-</td>\n",
       "      <td>$80.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gpt-4o-mini-audio-preview</td>\n",
       "      <td>gpt-4o-mini-audio-preview-2024-12-17</td>\n",
       "      <td>$10.00</td>\n",
       "      <td>-</td>\n",
       "      <td>$20.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gpt-4o-realtime-preview</td>\n",
       "      <td>gpt-4o-realtime-preview-2024-12-17</td>\n",
       "      <td>$40.00</td>\n",
       "      <td>$2.50</td>\n",
       "      <td>$80.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gpt-4o-mini-realtime-preview</td>\n",
       "      <td>gpt-4o-mini-realtime-preview-2024-12-17</td>\n",
       "      <td>$10.00</td>\n",
       "      <td>$0.30</td>\n",
       "      <td>$20.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Model                          Alternate_Model  \\\n",
       "0          gpt-4o-audio-preview          gpt-4o-audio-preview-2024-12-17   \n",
       "1     gpt-4o-mini-audio-preview     gpt-4o-mini-audio-preview-2024-12-17   \n",
       "2       gpt-4o-realtime-preview       gpt-4o-realtime-preview-2024-12-17   \n",
       "3  gpt-4o-mini-realtime-preview  gpt-4o-mini-realtime-preview-2024-12-17   \n",
       "\n",
       "    Input Cached input  Output  \n",
       "0  $40.00            -  $80.00  \n",
       "1  $10.00            -  $20.00  \n",
       "2  $40.00        $2.50  $80.00  \n",
       "3  $10.00        $0.30  $20.00  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd \n",
    "\n",
    "pd.DataFrame(pricing_info('audio'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all_prices_info.shape=(51, 13)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "category                                     text\n",
       "Model                             gpt-4.5-preview\n",
       "Alternate_Model        gpt-4.5-preview-2025-02-27\n",
       "Input                                      $75.00\n",
       "Cached input                               $37.50\n",
       "Output                                    $150.00\n",
       "Training                                      NaN\n",
       "Cost                                          NaN\n",
       "Search context size                           NaN\n",
       "Use case                                      NaN\n",
       "Quality                                       NaN\n",
       "1024x1024                                     NaN\n",
       "1024x1792                                     NaN\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd \n",
    "\n",
    "all_prices_info = pd.DataFrame(pricing_info())\n",
    "print(f\"{all_prices_info.shape=}\")\n",
    "all_prices_info.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DALL·E 3',\n",
       " 'babbage-002',\n",
       " 'davinci-002',\n",
       " 'gpt-3.5-turbo',\n",
       " 'gpt-4o or gpt-4o-search-preview',\n",
       " 'gpt-4o-audio-preview',\n",
       " 'gpt-4o-mini or gpt-4o-mini-search-preview',\n",
       " 'gpt-4o-mini-audio-preview',\n",
       " 'gpt-4o-mini-realtime-preview',\n",
       " 'gpt-4o-realtime-preview']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tabled\n",
    "\n",
    "duplicates = tabled.duplicate_groups(all_prices_info, 'Model')\n",
    "duplicate_groups = tabled.duplicate_groups(all_prices_info, 'Model', output='series')\n",
    "\n",
    "duplicates#.loc['gpt-4o-audio-preview']\n",
    "\n",
    "list(duplicate_groups.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A diagnoses function to compare the pricing info that was parsed to \n",
    "# the pricing info hard-coded in util.py dictionaries of model info\n",
    "from oa._resources import compare_pricing_info_to_model_info\n",
    "\n",
    "compare_pricing_info_to_model_info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using tabled.get_tables_from_url - but not sufficient for full information.\n",
    "\n",
    "This basically uses `pd.read_html`. But is not sufficient \n",
    "\n",
    "Here we have the tables, but:\n",
    "* Some of the Model column values have two mentioned models, and they are contatenated (so hard to separate)\n",
    "* The tables don't come with their headers (to know what they correspond to)\n",
    "* There are two versions of prices: With or with out Batch API\n",
    "\n",
    "So perhaps we need to parse the actual html instead of forwarding to `pd.read_html`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd \n",
    "import requests\n",
    "import tabled\n",
    "\n",
    "tables_pd = tabled.get_tables_from_url(pricing_url, url_to_html=('chrome', dict(wait=10)))\n",
    "len(tables_pd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Input</th>\n",
       "      <th>Cached input</th>\n",
       "      <th>Output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gpt-4.5-previewgpt-4.5-preview-2025-02-27</td>\n",
       "      <td>$75.00</td>\n",
       "      <td>$37.50</td>\n",
       "      <td>$150.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gpt-4ogpt-4o-2024-08-06</td>\n",
       "      <td>$2.50</td>\n",
       "      <td>$1.25</td>\n",
       "      <td>$10.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gpt-4o-audio-previewgpt-4o-audio-preview-2024-...</td>\n",
       "      <td>$2.50</td>\n",
       "      <td>-</td>\n",
       "      <td>$10.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gpt-4o-realtime-previewgpt-4o-realtime-preview...</td>\n",
       "      <td>$5.00</td>\n",
       "      <td>$2.50</td>\n",
       "      <td>$20.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gpt-4o-minigpt-4o-mini-2024-07-18</td>\n",
       "      <td>$0.15</td>\n",
       "      <td>$0.075</td>\n",
       "      <td>$0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>gpt-4o-mini-audio-previewgpt-4o-mini-audio-pre...</td>\n",
       "      <td>$0.15</td>\n",
       "      <td>-</td>\n",
       "      <td>$0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>gpt-4o-mini-realtime-previewgpt-4o-mini-realti...</td>\n",
       "      <td>$0.60</td>\n",
       "      <td>$0.30</td>\n",
       "      <td>$2.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>o1o1-2024-12-17</td>\n",
       "      <td>$15.00</td>\n",
       "      <td>$7.50</td>\n",
       "      <td>$60.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>o3-minio3-mini-2025-01-31</td>\n",
       "      <td>$1.10</td>\n",
       "      <td>$0.55</td>\n",
       "      <td>$4.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>o1-minio1-mini-2024-09-12</td>\n",
       "      <td>$1.10</td>\n",
       "      <td>$0.55</td>\n",
       "      <td>$4.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>gpt-4o-mini-search-previewgpt-4o-mini-search-p...</td>\n",
       "      <td>$0.15</td>\n",
       "      <td>-</td>\n",
       "      <td>$0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>gpt-4o-search-previewgpt-4o-search-preview-202...</td>\n",
       "      <td>$2.50</td>\n",
       "      <td>-</td>\n",
       "      <td>$10.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>computer-use-previewcomputer-use-preview-2025-...</td>\n",
       "      <td>$3.00</td>\n",
       "      <td>-</td>\n",
       "      <td>$12.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Model   Input Cached input  \\\n",
       "0           gpt-4.5-previewgpt-4.5-preview-2025-02-27  $75.00       $37.50   \n",
       "1                             gpt-4ogpt-4o-2024-08-06   $2.50        $1.25   \n",
       "2   gpt-4o-audio-previewgpt-4o-audio-preview-2024-...   $2.50            -   \n",
       "3   gpt-4o-realtime-previewgpt-4o-realtime-preview...   $5.00        $2.50   \n",
       "4                   gpt-4o-minigpt-4o-mini-2024-07-18   $0.15       $0.075   \n",
       "5   gpt-4o-mini-audio-previewgpt-4o-mini-audio-pre...   $0.15            -   \n",
       "6   gpt-4o-mini-realtime-previewgpt-4o-mini-realti...   $0.60        $0.30   \n",
       "7                                     o1o1-2024-12-17  $15.00        $7.50   \n",
       "8                           o3-minio3-mini-2025-01-31   $1.10        $0.55   \n",
       "9                           o1-minio1-mini-2024-09-12   $1.10        $0.55   \n",
       "10  gpt-4o-mini-search-previewgpt-4o-mini-search-p...   $0.15            -   \n",
       "11  gpt-4o-search-previewgpt-4o-search-preview-202...   $2.50            -   \n",
       "12  computer-use-previewcomputer-use-preview-2025-...   $3.00            -   \n",
       "\n",
       "     Output  \n",
       "0   $150.00  \n",
       "1    $10.00  \n",
       "2    $10.00  \n",
       "3    $20.00  \n",
       "4     $0.60  \n",
       "5     $0.60  \n",
       "6     $2.40  \n",
       "7    $60.00  \n",
       "8     $4.40  \n",
       "9     $4.40  \n",
       "10    $0.60  \n",
       "11   $10.00  \n",
       "12   $12.00  "
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tables_pd[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Input</th>\n",
       "      <th>Cached input</th>\n",
       "      <th>Output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gpt-4o-audio-previewgpt-4o-audio-preview-2024-...</td>\n",
       "      <td>$40.00</td>\n",
       "      <td>-</td>\n",
       "      <td>$80.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gpt-4o-mini-audio-previewgpt-4o-mini-audio-pre...</td>\n",
       "      <td>$10.00</td>\n",
       "      <td>-</td>\n",
       "      <td>$20.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gpt-4o-realtime-previewgpt-4o-realtime-preview...</td>\n",
       "      <td>$40.00</td>\n",
       "      <td>$2.50</td>\n",
       "      <td>$80.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gpt-4o-mini-realtime-previewgpt-4o-mini-realti...</td>\n",
       "      <td>$10.00</td>\n",
       "      <td>$0.30</td>\n",
       "      <td>$20.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Model   Input Cached input  \\\n",
       "0  gpt-4o-audio-previewgpt-4o-audio-preview-2024-...  $40.00            -   \n",
       "1  gpt-4o-mini-audio-previewgpt-4o-mini-audio-pre...  $10.00            -   \n",
       "2  gpt-4o-realtime-previewgpt-4o-realtime-preview...  $40.00        $2.50   \n",
       "3  gpt-4o-mini-realtime-previewgpt-4o-mini-realti...  $10.00        $0.30   \n",
       "\n",
       "   Output  \n",
       "0  $80.00  \n",
       "1  $20.00  \n",
       "2  $80.00  \n",
       "3  $20.00  "
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tables_pd[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using AI to parse the table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "openai_api_pricing_schema_description_key = 'data/openai_api_pricing_schema_description.txt'\n",
    "openai_api_pricing_schema_key = 'data/api_pricing_schema.json'\n",
    "openai_api_pricing_info_from_ai_key = 'data/openai_api_pricing_info_from_ai.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading schema_description from data/openai_api_pricing_schema_description.txt...\n"
     ]
    }
   ],
   "source": [
    "import pathlib \n",
    "import os \n",
    "\n",
    "if not os.path.isfile(openai_api_pricing_schema_description_key):\n",
    "    print(f\"Saving a schema_description to {openai_api_pricing_schema_description_key}...\")\n",
    "\n",
    "    schema_description = \"\"\"\n",
    "    A schema to contain the pricing information for OpenAI APIs.\n",
    "    The first level field should be the name category of API, (things like \"Text Tokens\", \"Audio Tokens\", \"Fine Tuning\", etc.)\n",
    "    The value of the first level field should be a dictionary with the following fields:\n",
    "    - \"pricing_table_schema\": A schema for the pricing table. That is, the list of the columns\n",
    "    - \"pricing_table\": A list of dicts, corresponding to the table of pricing information for the API category. \n",
    "        For example, ```[{\"Model\": ..., \"Input\": ..., ...,}, {\"Model\": ..., \"Input\": ..., ...,}, ...]```\n",
    "        Note that Sometimes there is more than one model name, so we should also have, in there, a field for \"Alternate Model Names\"\n",
    "    - \"extras\": A dictionary with extra information that might be parsed out of each category. \n",
    "    For example, there's often some text that gives a description of the API category, and/or specifies \"Price per 1M tokens\" etc.\n",
    "        \n",
    "    Note that some of these tables have a \"Batch API\" version too. \n",
    "    In this case, there should be extra fields that have the same name as the fields above, but with \"- Batch\" appended to the name.\n",
    "    For example, \"Text Tokens - Batch\", \"Fine Tuning - Batch\", etc.\n",
    "\n",
    "    It is important to note: These first level fields are not determined in advance. \n",
    "    They are determined by the data that is scraped from the page.\n",
    "    Therefore, these names should not be in the schema. \n",
    "    What should be in the schema is the fact that the data should be a JSON whose first \n",
    "    level field describes a category, and whose value specifies information about these category.\n",
    "    \"\"\"\n",
    "\n",
    "    pathlib.Path(openai_api_pricing_schema_description_key).write_text(schema_description)\n",
    "else:\n",
    "    print(f\"Loading schema_description from {openai_api_pricing_schema_description_key}...\")\n",
    "    schema_description = pathlib.Path(openai_api_pricing_schema_description_key).read_text()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Making and saving a schema to data/api_pricing_schema.json...\n",
      "{\n",
      "  \"name\": \"OpenAI_API_Pricing_Schema\",\n",
      "  \"type\": \"object\",\n",
      "  \"properties\": {\n",
      "    \"patternProperties\": {\n",
      "      \"^.+$\": {\n",
      "        \"type\": \"object\",\n",
      "        \"properties\": {\n",
      "          \"pricing_table_schema\": {\n",
      "            \"type\": \"object\",\n",
      "            \"properties\": {\n",
      "              \"columns\": {\n",
      "                \"type\": \"array\",\n",
      "                \"items\": {\n",
      "                  \"type\": \"string\"\n",
      "                }\n",
      "              }\n",
      "            },\n",
      "            \"required\": [\n",
      "              \"columns\"\n",
      "            ]\n",
      "...\n"
     ]
    }
   ],
   "source": [
    "import json, pathlib\n",
    "\n",
    "if not os.path.isfile(openai_api_pricing_schema_key):\n",
    "    import oa \n",
    "\n",
    "    print(f\"Making and saving a schema to {openai_api_pricing_schema_key}...\")\n",
    "\n",
    "    schema = oa.infer_schema_from_verbal_description(schema_description)\n",
    "    pathlib.Path(openai_api_pricing_schema_key).write_text(json.dumps(schema, indent=2))\n",
    "else:\n",
    "    schema = json.loads(pathlib.Path(openai_api_pricing_schema_key).read_text())\n",
    "\n",
    "print(json.dumps(lkj.truncate_dict_values(schema), indent=2)[:500] + '\\n...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompting for pricing information and saving to data/openai_api_pricing_info.json...\n"
     ]
    }
   ],
   "source": [
    "import pathlib, json \n",
    "\n",
    "if not os.path.isfile(openai_api_pricing_info_from_ai_key):\n",
    "    import oa \n",
    "\n",
    "    print(f\"Prompting for pricing information and saving to {openai_api_pricing_info_from_ai_key}...\")\n",
    "    \n",
    "    prompt = \"\"\"\n",
    "    Parse through this html and extract the pricing information for the OpenAI APIs.\n",
    "    The pricing information should be structured according to the schema described below:\n",
    "\n",
    "    \"\"\" + schema_description + \"\"\" \n",
    "\n",
    "    Here is the html to parse:\n",
    "\n",
    "\n",
    "    {html}\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    assert 'html' in locals(), \"Please load the html into the variable 'html' before running this cell.\"\n",
    "    \n",
    "    get_pricing_info = oa.prompt_json_function(prompt, schema)\n",
    "\n",
    "    pricing_info = get_pricing_info(html)\n",
    "\n",
    "    pathlib.Path(openai_api_pricing_info_from_ai_key).write_text(json.dumps(pricing_info, indent=2))\n",
    "else:\n",
    "    print(f\"Loading pricing information from {openai_api_pricing_info_from_ai_key}...\")\n",
    "    pricing_info = json.loads(pathlib.Path(openai_api_pricing_info_from_ai_key).read_text())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Text Tokens',\n",
       " 'Audio Tokens',\n",
       " 'Fine Tuning',\n",
       " 'Built-in Tools',\n",
       " 'Moderation',\n",
       " 'Image Generation',\n",
       " 'Embeddings',\n",
       " 'Other Models']"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(pricing_info)\n",
    "t = pricing_info['OpenAI_API_Pricing_Schema']\n",
    "list(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Text Tokens',\n",
       " 'Audio Tokens',\n",
       " 'Fine Tuning',\n",
       " 'Built-in Tools',\n",
       " 'Moderation',\n",
       " 'Image Generation',\n",
       " 'Embeddings',\n",
       " 'Other Models']"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import dol \n",
    "\n",
    "tables = dol.add_ipython_key_completions(dol.wrap_kvs(\n",
    "    pricing_info['OpenAI_API_Pricing_Schema'],\n",
    "    value_decoder=lambda x: pd.DataFrame(x['pricing_table'])\n",
    "))\n",
    "list(tables)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_models(tables):\n",
    "    for name, table in tables.items():\n",
    "        # print(name)\n",
    "        for model in list(table.get('Model', [])):\n",
    "            yield model\n",
    "        for model in list(table.get('Alternate Model Names', [])):\n",
    "            yield model\n",
    "            \n",
    "models = list(get_models(tables))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Data Access class for OpenAI pricing information.\n",
    "Provides a caching layer over various data sources and computations.\n",
    "\"\"\"\n",
    "\n",
    "from dataclasses import dataclass, field\n",
    "import json\n",
    "import pathlib\n",
    "from functools import partial\n",
    "from typing import Dict, Any, Optional, Callable, Union, List\n",
    "\n",
    "import pandas as pd\n",
    "import dol\n",
    "\n",
    "from oa._resources import resources_dir\n",
    "\n",
    "\n",
    "def _ensure_dir_exists(path: Union[str, pathlib.Path]) -> None:\n",
    "    \"\"\"Ensure the directory for the given path exists.\"\"\"\n",
    "    pathlib.Path(path).parent.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class Dacc:\n",
    "    \"\"\"\n",
    "    Data Access class for OpenAI pricing information.\n",
    "    \n",
    "    This class manages the retrieval, computation, and caching of\n",
    "    OpenAI pricing data through a chain of computations.\n",
    "    \n",
    "    >>> dacc = Dacc()\n",
    "    >>> dacc.resources_dir  # doctest: +ELLIPSIS\n",
    "    '...misc/data'\n",
    "    \"\"\"\n",
    "    resources_dir: str = resources_dir\n",
    "    pricing_url: str = 'https://platform.openai.com/docs/pricing'\n",
    "    \n",
    "    schema_description_key: str = 'openai_api_pricing_schema_description.txt'\n",
    "    schema_key: str = 'api_pricing_schema.json'\n",
    "    pricing_html_key: str = 'openai_api_pricing.html'\n",
    "    pricing_info_key: str = 'openai_api_pricing_info.json'\n",
    "    pricing_info_from_ai_key: str = 'openai_api_pricing_info_from_ai.json'\n",
    "    \n",
    "    # Dependencies that can be injected\n",
    "    get_pricing_page_html: Optional[Callable[[], str]] = None\n",
    "    infer_schema_from_verbal_description: Optional[Callable[[str], Dict[str, Any]]] = None\n",
    "    prompt_json_function: Optional[Callable[[str, Dict[str, Any]], Callable[[str], Dict[str, Any]]]] = None\n",
    "    \n",
    "    def __post_init__(self):\n",
    "        \"\"\"Initialize stores and ensure dependencies are available.\"\"\"\n",
    "        # Set up storage\n",
    "        _ensure_dir_exists(f\"{self.resources_dir}/{self.schema_description_key}\")\n",
    "        \n",
    "        self.json_store = dol.JsonFiles(self.resources_dir)\n",
    "        self.text_store = dol.TextFiles(self.resources_dir)\n",
    "        \n",
    "        # Import dependencies if not provided\n",
    "        if self.get_pricing_page_html is None:\n",
    "            from oa._resources import get_pricing_page_html\n",
    "            self.get_pricing_page_html = get_pricing_page_html\n",
    "            \n",
    "        if self.infer_schema_from_verbal_description is None or self.prompt_json_function is None:\n",
    "            import oa\n",
    "            \n",
    "            if self.infer_schema_from_verbal_description is None:\n",
    "                self.infer_schema_from_verbal_description = oa.infer_schema_from_verbal_description\n",
    "                \n",
    "            if self.prompt_json_function is None:\n",
    "                self.prompt_json_function = oa.prompt_json_function\n",
    "    \n",
    "    @dol.cache_this(cache='text_store', key=lambda self: self.schema_description_key)\n",
    "    def schema_description(self) -> str:\n",
    "        \"\"\"Generate and retrieve the schema description for OpenAI API pricing.\"\"\"\n",
    "        return \"\"\"\n",
    "        A schema to contain the pricing information for OpenAI APIs.\n",
    "        The first level field should be the name category of API, (things like \"Text Tokens\", \"Audio Tokens\", \"Fine Tuning\", etc.)\n",
    "        The value of the first level field should be a dictionary with the following fields:\n",
    "        - \"pricing_table_schema\": A schema for the pricing table. That is, the list of the columns\n",
    "        - \"pricing_table\": A list of dicts, corresponding to the table of pricing information for the API category. \n",
    "            For example, ```[{\"Model\": ..., \"Input\": ..., ...,}, {\"Model\": ..., \"Input\": ..., ...,}, ...]```\n",
    "            Note that Sometimes there is more than one model name, so we should also have, in there, a field for \"Alternate Model Names\"\n",
    "        - \"extras\": A dictionary with extra information that might be parsed out of each category. \n",
    "        For example, there's often some text that gives a description of the API category, and/or specifies \"Price per 1M tokens\" etc.\n",
    "            \n",
    "        Note that some of these tables have a \"Batch API\" version too. \n",
    "        In this case, there should be extra fields that have the same name as the fields above, but with \"- Batch\" appended to the name.\n",
    "        For example, \"Text Tokens - Batch\", \"Fine Tuning - Batch\", etc.\n",
    "\n",
    "        It is important to note: These first level fields are not determined in advance. \n",
    "        They are determined by the data that is scraped from the page.\n",
    "        Therefore, these names should not be in the schema. \n",
    "        What should be in the schema is the fact that the data should be a JSON whose first \n",
    "        level field describes a category, and whose value specifies information about these category.\n",
    "        \"\"\"\n",
    "    \n",
    "    @dol.cache_this(cache='text_store', key=lambda self: self.pricing_html_key)\n",
    "    def pricing_page_html(self) -> str:\n",
    "        \"\"\"Retrieve the HTML for the OpenAI pricing page.\"\"\"\n",
    "        return self.get_pricing_page_html()\n",
    "    \n",
    "    @dol.cache_this(cache='json_store', key=lambda self: self.schema_key) \n",
    "    def schema(self) -> Dict[str, Any]:\n",
    "        \"\"\"Generate or retrieve the schema for OpenAI API pricing.\"\"\"\n",
    "        return self.infer_schema_from_verbal_description(self.schema_description)\n",
    "    \n",
    "    @dol.cache_this(cache='json_store', key=lambda self: self.pricing_info_from_ai_key)\n",
    "    def pricing_info(self) -> Dict[str, Any]:\n",
    "        \"\"\"Extract pricing information from the HTML using AI.\"\"\"\n",
    "        prompt = f\"\"\"\n",
    "        Parse through this html and extract the pricing information for the OpenAI APIs.\n",
    "        The pricing information should be structured according to the schema described below:\n",
    "\n",
    "        {self.schema_description}\n",
    "\n",
    "        Here is the html to parse:\n",
    "\n",
    "        {{html}}\n",
    "        \"\"\"\n",
    "        \n",
    "        get_pricing_info = self.prompt_json_function(prompt, self.schema)\n",
    "        return get_pricing_info(self.pricing_page_html)\n",
    "    \n",
    "    @property\n",
    "    def pricing_tables(self) -> Dict[str, pd.DataFrame]:\n",
    "        \"\"\"Convert pricing tables to pandas DataFrames.\"\"\"\n",
    "        tables = self.pricing_info.get('OpenAI_API_Pricing_Schema', {})\n",
    "        \n",
    "        # Create a mapping interface that transforms table data to DataFrames\n",
    "        return dol.add_ipython_key_completions(\n",
    "            dol.wrap_kvs(\n",
    "                tables,\n",
    "                value_decoder=lambda x: pd.DataFrame(x.get('pricing_table', []))\n",
    "            )\n",
    "        )\n",
    "    \n",
    "    def list_pricing_categories(self) -> List[str]:\n",
    "        \"\"\"List available pricing categories.\"\"\"\n",
    "        return list(self.pricing_info.get('OpenAI_API_Pricing_Schema', {}))\n",
    "    \n",
    "    def get_pricing_table(self, category: str) -> pd.DataFrame:\n",
    "        \"\"\"Get a specific pricing table as a pandas DataFrame.\"\"\"\n",
    "        return self.pricing_tables[category]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['api_pricing_schema.json',\n",
       " 'openai_api_pricing_schema_description.txt',\n",
       " 'openai_api_pricing_info_from_ai.json',\n",
       " 'openai_api_pricing_info.json',\n",
       " 'openai_api_pricing.html']"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dacc = Dacc()\n",
    "list(dacc.json_store)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'JsonFiles'"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ChatDacc: Demo\n",
    "\n",
    "For background see this discussion comment: [Accessing OpenAI Chat Data and Shared Conversations\n",
    "](https://github.com/thorwhalen/oa/discussions/11#discussioncomment-11852719)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "from oa.chats import ChatDacc\n",
    "from pprint import pprint\n",
    "import pandas as pd\n",
    "\n",
    "url = 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5'\n",
    "dacc = ChatDacc(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'content': 'This conversation is meant to be used as an example, for '\n",
      "             'testing, and/or for figuring out how to parse the html and json '\n",
      "             'of a conversation. \\n'\n",
      "             '\\n'\n",
      "             \"As such, we'd like to keep it short. \\n\"\n",
      "             '\\n'\n",
      "             'Just say \"Hello World!\" back to me for now, and then in a second '\n",
      "             'line write 10 random words. ',\n",
      "  'id': '3b469b70-b069-4640-98af-5417491bb626',\n",
      "  'role': 'user',\n",
      "  'time': 1737020650.866734},\n",
      " {'content': 'Hello World!  \\n'\n",
      "             'apple, river, galaxy, chair, melody, shadow, cloud, puzzle, '\n",
      "             'flame, whisper.',\n",
      "  'id': '38ee4a3f-8487-4b35-9f92-ddee57c25d0a',\n",
      "  'role': 'assistant',\n",
      "  'time': 1737020652.436654},\n",
      " {'content': 'Now, so we get code blocks (and a second user query), write a '\n",
      "             'python code block containing a hello world print.',\n",
      "  'id': '01808c08-dc33-4932-bc12-64bd7e936760',\n",
      "  'role': 'user',\n",
      "  'time': 1737020700.676552},\n",
      " {'content': '```python\\nprint(\"Hello, World!\")\\n```',\n",
      "  'id': 'be4486db-894f-4e6f-bd0a-22d9d2facf69',\n",
      "  'role': 'assistant',\n",
      "  'time': 1737020701.381287}]\n"
     ]
    }
   ],
   "source": [
    "pprint(dacc.basic_turns_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>role</th>\n",
       "      <th>content</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3b469b70-b069-4640-98af-5417491bb626</td>\n",
       "      <td>user</td>\n",
       "      <td>This conversation is meant to be used as an ex...</td>\n",
       "      <td>2025-01-16 09:44:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>38ee4a3f-8487-4b35-9f92-ddee57c25d0a</td>\n",
       "      <td>assistant</td>\n",
       "      <td>Hello World!  \\napple, river, galaxy, chair, m...</td>\n",
       "      <td>2025-01-16 09:44:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01808c08-dc33-4932-bc12-64bd7e936760</td>\n",
       "      <td>user</td>\n",
       "      <td>Now, so we get code blocks (and a second user ...</td>\n",
       "      <td>2025-01-16 09:45:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>be4486db-894f-4e6f-bd0a-22d9d2facf69</td>\n",
       "      <td>assistant</td>\n",
       "      <td>```python\\nprint(\"Hello, World!\")\\n```</td>\n",
       "      <td>2025-01-16 09:45:01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     id       role  \\\n",
       "0  3b469b70-b069-4640-98af-5417491bb626       user   \n",
       "1  38ee4a3f-8487-4b35-9f92-ddee57c25d0a  assistant   \n",
       "2  01808c08-dc33-4932-bc12-64bd7e936760       user   \n",
       "3  be4486db-894f-4e6f-bd0a-22d9d2facf69  assistant   \n",
       "\n",
       "                                             content                time  \n",
       "0  This conversation is meant to be used as an ex... 2025-01-16 09:44:10  \n",
       "1  Hello World!  \\napple, river, galaxy, chair, m... 2025-01-16 09:44:12  \n",
       "2  Now, so we get code blocks (and a second user ... 2025-01-16 09:45:00  \n",
       "3             ```python\\nprint(\"Hello, World!\")\\n``` 2025-01-16 09:45:01  "
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dacc.basic_turns_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There's more turn keys than those that are shown here. \n",
    "The basic turns data just shows turns that have non-null roles and content. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['adff303b-75cc-493c-b757-605adadb8e56',\n",
       " '1473f2d9-ba09-4cd7-90c4-1452898676de',\n",
       " '40c5ed53-2b82-4e38-a4ec-dabf8f589553',\n",
       " '3b469b70-b069-4640-98af-5417491bb626',\n",
       " '94469d89-ff37-48c1-bdab-44b27299d79b',\n",
       " '38ee4a3f-8487-4b35-9f92-ddee57c25d0a',\n",
       " '01808c08-dc33-4932-bc12-64bd7e936760',\n",
       " 'be4486db-894f-4e6f-bd0a-22d9d2facf69']"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dacc.turns_data_keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'children': ['94469d89-ff37-48c1-bdab-44b27299d79b'],\n",
      " 'id': '3b469b70-b069-4640-98af-5417491bb626',\n",
      " 'message': {'author': {'metadata': {}, 'role': 'user'},\n",
      "             'content': {'content_type': 'text',\n",
      "                         'parts': ['This conversation is meant to be used as '\n",
      "                                   'an example, for testing, and/or for '\n",
      "                                   'figuring out how to parse the html and '\n",
      "                                   'json of a conversation. \\n'\n",
      "                                   '\\n'\n",
      "                                   \"As such, we'd like to keep it short. \\n\"\n",
      "                                   '\\n'\n",
      "                                   'Just say \"Hello World!\" back to me for '\n",
      "                                   'now, and then in a second line write 10 '\n",
      "                                   'random words. ']},\n",
      "             'create_time': 1737020650.866734,\n",
      "             'id': '3b469b70-b069-4640-98af-5417491bb626',\n",
      "             'metadata': {'message_source': None,\n",
      "                          'message_type': None,\n",
      "                          'request_id': '902d2a5a0ed1e15c-MRS',\n",
      "                          'shared_conversation_id': '6788d539-0f2c-8013-9535-889bf344d7d5',\n",
      "                          'timestamp_': 'absolute'},\n",
      "             'recipient': 'all',\n",
      "             'status': 'finished_successfully',\n",
      "             'weight': 1},\n",
      " 'parent': '40c5ed53-2b82-4e38-a4ec-dabf8f589553'}\n"
     ]
    }
   ],
   "source": [
    "pprint(dacc.turns_data[dacc.turns_data_keys[3]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can get a list of all possible fields (up to the time of writing this, up to our knowledge)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'adff303b-75cc-493c-b757-605adadb8e56': {'description': 'Unique identifier for the conversation turn.',\n",
       "  'example': 'adff303b-75cc-493c-b757-605adadb8e56'},\n",
       " 'id': {'description': 'The unique identifier for a message or turn in the conversation.',\n",
       "  'example': 'adff303b-75cc-493c-b757-605adadb8e56'},\n",
       " 'children': {'description': 'An array of identifiers for the child messages that are part of this turn.',\n",
       "  'example': ['1473f2d9-ba09-4cd7-90c4-1452898676de']},\n",
       " 'message': {'description': 'An object containing details about the message sent during this turn.',\n",
       "  'example': {'id': '1473f2d9-ba09-4cd7-90c4-1452898676de',\n",
       "   'author': {'role': 'system', 'metadata': {}},\n",
       "   'content': {'content_type': 'text', 'parts': ['']},\n",
       "   'status': 'finished_successfully',\n",
       "   'end_turn': True,\n",
       "   'weight': 0,\n",
       "   'metadata': {'is_visually_hidden_from_conversation': True,\n",
       "    'shared_conversation_id': '6788d539-0f2c-8013-9535-889bf344d7d5'},\n",
       "   'recipient': 'all'}},\n",
       " 'parent': {'description': 'The identifier of the parent message in the conversation tree.',\n",
       "  'example': 'adff303b-75cc-493c-b757-605adadb8e56'},\n",
       " 'create_time': {'description': 'Timestamp indicating when the message was created, typically represented as a Unix timestamp.',\n",
       "  'example': 1737020650.866734},\n",
       " 'author': {'description': 'An object containing details about the author of the message, including role and any additional metadata.',\n",
       "  'example': {'role': 'system', 'metadata': {}}},\n",
       " 'content_type': {'description': 'The type of content included in the message (e.g., text, model_editable_context).',\n",
       "  'example': 'text'},\n",
       " 'parts': {'description': 'An array containing the actual message content as individual parts.',\n",
       "  'example': ['Original custom instructions no longer available']},\n",
       " 'status': {'description': 'The status of the message processing (e.g., finished_successfully, error).',\n",
       "  'example': 'finished_successfully'},\n",
       " 'end_turn': {'description': 'Indicates whether this message concludes the current turn in the conversation.',\n",
       "  'example': True},\n",
       " 'weight': {'description': 'A numeric value representing the weight or importance of the message.',\n",
       "  'example': 0},\n",
       " 'metadata': {'description': 'An object containing additional metadata about the message, such as visibility and contextual data.',\n",
       "  'example': {'is_visually_hidden_from_conversation': True,\n",
       "   'shared_conversation_id': '6788d539-0f2c-8013-9535-889bf344d7d5'}},\n",
       " 'recipient': {'description': 'Indicates who the recipient of the message is (e.g., all, specific user).',\n",
       "  'example': 'all'},\n",
       " 'finish_details': {'description': 'An object detailing how the message processing finished, including any stop tokens that could affect subsequent content.',\n",
       "  'example': {'type': 'stop', 'stop_tokens': [200002]}},\n",
       " 'is_complete': {'description': 'Indicates whether the message processing is complete.',\n",
       "  'example': True},\n",
       " 'citations': {'description': 'An array of citations (if any) included with the message.',\n",
       "  'example': []},\n",
       " 'content_references': {'description': 'An array of content references associated with the message, can be empty.',\n",
       "  'example': []},\n",
       " 'model_slug': {'description': 'Indicates the model used to generate the response, if applicable.',\n",
       "  'example': 'gpt-4o'},\n",
       " 'default_model_slug': {'description': 'The default model slug that was utilized for generating responses.',\n",
       "  'example': 'gpt-4o'},\n",
       " 'parent_id': {'description': 'Identifier of the parent message for hierarchical relationships in the conversation.',\n",
       "  'example': '3b469b70-b069-4640-98af-5417491bb626'},\n",
       " 'request_id': {'description': 'Unique identifier for the request made to the model during the interaction.',\n",
       "  'example': '902d2b91cd44e15c-MRS'},\n",
       " 'timestamp_': {'description': 'Timestamp type indicating whether it is relative or absolute, often used for synchronization purposes.',\n",
       "  'example': 'absolute'},\n",
       " 'shared_conversation_id': {'description': 'Identifier for the shared conversation session to track context.',\n",
       "  'example': '6788d539-0f2c-8013-9535-889bf344d7d5'},\n",
       " 'is_redacted': {'description': 'Indicates whether the message content has been redacted for privacy or security reasons.',\n",
       "  'example': True}}"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(iter(dacc.turns_data_ssot.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>example</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <td>The unique identifier for a message or turn in...</td>\n",
       "      <td>adff303b-75cc-493c-b757-605adadb8e56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>children</th>\n",
       "      <td>An array of identifiers for the child messages...</td>\n",
       "      <td>[1473f2d9-ba09-4cd7-90c4-1452898676de]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>message</th>\n",
       "      <td>An object containing details about the message...</td>\n",
       "      <td>{'id': '1473f2d9-ba09-4cd7-90c4-1452898676de',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>parent</th>\n",
       "      <td>The identifier of the parent message in the co...</td>\n",
       "      <td>adff303b-75cc-493c-b757-605adadb8e56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>create_time</th>\n",
       "      <td>Timestamp indicating when the message was crea...</td>\n",
       "      <td>1737020650.866734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>author</th>\n",
       "      <td>An object containing details about the author ...</td>\n",
       "      <td>{'role': 'system', 'metadata': {}}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>content_type</th>\n",
       "      <td>The type of content included in the message (e...</td>\n",
       "      <td>text</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>parts</th>\n",
       "      <td>An array containing the actual message content...</td>\n",
       "      <td>[Original custom instructions no longer availa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>status</th>\n",
       "      <td>The status of the message processing (e.g., fi...</td>\n",
       "      <td>finished_successfully</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>end_turn</th>\n",
       "      <td>Indicates whether this message concludes the c...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weight</th>\n",
       "      <td>A numeric value representing the weight or imp...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metadata</th>\n",
       "      <td>An object containing additional metadata about...</td>\n",
       "      <td>{'is_visually_hidden_from_conversation': True,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recipient</th>\n",
       "      <td>Indicates who the recipient of the message is ...</td>\n",
       "      <td>all</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>finish_details</th>\n",
       "      <td>An object detailing how the message processing...</td>\n",
       "      <td>{'type': 'stop', 'stop_tokens': [200002]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_complete</th>\n",
       "      <td>Indicates whether the message processing is co...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>citations</th>\n",
       "      <td>An array of citations (if any) included with t...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>content_references</th>\n",
       "      <td>An array of content references associated with...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model_slug</th>\n",
       "      <td>Indicates the model used to generate the respo...</td>\n",
       "      <td>gpt-4o</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>default_model_slug</th>\n",
       "      <td>The default model slug that was utilized for g...</td>\n",
       "      <td>gpt-4o</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>parent_id</th>\n",
       "      <td>Identifier of the parent message for hierarchi...</td>\n",
       "      <td>3b469b70-b069-4640-98af-5417491bb626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>request_id</th>\n",
       "      <td>Unique identifier for the request made to the ...</td>\n",
       "      <td>902d2b91cd44e15c-MRS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>timestamp_</th>\n",
       "      <td>Timestamp type indicating whether it is relati...</td>\n",
       "      <td>absolute</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shared_conversation_id</th>\n",
       "      <td>Identifier for the shared conversation session...</td>\n",
       "      <td>6788d539-0f2c-8013-9535-889bf344d7d5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_redacted</th>\n",
       "      <td>Indicates whether the message content has been...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                              description  \\\n",
       "id                      The unique identifier for a message or turn in...   \n",
       "children                An array of identifiers for the child messages...   \n",
       "message                 An object containing details about the message...   \n",
       "parent                  The identifier of the parent message in the co...   \n",
       "create_time             Timestamp indicating when the message was crea...   \n",
       "author                  An object containing details about the author ...   \n",
       "content_type            The type of content included in the message (e...   \n",
       "parts                   An array containing the actual message content...   \n",
       "status                  The status of the message processing (e.g., fi...   \n",
       "end_turn                Indicates whether this message concludes the c...   \n",
       "weight                  A numeric value representing the weight or imp...   \n",
       "metadata                An object containing additional metadata about...   \n",
       "recipient               Indicates who the recipient of the message is ...   \n",
       "finish_details          An object detailing how the message processing...   \n",
       "is_complete             Indicates whether the message processing is co...   \n",
       "citations               An array of citations (if any) included with t...   \n",
       "content_references      An array of content references associated with...   \n",
       "model_slug              Indicates the model used to generate the respo...   \n",
       "default_model_slug      The default model slug that was utilized for g...   \n",
       "parent_id               Identifier of the parent message for hierarchi...   \n",
       "request_id              Unique identifier for the request made to the ...   \n",
       "timestamp_              Timestamp type indicating whether it is relati...   \n",
       "shared_conversation_id  Identifier for the shared conversation session...   \n",
       "is_redacted             Indicates whether the message content has been...   \n",
       "\n",
       "                                                                  example  \n",
       "id                                   adff303b-75cc-493c-b757-605adadb8e56  \n",
       "children                           [1473f2d9-ba09-4cd7-90c4-1452898676de]  \n",
       "message                 {'id': '1473f2d9-ba09-4cd7-90c4-1452898676de',...  \n",
       "parent                               adff303b-75cc-493c-b757-605adadb8e56  \n",
       "create_time                                             1737020650.866734  \n",
       "author                                 {'role': 'system', 'metadata': {}}  \n",
       "content_type                                                         text  \n",
       "parts                   [Original custom instructions no longer availa...  \n",
       "status                                              finished_successfully  \n",
       "end_turn                                                             True  \n",
       "weight                                                                  0  \n",
       "metadata                {'is_visually_hidden_from_conversation': True,...  \n",
       "recipient                                                             all  \n",
       "finish_details                  {'type': 'stop', 'stop_tokens': [200002]}  \n",
       "is_complete                                                          True  \n",
       "citations                                                              []  \n",
       "content_references                                                     []  \n",
       "model_slug                                                         gpt-4o  \n",
       "default_model_slug                                                 gpt-4o  \n",
       "parent_id                            3b469b70-b069-4640-98af-5417491bb626  \n",
       "request_id                                           902d2b91cd44e15c-MRS  \n",
       "timestamp_                                                       absolute  \n",
       "shared_conversation_id               6788d539-0f2c-8013-9535-889bf344d7d5  \n",
       "is_redacted                                                          True  "
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(index=dacc.turns_data_ssot.keys(), data=dacc.turns_data_ssot.values())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You have a tool to extract data from the turns data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>role</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>adff303b-75cc-493c-b757-605adadb8e56</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3b469b70-b069-4640-98af-5417491bb626</td>\n",
       "      <td>user</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>94469d89-ff37-48c1-bdab-44b27299d79b</td>\n",
       "      <td>assistant</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>38ee4a3f-8487-4b35-9f92-ddee57c25d0a</td>\n",
       "      <td>assistant</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01808c08-dc33-4932-bc12-64bd7e936760</td>\n",
       "      <td>user</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>be4486db-894f-4e6f-bd0a-22d9d2facf69</td>\n",
       "      <td>assistant</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     id       role  weight\n",
       "0  adff303b-75cc-493c-b757-605adadb8e56       None     NaN\n",
       "1  3b469b70-b069-4640-98af-5417491bb626       user     1.0\n",
       "2  94469d89-ff37-48c1-bdab-44b27299d79b  assistant     1.0\n",
       "3  38ee4a3f-8487-4b35-9f92-ddee57c25d0a  assistant     1.0\n",
       "4  01808c08-dc33-4932-bc12-64bd7e936760       user     1.0\n",
       "5  be4486db-894f-4e6f-bd0a-22d9d2facf69  assistant     1.0"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = dacc.extract_turns(\n",
    "    {\n",
    "        'id': 'id', 'role': \n",
    "        'message.author.role', \n",
    "        'weight': 'message.weight'\n",
    "    }\n",
    ")\n",
    "pd.DataFrame(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And there's also `metadata` (which in the original json, contains `mappings` and l`inear_conversation` fields, which are two fields that contain the turns data (the `mapping` maps them by ids, the `linear_conversation` fields are just a list of the values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': 'Test Chat 1',\n",
       " 'create_time': 1737020729.060687,\n",
       " 'update_time': 1737020733.031014,\n",
       " 'moderation_results': [],\n",
       " 'current_node': 'be4486db-894f-4e6f-bd0a-22d9d2facf69',\n",
       " 'conversation_id': '6788d539-0f2c-8013-9535-889bf344d7d5',\n",
       " 'is_archived': False,\n",
       " 'safe_urls': [],\n",
       " 'default_model_slug': 'gpt-4o',\n",
       " 'disabled_tool_ids': [],\n",
       " 'is_public': True,\n",
       " 'has_user_editable_context': False,\n",
       " 'continue_conversation_url': 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5/continue',\n",
       " 'moderation_state': {'has_been_moderated': False,\n",
       "  'has_been_blocked': False,\n",
       "  'has_been_accepted': False,\n",
       "  'has_been_auto_blocked': False,\n",
       "  'has_been_auto_moderated': False},\n",
       " 'is_indexable': False,\n",
       " 'is_better_metatags_enabled': True}"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dacc.metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>example</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <td>The title of the conversation.</td>\n",
       "      <td>Test Chat 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>create_time</th>\n",
       "      <td>The timestamp (in seconds since epoch) when th...</td>\n",
       "      <td>1737020729.060687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>update_time</th>\n",
       "      <td>The timestamp (in seconds since epoch) when th...</td>\n",
       "      <td>1737020733.031014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>moderation_results</th>\n",
       "      <td>An array that holds results from any moderatio...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>current_node</th>\n",
       "      <td>The unique identifier of the current node in t...</td>\n",
       "      <td>be4486db-894f-4e6f-bd0a-22d9d2facf69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>conversation_id</th>\n",
       "      <td>A unique identifier for the entire conversation.</td>\n",
       "      <td>6788d539-0f2c-8013-9535-889bf344d7d5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_archived</th>\n",
       "      <td>A boolean indicating whether the conversation ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>safe_urls</th>\n",
       "      <td>An array of URLs deemed safe within the contex...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>default_model_slug</th>\n",
       "      <td>The identifier for the default model used in t...</td>\n",
       "      <td>gpt-4o</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disabled_tool_ids</th>\n",
       "      <td>An array of tool identifiers that have been di...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_public</th>\n",
       "      <td>A boolean indicating whether the conversation ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>linear_conversation</th>\n",
       "      <td>An array representing the sequential flow of t...</td>\n",
       "      <td>[{'id': 'adff303b-75cc-493c-b757-605adadb8e56'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>has_user_editable_context</th>\n",
       "      <td>A boolean indicating if the user can edit the ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>continue_conversation_url</th>\n",
       "      <td>A URL that allows users to continue the conver...</td>\n",
       "      <td>https://chatgpt.com/share/6788d539-0f2c-8013-9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>moderation_state</th>\n",
       "      <td>An object that encapsulates the moderation sta...</td>\n",
       "      <td>{'has_been_moderated': False, 'has_been_blocke...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_indexable</th>\n",
       "      <td>A boolean indicating if this conversation can ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_better_metatags_enabled</th>\n",
       "      <td>A boolean indicating if better metatags are en...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                  description  \\\n",
       "title                                          The title of the conversation.   \n",
       "create_time                 The timestamp (in seconds since epoch) when th...   \n",
       "update_time                 The timestamp (in seconds since epoch) when th...   \n",
       "moderation_results          An array that holds results from any moderatio...   \n",
       "current_node                The unique identifier of the current node in t...   \n",
       "conversation_id              A unique identifier for the entire conversation.   \n",
       "is_archived                 A boolean indicating whether the conversation ...   \n",
       "safe_urls                   An array of URLs deemed safe within the contex...   \n",
       "default_model_slug          The identifier for the default model used in t...   \n",
       "disabled_tool_ids           An array of tool identifiers that have been di...   \n",
       "is_public                   A boolean indicating whether the conversation ...   \n",
       "linear_conversation         An array representing the sequential flow of t...   \n",
       "has_user_editable_context   A boolean indicating if the user can edit the ...   \n",
       "continue_conversation_url   A URL that allows users to continue the conver...   \n",
       "moderation_state            An object that encapsulates the moderation sta...   \n",
       "is_indexable                A boolean indicating if this conversation can ...   \n",
       "is_better_metatags_enabled  A boolean indicating if better metatags are en...   \n",
       "\n",
       "                                                                      example  \n",
       "title                                                             Test Chat 1  \n",
       "create_time                                                 1737020729.060687  \n",
       "update_time                                                 1737020733.031014  \n",
       "moderation_results                                                         []  \n",
       "current_node                             be4486db-894f-4e6f-bd0a-22d9d2facf69  \n",
       "conversation_id                          6788d539-0f2c-8013-9535-889bf344d7d5  \n",
       "is_archived                                                             False  \n",
       "safe_urls                                                                  []  \n",
       "default_model_slug                                                     gpt-4o  \n",
       "disabled_tool_ids                                                          []  \n",
       "is_public                                                                True  \n",
       "linear_conversation         [{'id': 'adff303b-75cc-493c-b757-605adadb8e56'...  \n",
       "has_user_editable_context                                               False  \n",
       "continue_conversation_url   https://chatgpt.com/share/6788d539-0f2c-8013-9...  \n",
       "moderation_state            {'has_been_moderated': False, 'has_been_blocke...  \n",
       "is_indexable                                                            False  \n",
       "is_better_metatags_enabled                                               True  "
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(index=dacc.metadata_ssot.keys(), data=dacc.metadata_ssot.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And then, there's where all of this came from: The base json that was extracted from the conversations html:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['basename', 'future', 'isSpaMode', 'state']"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(dacc.full_json_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a large nested json. The following is the path to the metadata (that contains the turns data etc.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('state',\n",
       " 'loaderData',\n",
       " 'routes/share.$shareId.($action)',\n",
       " 'serverResponse',\n",
       " 'data')"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dacc.metadata_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['title',\n",
       " 'create_time',\n",
       " 'update_time',\n",
       " 'mapping',\n",
       " 'moderation_results',\n",
       " 'current_node',\n",
       " 'conversation_id',\n",
       " 'is_archived',\n",
       " 'safe_urls',\n",
       " 'default_model_slug',\n",
       " 'disabled_tool_ids',\n",
       " 'is_public',\n",
       " 'linear_conversation',\n",
       " 'has_user_editable_context',\n",
       " 'continue_conversation_url',\n",
       " 'moderation_state',\n",
       " 'is_indexable',\n",
       " 'is_better_metatags_enabled']"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dol import path_get\n",
    "\n",
    "data = path_get(dacc.full_json_dict, dacc.metadata_path)\n",
    "list(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': 'adff303b-75cc-493c-b757-605adadb8e56', 'role': None, 'content': None},\n",
       " {'id': '3b469b70-b069-4640-98af-5417491bb626',\n",
       "  'role': 'user',\n",
       "  'content': ['This conversation is meant to be used as an example, for testing, and/or for figuring out how to parse the html and json of a conversation. \\n\\nAs such, we\\'d like to keep it short. \\n\\nJust say \"Hello World!\" back to me for now, and then in a second line write 10 random words. ']},\n",
       " {'id': '94469d89-ff37-48c1-bdab-44b27299d79b',\n",
       "  'role': 'assistant',\n",
       "  'content': None},\n",
       " {'id': '38ee4a3f-8487-4b35-9f92-ddee57c25d0a',\n",
       "  'role': 'assistant',\n",
       "  'content': ['Hello World!  \\napple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper.']},\n",
       " {'id': '01808c08-dc33-4932-bc12-64bd7e936760',\n",
       "  'role': 'user',\n",
       "  'content': ['Now, so we get code blocks (and a second user query), write a python code block containing a hello world print.']},\n",
       " {'id': 'be4486db-894f-4e6f-bd0a-22d9d2facf69',\n",
       "  'role': 'assistant',\n",
       "  'content': ['```python\\nprint(\"Hello, World!\")\\n```']}]"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = dacc.extract_turns(\n",
    "    {'id': 'id', 'role': 'message.author.role', 'content': 'message.content.parts'}\n",
    ")\n",
    "list(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>role</th>\n",
       "      <th>content</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3b469b70-b069-4640-98af-5417491bb626</th>\n",
       "      <td>user</td>\n",
       "      <td>This conversation is meant to be used as an ex...</td>\n",
       "      <td>2025-01-16 09:44:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38ee4a3f-8487-4b35-9f92-ddee57c25d0a</th>\n",
       "      <td>assistant</td>\n",
       "      <td>Hello World!  \\napple, river, galaxy, chair, m...</td>\n",
       "      <td>2025-01-16 09:44:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01808c08-dc33-4932-bc12-64bd7e936760</th>\n",
       "      <td>user</td>\n",
       "      <td>Now, so we get code blocks (and a second user ...</td>\n",
       "      <td>2025-01-16 09:45:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>be4486db-894f-4e6f-bd0a-22d9d2facf69</th>\n",
       "      <td>assistant</td>\n",
       "      <td>```python\\nprint(\"Hello, World!\")\\n```</td>\n",
       "      <td>2025-01-16 09:45:01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           role  \\\n",
       "id                                                \n",
       "3b469b70-b069-4640-98af-5417491bb626       user   \n",
       "38ee4a3f-8487-4b35-9f92-ddee57c25d0a  assistant   \n",
       "01808c08-dc33-4932-bc12-64bd7e936760       user   \n",
       "be4486db-894f-4e6f-bd0a-22d9d2facf69  assistant   \n",
       "\n",
       "                                                                                content  \\\n",
       "id                                                                                        \n",
       "3b469b70-b069-4640-98af-5417491bb626  This conversation is meant to be used as an ex...   \n",
       "38ee4a3f-8487-4b35-9f92-ddee57c25d0a  Hello World!  \\napple, river, galaxy, chair, m...   \n",
       "01808c08-dc33-4932-bc12-64bd7e936760  Now, so we get code blocks (and a second user ...   \n",
       "be4486db-894f-4e6f-bd0a-22d9d2facf69             ```python\\nprint(\"Hello, World!\")\\n```   \n",
       "\n",
       "                                                    time  \n",
       "id                                                        \n",
       "3b469b70-b069-4640-98af-5417491bb626 2025-01-16 09:44:10  \n",
       "38ee4a3f-8487-4b35-9f92-ddee57c25d0a 2025-01-16 09:44:12  \n",
       "01808c08-dc33-4932-bc12-64bd7e936760 2025-01-16 09:45:00  \n",
       "be4486db-894f-4e6f-bd0a-22d9d2facf69 2025-01-16 09:45:01  "
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dacc.turns_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bootstrapping the parsing (and maintaining the parser)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['basename', 'future', 'isSpaMode', 'state']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from oa.chats import url_to_html, parse_shared_chatGPT_chat\n",
    "\n",
    "url = 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5'\n",
    "chat_html = url_to_html(url)\n",
    "chat_json_dict = parse_shared_chatGPT_chat(chat_html)\n",
    "list(chat_json_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you have a look at the `parse_shared_chatGPT_chat` you'll notice it's parametrized by several arguments you didn't have to specify.\n",
    "\n",
    "```python\n",
    "def parse_shared_chatGPT_chat(\n",
    "    html: str,\n",
    "    is_target_string: Union[str, Callable] = 'data.*mapping.*message',\n",
    "    *,\n",
    "    variable_name: str = 'window.__remixContext',\n",
    "    json_pattern: str = '(\\\\{.*?\\\\});') -> dict\n",
    ") -> dict:\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(html: str,\n",
      " is_target_string: Union[str,\n",
      " Callable] = 'data.*mapping.*message',\n",
      " *,\n",
      " variable_name: str = 'window.__remixContext',\n",
      " json_pattern: str = '(\\\\{.*?\\\\});') -> dict\n"
     ]
    }
   ],
   "source": [
    "from i2 import Sig\n",
    "\n",
    "print(',\\n'.join(str(Sig(parse_shared_chatGPT_chat)).split(',')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We did this so that we can easily maintain the parser's definition. \n",
    "\n",
    "At the time of writing this, we've noticed that the json we want is assigned to a \n",
    "variable named `window.__remixContext` and has fields `data`, `mapping`, and `message` in that order. \n",
    "\n",
    "But to be able to get that in the first place, we used `parse_shared_chatGPT_chat` with different parameters. \n",
    "\n",
    "We tried it with a chat we knew the contents of, and asked `parse_shared_chatGPT_chat` to find the json \n",
    "based on a substring we knew should be in there:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['basename', 'future', 'isSpaMode', 'state']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from oa.chats import url_to_html, parse_shared_chatGPT_chat\n",
    "\n",
    "url = 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5'\n",
    "string_contained_in_conversation = 'apple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper'\n",
    "chat_html = url_to_html(url)\n",
    "chat_json_dict = parse_shared_chatGPT_chat(chat_html, is_target_string=string_contained_in_conversation)\n",
    "list(chat_json_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's get a truncated version of this dict, to make sure it's not too big to study..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['basename', 'future', 'isSpaMode', 'state']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from lkj import truncate_dict_values\n",
    "from pprint import pprint\n",
    "from pyperclip import copy\n",
    "\n",
    "\n",
    "t = truncate_dict_values(chat_json_dict)\n",
    "list(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "330325"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save this truncated dict to a file\n",
    "import tempfile\n",
    "import json\n",
    "from pathlib import Path\n",
    "\n",
    "tmp_filepath = tempfile.mktemp()\n",
    "Path\n",
    "\n",
    "\n",
    "len(json.dumps(t))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parsing out the json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is probably the most fragile part of the process. \n",
    "It's the part that broke previous tools I found to solve my problem.\n",
    "\n",
    "The approach I'm taking here is to identify a part of the conversation that I know should be in the conversation, and using that to find what I'm looking for. \n",
    "\n",
    "So what I did here is copy the html of a conversation and used chatGPT (o1) to help me out. \n",
    "[Here's the conversation](https://chatgpt.com/share/6788d964-9e60-8013-a2b3-e191d567c8ad)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['basename', 'future', 'isSpaMode', 'state']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Bootstrapping (when you don't know the structure, but you do know of a string that is contained in the conversation)\n",
    "\n",
    "from oa.chats import parse_shared_chatGPT_chat, url_to_html\n",
    "\n",
    "url = 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5'\n",
    "string_contained_in_conversation = 'apple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper'\n",
    "\n",
    "chat_html = url_to_html(url)\n",
    "chat_json_dict = parse_shared_chatGPT_chat(chat_html, is_target_string=string_contained_in_conversation)\n",
    "list(chat_json_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get a truncated version of the chat_json_dict (too big to study!)\n",
    "from lkj import truncate_dict_values\n",
    "import json \n",
    "\n",
    "truncated_json_dict = truncate_dict_values(chat_json_dict, max_list_size=1, max_string_size=90)\n",
    "truncated_json_dict_string = json.dumps(truncated_json_dict, indent=2)\n",
    "assert string_contained_in_conversation in truncated_json_dict_string, (\n",
    "    \"The string_contained_in_conversation is not in the truncated_json_dict_string. Increase the max_* parameters.\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "584063 characters written to /var/folders/mc/c070wfh51kxd9lft8dl74q1r0000gn/T/tmph5v0uvws.json\n"
     ]
    }
   ],
   "source": [
    "# Save to a file to go study it...\n",
    "\n",
    "import tempfile\n",
    "from pathlib import Path\n",
    "\n",
    "tmp_filepath = tempfile.mktemp() + '.json'\n",
    "Path(tmp_filepath).write_text(truncated_json_dict_string)\n",
    "print(f\"{len(truncated_json_dict_string)} characters written to {tmp_filepath}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parsing out the conversation part of the json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('state',\n",
       " 'loaderData',\n",
       " 'routes/share.$shareId.($action)',\n",
       " 'serverResponse',\n",
       " 'data',\n",
       " 'mapping',\n",
       " '38ee4a3f-8487-4b35-9f92-ddee57c25d0a',\n",
       " 'message',\n",
       " 'content',\n",
       " 'parts')"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from oa.chats import find_all_matching_paths_in_list_values\n",
    "from dol import path_get\n",
    "\n",
    "paths = find_all_matching_paths_in_list_values(truncated_json_dict, target_value=string_contained_in_conversation)\n",
    "path = next(paths)\n",
    "path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So now we know the path to where our target string can be found:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hello World!  \\napple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper.']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_get(truncated_json_dict, path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also look around it to get a better sense of the relevant json structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'content_type': 'text',\n",
       " 'parts': ['Hello World!  \\napple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper.']}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_get(truncated_json_dict, path[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '38ee4a3f-8487-4b35-9f92-ddee57c25d0a',\n",
       " 'author': {'role': 'assistant', 'metadata': {}},\n",
       " 'create_time': 1737020652.436654,\n",
       " 'content': {'content_type': 'text',\n",
       "  'parts': ['Hello World!  \\napple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper.']},\n",
       " 'status': 'finished_successfully',\n",
       " 'end_turn': True,\n",
       " 'weight': 1,\n",
       " 'metadata': {'finish_details': {'type': 'stop', 'stop_tokens': [200002]},\n",
       "  'is_complete': True,\n",
       "  'citations': [],\n",
       "  'content_references': [],\n",
       "  'message_type': None,\n",
       "  'model_slug': 'gpt-4o',\n",
       "  'default_model_slug': 'gpt-4o',\n",
       "  'parent_id': '94469d89-ff37-48c1-bdab-44b27299d79b',\n",
       "  'request_id': '902d2a5a0ed1e15c-MRS',\n",
       "  'timestamp_': 'absolute',\n",
       "  'shared_conversation_id': '6788d539-0f2c-8013-9535-889bf344d7d5'},\n",
       " 'recipient': 'all'}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_get(truncated_json_dict, path[:-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('state', 'loaderData', 'routes/share.$shareId.($action)', 'serverResponse', 'data', 'mapping')\n"
     ]
    }
   ],
   "source": [
    "print(path[:-4])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A wild guess.\n",
    "`('state', 'loaderData', 'routes/share.$shareId.($action)', 'serverResponse', 'data', 'mapping')` is where the \"turns\" of the conversation are."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['adff303b-75cc-493c-b757-605adadb8e56',\n",
       " '1473f2d9-ba09-4cd7-90c4-1452898676de',\n",
       " '40c5ed53-2b82-4e38-a4ec-dabf8f589553',\n",
       " '3b469b70-b069-4640-98af-5417491bb626',\n",
       " '94469d89-ff37-48c1-bdab-44b27299d79b',\n",
       " '38ee4a3f-8487-4b35-9f92-ddee57c25d0a',\n",
       " '01808c08-dc33-4932-bc12-64bd7e936760',\n",
       " 'be4486db-894f-4e6f-bd0a-22d9d2facf69']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "turns_path = ('state', 'loaderData', 'routes/share.$shareId.($action)', 'serverResponse', 'data', 'mapping')\n",
    "turns_data = path_get(truncated_json_dict, turns_path)\n",
    "list(turns_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['title',\n",
       " 'create_time',\n",
       " 'update_time',\n",
       " 'mapping',\n",
       " 'moderation_results',\n",
       " 'current_node',\n",
       " 'conversation_id',\n",
       " 'is_archived',\n",
       " 'safe_urls',\n",
       " 'default_model_slug',\n",
       " 'disabled_tool_ids',\n",
       " 'is_public',\n",
       " 'linear_conversation',\n",
       " 'has_user_editable_context',\n",
       " 'continue_conversation_url',\n",
       " 'moderation_state',\n",
       " 'is_indexable',\n",
       " 'is_better_metatags_enabled']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chat_data_path = turns_path[:-1]\n",
    "list(path_get(truncated_json_dict, chat_data_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': 'Test Chat 1',\n",
       " 'create_time': 1737020729.060687,\n",
       " 'update_time': 1737020733.031014,\n",
       " 'moderation_results': [],\n",
       " 'current_node': 'be4486db-894f-4e6f-bd0a-22d9d2facf69',\n",
       " 'conversation_id': '6788d539-0f2c-8013-9535-889bf344d7d5',\n",
       " 'is_archived': False,\n",
       " 'safe_urls': [],\n",
       " 'default_model_slug': 'gpt-4o',\n",
       " 'disabled_tool_ids': [],\n",
       " 'is_public': True,\n",
       " 'linear_conversation': [{'id': 'adff303b-75cc-493c-b757-605adadb8e56',\n",
       "   'children': ['1473f2d9-ba09-4cd7-90c4-1452898676de']}],\n",
       " 'has_user_editable_context': False,\n",
       " 'continue_conversation_url': 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5/continue',\n",
       " 'moderation_state': {'has_been_moderated': False,\n",
       "  'has_been_blocked': False,\n",
       "  'has_been_accepted': False,\n",
       "  'has_been_auto_blocked': False,\n",
       "  'has_been_auto_moderated': False},\n",
       " 'is_indexable': False,\n",
       " 'is_better_metatags_enabled': True}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata = path_get(truncated_json_dict, chat_data_path)\n",
    "metadata = {k: v for k, v in metadata.items() if k != 'mapping'}\n",
    "metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make some documentation for this part of the chat json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function oa.tools.prompt_function.<locals>.ask_oa(example_json, *, field_name=' {\"description\": ..., \"example\": ...}, ...', context=' just a general context')>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from oa import prompt_json_function\n",
    "\n",
    "\n",
    "mk_json_field_doc_ssot = prompt_json_function(\"\"\"\n",
    "You are a technical writer specialized in documenting JSON fields. \n",
    "Below is a JSON object. I'd like you to document each field, giving me a json\n",
    "whose fields are the names of the fields and the values are the description and example.\n",
    "That is: {field_name: {\"description\": ..., \"example\": ...}, ...}\n",
    "                                              \n",
    "The context is:\n",
    "{context: just a general context}\n",
    "                                              \n",
    "Here's an example json object:\n",
    "\n",
    "{example_json}\n",
    "\"\"\")\n",
    "\n",
    "mk_json_field_doc_ssot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'adff303b-75cc-493c-b757-605adadb8e56': {'description': 'Unique identifier '\n",
      "                                                         'for the conversation '\n",
      "                                                         'turn.',\n",
      "                                          'example': 'adff303b-75cc-493c-b757-605adadb8e56'},\n",
      " 'author': {'description': 'An object containing details about the author of '\n",
      "                           'the message, including role and any additional '\n",
      "                           'metadata.',\n",
      "            'example': {'metadata': {}, 'role': 'system'}},\n",
      " 'children': {'description': 'An array of identifiers for the child messages '\n",
      "                             'that are part of this turn.',\n",
      "              'example': ['1473f2d9-ba09-4cd7-90c4-1452898676de']},\n",
      " 'citations': {'description': 'An array of citations (if any) included with '\n",
      "                              'the message.',\n",
      "               'example': []},\n",
      " 'content_references': {'description': 'An array of content references '\n",
      "                                       'associated with the message, can be '\n",
      "                                       'empty.',\n",
      "                        'example': []},\n",
      " 'content_type': {'description': 'The type of content included in the message '\n",
      "                                 '(e.g., text, model_editable_context).',\n",
      "                  'example': 'text'},\n",
      " 'create_time': {'description': 'Timestamp indicating when the message was '\n",
      "                                'created, typically represented as a Unix '\n",
      "                                'timestamp.',\n",
      "                 'example': 1737020650.866734},\n",
      " 'default_model_slug': {'description': 'The default model slug that was '\n",
      "                                       'utilized for generating responses.',\n",
      "                        'example': 'gpt-4o'},\n",
      " 'end_turn': {'description': 'Indicates whether this message concludes the '\n",
      "                             'current turn in the conversation.',\n",
      "              'example': True},\n",
      " 'finish_details': {'description': 'An object detailing how the message '\n",
      "                                   'processing finished, including any stop '\n",
      "                                   'tokens that could affect subsequent '\n",
      "                                   'content.',\n",
      "                    'example': {'stop_tokens': [200002], 'type': 'stop'}},\n",
      " 'id': {'description': 'The unique identifier for a message or turn in the '\n",
      "                       'conversation.',\n",
      "        'example': 'adff303b-75cc-493c-b757-605adadb8e56'},\n",
      " 'is_complete': {'description': 'Indicates whether the message processing is '\n",
      "                                'complete.',\n",
      "                 'example': True},\n",
      " 'is_redacted': {'description': 'Indicates whether the message content has '\n",
      "                                'been redacted for privacy or security '\n",
      "                                'reasons.',\n",
      "                 'example': True},\n",
      " 'message': {'description': 'An object containing details about the message '\n",
      "                            'sent during this turn.',\n",
      "             'example': {'author': {'metadata': {}, 'role': 'system'},\n",
      "                         'content': {'content_type': 'text', 'parts': ['']},\n",
      "                         'end_turn': True,\n",
      "                         'id': '1473f2d9-ba09-4cd7-90c4-1452898676de',\n",
      "                         'metadata': {'is_visually_hidden_from_conversation': True,\n",
      "                                      'shared_conversation_id': '6788d539-0f2c-8013-9535-889bf344d7d5'},\n",
      "                         'recipient': 'all',\n",
      "                         'status': 'finished_successfully',\n",
      "                         'weight': 0}},\n",
      " 'metadata': {'description': 'An object containing additional metadata about '\n",
      "                             'the message, such as visibility and contextual '\n",
      "                             'data.',\n",
      "              'example': {'is_visually_hidden_from_conversation': True,\n",
      "                          'shared_conversation_id': '6788d539-0f2c-8013-9535-889bf344d7d5'}},\n",
      " 'model_slug': {'description': 'Indicates the model used to generate the '\n",
      "                               'response, if applicable.',\n",
      "                'example': 'gpt-4o'},\n",
      " 'parent': {'description': 'The identifier of the parent message in the '\n",
      "                           'conversation tree.',\n",
      "            'example': 'adff303b-75cc-493c-b757-605adadb8e56'},\n",
      " 'parent_id': {'description': 'Identifier of the parent message for '\n",
      "                              'hierarchical relationships in the conversation.',\n",
      "               'example': '3b469b70-b069-4640-98af-5417491bb626'},\n",
      " 'parts': {'description': 'An array containing the actual message content as '\n",
      "                          'individual parts.',\n",
      "           'example': ['Original custom instructions no longer available']},\n",
      " 'recipient': {'description': 'Indicates who the recipient of the message is '\n",
      "                              '(e.g., all, specific user).',\n",
      "               'example': 'all'},\n",
      " 'request_id': {'description': 'Unique identifier for the request made to the '\n",
      "                               'model during the interaction.',\n",
      "                'example': '902d2b91cd44e15c-MRS'},\n",
      " 'shared_conversation_id': {'description': 'Identifier for the shared '\n",
      "                                           'conversation session to track '\n",
      "                                           'context.',\n",
      "                            'example': '6788d539-0f2c-8013-9535-889bf344d7d5'},\n",
      " 'status': {'description': 'The status of the message processing (e.g., '\n",
      "                           'finished_successfully, error).',\n",
      "            'example': 'finished_successfully'},\n",
      " 'timestamp_': {'description': 'Timestamp type indicating whether it is '\n",
      "                               'relative or absolute, often used for '\n",
      "                               'synchronization purposes.',\n",
      "                'example': 'absolute'},\n",
      " 'weight': {'description': 'A numeric value representing the weight or '\n",
      "                           'importance of the message.',\n",
      "            'example': 0}}\n"
     ]
    }
   ],
   "source": [
    "turns_data_ssot = mk_json_field_doc_ssot(\n",
    "    json.dumps(turns_data, indent=2), \n",
    "    context=\"The conversation 'turns' of the json object holding a shared chatGPT conversation.\"\n",
    ")\n",
    "pprint(turns_data_ssot['result'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'continue_conversation_url': {'description': 'A URL that allows users to '\n",
      "                                              'continue the conversation from '\n",
      "                                              'its current state.',\n",
      "                               'example': 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5/continue'},\n",
      " 'conversation_id': {'description': 'A unique identifier for the entire '\n",
      "                                    'conversation.',\n",
      "                     'example': '6788d539-0f2c-8013-9535-889bf344d7d5'},\n",
      " 'create_time': {'description': 'The timestamp (in seconds since epoch) when '\n",
      "                                'the conversation was created.',\n",
      "                 'example': 1737020729.060687},\n",
      " 'current_node': {'description': 'The unique identifier of the current node in '\n",
      "                                 'the conversation flow.',\n",
      "                  'example': 'be4486db-894f-4e6f-bd0a-22d9d2facf69'},\n",
      " 'default_model_slug': {'description': 'The identifier for the default model '\n",
      "                                       'used in the conversation.',\n",
      "                        'example': 'gpt-4o'},\n",
      " 'disabled_tool_ids': {'description': 'An array of tool identifiers that have '\n",
      "                                      'been disabled for this conversation. It '\n",
      "                                      'is empty if no tools are disabled.',\n",
      "                       'example': []},\n",
      " 'has_user_editable_context': {'description': 'A boolean indicating if the '\n",
      "                                              'user can edit the context of '\n",
      "                                              'the conversation.',\n",
      "                               'example': False},\n",
      " 'is_archived': {'description': 'A boolean indicating whether the conversation '\n",
      "                                'is archived.',\n",
      "                 'example': False},\n",
      " 'is_better_metatags_enabled': {'description': 'A boolean indicating if better '\n",
      "                                               'metatags are enabled for this '\n",
      "                                               'conversation.',\n",
      "                                'example': True},\n",
      " 'is_indexable': {'description': 'A boolean indicating if this conversation '\n",
      "                                 'can be indexed by search engines or other '\n",
      "                                 'indexing tools.',\n",
      "                  'example': False},\n",
      " 'is_public': {'description': 'A boolean indicating whether the conversation '\n",
      "                              'is accessible to the public.',\n",
      "               'example': True},\n",
      " 'linear_conversation': {'description': 'An array representing the sequential '\n",
      "                                        'flow of the conversation, where each '\n",
      "                                        'object contains an id and its '\n",
      "                                        'children (sub-sequent nodes).',\n",
      "                         'example': [{'children': ['1473f2d9-ba09-4cd7-90c4-1452898676de'],\n",
      "                                      'id': 'adff303b-75cc-493c-b757-605adadb8e56'}]},\n",
      " 'moderation_results': {'description': 'An array that holds results from any '\n",
      "                                       'moderation applied to the '\n",
      "                                       'conversation. It is empty if no '\n",
      "                                       'moderation has been done.',\n",
      "                        'example': []},\n",
      " 'moderation_state': {'description': 'An object that encapsulates the '\n",
      "                                     'moderation status of the conversation, '\n",
      "                                     'detailing various moderation flags.',\n",
      "                      'example': {'has_been_accepted': False,\n",
      "                                  'has_been_auto_blocked': False,\n",
      "                                  'has_been_auto_moderated': False,\n",
      "                                  'has_been_blocked': False,\n",
      "                                  'has_been_moderated': False}},\n",
      " 'safe_urls': {'description': 'An array of URLs deemed safe within the context '\n",
      "                              'of the conversation. It is empty if there are '\n",
      "                              'no safe URLs.',\n",
      "               'example': []},\n",
      " 'title': {'description': 'The title of the conversation.',\n",
      "           'example': 'Test Chat 1'},\n",
      " 'update_time': {'description': 'The timestamp (in seconds since epoch) when '\n",
      "                                'the conversation was last updated.',\n",
      "                 'example': 1737020733.031014}}\n"
     ]
    }
   ],
   "source": [
    "metadata_ssot = mk_json_field_doc_ssot(\n",
    "    metadata,\n",
    "    context=\"The metadata of the json object holding a shared chatGPT conversation.\"\n",
    ")\n",
    "pprint(metadata_ssot['result'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function oa.tools.prompt_function.<locals>.ask_oa(example_json, *, context=' just a general context')>"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from oa import prompt_function\n",
    "\n",
    "\n",
    "mk_json_field_documentation = prompt_function(\"\"\"\n",
    "You are a technical writer specialized in documenting JSON fields. \n",
    "Below is a JSON object. I'd like you to document each field in a markdown table.\n",
    "The table should contain the name, description, and example value of each field.\n",
    "                                              \n",
    "The context is:\n",
    "{context: just a general context}\n",
    "                                              \n",
    "Here's an example json object:\n",
    "\n",
    "{example_json}\n",
    "\"\"\")\n",
    "\n",
    "mk_json_field_documentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Here's the documentation for the provided JSON object, structured in a markdown table format:\n",
      "\n",
      "| Field Name                                         | Description                                                                                     | Example Value                                                                         |\n",
      "|---------------------------------------------------|-------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------|\n",
      "| `<Conversation ID>`                               | Unique identifier for each turn in the conversation.                                          | `adff303b-75cc-493c-b757-605adadb8e56`                                             |\n",
      "| `id`                                              | The unique identifier for the specific message/turn.                                         | `adff303b-75cc-493c-b757-605adadb8e56`                                             |\n",
      "| `children`                                        | An array of IDs representing subsequent messages/turns that are children of this turn.       | `[\"1473f2d9-ba09-4cd7-90c4-1452898676de\"]`                                         |\n",
      "| `message`                                         | Contains details about the message that was sent.                                            | `{...}` (Object with further nested fields)                                         |\n",
      "| `author`                                          | Specifies the author of the message, including their role and metadata.                      | `{ \"role\": \"system\", \"metadata\": {} }`                                            |\n",
      "| `content`                                         | Contains the content of the message, specifying the type and actual message parts.           | `{ \"content_type\": \"text\", \"parts\": [\"\"] }`                                       |\n",
      "| `status`                                         | The status of the message indicating the completion state.                                    | `finished_successfully`                                                              |\n",
      "| `end_turn`                                       | Boolean indicating whether this message marks the end of a conversation turn.                 | `true`                                                                               |\n",
      "| `weight`                                         | Numerical value representing the significance or order of the turn in the conversation.      | `0`                                                                                  |\n",
      "| `metadata`                                       | Contains additional information about the message, such as visibility and identifiers.         | `{ \"is_visually_hidden_from_conversation\": true, \"shared_conversation_id\": \"6788d539-0f2c-8013-9535-889bf344d7d5\" }` |\n",
      "| `recipient`                                       | The intended recipient(s) of the message, indicating audience scope.                         | `all`                                                                                |\n",
      "| `parent`                                          | ID of the parent turn/message to which this turn is a response.                              | `adff303b-75cc-493c-b757-605adadb8e56`                                             |\n",
      "| `create_time`                                     | Timestamp indicating the creation time of the message, in Unix time format.                   | `1737020650.866734`                                                                  |\n",
      "| `is_user_system_message`                          | Indicates if the message is a system message from the user.                                  | `true`                                                                               |\n",
      "| `is_redacted`                                     | Indicates if the message has been redacted for privacy or security reasons.                   | `true`                                                                               |\n",
      "| `finish_details`                                 | Provides details on how the message was concluded, including stop tokens.                     | `{ \"type\": \"stop\", \"stop_tokens\": [200002] }`                                     |\n",
      "| `model_slug`                                      | Identifies the model used for generating the message response.                                | `gpt-4o`                                                                             |\n",
      "| `default_model_slug`                              | The default model slug used for the current conversation context.                             | `gpt-4o`                                                                             |\n",
      "| `timestamp_`                                      | A string indicating the timestamp's nature (e.g., 'absolute').                               | `absolute`                                                                           |\n",
      "| `content_type`                                    | The type of content within the message, such as `text`, `model_editable_context`, etc.       | `text`                                                                               |\n",
      "| `parts`                                           | An array containing the actual parts of the message content.                                   | `[\"Hello World! \\napple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper.\"]` |\n",
      "\n",
      "This table documents the fields found within the JSON object representing a shared conversation of turns in a chat platform. Each field is described to inform developers or users of its structure and purpose, along with example values for clarity.\n"
     ]
    }
   ],
   "source": [
    "turn_data_docs = mk_json_field_documentation(\n",
    "    json.dumps(turns_data, indent=2), \n",
    "    context=\"the conversation 'turns' of the json object holding a shared chatGPT conversation\"\n",
    ")\n",
    "print(turn_data_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Here's the documentation for each field in the provided JSON object, formatted as a markdown table:\n",
      "\n",
      "| Field Name                                | Description                                                                                          | Example Value                                                      |\n",
      "|-------------------------------------------|------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------|\n",
      "| `title`                                   | The title of the shared chat GPT conversation.                                                      | \"Test Chat 1\"                                                    |\n",
      "| `create_time`                             | The timestamp when the conversation was created, represented in seconds since the Unix epoch.      | 1737020729.060687                                                |\n",
      "| `update_time`                             | The timestamp when the conversation was last updated, represented in seconds since the Unix epoch. | 1737020733.031014                                                |\n",
      "| `moderation_results`                      | An array containing results of moderation checks, if any.                                          | []                                                               |\n",
      "| `current_node`                            | The unique identifier of the current node in the conversation tree.                                 | \"be4486db-894f-4e6f-bd0a-22d9d2facf69\"                        |\n",
      "| `conversation_id`                         | A unique identifier for the conversation.                                                           | \"6788d539-0f2c-8013-9535-889bf344d7d5\"                         |\n",
      "| `is_archived`                             | A boolean value indicating whether the conversation is archived.                                     | false                                                             |\n",
      "| `safe_urls`                               | An array of URLs that have been deemed safe in the conversation.                                    | []                                                               |\n",
      "| `default_model_slug`                      | The slug representing the default model used for the conversation.                                  | \"gpt-4o\"                                                         |\n",
      "| `disabled_tool_ids`                       | An array of identifiers for tools that have been disabled for this conversation.                    | []                                                               |\n",
      "| `is_public`                               | A boolean value indicating whether the conversation is accessible to the public.                    | true                                                              |\n",
      "| `linear_conversation`                     | An array representing the conversation in a linear format with node identifiers.                    | [{\"id\": \"adff303b-75cc-493c-b757-605adadb8e56\", \"children\": [\"1473f2d9-ba09-4cd7-90c4-1452898676de\"]}] |\n",
      "| `has_user_editable_context`              | A boolean value indicating if the user can edit context for the conversation.                       | false                                                             |\n",
      "| `continue_conversation_url`               | A URL that allows the user to continue the conversation.                                            | \"https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5/continue\" |\n",
      "| `moderation_state`                        | An object containing the state of moderation checks applied to the conversation.                     | {\"has_been_moderated\": false, \"has_been_blocked\": false, \"has_been_accepted\": false, \"has_been_auto_blocked\": false, \"has_been_auto_moderated\": false} |\n",
      "| `is_indexable`                           | A boolean value indicating whether the conversation can be indexed for search purposes.             | false                                                             |\n",
      "| `is_better_metatags_enabled`             | A boolean value indicating whether enhanced metatags are enabled for this conversation.             | true                                                              |\n",
      "\n",
      "This table provides a concise overview of each field, its purpose, and an illustrative example.\n"
     ]
    }
   ],
   "source": [
    "metadata_field_docs = mk_json_field_documentation(\n",
    "    json.dumps(metadata, indent=2), \n",
    "    context=\"metadata of a shared chatGPT conversation\"\n",
    ")\n",
    "print(metadata_field_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import json\n",
    "from typing import Callable, Optional\n",
    "\n",
    "def raise_run_time_error(msg: str):\n",
    "    raise RuntimeError(msg)\n",
    "\n",
    "def extract_json_dict(\n",
    "    string: str,\n",
    "    object_filter: Callable,\n",
    "    *,\n",
    "    decoder: json.JSONDecoder = json.JSONDecoder(),\n",
    "    not_found_callback: Optional[\n",
    "        Callable\n",
    "    ] = lambda string, object_filter: raise_run_time_error(\n",
    "        \"Object not found in string\"\n",
    "    ),\n",
    ") -> dict:\n",
    "    \"\"\"\n",
    "    Searches `string` from the beginning, attempting to decode consecutive\n",
    "    JSON objects using `decoder.raw_decode`. When an object satisfies\n",
    "    `object_filter`, returns it as a Python dictionary.\n",
    "    \"\"\"\n",
    "    pos = 0\n",
    "    while pos < len(string):\n",
    "        try:\n",
    "            obj, pos = decoder.raw_decode(string, pos)\n",
    "            if object_filter(obj):\n",
    "                return obj\n",
    "        except json.JSONDecodeError:\n",
    "            pos += 1\n",
    "    return not_found_callback(string, object_filter)\n",
    "\n",
    "def parse_shared_chatGPT_chat(\n",
    "    html: str, \n",
    "    string_contained_in_conversation: str\n",
    ") -> dict:\n",
    "    \"\"\"\n",
    "    Locates the big JSON structure assigned to window.__remixContext, \n",
    "    checks that the conversation portion includes `string_contained_in_conversation`,\n",
    "    and returns the JSON as a dict.\n",
    "    \"\"\"\n",
    "    # Regex pattern to capture the object assigned to window.__remixContext = {...};\n",
    "    pattern = r\"window\\.__remixContext\\s*=\\s*(\\{.*?\\});\"\n",
    "    match = re.search(pattern, html, flags=re.DOTALL)\n",
    "    if not match:\n",
    "        raise RuntimeError(\"Could not locate the JSON assigned to window.__remixContext\")\n",
    "\n",
    "    # Extract the raw JSON text (removing the trailing semicolon if needed)\n",
    "    raw_json_text = match.group(1).strip()\n",
    "\n",
    "    # We'll define a filter that checks if the conversation data includes the target string.\n",
    "    # Because the JSON is quite large, we can simply check if the substring is present in the raw text:\n",
    "    # but if we want to be more precise, we can parse first and only return if there's conversation data.\n",
    "    def conversation_filter(obj):\n",
    "        # Convert to string once more, or deeply check \"routes/share.$shareId.($action)\" for the substring.\n",
    "        # For simplicity, do a textual check on the entire string representation. \n",
    "        # If needed, we can refine to parse a specific location in obj.\n",
    "        return string_contained_in_conversation in json.dumps(obj, ensure_ascii=False)\n",
    "\n",
    "    # Now parse the JSON, returning only if the filter passes\n",
    "    extracted_dict = extract_json_dict(\n",
    "        raw_json_text,\n",
    "        object_filter=conversation_filter,\n",
    "    )\n",
    "\n",
    "    return extracted_dict\n",
    "\n",
    "\n",
    "t = parse_shared_chatGPT_chat(html, existing_substring)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['basename', 'future', 'isSpaMode', 'state']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "p10",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
