{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Essentials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from oa.chats import get_rendered_html, reduce_chat_html, parse_chat_html\n",
    "\n",
    "url = 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5'\n",
    "\n",
    "html = get_rendered_html(url)\n",
    "parsed = parse_chat_html(html)\n",
    "len(parsed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': '3b469b70-b069-4640-98af-5417491bb626',\n",
       "  'role': 'user',\n",
       "  'content': 'This conversation is meant to be used as an example, for testing, and/or for figuring out how to parse the html and json of a conversation. \\n\\nAs such, we\\'d like to keep it short. \\n\\nJust say \"Hello World!\" back to me for now, and then in a second line write 10 random words.',\n",
       "  'message_id': '3b469b70-b069-4640-98af-5417491bb626'},\n",
       " {'id': '94469d89-ff37-48c1-bdab-44b27299d79b',\n",
       "  'role': 'assistant',\n",
       "  'content': 'Hello World!\\napple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper.',\n",
       "  'message_id': '38ee4a3f-8487-4b35-9f92-ddee57c25d0a'},\n",
       " {'id': '01808c08-dc33-4932-bc12-64bd7e936760',\n",
       "  'role': 'user',\n",
       "  'content': 'Now, so we get code blocks (and a second user query), write a python code block containing a hello world print.',\n",
       "  'message_id': '01808c08-dc33-4932-bc12-64bd7e936760'},\n",
       " {'id': 'be4486db-894f-4e6f-bd0a-22d9d2facf69',\n",
       "  'role': 'assistant',\n",
       "  'content': '```python\\nprint(\"Hello, World!\")\\n```',\n",
       "  'message_id': 'be4486db-894f-4e6f-bd0a-22d9d2facf69'}]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parsed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(reduced_html)=31412 versus len(html)=268816\n",
      "And yet, we still have the information we need to parse the conversaton\n"
     ]
    }
   ],
   "source": [
    "# NOTE: Useful to analyze the conversation part of the html (without blowing up your RAM or AI!!)\n",
    "reduced_html = reduce_chat_html(html)\n",
    "print(f\"{len(reduced_html)=} versus {len(html)=}\")\n",
    "print(\"And yet, we still have the information we need to parse the conversaton!\")\n",
    "assert parse_chat_html(reduced_html) == parsed, \"The reduced html doesn't lead to the same parsed anymore!\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A dacc for this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from oa.chats import ChatDacc\n",
    "\n",
    "url = 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5'\n",
    "dacc = ChatDacc(url)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': '3b469b70-b069-4640-98af-5417491bb626',\n",
       "  'role': 'user',\n",
       "  'content': 'This conversation is meant to be used as an example, for testing, and/or for figuring out how to parse the html and json of a conversation. \\n\\nAs such, we\\'d like to keep it short. \\n\\nJust say \"Hello World!\" back to me for now, and then in a second line write 10 random words.',\n",
       "  'message_id': '3b469b70-b069-4640-98af-5417491bb626'},\n",
       " {'id': '94469d89-ff37-48c1-bdab-44b27299d79b',\n",
       "  'role': 'assistant',\n",
       "  'content': 'Hello World!\\napple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper.',\n",
       "  'message_id': '38ee4a3f-8487-4b35-9f92-ddee57c25d0a'},\n",
       " {'id': '01808c08-dc33-4932-bc12-64bd7e936760',\n",
       "  'role': 'user',\n",
       "  'content': 'Now, so we get code blocks (and a second user query), write a python code block containing a hello world print.',\n",
       "  'message_id': '01808c08-dc33-4932-bc12-64bd7e936760'},\n",
       " {'id': 'be4486db-894f-4e6f-bd0a-22d9d2facf69',\n",
       "  'role': 'assistant',\n",
       "  'content': '```python\\nprint(\"Hello, World!\")\\n```',\n",
       "  'message_id': 'be4486db-894f-4e6f-bd0a-22d9d2facf69'}]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dacc.basic_turns_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>role</th>\n",
       "      <th>content</th>\n",
       "      <th>message_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3b469b70-b069-4640-98af-5417491bb626</td>\n",
       "      <td>user</td>\n",
       "      <td>This conversation is meant to be used as an ex...</td>\n",
       "      <td>3b469b70-b069-4640-98af-5417491bb626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>94469d89-ff37-48c1-bdab-44b27299d79b</td>\n",
       "      <td>assistant</td>\n",
       "      <td>Hello World!\\napple, river, galaxy, chair, mel...</td>\n",
       "      <td>38ee4a3f-8487-4b35-9f92-ddee57c25d0a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01808c08-dc33-4932-bc12-64bd7e936760</td>\n",
       "      <td>user</td>\n",
       "      <td>Now, so we get code blocks (and a second user ...</td>\n",
       "      <td>01808c08-dc33-4932-bc12-64bd7e936760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>be4486db-894f-4e6f-bd0a-22d9d2facf69</td>\n",
       "      <td>assistant</td>\n",
       "      <td>```python\\nprint(\"Hello, World!\")\\n```</td>\n",
       "      <td>be4486db-894f-4e6f-bd0a-22d9d2facf69</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     id       role  \\\n",
       "0  3b469b70-b069-4640-98af-5417491bb626       user   \n",
       "1  94469d89-ff37-48c1-bdab-44b27299d79b  assistant   \n",
       "2  01808c08-dc33-4932-bc12-64bd7e936760       user   \n",
       "3  be4486db-894f-4e6f-bd0a-22d9d2facf69  assistant   \n",
       "\n",
       "                                             content  \\\n",
       "0  This conversation is meant to be used as an ex...   \n",
       "1  Hello World!\\napple, river, galaxy, chair, mel...   \n",
       "2  Now, so we get code blocks (and a second user ...   \n",
       "3             ```python\\nprint(\"Hello, World!\")\\n```   \n",
       "\n",
       "                             message_id  \n",
       "0  3b469b70-b069-4640-98af-5417491bb626  \n",
       "1  38ee4a3f-8487-4b35-9f92-ddee57c25d0a  \n",
       "2  01808c08-dc33-4932-bc12-64bd7e936760  \n",
       "3  be4486db-894f-4e6f-bd0a-22d9d2facf69  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dacc.basic_turns_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dacc.url_data()  # when urls are there, you get them in a list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Note: Below this probably doesn't work any more since chatGPT changed it's way of serving shared conversations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extracting information from a shared chatGPT chat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For background see this discussion comment: [Accessing OpenAI Chat Data and Shared Conversations\n",
    "](https://github.com/thorwhalen/oa/discussions/11#discussioncomment-11852719)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from oa.chats import ChatDacc\n",
    "from pprint import pprint\n",
    "import pandas as pd\n",
    "\n",
    "url = 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5'\n",
    "dacc = ChatDacc(url)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### basic_turns_data\n",
    "\n",
    "If you're just interested in the text of the conversation, along with minimal meta data (e.g. time, role, id,...) then `basic_turns_data` is what you want"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'content': 'This conversation is meant to be used as an example, for '\n",
      "             'testing, and/or for figuring out how to parse the html and json '\n",
      "             'of a conversation. \\n'\n",
      "             '\\n'\n",
      "             \"As such, we'd like to keep it short. \\n\"\n",
      "             '\\n'\n",
      "             'Just say \"Hello World!\" back to me for now, and then in a second '\n",
      "             'line write 10 random words. ',\n",
      "  'id': '3b469b70-b069-4640-98af-5417491bb626',\n",
      "  'role': 'user',\n",
      "  'time': 1737020650.866734},\n",
      " {'content': 'Hello World!  \\n'\n",
      "             'apple, river, galaxy, chair, melody, shadow, cloud, puzzle, '\n",
      "             'flame, whisper.',\n",
      "  'id': '38ee4a3f-8487-4b35-9f92-ddee57c25d0a',\n",
      "  'role': 'assistant',\n",
      "  'time': 1737020652.436654},\n",
      " {'content': 'Now, so we get code blocks (and a second user query), write a '\n",
      "             'python code block containing a hello world print.',\n",
      "  'id': '01808c08-dc33-4932-bc12-64bd7e936760',\n",
      "  'role': 'user',\n",
      "  'time': 1737020700.676552},\n",
      " {'content': '```python\\nprint(\"Hello, World!\")\\n```',\n",
      "  'id': 'be4486db-894f-4e6f-bd0a-22d9d2facf69',\n",
      "  'role': 'assistant',\n",
      "  'time': 1737020701.381287}]\n"
     ]
    }
   ],
   "source": [
    "pprint(dacc.basic_turns_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If `pandas` is installed, you can get this directly as a pandas dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>role</th>\n",
       "      <th>content</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3b469b70-b069-4640-98af-5417491bb626</td>\n",
       "      <td>user</td>\n",
       "      <td>This conversation is meant to be used as an ex...</td>\n",
       "      <td>2025-01-16 09:44:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>38ee4a3f-8487-4b35-9f92-ddee57c25d0a</td>\n",
       "      <td>assistant</td>\n",
       "      <td>Hello World!  \\napple, river, galaxy, chair, m...</td>\n",
       "      <td>2025-01-16 09:44:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01808c08-dc33-4932-bc12-64bd7e936760</td>\n",
       "      <td>user</td>\n",
       "      <td>Now, so we get code blocks (and a second user ...</td>\n",
       "      <td>2025-01-16 09:45:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>be4486db-894f-4e6f-bd0a-22d9d2facf69</td>\n",
       "      <td>assistant</td>\n",
       "      <td>```python\\nprint(\"Hello, World!\")\\n```</td>\n",
       "      <td>2025-01-16 09:45:01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     id       role  \\\n",
       "0  3b469b70-b069-4640-98af-5417491bb626       user   \n",
       "1  38ee4a3f-8487-4b35-9f92-ddee57c25d0a  assistant   \n",
       "2  01808c08-dc33-4932-bc12-64bd7e936760       user   \n",
       "3  be4486db-894f-4e6f-bd0a-22d9d2facf69  assistant   \n",
       "\n",
       "                                             content                time  \n",
       "0  This conversation is meant to be used as an ex... 2025-01-16 09:44:10  \n",
       "1  Hello World!  \\napple, river, galaxy, chair, m... 2025-01-16 09:44:12  \n",
       "2  Now, so we get code blocks (and a second user ... 2025-01-16 09:45:00  \n",
       "3             ```python\\nprint(\"Hello, World!\")\\n``` 2025-01-16 09:45:01  "
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dacc.basic_turns_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### turns_data: All the turns and all the fields"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There's more turn keys than those that are shown here. \n",
    "The basic turns data just shows turns that have non-null roles and content. \n",
    "You'll find all the turns data (both all the turns and all their fields) in the \n",
    "`turns_data` attribute. It's a dict whose keys are turn ids and values are the contents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['adff303b-75cc-493c-b757-605adadb8e56',\n",
       " '1473f2d9-ba09-4cd7-90c4-1452898676de',\n",
       " '40c5ed53-2b82-4e38-a4ec-dabf8f589553',\n",
       " '3b469b70-b069-4640-98af-5417491bb626',\n",
       " '94469d89-ff37-48c1-bdab-44b27299d79b',\n",
       " '38ee4a3f-8487-4b35-9f92-ddee57c25d0a',\n",
       " '01808c08-dc33-4932-bc12-64bd7e936760',\n",
       " 'be4486db-894f-4e6f-bd0a-22d9d2facf69']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(dacc.turns_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To facilitate index-based access to turns, we have a `turns_data_keys` attribute in dacc:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['adff303b-75cc-493c-b757-605adadb8e56',\n",
       " '1473f2d9-ba09-4cd7-90c4-1452898676de',\n",
       " '40c5ed53-2b82-4e38-a4ec-dabf8f589553',\n",
       " '3b469b70-b069-4640-98af-5417491bb626',\n",
       " '94469d89-ff37-48c1-bdab-44b27299d79b',\n",
       " '38ee4a3f-8487-4b35-9f92-ddee57c25d0a',\n",
       " '01808c08-dc33-4932-bc12-64bd7e936760',\n",
       " 'be4486db-894f-4e6f-bd0a-22d9d2facf69']"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dacc.turns_data_keys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "... which allows you to do this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'children': ['94469d89-ff37-48c1-bdab-44b27299d79b'],\n",
      " 'id': '3b469b70-b069-4640-98af-5417491bb626',\n",
      " 'message': {'author': {'metadata': {}, 'role': 'user'},\n",
      "             'content': {'content_type': 'text',\n",
      "                         'parts': ['This conversation is meant to be used as '\n",
      "                                   'an example, for testing, and/or for '\n",
      "                                   'figuring out how to parse the html and '\n",
      "                                   'json of a conversation. \\n'\n",
      "                                   '\\n'\n",
      "                                   \"As such, we'd like to keep it short. \\n\"\n",
      "                                   '\\n'\n",
      "                                   'Just say \"Hello World!\" back to me for '\n",
      "                                   'now, and then in a second line write 10 '\n",
      "                                   'random words. ']},\n",
      "             'create_time': 1737020650.866734,\n",
      "             'id': '3b469b70-b069-4640-98af-5417491bb626',\n",
      "             'metadata': {'message_source': None,\n",
      "                          'message_type': None,\n",
      "                          'request_id': '902d2a5a0ed1e15c-MRS',\n",
      "                          'shared_conversation_id': '6788d539-0f2c-8013-9535-889bf344d7d5',\n",
      "                          'timestamp_': 'absolute'},\n",
      "             'recipient': 'all',\n",
      "             'status': 'finished_successfully',\n",
      "             'weight': 1},\n",
      " 'parent': '40c5ed53-2b82-4e38-a4ec-dabf8f589553'}\n"
     ]
    }
   ],
   "source": [
    "turn_data_for_index = lambda i: dacc.turns_data[dacc.turns_data_keys[i]]\n",
    "pprint(turn_data_for_index(3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have a `turns_data_ssot` attribute which lists name, description and examples of a bunch (but not necessarily all) fiels you can find in turn data.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>example</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>continue_conversation_url</th>\n",
       "      <td>A URL that allows users to continue the conver...</td>\n",
       "      <td>https://chatgpt.com/share/6788d539-0f2c-8013-9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>conversation_id</th>\n",
       "      <td>A unique identifier for the entire conversation.</td>\n",
       "      <td>6788d539-0f2c-8013-9535-889bf344d7d5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>create_time</th>\n",
       "      <td>The timestamp (in seconds since epoch) when th...</td>\n",
       "      <td>1737020729.060687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>current_node</th>\n",
       "      <td>The unique identifier of the current node in t...</td>\n",
       "      <td>be4486db-894f-4e6f-bd0a-22d9d2facf69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>default_model_slug</th>\n",
       "      <td>The identifier for the default model used in t...</td>\n",
       "      <td>gpt-4o</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disabled_tool_ids</th>\n",
       "      <td>An array of tool identifiers that have been di...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>has_user_editable_context</th>\n",
       "      <td>A boolean indicating if the user can edit the ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_archived</th>\n",
       "      <td>A boolean indicating whether the conversation ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_better_metatags_enabled</th>\n",
       "      <td>A boolean indicating if better metatags are en...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_indexable</th>\n",
       "      <td>A boolean indicating if this conversation can ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_public</th>\n",
       "      <td>A boolean indicating whether the conversation ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>linear_conversation</th>\n",
       "      <td>An array representing the sequential flow of t...</td>\n",
       "      <td>[{'children': ['1473f2d9-ba09-4cd7-90c4-145289...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>moderation_results</th>\n",
       "      <td>An array that holds results from any moderatio...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>moderation_state</th>\n",
       "      <td>An object that encapsulates the moderation sta...</td>\n",
       "      <td>{'has_been_accepted': False, 'has_been_auto_bl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>safe_urls</th>\n",
       "      <td>An array of URLs deemed safe within the contex...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <td>The title of the conversation.</td>\n",
       "      <td>Test Chat 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>update_time</th>\n",
       "      <td>The timestamp (in seconds since epoch) when th...</td>\n",
       "      <td>1737020733.031014</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                  description  \\\n",
       "continue_conversation_url   A URL that allows users to continue the conver...   \n",
       "conversation_id              A unique identifier for the entire conversation.   \n",
       "create_time                 The timestamp (in seconds since epoch) when th...   \n",
       "current_node                The unique identifier of the current node in t...   \n",
       "default_model_slug          The identifier for the default model used in t...   \n",
       "disabled_tool_ids           An array of tool identifiers that have been di...   \n",
       "has_user_editable_context   A boolean indicating if the user can edit the ...   \n",
       "is_archived                 A boolean indicating whether the conversation ...   \n",
       "is_better_metatags_enabled  A boolean indicating if better metatags are en...   \n",
       "is_indexable                A boolean indicating if this conversation can ...   \n",
       "is_public                   A boolean indicating whether the conversation ...   \n",
       "linear_conversation         An array representing the sequential flow of t...   \n",
       "moderation_results          An array that holds results from any moderatio...   \n",
       "moderation_state            An object that encapsulates the moderation sta...   \n",
       "safe_urls                   An array of URLs deemed safe within the contex...   \n",
       "title                                          The title of the conversation.   \n",
       "update_time                 The timestamp (in seconds since epoch) when th...   \n",
       "\n",
       "                                                                      example  \n",
       "continue_conversation_url   https://chatgpt.com/share/6788d539-0f2c-8013-9...  \n",
       "conversation_id                          6788d539-0f2c-8013-9535-889bf344d7d5  \n",
       "create_time                                                 1737020729.060687  \n",
       "current_node                             be4486db-894f-4e6f-bd0a-22d9d2facf69  \n",
       "default_model_slug                                                     gpt-4o  \n",
       "disabled_tool_ids                                                          []  \n",
       "has_user_editable_context                                               False  \n",
       "is_archived                                                             False  \n",
       "is_better_metatags_enabled                                               True  \n",
       "is_indexable                                                            False  \n",
       "is_public                                                                True  \n",
       "linear_conversation         [{'children': ['1473f2d9-ba09-4cd7-90c4-145289...  \n",
       "moderation_results                                                         []  \n",
       "moderation_state            {'has_been_accepted': False, 'has_been_auto_bl...  \n",
       "safe_urls                                                                  []  \n",
       "title                                                             Test Chat 1  \n",
       "update_time                                                 1737020733.031014  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(index=dacc.turns_data_ssot.keys(), data=dacc.turns_data_ssot.values())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You have a tool to extract data from the turns data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>role</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>adff303b-75cc-493c-b757-605adadb8e56</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3b469b70-b069-4640-98af-5417491bb626</td>\n",
       "      <td>user</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>94469d89-ff37-48c1-bdab-44b27299d79b</td>\n",
       "      <td>assistant</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>38ee4a3f-8487-4b35-9f92-ddee57c25d0a</td>\n",
       "      <td>assistant</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01808c08-dc33-4932-bc12-64bd7e936760</td>\n",
       "      <td>user</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>be4486db-894f-4e6f-bd0a-22d9d2facf69</td>\n",
       "      <td>assistant</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     id       role  weight\n",
       "0  adff303b-75cc-493c-b757-605adadb8e56       None     NaN\n",
       "1  3b469b70-b069-4640-98af-5417491bb626       user     1.0\n",
       "2  94469d89-ff37-48c1-bdab-44b27299d79b  assistant     1.0\n",
       "3  38ee4a3f-8487-4b35-9f92-ddee57c25d0a  assistant     1.0\n",
       "4  01808c08-dc33-4932-bc12-64bd7e936760       user     1.0\n",
       "5  be4486db-894f-4e6f-bd0a-22d9d2facf69  assistant     1.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = dacc.extract_turns(\n",
    "    {\n",
    "        'id': 'id', 'role': \n",
    "        'message.author.role', \n",
    "        'weight': 'message.weight'\n",
    "    }\n",
    ")\n",
    "pd.DataFrame(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And there's also `metadata` (which in the original json, contains `mappings` and l`inear_conversation` fields, which are two fields that contain the turns data (the `mapping` maps them by ids, the `linear_conversation` fields are just a list of the values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': 'Test Chat 1',\n",
       " 'create_time': 1737020729.060687,\n",
       " 'update_time': 1737020733.031014,\n",
       " 'moderation_results': [],\n",
       " 'current_node': 'be4486db-894f-4e6f-bd0a-22d9d2facf69',\n",
       " 'conversation_id': '6788d539-0f2c-8013-9535-889bf344d7d5',\n",
       " 'is_archived': False,\n",
       " 'safe_urls': [],\n",
       " 'default_model_slug': 'gpt-4o',\n",
       " 'disabled_tool_ids': [],\n",
       " 'is_public': True,\n",
       " 'has_user_editable_context': False,\n",
       " 'continue_conversation_url': 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5/continue',\n",
       " 'moderation_state': {'has_been_moderated': False,\n",
       "  'has_been_blocked': False,\n",
       "  'has_been_accepted': False,\n",
       "  'has_been_auto_blocked': False,\n",
       "  'has_been_auto_moderated': False},\n",
       " 'is_indexable': False,\n",
       " 'is_better_metatags_enabled': True}"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dacc.metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "metadata also has a dict that describes the fields:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>example</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <td>The title of the conversation.</td>\n",
       "      <td>Test Chat 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>create_time</th>\n",
       "      <td>The timestamp (in seconds since epoch) when th...</td>\n",
       "      <td>1737020729.060687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>update_time</th>\n",
       "      <td>The timestamp (in seconds since epoch) when th...</td>\n",
       "      <td>1737020733.031014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>moderation_results</th>\n",
       "      <td>An array that holds results from any moderatio...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>current_node</th>\n",
       "      <td>The unique identifier of the current node in t...</td>\n",
       "      <td>be4486db-894f-4e6f-bd0a-22d9d2facf69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>conversation_id</th>\n",
       "      <td>A unique identifier for the entire conversation.</td>\n",
       "      <td>6788d539-0f2c-8013-9535-889bf344d7d5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_archived</th>\n",
       "      <td>A boolean indicating whether the conversation ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>safe_urls</th>\n",
       "      <td>An array of URLs deemed safe within the contex...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>default_model_slug</th>\n",
       "      <td>The identifier for the default model used in t...</td>\n",
       "      <td>gpt-4o</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disabled_tool_ids</th>\n",
       "      <td>An array of tool identifiers that have been di...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_public</th>\n",
       "      <td>A boolean indicating whether the conversation ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>linear_conversation</th>\n",
       "      <td>An array representing the sequential flow of t...</td>\n",
       "      <td>[{'id': 'adff303b-75cc-493c-b757-605adadb8e56'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>has_user_editable_context</th>\n",
       "      <td>A boolean indicating if the user can edit the ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>continue_conversation_url</th>\n",
       "      <td>A URL that allows users to continue the conver...</td>\n",
       "      <td>https://chatgpt.com/share/6788d539-0f2c-8013-9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>moderation_state</th>\n",
       "      <td>An object that encapsulates the moderation sta...</td>\n",
       "      <td>{'has_been_moderated': False, 'has_been_blocke...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_indexable</th>\n",
       "      <td>A boolean indicating if this conversation can ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_better_metatags_enabled</th>\n",
       "      <td>A boolean indicating if better metatags are en...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                  description  \\\n",
       "title                                          The title of the conversation.   \n",
       "create_time                 The timestamp (in seconds since epoch) when th...   \n",
       "update_time                 The timestamp (in seconds since epoch) when th...   \n",
       "moderation_results          An array that holds results from any moderatio...   \n",
       "current_node                The unique identifier of the current node in t...   \n",
       "conversation_id              A unique identifier for the entire conversation.   \n",
       "is_archived                 A boolean indicating whether the conversation ...   \n",
       "safe_urls                   An array of URLs deemed safe within the contex...   \n",
       "default_model_slug          The identifier for the default model used in t...   \n",
       "disabled_tool_ids           An array of tool identifiers that have been di...   \n",
       "is_public                   A boolean indicating whether the conversation ...   \n",
       "linear_conversation         An array representing the sequential flow of t...   \n",
       "has_user_editable_context   A boolean indicating if the user can edit the ...   \n",
       "continue_conversation_url   A URL that allows users to continue the conver...   \n",
       "moderation_state            An object that encapsulates the moderation sta...   \n",
       "is_indexable                A boolean indicating if this conversation can ...   \n",
       "is_better_metatags_enabled  A boolean indicating if better metatags are en...   \n",
       "\n",
       "                                                                      example  \n",
       "title                                                             Test Chat 1  \n",
       "create_time                                                 1737020729.060687  \n",
       "update_time                                                 1737020733.031014  \n",
       "moderation_results                                                         []  \n",
       "current_node                             be4486db-894f-4e6f-bd0a-22d9d2facf69  \n",
       "conversation_id                          6788d539-0f2c-8013-9535-889bf344d7d5  \n",
       "is_archived                                                             False  \n",
       "safe_urls                                                                  []  \n",
       "default_model_slug                                                     gpt-4o  \n",
       "disabled_tool_ids                                                          []  \n",
       "is_public                                                                True  \n",
       "linear_conversation         [{'id': 'adff303b-75cc-493c-b757-605adadb8e56'...  \n",
       "has_user_editable_context                                               False  \n",
       "continue_conversation_url   https://chatgpt.com/share/6788d539-0f2c-8013-9...  \n",
       "moderation_state            {'has_been_moderated': False, 'has_been_blocke...  \n",
       "is_indexable                                                            False  \n",
       "is_better_metatags_enabled                                               True  "
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(index=dacc.metadata_ssot.keys(), data=dacc.metadata_ssot.values())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### full_json_dict: The full json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And then, there's where all of this came from: The base json that was extracted from the conversations html:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['basename', 'future', 'isSpaMode', 'state']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(dacc.full_json_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a large nested json. The following is the path to the metadata (that contains the turns data etc.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('state',\n",
       " 'loaderData',\n",
       " 'routes/share.$shareId.($action)',\n",
       " 'serverResponse',\n",
       " 'data')"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dacc.metadata_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['title',\n",
       " 'create_time',\n",
       " 'update_time',\n",
       " 'mapping',\n",
       " 'moderation_results',\n",
       " 'current_node',\n",
       " 'conversation_id',\n",
       " 'is_archived',\n",
       " 'safe_urls',\n",
       " 'default_model_slug',\n",
       " 'disabled_tool_ids',\n",
       " 'is_public',\n",
       " 'linear_conversation',\n",
       " 'has_user_editable_context',\n",
       " 'continue_conversation_url',\n",
       " 'moderation_state',\n",
       " 'is_indexable',\n",
       " 'is_better_metatags_enabled']"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dol import path_get\n",
    "\n",
    "data = path_get(dacc.full_json_dict, dacc.metadata_path)\n",
    "list(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting the urls\n",
    "\n",
    "Sometimes, you want to extract some (or all) of the urls you have in a conversation. \n",
    "These urls could be quoted in the messages' text itself, but could also be somewhere else \n",
    "in the turns data (for example, when doing a chatGPT search, you have a list of \"sources\"). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from oa import ChatDacc\n",
    "\n",
    "url = 'https://chatgpt.com/share/67877101-6708-8013-ba9e-2e770186db58'  # url with some searches\n",
    "dacc = ChatDacc(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dacc.basic_turns_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### url_paths\n",
    "\n",
    "The `url_paths` attribute gives you a list of all the LEAF turns data paths to values that are strings and have urls in them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(url_paths)=204\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['68148933-cbc2-43ef-9b96-6a8ff7a2159c.message.metadata.search_result_groups[0].entries[0].url',\n",
       " '68148933-cbc2-43ef-9b96-6a8ff7a2159c.message.metadata.search_result_groups[1].entries[0].url',\n",
       " '68148933-cbc2-43ef-9b96-6a8ff7a2159c.message.metadata.search_result_groups[2].entries[0].url',\n",
       " '68148933-cbc2-43ef-9b96-6a8ff7a2159c.message.metadata.search_result_groups[3].entries[0].url',\n",
       " '68148933-cbc2-43ef-9b96-6a8ff7a2159c.message.metadata.search_result_groups[4].entries[0].url']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Paths to the URLs identified in the chat (uses find_url_keys())\n",
    "url_paths = dacc.url_paths\n",
    "print(f\"{len(url_paths)=}\")\n",
    "url_paths[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### url_data(...)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(all_urls)=204\n",
      "len(unique_urls)=132\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://www.rwdigital.ca/blog/how-much-energy-do-google-search-and-chatgpt-use/'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_urls = dacc.url_data() \n",
    "print(f\"{len(all_urls)=}\")\n",
    "unique_urls = sorted(set(all_urls))\n",
    "print(f\"{len(unique_urls)=}\")\n",
    "t[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['([AffMaven](https://affmaven.com/google-search-statistics/))',\n",
       " '([DemandSage](https://www.demandsage.com/chatgpt-statistics/))',\n",
       " '([DemandSage](https://www.demandsage.com/google-search-statistics/))',\n",
       " '([EIA](https://www.eia.gov/tools/faqs/faq.php?id=85&t=1))',\n",
       " '([Limited Systems](https://limited.systems/articles/google-search-vs-chatgpt-emissions/))',\n",
       " '([RW Digital](https://www.rwdigital.ca/blog/how-much-energy-do-google-search-and-chatgpt-use/))']"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_urls[:6]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By default, `dacc.url_data` uses `prior_levels_to_include=0, remove_chatgpt_utm=True`. \n",
    "If you want to get more information on the urls, you can ask for `prior_levels_to_include=1` or \n",
    "even more levels upward from where the url was found. \n",
    "Know that by doing so, you may have less items than before (but won't lose any urls) because sometimes, there are \n",
    "more than one url under some level.\n",
    "\n",
    "Also, if you want to leave the `utm_source=chatgpt.com` prefix from urls, you can say so via \n",
    "`remove_chatgpt_utm=True`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(urls_in_context)=199\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'matched_text': '\\ue200cite\\ue202turn0search7\\ue201',\n",
       " 'start_idx': 125,\n",
       " 'end_idx': 144,\n",
       " 'alt': '([Search Engine Land](https://searchengineland.com/calculating-the-carbon-footprint-of-a-google-search-16105?utm_source=chatgpt.com))',\n",
       " 'prompt_text': None,\n",
       " 'type': 'grouped_webpages',\n",
       " 'items': [{'title': 'Calculating The Carbon Footprint Of A Google Search - Search Engine Land',\n",
       "   'url': 'https://searchengineland.com/calculating-the-carbon-footprint-of-a-google-search-16105?utm_source=chatgpt.com',\n",
       "   'pub_date': 1231718400,\n",
       "   'snippet': 'Queries vary in degree of difficulty, but for the average query, the servers it touches each work on it for just a few thousandths of a second. ... this amounts to 0.0003 kWh of energy per search ...',\n",
       "   'attribution_segments': None,\n",
       "   'supporting_websites': [],\n",
       "   'hue': None,\n",
       "   'attributions': None,\n",
       "   'attribution': 'Search Engine Land'}],\n",
       " 'status': 'done',\n",
       " 'error': None,\n",
       " 'style': None}"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "urls_in_context = dacc.url_data(prior_levels_to_include=1, remove_chatgpt_utm=False)\n",
    "print(f\"{len(urls_in_context)=}\")  # note: some levels have several urls, so you'll get less items here than previously\n",
    "urls_in_context[12]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### See what path patterns contain urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['message.metadata.content_references[*].alt',\n",
       " 'message.metadata.content_references[*].items[*].thumbnail_url',\n",
       " 'message.metadata.content_references[*].items[*].url',\n",
       " 'message.metadata.content_references[*].sources[*].url',\n",
       " 'message.metadata.search_result_groups[*].entries[*].url']"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from functools import partial\n",
    "from oa.chats import find_url_keys\n",
    "from dol import Pipe\n",
    "import re \n",
    "\n",
    "url_paths = list(find_url_keys(dacc.turns_data))\n",
    "\n",
    "replace_array_index_with_star = partial(re.sub, '\\[\\d+\\]', '[*]')\n",
    "ignore_first_part_of_path = lambda x: '.'.join(x.split('.')[1:])\n",
    "transform = Pipe(replace_array_index_with_star, ignore_first_part_of_path)\n",
    "paths_containing_urls = sorted(set(map(transform, url_paths)))\n",
    "paths_containing_urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bootstrapping the parsing (and maintaining the parser)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['basename', 'future', 'isSpaMode', 'state']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from oa.chats import url_to_html, parse_shared_chatGPT_chat\n",
    "\n",
    "url = 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5'\n",
    "chat_html = url_to_html(url)\n",
    "chat_json_dict = parse_shared_chatGPT_chat(chat_html)\n",
    "list(chat_json_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you have a look at the `parse_shared_chatGPT_chat` you'll notice it's parametrized by several arguments you didn't have to specify.\n",
    "\n",
    "```python\n",
    "def parse_shared_chatGPT_chat(\n",
    "    html: str,\n",
    "    is_target_string: Union[str, Callable] = 'data.*mapping.*message',\n",
    "    *,\n",
    "    variable_name: str = 'window.__remixContext',\n",
    "    json_pattern: str = '(\\\\{.*?\\\\});') -> dict\n",
    ") -> dict:\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(html: str,\n",
      " is_target_string: Union[str,\n",
      " Callable] = 'data.*mapping.*message',\n",
      " *,\n",
      " variable_name: str = 'window.__remixContext',\n",
      " json_pattern: str = '(\\\\{.*?\\\\});') -> dict\n"
     ]
    }
   ],
   "source": [
    "from i2 import Sig\n",
    "\n",
    "print(',\\n'.join(str(Sig(parse_shared_chatGPT_chat)).split(',')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We did this so that we can easily maintain the parser's definition. \n",
    "\n",
    "At the time of writing this, we've noticed that the json we want is assigned to a \n",
    "variable named `window.__remixContext` and has fields `data`, `mapping`, and `message` in that order. \n",
    "\n",
    "But to be able to get that in the first place, we used `parse_shared_chatGPT_chat` with different parameters. \n",
    "\n",
    "We tried it with a chat we knew the contents of, and asked `parse_shared_chatGPT_chat` to find the json \n",
    "based on a substring we knew should be in there:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['basename', 'future', 'isSpaMode', 'state']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from oa.chats import url_to_html, parse_shared_chatGPT_chat\n",
    "\n",
    "url = 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5'\n",
    "string_contained_in_conversation = 'apple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper'\n",
    "chat_html = url_to_html(url)\n",
    "chat_json_dict = parse_shared_chatGPT_chat(chat_html, is_target_string=string_contained_in_conversation)\n",
    "list(chat_json_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's get a truncated version of this dict, to make sure it's not too big to study..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['basename', 'future', 'isSpaMode', 'state']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from lkj import truncate_dict_values\n",
    "from pprint import pprint\n",
    "from pyperclip import copy\n",
    "\n",
    "\n",
    "t = truncate_dict_values(chat_json_dict)\n",
    "list(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "330325"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save this truncated dict to a file\n",
    "import tempfile\n",
    "import json\n",
    "from pathlib import Path\n",
    "\n",
    "tmp_filepath = tempfile.mktemp()\n",
    "Path\n",
    "\n",
    "\n",
    "len(json.dumps(t))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parsing out the json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is probably the most fragile part of the process. \n",
    "It's the part that broke previous tools I found to solve my problem.\n",
    "\n",
    "The approach I'm taking here is to identify a part of the conversation that I know should be in the conversation, and using that to find what I'm looking for. \n",
    "\n",
    "So what I did here is copy the html of a conversation and used chatGPT (o1) to help me out. \n",
    "[Here's the conversation](https://chatgpt.com/share/6788d964-9e60-8013-a2b3-e191d567c8ad)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['basename', 'future', 'isSpaMode', 'state']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Bootstrapping (when you don't know the structure, but you do know of a string that is contained in the conversation)\n",
    "\n",
    "from oa.chats import parse_shared_chatGPT_chat, url_to_html\n",
    "\n",
    "url = 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5'\n",
    "string_contained_in_conversation = 'apple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper'\n",
    "\n",
    "chat_html = url_to_html(url)\n",
    "chat_json_dict = parse_shared_chatGPT_chat(chat_html, is_target_string=string_contained_in_conversation)\n",
    "list(chat_json_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get a truncated version of the chat_json_dict (too big to study!)\n",
    "from lkj import truncate_dict_values\n",
    "import json \n",
    "\n",
    "truncated_json_dict = truncate_dict_values(chat_json_dict, max_list_size=1, max_string_size=90)\n",
    "truncated_json_dict_string = json.dumps(truncated_json_dict, indent=2)\n",
    "assert string_contained_in_conversation in truncated_json_dict_string, (\n",
    "    \"The string_contained_in_conversation is not in the truncated_json_dict_string. Increase the max_* parameters.\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "584063 characters written to /var/folders/mc/c070wfh51kxd9lft8dl74q1r0000gn/T/tmph5v0uvws.json\n"
     ]
    }
   ],
   "source": [
    "# Save to a file to go study it...\n",
    "\n",
    "import tempfile\n",
    "from pathlib import Path\n",
    "\n",
    "tmp_filepath = tempfile.mktemp() + '.json'\n",
    "Path(tmp_filepath).write_text(truncated_json_dict_string)\n",
    "print(f\"{len(truncated_json_dict_string)} characters written to {tmp_filepath}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parsing out the conversation part of the json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('state',\n",
       " 'loaderData',\n",
       " 'routes/share.$shareId.($action)',\n",
       " 'serverResponse',\n",
       " 'data',\n",
       " 'mapping',\n",
       " '38ee4a3f-8487-4b35-9f92-ddee57c25d0a',\n",
       " 'message',\n",
       " 'content',\n",
       " 'parts')"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from oa.chats import find_all_matching_paths_in_list_values\n",
    "from dol import path_get\n",
    "\n",
    "paths = find_all_matching_paths_in_list_values(truncated_json_dict, target_value=string_contained_in_conversation)\n",
    "path = next(paths)\n",
    "path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So now we know the path to where our target string can be found:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hello World!  \\napple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper.']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_get(truncated_json_dict, path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also look around it to get a better sense of the relevant json structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'content_type': 'text',\n",
       " 'parts': ['Hello World!  \\napple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper.']}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_get(truncated_json_dict, path[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '38ee4a3f-8487-4b35-9f92-ddee57c25d0a',\n",
       " 'author': {'role': 'assistant', 'metadata': {}},\n",
       " 'create_time': 1737020652.436654,\n",
       " 'content': {'content_type': 'text',\n",
       "  'parts': ['Hello World!  \\napple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper.']},\n",
       " 'status': 'finished_successfully',\n",
       " 'end_turn': True,\n",
       " 'weight': 1,\n",
       " 'metadata': {'finish_details': {'type': 'stop', 'stop_tokens': [200002]},\n",
       "  'is_complete': True,\n",
       "  'citations': [],\n",
       "  'content_references': [],\n",
       "  'message_type': None,\n",
       "  'model_slug': 'gpt-4o',\n",
       "  'default_model_slug': 'gpt-4o',\n",
       "  'parent_id': '94469d89-ff37-48c1-bdab-44b27299d79b',\n",
       "  'request_id': '902d2a5a0ed1e15c-MRS',\n",
       "  'timestamp_': 'absolute',\n",
       "  'shared_conversation_id': '6788d539-0f2c-8013-9535-889bf344d7d5'},\n",
       " 'recipient': 'all'}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_get(truncated_json_dict, path[:-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('state', 'loaderData', 'routes/share.$shareId.($action)', 'serverResponse', 'data', 'mapping')\n"
     ]
    }
   ],
   "source": [
    "print(path[:-4])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A wild guess.\n",
    "`('state', 'loaderData', 'routes/share.$shareId.($action)', 'serverResponse', 'data', 'mapping')` is where the \"turns\" of the conversation are."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['adff303b-75cc-493c-b757-605adadb8e56',\n",
       " '1473f2d9-ba09-4cd7-90c4-1452898676de',\n",
       " '40c5ed53-2b82-4e38-a4ec-dabf8f589553',\n",
       " '3b469b70-b069-4640-98af-5417491bb626',\n",
       " '94469d89-ff37-48c1-bdab-44b27299d79b',\n",
       " '38ee4a3f-8487-4b35-9f92-ddee57c25d0a',\n",
       " '01808c08-dc33-4932-bc12-64bd7e936760',\n",
       " 'be4486db-894f-4e6f-bd0a-22d9d2facf69']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "turns_path = ('state', 'loaderData', 'routes/share.$shareId.($action)', 'serverResponse', 'data', 'mapping')\n",
    "turns_data = path_get(truncated_json_dict, turns_path)\n",
    "list(turns_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['title',\n",
       " 'create_time',\n",
       " 'update_time',\n",
       " 'mapping',\n",
       " 'moderation_results',\n",
       " 'current_node',\n",
       " 'conversation_id',\n",
       " 'is_archived',\n",
       " 'safe_urls',\n",
       " 'default_model_slug',\n",
       " 'disabled_tool_ids',\n",
       " 'is_public',\n",
       " 'linear_conversation',\n",
       " 'has_user_editable_context',\n",
       " 'continue_conversation_url',\n",
       " 'moderation_state',\n",
       " 'is_indexable',\n",
       " 'is_better_metatags_enabled']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chat_data_path = turns_path[:-1]\n",
    "list(path_get(truncated_json_dict, chat_data_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': 'Test Chat 1',\n",
       " 'create_time': 1737020729.060687,\n",
       " 'update_time': 1737020733.031014,\n",
       " 'moderation_results': [],\n",
       " 'current_node': 'be4486db-894f-4e6f-bd0a-22d9d2facf69',\n",
       " 'conversation_id': '6788d539-0f2c-8013-9535-889bf344d7d5',\n",
       " 'is_archived': False,\n",
       " 'safe_urls': [],\n",
       " 'default_model_slug': 'gpt-4o',\n",
       " 'disabled_tool_ids': [],\n",
       " 'is_public': True,\n",
       " 'linear_conversation': [{'id': 'adff303b-75cc-493c-b757-605adadb8e56',\n",
       "   'children': ['1473f2d9-ba09-4cd7-90c4-1452898676de']}],\n",
       " 'has_user_editable_context': False,\n",
       " 'continue_conversation_url': 'https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5/continue',\n",
       " 'moderation_state': {'has_been_moderated': False,\n",
       "  'has_been_blocked': False,\n",
       "  'has_been_accepted': False,\n",
       "  'has_been_auto_blocked': False,\n",
       "  'has_been_auto_moderated': False},\n",
       " 'is_indexable': False,\n",
       " 'is_better_metatags_enabled': True}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata = path_get(truncated_json_dict, chat_data_path)\n",
    "metadata = {k: v for k, v in metadata.items() if k != 'mapping'}\n",
    "metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make some documentation for this part of the chat json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge several examples of turns data to get a fuller sense of the possible fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = dict(\n",
    "    simple_url='https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5',\n",
    "    url_with_searches='https://chatgpt.com/share/67877101-6708-8013-ba9e-2e770186db58',\n",
    "    url_with_image_gen='https://chatgpt.com/share/678a1339-d14c-8013-bfcb-288d367a9079',\n",
    ")\n",
    "\n",
    "# sharing_chat_with_image_upload_not_yet_supported_by_openai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from oa.chats import ChatDacc\n",
    "from lkj import merge_dicts\n",
    "\n",
    "# Note, the turns_data is a dict whose keys are \"turn ids\" and values are the metadata for that turn, so we want to merge those\n",
    "chat_daccs = {\n",
    "    k: ChatDacc(url) for k, url in urls.items()\n",
    "}\n",
    "turn_datas = {\n",
    "    k: merge_dicts(*_dacc.turns_data.values()) for k, _dacc in chat_daccs.items()\n",
    "}\n",
    "merged_turn_data = merge_dicts(*turn_datas.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of first level fields: 4\n",
      "Number of leaf values: 42\n",
      "Number of json characters: 4961\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "import json\n",
    "from lkj import truncate_dict_values\n",
    "from dol import flatten_dict\n",
    "\n",
    "truncated_merged_turn_data = truncate_dict_values(merged_turn_data)\n",
    "flat_truncated_merged_turn_data = flatten_dict(truncated_merged_turn_data)\n",
    "\n",
    "print(f\"Number of first level fields: {len(truncated_merged_turn_data)}\")\n",
    "print(f\"Number of leaf values: {len(flat_truncated_merged_turn_data)}\")\n",
    "print(f\"Number of json characters: {len(json.dumps(truncated_merged_turn_data, indent=4))}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_dacc = next(iter(chat_daccs.values()))\n",
    "metadata_example = first_dacc.metadata\n",
    "metadata_fields = list(metadata_example)\n",
    "assert all(list(_dacc.metadata) == metadata_fields for _dacc in chat_daccs.values())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Have AI guess what each field is for\n",
    "(since we can't find any documentation for this)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function oa.tools.prompt_function.<locals>.ask_oa(example_json, *, context=' just a general context')>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from oa import prompt_json_function\n",
    "\n",
    "\n",
    "mk_json_field_doc_ssot = prompt_json_function(\n",
    "    \"\"\"\n",
    "You are a technical writer specialized in documenting JSON fields. \n",
    "Below is a JSON object. I'd like you to document each field, giving me a json\n",
    "whose fields are the names of the fields and the values are the description and example.\n",
    "Something like this: \n",
    "```{\"message.author.role\": {\"description\": ..., \"example\", ...}, \"message.author.name\": ...}```\n",
    "\n",
    "                                              \n",
    "The context is:\n",
    "{context: just a general context}\n",
    "                                              \n",
    "Here's an example json object:\n",
    "\n",
    "{example_json}\n",
    "\"\"\",\n",
    "    egress=lambda x: x['result'],\n",
    ")\n",
    "\n",
    "mk_json_field_doc_ssot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Have AI get parameter descriptions for turn data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': {'description': 'A unique identifier for the conversation turn.', 'example': '1fc35aa7-6b7a-4dae-9838-ead52c6d4793'}, 'children': {'description': 'An array of child conversation turns, which can hold additional messages in the conversation thread.', 'example': '[]'}, 'message.id': {'description': 'A unique identifier for the message within the conversation turn.', 'example': '1fc35aa7-6b7a-4dae-9838-ead52c6d4793'}, 'message.author.role': {'description': 'The role of the author of the message (e.g., user, assistant).', 'example': 'assistant'}, 'message.author.metadata.real_author': {'description': 'Metadata indicating the real author or source of the message.', 'example': 'tool:web'}, 'message.author.name': {'description': 'The name of the author of the message.', 'example': 'dalle.text2im'}, 'message.content.content_type': {'description': 'The type of content in the message (e.g., text, image, etc.).', 'example': 'text'}, 'message.content.parts': {'description': 'An array containing parts or segments of the message content, typically for handling long messages.', 'example': ['The generated image visually ca...lements in a futuristic design.']}, 'message.content.model_set_context': {'description': 'Context information related to the model used for content generation.', 'example': ''}, 'message.content.language': {'description': 'The language of the message content, represented in standard language codes.', 'example': 'unknown'}, 'message.content.text': {'description': 'The actual text of the message.', 'example': 'search(\"Please give me an estim...rian diet and a omnivore diet\")'}, 'message.status': {'description': 'The processing status of the message (e.g., finished, in-progress).', 'example': 'finished_successfully'}, 'message.end_turn': {'description': 'A boolean indicating if this is the last message in the conversation turn.', 'example': True}, 'message.weight': {'description': \"A numeric value representing the message's importance or relevance in the conversation.\", 'example': 1}, 'message.metadata.is_visually_hidden_from_conversation': {'description': 'A boolean indicating if the message is hidden from the visible conversation stream.', 'example': True}, 'message.metadata.shared_conversation_id': {'description': 'An identifier for the shared context of the conversation, if applicable.', 'example': '678a1339-d14c-8013-bfcb-288d367a9079'}, 'message.metadata.user_context_message_data': {'description': \"Contextual data related to the user's message, if applicable.\", 'example': None}, 'message.metadata.is_user_system_message': {'description': 'A boolean indicating if the message is generated as a system message for the user.', 'example': True}, 'message.metadata.is_redacted': {'description': 'A boolean indicating if the message content has been redacted for privacy or security reasons.', 'example': True}, 'message.metadata.request_id': {'description': 'An identifier for the request associated with the message, useful for debugging.', 'example': '9034eeef6e62e209-MRS'}, 'message.metadata.message_source': {'description': 'Information about the source of the message, if applicable.', 'example': None}, 'message.metadata.timestamp_': {'description': \"The timestamp format type for message creation, indicating if it's absolute or relative.\", 'example': 'absolute'}, 'message.metadata.message_type': {'description': 'The type/category of message, useful for filtering or processing messages.', 'example': None}, 'message.metadata.model_slug': {'description': 'A slug representing the model used to generate the response.', 'example': 'gpt-4o'}, 'message.metadata.default_model_slug': {'description': 'The default model slug for the message, representing the standard model used.', 'example': 'gpt-4o'}, 'message.metadata.parent_id': {'description': 'The ID of the parent message for threading purposes.', 'example': '073e2336-5c95-434e-a0d2-74a58b68f8e0'}, 'message.metadata.finish_details.type': {'description': 'The type of finish that occurred for the message processing (e.g., stop, timeout).', 'example': 'stop'}, 'message.metadata.finish_details.stop_tokens': {'description': 'An array of token IDs that indicate where the message generation stopped.', 'example': [200002, 200007]}, 'message.metadata.is_complete': {'description': 'A boolean indicating if the message generation process was completed successfully.', 'example': True}, 'message.metadata.citations': {'description': 'An array of citations included in the message, if applicable.', 'example': '[]'}, 'message.metadata.content_references': {'description': 'References to additional content used in the message, if any.', 'example': '[]'}, 'message.metadata.command': {'description': 'The command issued by the user that generated this message.', 'example': 'search'}, 'message.metadata.status': {'description': 'The status of the message at the time of capture (completed, in-progress, etc.).', 'example': 'finished'}, 'message.metadata.search_source': {'description': 'The source from which search results were derived, if applicable.', 'example': 'composer_search'}, 'message.metadata.client_reported_search_source': {'description': 'The source reported by the client regarding the search origin.', 'example': 'conversation_composer_previous_web_mode'}, 'message.metadata.search_result_groups': {'description': \"An array of search result groups that provide relevant information based on the user's query.\", 'example': [{'type': 'search_result_group', 'domain': 'learnmetrics.com', 'entries': [{'type': 'search_result', 'url': 'https://learnmetrics.com/how-ma...average-home-electricity-usage/', 'title': 'How Many kWh Per Day Is Normal? Average 1-6 Person Home kWh Usage', 'snippet': '7,340 kWh Per Year: 2 Person Ho...: 4 Person Home: 36.58 kWh P...', 'ref_id': {'turn_index': 0, 'ref_type': 'search', 'ref_index': 0}, 'content_type': None, 'pub_date': None, 'attributions': None}]}]}, 'message.metadata.safe_urls': {'description': 'An array of URLs considered safe for sharing, derived from the content.', 'example': ['https://www.sciencing.com/being...ls-3342/?utm_source=chatgpt.com']}, 'message.metadata.message_locale': {'description': 'The locale in which the message was generated, formatted as a language-country code.', 'example': 'en-US'}, 'message.metadata.image_results': {'description': 'An array of generated image results related to the conversation, if any.', 'example': '[]'}, 'message.recipient': {'description': 'The intended recipient of the message, indicating if it was meant for a specific user or a group.', 'example': 'all'}, 'message.create_time': {'description': 'The creation timestamp of the message, represented as a float for more precision.', 'example': 1737102115.45064}, 'parent': {'description': 'The ID of the parent turn of the conversation, refers to the context or previous message.', 'example': '073e2336-5c95-434e-a0d2-74a58b68f8e0'}}\n"
     ]
    }
   ],
   "source": [
    "turns_data_ssot = mk_json_field_doc_ssot(\n",
    "    flat_truncated_merged_turn_data, \n",
    "    context=\"The conversation 'turns' of the json object holding a shared chatGPT conversation.\"\n",
    ")\n",
    "# turns_data_ssot = turns_data_ssot['result']\n",
    "print(turns_data_ssot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Have AI get parameter descriptions for metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from oa import ChatDacc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"title\": {\n",
      "    \"description\": \"The title of the chat conversation.\",\n",
      "    \"example\": \"Test Chat 1\"\n",
      "  },\n",
      "  \"create_time\": {\n",
      "    \"description\": \"A timestamp indicating when the chat conversation was created, represented in Unix time format.\",\n",
      "    \"example\": 1737020729.060687\n",
      "  },\n",
      "  \"update_time\": {\n",
      "    \"description\": \"A timestamp indicating the last time the chat conversation was updated, represented in Unix time format.\",\n",
      "    \"example\": 1737020733.031014\n",
      "  },\n",
      "  \"moderation_results\": {\n",
      "    \"description\": \"An array holding the results of moderation checks applied to the conversation. If no moderation has taken place, this array will be empty.\",\n",
      "    \"example\": []\n",
      "  },\n",
      "  \"current_node\": {\n",
      "    \"description\": \"The unique identifier for the current state or node in the conversation flow, typically in UUID format.\",\n",
      "    \"example\": \"be4486db-894f-4e6f-bd0a-22d9d2facf69\"\n",
      "  },\n",
      "  \"conversation_id\": {\n",
      "    \"description\": \"A unique identifier for the conversation as a whole, typically in UUID format.\",\n",
      "    \"example\": \"6788d539-0f2c-8013-9535-889bf344d7d5\"\n",
      "  },\n",
      "  \"is_archived\": {\n",
      "    \"description\": \"A boolean indicating whether the chat conversation has been archived. True means it is archived, false means it is active.\",\n",
      "    \"example\": false\n",
      "  },\n",
      "  \"safe_urls\": {\n",
      "    \"description\": \"An array of safe URLs that were included in the conversation. If there are no safe URLs, this array will be empty.\",\n",
      "    \"example\": []\n",
      "  },\n",
      "  \"default_model_slug\": {\n",
      "    \"description\": \"A string representing the default model used during the conversation, which specifies the AI language model employed.\",\n",
      "    \"example\": \"gpt-4o\"\n",
      "  },\n",
      "  \"disabled_tool_ids\": {\n",
      "    \"description\": \"An array containing the identifiers of any tools that have been disabled during the conversation. If none are disabled, this array will be empty.\",\n",
      "    \"example\": []\n",
      "  },\n",
      "  \"is_public\": {\n",
      "    \"description\": \"A boolean indicating whether the conversation is accessible to the public. True means it is public, false means it is private.\",\n",
      "    \"example\": true\n",
      "  },\n",
      "  \"has_user_editable_context\": {\n",
      "    \"description\": \"A boolean indicating whether the user can modify the context of the conversation. True means editable, false means not editable.\",\n",
      "    \"example\": false\n",
      "  },\n",
      "  \"continue_conversation_url\": {\n",
      "    \"description\": \"A URL that allows users to continue the conversation from a specific point. This link will redirect to the conversation session.\",\n",
      "    \"example\": \"https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5/continue\"\n",
      "  },\n",
      "  \"moderation_state\": {\n",
      "    \"description\": \"An object holding the state of moderation checks applied to the conversation, providing details on whether different moderation actions have taken place.\",\n",
      "    \"example\": {\n",
      "      \"has_been_moderated\": false,\n",
      "      \"has_been_blocked\": false,\n",
      "      \"has_been_accepted\": false,\n",
      "      \"has_been_auto_blocked\": false,\n",
      "      \"has_been_auto_moderated\": false\n",
      "    }\n",
      "  },\n",
      "  \"is_indexable\": {\n",
      "    \"description\": \"A boolean indicating whether the chat conversation can be indexed for search purposes. True means it is indexable, false means it is not.\",\n",
      "    \"example\": false\n",
      "  },\n",
      "  \"is_better_metatags_enabled\": {\n",
      "    \"description\": \"A boolean indicating whether enhanced metatags are enabled for the conversation. True means better metatags are enabled, implying improved discoverability, while false means they are not.\",\n",
      "    \"example\": true\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "metadata_ssot = mk_json_field_doc_ssot(\n",
    "    metadata_example,\n",
    "    context=\"The metadata of the json object holding a shared chatGPT conversation.\"\n",
    ")\n",
    "print(metadata_ssot)  # then copy this and paste it into the chats.py file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'dict' object has no attribute 'metadata'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[49], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mturn_datas\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43msimple_url\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmetadata\u001b[49m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'dict' object has no attribute 'metadata'"
     ]
    }
   ],
   "source": [
    "ChatDacc('https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5').metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|                                                       | description                                                                                                          | example                                                                                                                                                                                                      |\n",
      "|:------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n",
      "| id                                                    | A unique identifier for the conversation turn.                                                                       | 1fc35aa7-6b7a-4dae-9838-ead52c6d4793                                                                                                                                                                         |\n",
      "| children                                              | An array of child conversation turns, which can contain nested messages.                                             | []                                                                                                                                                                                                           |\n",
      "| message                                               | The message object that contains details about the conversation turn.                                                | {\"id\": \"1fc35aa7-6b7a-4dae-9838-ead52c6d4793\", \"author\": {...}, \"content\": {...}, ... }                                                                                                                      |\n",
      "| recipient                                             | Indicates the recipient of the message. Possible values include 'all' indicating all participants.                   | \"all\"                                                                                                                                                                                                        |\n",
      "| create_time                                           | A timestamp indicating when the conversation turn was created, represented in Unix time format.                      | 1737102115.45064                                                                                                                                                                                             |\n",
      "| parent                                                | The unique identifier of the parent message from which this conversation turn is derived.                            | 073e2336-5c95-434e-a0d2-74a58b68f8e0                                                                                                                                                                         |\n",
      "| message.id                                            | A unique identifier for the message within the conversation turn.                                                    | 1fc35aa7-6b7a-4dae-9838-ead52c6d4793                                                                                                                                                                         |\n",
      "| message.author                                        | An object representing the author of the message, containing role and metadata.                                      | {\"role\": \"assistant\", \"metadata\": {\"real_author\": \"tool:web\"}, \"name\": \"dalle.text2im\"}                                                                                                                      |\n",
      "| message.content                                       | An object containing the content of the message, including content type and parts.                                   | {\"content_type\": \"text\", \"parts\": [\"The generated image visually ca...lements in a futuristic design.\"], \"language\": \"unknown\", \"text\": \"search(\"Please give me an estim...rian diet and a omnivore diet\")\"} |\n",
      "| message.status                                        | The status of the message processing, indicating whether it completed successfully or with issues.                   | \"finished_successfully\"                                                                                                                                                                                      |\n",
      "| message.end_turn                                      | A boolean indicating whether this message marks the end of a conversation turn.                                      | True                                                                                                                                                                                                         |\n",
      "| message.weight                                        | An integer representing the weight of the message, affecting its significance or priority in the conversation.       | 1                                                                                                                                                                                                            |\n",
      "| message.metadata                                      | Additional metadata about the message, including visibility and source information.                                  | {\"is_visually_hidden_from_conversation\": true, ... }                                                                                                                                                         |\n",
      "| message.metadata.is_visually_hidden_from_conversation | Indicates if the message is hidden from the visual conversation view.                                                | True                                                                                                                                                                                                         |\n",
      "| message.metadata.shared_conversation_id               | The unique identifier for the shared conversation this message belongs to.                                           | 678a1339-d14c-8013-bfcb-288d367a9079                                                                                                                                                                         |\n",
      "| message.metadata.user_context_message_data            | Holds user context message data, which may contain additional details about the user.                                | null                                                                                                                                                                                                         |\n",
      "| message.metadata.is_user_system_message               | A boolean indicating whether the message is a system message issued by the user.                                     | True                                                                                                                                                                                                         |\n",
      "| message.metadata.is_redacted                          | A boolean indicating whether the message has been redacted for confidentiality.                                      | True                                                                                                                                                                                                         |\n",
      "| message.metadata.request_id                           | A unique identifier for the request related to this message.                                                         | 9034eeef6e62e209-MRS                                                                                                                                                                                         |\n",
      "| message.metadata.timestamp_                           | Timestamp indicating when the message was created in a specific format.                                              | \"absolute\"                                                                                                                                                                                                   |\n",
      "| message.metadata.message_type                         | Type of the message, which may vary in significance or content.                                                      | null                                                                                                                                                                                                         |\n",
      "| message.metadata.model_slug                           | Slug identifier for the model used to generate the message.                                                          | gpt-4o                                                                                                                                                                                                       |\n",
      "| message.metadata.default_model_slug                   | Slug identifier for the default model for this conversation.                                                         | gpt-4o                                                                                                                                                                                                       |\n",
      "| message.metadata.parent_id                            | The unique identifier of the parent message in the conversation thread.                                              | 073e2336-5c95-434e-a0d2-74a58b68f8e0                                                                                                                                                                         |\n",
      "| message.metadata.finish_details                       | An object describing the details of how the message finished processing, including type and tokens.                  | {\"type\": \"stop\", \"stop_tokens\": [200002, 200007]}                                                                                                                                                            |\n",
      "| message.metadata.is_complete                          | A boolean indicating whether the message processing is complete.                                                     | True                                                                                                                                                                                                         |\n",
      "| message.metadata.citations                            | An array holding citations related to the content of the message, if applicable.                                     | []                                                                                                                                                                                                           |\n",
      "| message.metadata.content_references                   | Holds references to content that is related to the message, if any.                                                  | []                                                                                                                                                                                                           |\n",
      "| message.metadata.command                              | The command issued in the message that triggered the response.                                                       | \"search\"                                                                                                                                                                                                     |\n",
      "| message.metadata.status                               | The current status of the message processing.                                                                        | \"finished\"                                                                                                                                                                                                   |\n",
      "| message.metadata.search_source                        | Source of the search that was executed, linked to the message content.                                               | \"composer_search\"                                                                                                                                                                                            |\n",
      "| message.metadata.client_reported_search_source        | The search source reported by the client application.                                                                | \"conversation_composer_previous_web_mode\"                                                                                                                                                                    |\n",
      "| message.metadata.search_result_groups                 | An array of search result groups contained within the message, each representing a separate group of search results. | [{\"type\": \"search_result_group\", \"domain\": \"learnmetrics.com\", \"entries\": [...]}, {\"type\": \"search_result_group\", ...} ]                                                                                     |\n",
      "| message.metadata.safe_urls                            | An array of safe URLs included in the message, ensuring they can be shared without risk.                             | [\"https://www.sciencing.com/...\"]                                                                                                                                                                            |\n",
      "| message.metadata.message_locale                       | The locale in which the message was created, typically following language-region format.                             | \"en-US\"                                                                                                                                                                                                      |\n",
      "| message.metadata.image_results                        | An array of results for any images generated or referenced in the message.                                           | []                                                                                                                                                                                                           |\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "print(pd.DataFrame(index=turns_data_ssot.keys(), data=turns_data_ssot.values()).to_markdown())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Here's the documentation for the provided JSON object, structured in a markdown table format:\n",
      "\n",
      "| Field Name                                         | Description                                                                                     | Example Value                                                                         |\n",
      "|---------------------------------------------------|-------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------|\n",
      "| `<Conversation ID>`                               | Unique identifier for each turn in the conversation.                                          | `adff303b-75cc-493c-b757-605adadb8e56`                                             |\n",
      "| `id`                                              | The unique identifier for the specific message/turn.                                         | `adff303b-75cc-493c-b757-605adadb8e56`                                             |\n",
      "| `children`                                        | An array of IDs representing subsequent messages/turns that are children of this turn.       | `[\"1473f2d9-ba09-4cd7-90c4-1452898676de\"]`                                         |\n",
      "| `message`                                         | Contains details about the message that was sent.                                            | `{...}` (Object with further nested fields)                                         |\n",
      "| `author`                                          | Specifies the author of the message, including their role and metadata.                      | `{ \"role\": \"system\", \"metadata\": {} }`                                            |\n",
      "| `content`                                         | Contains the content of the message, specifying the type and actual message parts.           | `{ \"content_type\": \"text\", \"parts\": [\"\"] }`                                       |\n",
      "| `status`                                         | The status of the message indicating the completion state.                                    | `finished_successfully`                                                              |\n",
      "| `end_turn`                                       | Boolean indicating whether this message marks the end of a conversation turn.                 | `true`                                                                               |\n",
      "| `weight`                                         | Numerical value representing the significance or order of the turn in the conversation.      | `0`                                                                                  |\n",
      "| `metadata`                                       | Contains additional information about the message, such as visibility and identifiers.         | `{ \"is_visually_hidden_from_conversation\": true, \"shared_conversation_id\": \"6788d539-0f2c-8013-9535-889bf344d7d5\" }` |\n",
      "| `recipient`                                       | The intended recipient(s) of the message, indicating audience scope.                         | `all`                                                                                |\n",
      "| `parent`                                          | ID of the parent turn/message to which this turn is a response.                              | `adff303b-75cc-493c-b757-605adadb8e56`                                             |\n",
      "| `create_time`                                     | Timestamp indicating the creation time of the message, in Unix time format.                   | `1737020650.866734`                                                                  |\n",
      "| `is_user_system_message`                          | Indicates if the message is a system message from the user.                                  | `true`                                                                               |\n",
      "| `is_redacted`                                     | Indicates if the message has been redacted for privacy or security reasons.                   | `true`                                                                               |\n",
      "| `finish_details`                                 | Provides details on how the message was concluded, including stop tokens.                     | `{ \"type\": \"stop\", \"stop_tokens\": [200002] }`                                     |\n",
      "| `model_slug`                                      | Identifies the model used for generating the message response.                                | `gpt-4o`                                                                             |\n",
      "| `default_model_slug`                              | The default model slug used for the current conversation context.                             | `gpt-4o`                                                                             |\n",
      "| `timestamp_`                                      | A string indicating the timestamp's nature (e.g., 'absolute').                               | `absolute`                                                                           |\n",
      "| `content_type`                                    | The type of content within the message, such as `text`, `model_editable_context`, etc.       | `text`                                                                               |\n",
      "| `parts`                                           | An array containing the actual parts of the message content.                                   | `[\"Hello World! \\napple, river, galaxy, chair, melody, shadow, cloud, puzzle, flame, whisper.\"]` |\n",
      "\n",
      "This table documents the fields found within the JSON object representing a shared conversation of turns in a chat platform. Each field is described to inform developers or users of its structure and purpose, along with example values for clarity.\n"
     ]
    }
   ],
   "source": [
    "turn_data_docs = mk_json_field_documentation(\n",
    "    json.dumps(turns_data, indent=2), \n",
    "    context=\"the conversation 'turns' of the json object holding a shared chatGPT conversation\"\n",
    ")\n",
    "print(turn_data_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Here's the documentation for each field in the provided JSON object, formatted as a markdown table:\n",
      "\n",
      "| Field Name                                | Description                                                                                          | Example Value                                                      |\n",
      "|-------------------------------------------|------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------|\n",
      "| `title`                                   | The title of the shared chat GPT conversation.                                                      | \"Test Chat 1\"                                                    |\n",
      "| `create_time`                             | The timestamp when the conversation was created, represented in seconds since the Unix epoch.      | 1737020729.060687                                                |\n",
      "| `update_time`                             | The timestamp when the conversation was last updated, represented in seconds since the Unix epoch. | 1737020733.031014                                                |\n",
      "| `moderation_results`                      | An array containing results of moderation checks, if any.                                          | []                                                               |\n",
      "| `current_node`                            | The unique identifier of the current node in the conversation tree.                                 | \"be4486db-894f-4e6f-bd0a-22d9d2facf69\"                        |\n",
      "| `conversation_id`                         | A unique identifier for the conversation.                                                           | \"6788d539-0f2c-8013-9535-889bf344d7d5\"                         |\n",
      "| `is_archived`                             | A boolean value indicating whether the conversation is archived.                                     | false                                                             |\n",
      "| `safe_urls`                               | An array of URLs that have been deemed safe in the conversation.                                    | []                                                               |\n",
      "| `default_model_slug`                      | The slug representing the default model used for the conversation.                                  | \"gpt-4o\"                                                         |\n",
      "| `disabled_tool_ids`                       | An array of identifiers for tools that have been disabled for this conversation.                    | []                                                               |\n",
      "| `is_public`                               | A boolean value indicating whether the conversation is accessible to the public.                    | true                                                              |\n",
      "| `linear_conversation`                     | An array representing the conversation in a linear format with node identifiers.                    | [{\"id\": \"adff303b-75cc-493c-b757-605adadb8e56\", \"children\": [\"1473f2d9-ba09-4cd7-90c4-1452898676de\"]}] |\n",
      "| `has_user_editable_context`              | A boolean value indicating if the user can edit context for the conversation.                       | false                                                             |\n",
      "| `continue_conversation_url`               | A URL that allows the user to continue the conversation.                                            | \"https://chatgpt.com/share/6788d539-0f2c-8013-9535-889bf344d7d5/continue\" |\n",
      "| `moderation_state`                        | An object containing the state of moderation checks applied to the conversation.                     | {\"has_been_moderated\": false, \"has_been_blocked\": false, \"has_been_accepted\": false, \"has_been_auto_blocked\": false, \"has_been_auto_moderated\": false} |\n",
      "| `is_indexable`                           | A boolean value indicating whether the conversation can be indexed for search purposes.             | false                                                             |\n",
      "| `is_better_metatags_enabled`             | A boolean value indicating whether enhanced metatags are enabled for this conversation.             | true                                                              |\n",
      "\n",
      "This table provides a concise overview of each field, its purpose, and an illustrative example.\n"
     ]
    }
   ],
   "source": [
    "metadata_field_docs = mk_json_field_documentation(\n",
    "    json.dumps(metadata, indent=2), \n",
    "    context=\"metadata of a shared chatGPT conversation\"\n",
    ")\n",
    "print(metadata_field_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import json\n",
    "from typing import Callable, Optional\n",
    "\n",
    "def raise_run_time_error(msg: str):\n",
    "    raise RuntimeError(msg)\n",
    "\n",
    "def extract_json_dict(\n",
    "    string: str,\n",
    "    object_filter: Callable,\n",
    "    *,\n",
    "    decoder: json.JSONDecoder = json.JSONDecoder(),\n",
    "    not_found_callback: Optional[\n",
    "        Callable\n",
    "    ] = lambda string, object_filter: raise_run_time_error(\n",
    "        \"Object not found in string\"\n",
    "    ),\n",
    ") -> dict:\n",
    "    \"\"\"\n",
    "    Searches `string` from the beginning, attempting to decode consecutive\n",
    "    JSON objects using `decoder.raw_decode`. When an object satisfies\n",
    "    `object_filter`, returns it as a Python dictionary.\n",
    "    \"\"\"\n",
    "    pos = 0\n",
    "    while pos < len(string):\n",
    "        try:\n",
    "            obj, pos = decoder.raw_decode(string, pos)\n",
    "            if object_filter(obj):\n",
    "                return obj\n",
    "        except json.JSONDecodeError:\n",
    "            pos += 1\n",
    "    return not_found_callback(string, object_filter)\n",
    "\n",
    "def parse_shared_chatGPT_chat(\n",
    "    html: str, \n",
    "    string_contained_in_conversation: str\n",
    ") -> dict:\n",
    "    \"\"\"\n",
    "    Locates the big JSON structure assigned to window.__remixContext, \n",
    "    checks that the conversation portion includes `string_contained_in_conversation`,\n",
    "    and returns the JSON as a dict.\n",
    "    \"\"\"\n",
    "    # Regex pattern to capture the object assigned to window.__remixContext = {...};\n",
    "    pattern = r\"window\\.__remixContext\\s*=\\s*(\\{.*?\\});\"\n",
    "    match = re.search(pattern, html, flags=re.DOTALL)\n",
    "    if not match:\n",
    "        raise RuntimeError(\"Could not locate the JSON assigned to window.__remixContext\")\n",
    "\n",
    "    # Extract the raw JSON text (removing the trailing semicolon if needed)\n",
    "    raw_json_text = match.group(1).strip()\n",
    "\n",
    "    # We'll define a filter that checks if the conversation data includes the target string.\n",
    "    # Because the JSON is quite large, we can simply check if the substring is present in the raw text:\n",
    "    # but if we want to be more precise, we can parse first and only return if there's conversation data.\n",
    "    def conversation_filter(obj):\n",
    "        # Convert to string once more, or deeply check \"routes/share.$shareId.($action)\" for the substring.\n",
    "        # For simplicity, do a textual check on the entire string representation. \n",
    "        # If needed, we can refine to parse a specific location in obj.\n",
    "        return string_contained_in_conversation in json.dumps(obj, ensure_ascii=False)\n",
    "\n",
    "    # Now parse the JSON, returning only if the filter passes\n",
    "    extracted_dict = extract_json_dict(\n",
    "        raw_json_text,\n",
    "        object_filter=conversation_filter,\n",
    "    )\n",
    "\n",
    "    return extracted_dict\n",
    "\n",
    "\n",
    "t = parse_shared_chatGPT_chat(html, existing_substring)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['basename', 'future', 'isSpaMode', 'state']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "p10",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
