{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c1fe7a6b-61a6-4a53-8503-9d83c5217532",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "from pycharter.etl_generator import Pipeline\n",
    "\n",
    "ROOT_DIR = os.path.join('..')\n",
    "DATA_DIR = os.path.join(ROOT_DIR, 'data')\n",
    "\n",
    "sys.path.append(ROOT_DIR)\n",
    "\n",
    "p = Pipeline.from_config_files(\n",
    "    extract='../data/etl_examples/extract.yaml',    \n",
    "    transform='../data/etl_examples/transform.yaml',\n",
    "    load='../data/etl_examples/load.yaml',\n",
    "    variables={\n",
    "        \"INPUT_PATH\": str(os.path.join(ROOT_DIR,\"data/etl_examples/input.csv\")),\n",
    "        \"OUTPUT_PATH\": str(os.path.join(ROOT_DIR,\"data/etl_examples/output2.csv\")),\n",
    "    }\n",
    ")\n",
    "r = await p.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "02980ae7-70bd-431f-ad72-c9a1843175c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PipelineResult(success=True, rows_extracted=5, rows_transformed=5, rows_loaded=5, rows_failed=0, rows_quarantined_extract=0, rows_quarantined_load=0, validation_errors_extract=[], validation_errors_load=[], start_time=datetime.datetime(2026, 3, 3, 0, 49, 43, 60132, tzinfo=datetime.timezone.utc), end_time=datetime.datetime(2026, 3, 3, 0, 49, 43, 67964, tzinfo=datetime.timezone.utc), duration_seconds=0.007832, batches_processed=1, batch_results=[BatchResult(batch_index=0, rows_in=5, rows_out=5, rows_failed=0, errors=[])], errors=[], pipeline_name=None, run_id='38ae0c9a')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "43a1c514-1afb-4277-a027-5ae1f61d11b8",
   "metadata": {},
   "outputs": [
    {
     "ename": "ConfigValidationError",
     "evalue": "Invalid transform configuration\n\nValidation errors:\n  1. transform.SimpleOpsConfig.transform: Unknown field: 'transform'\n  2. transform.list[dict[str,any]]: Input should be a valid list",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mConfigValidationError\u001b[39m                     Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 31\u001b[39m\n\u001b[32m     15\u001b[39m config = {\n\u001b[32m     16\u001b[39m     \u001b[33m\"\u001b[39m\u001b[33mextract\u001b[39m\u001b[33m\"\u001b[39m: load_yaml(os.path.join(ETL_DIR, \u001b[33m\"\u001b[39m\u001b[33mextract.yaml\u001b[39m\u001b[33m\"\u001b[39m)),\n\u001b[32m     17\u001b[39m     \u001b[33m\"\u001b[39m\u001b[33mload\u001b[39m\u001b[33m\"\u001b[39m: load_yaml(os.path.join(ETL_DIR, \u001b[33m\"\u001b[39m\u001b[33mload.yaml\u001b[39m\u001b[33m\"\u001b[39m)),\n\u001b[32m   (...)\u001b[39m\u001b[32m     23\u001b[39m     },\n\u001b[32m     24\u001b[39m }\n\u001b[32m     26\u001b[39m variables = {\n\u001b[32m     27\u001b[39m     \u001b[33m\"\u001b[39m\u001b[33mINPUT_PATH\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mstr\u001b[39m(os.path.join(ETL_DIR, \u001b[33m\"\u001b[39m\u001b[33minput.csv\u001b[39m\u001b[33m\"\u001b[39m)),\n\u001b[32m     28\u001b[39m     \u001b[33m\"\u001b[39m\u001b[33mOUTPUT_PATH\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mstr\u001b[39m(os.path.join(ETL_DIR, \u001b[33m\"\u001b[39m\u001b[33moutput2.csv\u001b[39m\u001b[33m\"\u001b[39m)),\n\u001b[32m     29\u001b[39m }\n\u001b[32m---> \u001b[39m\u001b[32m31\u001b[39m p = \u001b[43mPipeline\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfrom_dict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvariables\u001b[49m\u001b[43m=\u001b[49m\u001b[43mvariables\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     32\u001b[39m r = asyncio.run(p.run())\n\u001b[32m     33\u001b[39m r\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/statfyi/pycharter/src/pycharter/etl_generator/pipeline.py:787\u001b[39m, in \u001b[36mPipeline.from_dict\u001b[39m\u001b[34m(cls, config, variables, validate, base_dir)\u001b[39m\n\u001b[32m    784\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m config.get(\u001b[33m\"\u001b[39m\u001b[33mcontract\u001b[39m\u001b[33m\"\u001b[39m):\n\u001b[32m    785\u001b[39m     settings[\u001b[33m\"\u001b[39m\u001b[33mcontract\u001b[39m\u001b[33m\"\u001b[39m] = config[\u001b[33m\"\u001b[39m\u001b[33mcontract\u001b[39m\u001b[33m\"\u001b[39m]\n\u001b[32m--> \u001b[39m\u001b[32m787\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_build_from_configs\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    788\u001b[39m \u001b[43m    \u001b[49m\u001b[43mextract_config\u001b[49m\u001b[43m=\u001b[49m\u001b[43mextract_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    789\u001b[39m \u001b[43m    \u001b[49m\u001b[43mtransform_config\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtransform_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    790\u001b[39m \u001b[43m    \u001b[49m\u001b[43mload_config\u001b[49m\u001b[43m=\u001b[49m\u001b[43mload_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    791\u001b[39m \u001b[43m    \u001b[49m\u001b[43mvariables\u001b[49m\u001b[43m=\u001b[49m\u001b[43mvariables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    792\u001b[39m \u001b[43m    \u001b[49m\u001b[43mvalidate\u001b[49m\u001b[43m=\u001b[49m\u001b[43mvalidate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    793\u001b[39m \u001b[43m    \u001b[49m\u001b[43mname\u001b[49m\u001b[43m=\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mname\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    794\u001b[39m \u001b[43m    \u001b[49m\u001b[43mextract_validation_config\u001b[49m\u001b[43m=\u001b[49m\u001b[43mextract_validation_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    795\u001b[39m \u001b[43m    \u001b[49m\u001b[43mload_validation_config\u001b[49m\u001b[43m=\u001b[49m\u001b[43mload_validation_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    796\u001b[39m \u001b[43m    \u001b[49m\u001b[43msettings\u001b[49m\u001b[43m=\u001b[49m\u001b[43msettings\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    797\u001b[39m \u001b[43m    \u001b[49m\u001b[43mbase_dir\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbase_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    798\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/statfyi/pycharter/src/pycharter/etl_generator/pipeline.py:837\u001b[39m, in \u001b[36mPipeline._build_from_configs\u001b[39m\u001b[34m(cls, extract_config, transform_config, load_config, variables, validate, name, extract_validation_config, load_validation_config, settings, base_dir)\u001b[39m\n\u001b[32m    835\u001b[39m             validator.validate_transform({\u001b[33m\"\u001b[39m\u001b[33mtransform\u001b[39m\u001b[33m\"\u001b[39m: transform_config})\n\u001b[32m    836\u001b[39m         \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m837\u001b[39m             \u001b[43mvalidator\u001b[49m\u001b[43m.\u001b[49m\u001b[43mvalidate_transform\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtransform_config\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    838\u001b[39m     validator.validate_load(load_config)\n\u001b[32m    840\u001b[39m \u001b[38;5;66;03m# Create context\u001b[39;00m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/statfyi/pycharter/src/pycharter/etl_generator/config_validator.py:184\u001b[39m, in \u001b[36mConfigValidator.validate_transform\u001b[39m\u001b[34m(self, config, config_path)\u001b[39m\n\u001b[32m    179\u001b[39m is_valid, errors = \u001b[38;5;28mself\u001b[39m._validate_with_pydantic(\n\u001b[32m    180\u001b[39m     config, \u001b[33m\"\u001b[39m\u001b[33mtransform\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28;01mlambda\u001b[39;00m c: TransformConfig(**c)\n\u001b[32m    181\u001b[39m )\n\u001b[32m    183\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_valid \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m.strict:\n\u001b[32m--> \u001b[39m\u001b[32m184\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m ConfigValidationError(\n\u001b[32m    185\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mInvalid transform configuration\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m    186\u001b[39m         errors=errors,\n\u001b[32m    187\u001b[39m         config_type=\u001b[33m\"\u001b[39m\u001b[33mtransform\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m    188\u001b[39m         config_path=config_path,\n\u001b[32m    189\u001b[39m     )\n",
      "\u001b[31mConfigValidationError\u001b[39m: Invalid transform configuration\n\nValidation errors:\n  1. transform.SimpleOpsConfig.transform: Unknown field: 'transform'\n  2. transform.list[dict[str,any]]: Input should be a valid list"
     ]
    }
   ],
   "source": [
    "import asyncio\n",
    "import os\n",
    "import yaml\n",
    "from pathlib import Path\n",
    "from pycharter.etl_generator import Pipeline\n",
    "\n",
    "ROOT_DIR = os.path.join('..')\n",
    "DATA_DIR = os.path.join(ROOT_DIR, 'data')\n",
    "ETL_DIR = os.path.join(DATA_DIR, 'etl_examples')\n",
    "\n",
    "def load_yaml(path):\n",
    "    with open(path) as f:\n",
    "        return yaml.safe_load(f)\n",
    "\n",
    "config = {\n",
    "    \"extract\": load_yaml(os.path.join(ETL_DIR, \"extract.yaml\")),\n",
    "    \"load\": load_yaml(os.path.join(ETL_DIR, \"load.yaml\")),\n",
    "    \"transform\": load_yaml(os.path.join(ETL_DIR, \"transform.yaml\")),\n",
    "    \"lineage\": {\n",
    "        \"enabled\": True,\n",
    "        \"namespace\": \"pycharter\",\n",
    "        \"backend_url\": \"http://localhost:5000/api/v1/lineage\",\n",
    "    },\n",
    "}\n",
    "\n",
    "variables = {\n",
    "    \"INPUT_PATH\": str(os.path.join(ETL_DIR, \"input.csv\")),\n",
    "    \"OUTPUT_PATH\": str(os.path.join(ETL_DIR, \"output2.csv\")),\n",
    "}\n",
    "\n",
    "p = Pipeline.from_dict(config, variables=variables)\n",
    "r = asyncio.run(p.run())\n",
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "08fa056f-9188-457e-981c-bf6a75707a44",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycharter.runtime_validator import Validator\n",
    "\n",
    "contract = \"../data/aviation_examples/aircraft_configuration/aircraft_configuration_contract.yaml\"\n",
    "validator = Validator(contract_file=str(contract))\n",
    "result = validator.validate({\n",
    "    \"metadata\": {},\n",
    "    \"registration\": \"CX77G\",\n",
    "    \"valid_since\": \"1996-01-01T00:00:00Z\",\n",
    "    \"valid_until\": \"2013-12-31T00:00:00Z\",\n",
    "    \"ac_operator\": \"CX\",\n",
    "    \"ac_owner\": \"CX\",\n",
    "    \"ac_subtype\": \"77G\",\n",
    "    \"ac_logical_no\": 0,\n",
    "    \"ac_state\": \"O\",\n",
    "    \"dry_operating_wgt\": 0,\n",
    "    \"max_takeoff_wgt\": 0,\n",
    "    \"cargo_capacity\": 0,\n",
    "    \"fuel_capacity\": 0,\n",
    "    \"avg_fuel_consump\": 0,\n",
    "    \"ac_index\": 100,\n",
    "    \"crewsize_cockpit\": 0,\n",
    "    \"crewsize_cabin\": 0,\n",
    "    \"std_version\": \"STN\",\n",
    "    \"ap_restriction\": \"N\",\n",
    "    \"ac_owner_name\": \"CX\",\n",
    "    \"ac_subtype_name\": \"77G\",\n",
    "    \"ac_category\": \"J\",\n",
    "    \"fuel_measure_unit\": \"kg\",\n",
    "})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f59c3bfb-9288-4b73-bed3-0eb0f85db10c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# result.data\n",
    "result.errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6999157a-bd71-4db9-993d-55a98e9b1745",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9f4167f6-4ec5-4c17-a58b-07ebdd7b90e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymongo import MongoClient\n",
    "import psycopg2\n",
    "connection_string='postgresql://postgres:1234567890@ioc-de-delete-me-3-instance-1.cvpchc1tnb8b.ap-southeast-1.rds.amazonaws.com'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4276a85a-0f18-426f-a9fc-53f5cf66220e",
   "metadata": {},
   "outputs": [
    {
     "ename": "OperationalError",
     "evalue": "connection to server at \"ioc-de-delete-me-3-instance-1.cvpchc1tnb8b.ap-southeast-1.rds.amazonaws.com\" (172.31.21.157), port 5432 failed: Operation timed out\n\tIs the server running on that host and accepting TCP/IP connections?\n",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mOperationalError\u001b[39m                          Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[43mpsycopg2\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconnection_string\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/akc/votedrop/pycharter/venv/lib/python3.12/site-packages/psycopg2/__init__.py:122\u001b[39m, in \u001b[36mconnect\u001b[39m\u001b[34m(dsn, connection_factory, cursor_factory, **kwargs)\u001b[39m\n\u001b[32m    119\u001b[39m     kwasync[\u001b[33m'\u001b[39m\u001b[33masync_\u001b[39m\u001b[33m'\u001b[39m] = kwargs.pop(\u001b[33m'\u001b[39m\u001b[33masync_\u001b[39m\u001b[33m'\u001b[39m)\n\u001b[32m    121\u001b[39m dsn = _ext.make_dsn(dsn, **kwargs)\n\u001b[32m--> \u001b[39m\u001b[32m122\u001b[39m conn = \u001b[43m_connect\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdsn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconnection_factory\u001b[49m\u001b[43m=\u001b[49m\u001b[43mconnection_factory\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwasync\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    123\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m cursor_factory \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m    124\u001b[39m     conn.cursor_factory = cursor_factory\n",
      "\u001b[31mOperationalError\u001b[39m: connection to server at \"ioc-de-delete-me-3-instance-1.cvpchc1tnb8b.ap-southeast-1.rds.amazonaws.com\" (172.31.21.157), port 5432 failed: Operation timed out\n\tIs the server running on that host and accepting TCP/IP connections?\n"
     ]
    }
   ],
   "source": [
    "connect = psycopg2.connect(connection_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "02aa5ba5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✓ Retrieved 20 documents from MongoDB\n",
      "✓ Saved 20 documents to mel_to_constraints_info_data.json\n",
      "✓ Saved clean JSON to mel_to_constraints_info_data_clean.json\n",
      "✓ Saved 20 rows to mel_to_constraints_info_data.csv\n",
      "  Columns: 9\n",
      "  Shape: (20, 9)\n",
      "\n",
      "Data preview:\n",
      "                        _id ata_chapter ata_subchapter ata_section ata_item  \\\n",
      "0  691162b1952cd10782bf6439          26             22         02B            \n",
      "1  691162b1952cd10782bf641a          22             70         01D            \n",
      "2  691162b1952cd10782bf6406          22             10         02A            \n",
      "3  691162b1952cd10782bf6437          26             22         01B            \n",
      "4  691162b1952cd10782bf6420          23             10         01A            \n",
      "\n",
      "  constraint_code ac_owner ac_subtype  \\\n",
      "0             APU       CX        32P   \n",
      "1             FMC       CX        32P   \n",
      "2            LAND       CX        32P   \n",
      "3             APU       CX        32P   \n",
      "4              HF       CX        32P   \n",
      "\n",
      "                                constraint_code_info  \n",
      "0  {'description': '- APU INOP (LESS PREFERRED DU...  \n",
      "1  {'description': '- FMC/FMS INOP', 'usage': 'R'...  \n",
      "2  {'description': '- AUTOLAND DOWNGRADED/INOP', ...  \n",
      "3  {'description': '- APU INOP (LESS PREFERRED DU...  \n",
      "4  {'description': '- HF RADIO INOP', 'usage': 'R...  \n",
      "\n",
      "✓ Connection closed\n",
      "\n",
      "Files saved:\n",
      "  - mel_to_constraints_info_data.json (MongoDB format with $oid, $date)\n",
      "  - mel_to_constraints_info_data_clean.json (Clean JSON format)\n",
      "  - mel_to_constraints_info_data.csv\n"
     ]
    }
   ],
   "source": [
    "import pymongo\n",
    "import pandas as pd\n",
    "import json\n",
    "from datetime import datetime\n",
    "from bson import ObjectId\n",
    "from bson.json_util import dumps\n",
    "\n",
    "# Set up connection parameters\n",
    "# NEVER hardcode credentials in production code. Use environment variables.\n",
    "CONNECTION_STRING = \"mongodb+srv://ioc_de_ete:RjL7xac3g2U6RULZ@ioc-de-ete.oijg8.mongodb.net/?authMechanism=SCRAM-SHA-1&tls=true\" \n",
    "DB_NAME = \"external\"\n",
    "COLLECTION_NAME = \"replica_nlo_mel_to_constraints_info_serve\"\n",
    "\n",
    "# Output file paths\n",
    "JSON_OUTPUT = \"mel_to_constraints_info_data.json\"\n",
    "CSV_OUTPUT = \"mel_to_constraints_info_data.csv\"\n",
    "\n",
    "try:\n",
    "    # Create a MongoDB client\n",
    "    client = pymongo.MongoClient(CONNECTION_STRING)\n",
    "\n",
    "    # Access the database and collection\n",
    "    database = client[DB_NAME]\n",
    "    collection = database[COLLECTION_NAME]\n",
    "\n",
    "    # Retrieve all documents (empty filter {}) using find() method\n",
    "    cursor = collection.find({}).sort(\"created_at\", -1).limit(20)\n",
    "\n",
    "    # Convert the cursor results to a Python list of dictionaries\n",
    "    list_of_documents = list(cursor)\n",
    "    \n",
    "    print(f\"✓ Retrieved {len(list_of_documents)} documents from MongoDB\")\n",
    "\n",
    "    # ============================================================\n",
    "    # Save to JSON\n",
    "    # ============================================================\n",
    "    # Method 1: Using bson.json_util (handles ObjectId, datetime automatically)\n",
    "    with open(JSON_OUTPUT, 'w') as json_file:\n",
    "        json_file.write(dumps(list_of_documents, indent=2, default=str))\n",
    "    \n",
    "    print(f\"✓ Saved {len(list_of_documents)} documents to {JSON_OUTPUT}\")\n",
    "    \n",
    "    # Method 2: Manual conversion (cleaner format for validation)\n",
    "    def convert_mongodb_to_json(obj):\n",
    "        \"\"\"Convert MongoDB types to JSON-serializable types.\"\"\"\n",
    "        if isinstance(obj, ObjectId):\n",
    "            return str(obj)\n",
    "        elif isinstance(obj, datetime):\n",
    "            return obj.isoformat()\n",
    "        elif isinstance(obj, dict):\n",
    "            return {k: convert_mongodb_to_json(v) for k, v in obj.items()}\n",
    "        elif isinstance(obj, list):\n",
    "            return [convert_mongodb_to_json(item) for item in obj]\n",
    "        else:\n",
    "            return obj\n",
    "    \n",
    "    json_data = convert_mongodb_to_json(list_of_documents)\n",
    "    clean_json_output = JSON_OUTPUT.replace('.json', '_clean.json')\n",
    "    with open(clean_json_output, 'w') as json_file:\n",
    "        json.dump(json_data, json_file, indent=2, ensure_ascii=False)\n",
    "    \n",
    "    print(f\"✓ Saved clean JSON to {clean_json_output}\")\n",
    "\n",
    "    # ============================================================\n",
    "    # Save to CSV\n",
    "    # ============================================================\n",
    "    # Convert to pandas DataFrame\n",
    "    df = pd.DataFrame(list_of_documents)\n",
    "    \n",
    "    # Handle MongoDB-specific types for CSV export\n",
    "    # Convert ObjectId columns to strings\n",
    "    for col in df.columns:\n",
    "        if df[col].dtype == 'object':\n",
    "            # Check if column contains ObjectId\n",
    "            sample = df[col].dropna()\n",
    "            if len(sample) > 0 and isinstance(sample.iloc[0], ObjectId):\n",
    "                df[col] = df[col].astype(str)\n",
    "            # Convert datetime to string\n",
    "            elif len(sample) > 0 and isinstance(sample.iloc[0], datetime):\n",
    "                df[col] = df[col].apply(lambda x: x.isoformat() if pd.notna(x) and isinstance(x, datetime) else x)\n",
    "    \n",
    "    # Save to CSV\n",
    "    df.to_csv(CSV_OUTPUT, index=False, encoding='utf-8')\n",
    "    print(f\"✓ Saved {len(df)} rows to {CSV_OUTPUT}\")\n",
    "    print(f\"  Columns: {len(df.columns)}\")\n",
    "    print(f\"  Shape: {df.shape}\")\n",
    "    \n",
    "    # Display preview\n",
    "    print(\"\\nData preview:\")\n",
    "    print(df.head())\n",
    "\n",
    "    # Close the connection\n",
    "    client.close()\n",
    "    print(f\"\\n✓ Connection closed\")\n",
    "    print(f\"\\nFiles saved:\")\n",
    "    print(f\"  - {JSON_OUTPUT} (MongoDB format with $oid, $date)\")\n",
    "    print(f\"  - {clean_json_output} (Clean JSON format)\")\n",
    "    print(f\"  - {CSV_OUTPUT}\")\n",
    "\n",
    "except Exception as e:\n",
    "    print(f\"An error occurred: {e}\")\n",
    "    import traceback\n",
    "    traceback.print_exc()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a7553a76-1065-4312-ab1b-a0bcbdc968df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3dc17d63",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (pycharter-dev)",
   "language": "python",
   "name": "pycharter-dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
