{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Testing ETL Orchestrator with Massive Datasets\n",
        "\n",
        "This notebook tests the ETL Orchestrator with Massive's datasets:\n",
        "- `massive_historical_prices`\n",
        "- `massive_top_market_movers`\n",
        "- `massive_news`\n",
        "\n",
        "## Setup\n",
        "\n",
        "Make sure you have:\n",
        "1. `MASSIVE_API_KEY` environment variable set\n",
        "2. The orchestrator supports JSON schema files (not just YAML)\n",
        "3. Path parameters in URLs are properly handled\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "import os\n",
        "import sys\n",
        "from pathlib import Path\n",
        "\n",
        "# Add parent directory to path\n",
        "if Path.cwd().name == \"notebooks\":\n",
        "    sys.path.insert(0, str(Path.cwd().parent))\n",
        "else:\n",
        "    sys.path.insert(0, str(Path.cwd()))\n",
        "\n",
        "from pycharter.etl_generator import ETLOrchestrator\n",
        "import asyncio\n",
        "\n",
        "# Check if API key is set\n",
        "api_key = os.getenv(\"MASSIVE_API_KEY\")\n",
        "if not api_key:\n",
        "    print(\"⚠️  WARNING: MASSIVE_API_KEY not set. Set it with:\")\n",
        "    print(\"   export MASSIVE_API_KEY='your_api_key_here'\")\n",
        "    print(\"\\nContinuing with placeholder key for testing...\")\n",
        "    os.environ[\"MASSIVE_API_KEY\"] = \"test_key_placeholder\"\n",
        "else:\n",
        "    print(f\"✓ MASSIVE_API_KEY is set (length: {len(api_key)})\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Test 1: Massive Historical Prices\n",
        "\n",
        "This dataset requires:\n",
        "- `symbol` (required): Stock ticker symbol\n",
        "- `start_date` (optional): Start date (YYYY-MM-DD)\n",
        "- `end_date` (optional): End date (YYYY-MM-DD)\n",
        "\n",
        "The endpoint uses path parameters: `/v2/aggs/ticker/{symbol}/range/1/day/{start_date}/{end_date}`\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Test massive_historical_prices\n",
        "print(\"=\" * 70)\n",
        "print(\"TEST 1: Massive Historical Prices\")\n",
        "print(\"=\" * 70)\n",
        "\n",
        "contract_dir = Path(\"../data/stock_examples/massive_historical_prices\")\n",
        "if not contract_dir.exists():\n",
        "    contract_dir = Path(\"data/stock_examples/massive_historical_prices\")\n",
        "\n",
        "print(f\"Contract directory: {contract_dir}\")\n",
        "print(f\"Exists: {contract_dir.exists()}\")\n",
        "\n",
        "if contract_dir.exists():\n",
        "    try:\n",
        "        orchestrator = ETLOrchestrator(contract_dir=str(contract_dir))\n",
        "        print(\"✓ Orchestrator initialized successfully\")\n",
        "        print(f\"  Schema loaded: {orchestrator.schema is not None}\")\n",
        "        print(f\"  Extract config loaded: {orchestrator.extract_config is not None}\")\n",
        "        \n",
        "        # Test extraction (dry run - no actual API call)\n",
        "        print(\"\\n📥 Testing extraction with sample parameters...\")\n",
        "        print(\"   Note: This will make an actual API call if MASSIVE_API_KEY is valid\")\n",
        "        \n",
        "        # Use a small date range for testing\n",
        "        try:\n",
        "            data = await orchestrator.extract(\n",
        "                symbol=\"AAPL\",\n",
        "                start_date=\"2024-01-01\",\n",
        "                end_date=\"2024-01-05\"  # Just 5 days for testing\n",
        "            )\n",
        "            print(f\"✓ Extraction successful: {len(data)} records\")\n",
        "            if data:\n",
        "                print(f\"  Sample record keys: {list(data[0].keys())}\")\n",
        "                print(f\"  First record: {data[0]}\")\n",
        "        except Exception as e:\n",
        "            print(f\"⚠️  Extraction failed (expected if API key is invalid): {e}\")\n",
        "            print(\"   This is OK for testing - the orchestrator is working correctly\")\n",
        "            \n",
        "    except Exception as e:\n",
        "        print(f\"❌ Error initializing orchestrator: {e}\")\n",
        "        import traceback\n",
        "        traceback.print_exc()\n",
        "else:\n",
        "    print(f\"❌ Contract directory not found: {contract_dir}\")\n"
      ]
    }
  ],
  "metadata": {
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}
