"""
Persistent SQLite-based cache with incremental sync support.

This module provides a persistent cache implementation using SQLite with:
- Incremental sync algorithm (stops at first known test ID + 1 page)
- Multi-tenant design (customer_id isolation)
- WAL mode for concurrent reads during background writes
- Async operations via aiosqlite
- Query interface with filtering and pagination

**Performance:** ~10ms queries at our scale, imperceptible to users.
**Simplicity:** No TTL management, no memory overhead, no stampede protection needed.

Reference: STORY-021 (Local Data Store with Incremental Sync), AC5
"""

import asyncio
import json
import logging
import math
import os
from collections.abc import Iterator
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

import aiosqlite
import psutil
from filelock import FileLock, Timeout

from testio_mcp.client import TestIOClient
from testio_mcp.repositories.test_repository import TestRepository

logger = logging.getLogger(__name__)


@contextmanager
def suppress_httpx_logs() -> Iterator[None]:
    """Temporarily suppress httpx INFO logs during bulk operations.

    Use this context manager during refresh operations to avoid flooding
    logs with individual HTTP request lines. Summary statistics are shown
    instead after the operation completes.

    Example:
        with suppress_httpx_logs():
            # Perform bulk refresh - no individual request logs
            result = await cache.refresh_active_tests(product_id)
        # Summary shown here
    """
    httpx_logger = logging.getLogger("httpx")
    original_level = httpx_logger.level

    try:
        # Suppress httpx logs (only show WARNING and above)
        httpx_logger.setLevel(logging.WARNING)
        yield
    finally:
        # Restore original level
        httpx_logger.setLevel(original_level)


@dataclass
class SyncResult:
    """Result of syncing product tests from API.

    Attributes:
        new_tests_count: Number of new tests inserted
        tests_checked: Total number of tests checked/processed from API
        skipped_tests: Tests that couldn't be fetched (500 errors with boundary info)
        completed: True if no tests were skipped
        boundary_info: Last successful page boundary for debugging
        recovery_attempts: Number of adaptive paging recovery attempts
    """

    new_tests_count: int
    tests_checked: int
    skipped_tests: list[dict[str, Any]]
    completed: bool
    boundary_info: dict[str, Any]
    recovery_attempts: int


@dataclass
class RefreshResult:
    """Result from refresh_all_tests (upsert strategy).

    Attributes:
        tests_updated: Existing tests updated
        tests_added: New tests added
        total_tests: Total tests after refresh
        errors: List of errors encountered during refresh
    """

    tests_updated: int
    tests_added: int
    total_tests: int
    errors: list[str]


@dataclass
class RebuildResult:
    """Result from nuke_and_rebuild (destructive).

    Attributes:
        tests_synced: Number of tests synced after deletion
        database_recreated: True if entire database was recreated
    """

    tests_synced: int
    database_recreated: bool


class PersistentCache:
    """SQLite-based persistent cache with in-memory caching layer.

    Performance: ~10ms queries at our scale, imperceptible to users.

    Architecture:
    - SQLite: Single source of truth for all data (products, tests, bugs)
    - Repositories: Clean data access layer (no caching logic)
    - Background sync: Keeps SQLite fresh (incremental sync every 5 minutes)

    MVP: Uses self.customer_id for all DB operations (single customer).
    Future (STORY-010): Will accept customer_id parameter for multi-customer support.

    Attributes:
        db_path: Path to SQLite database file
        db: aiosqlite connection (initialized in initialize())
        client: TestIO API client for fetching data
        customer_id: Stable customer identifier (from TestIO/Cirro system)

    Note (STORY-023c):
        In-memory cache removed. All data access now goes through SQLite via repositories.
        Services use repositories directly instead of cache.get/set pattern.
    """

    # File lock configuration (STORY-021e)
    LOCK_FILE = Path.home() / ".testio-mcp" / "sync.lock"

    def __init__(
        self,
        db_path: str,
        client: TestIOClient,
        customer_id: int,
        customer_name: str | None = None,
    ) -> None:
        """Initialize persistent cache.

        Args:
            db_path: Path to SQLite database (e.g., "~/.testio-mcp/cache.db")
            client: TestIO API client for fetching data
            customer_id: Stable customer identifier from TestIO/Cirro system
            customer_name: Optional customer name for display purposes
        """
        self.db_path = Path(db_path).expanduser()
        self.db: aiosqlite.Connection | None = None
        self.client = client
        self.customer_id = customer_id
        self.customer_name = customer_name
        self.repo: TestRepository | None = None

    @property
    def _db(self) -> aiosqlite.Connection:
        """Get database connection (raises if not initialized).

        Returns:
            Active database connection

        Raises:
            AssertionError: If database not initialized (call initialize() first)
        """
        assert self.db is not None, "Database not initialized. Call initialize() first."
        return self.db

    @property
    def _repo(self) -> TestRepository:
        """Get repository (raises if not initialized).

        Returns:
            Active test repository

        Raises:
            AssertionError: If repository not initialized (call initialize() first)
        """
        assert self.repo is not None, "Repository not initialized. Call initialize() first."
        return self.repo

    def _acquire_sync_lock(self) -> FileLock:
        """Acquire cross-process sync lock with stale detection (STORY-021e).

        Returns:
            Acquired file lock

        Raises:
            RuntimeError: If lock is held by another active process
        """
        # Ensure directory exists
        self.LOCK_FILE.parent.mkdir(parents=True, exist_ok=True)

        lock = FileLock(self.LOCK_FILE, timeout=1)  # Non-blocking check

        try:
            lock.acquire(timeout=1)
            # Write current PID to lock file for stale detection
            with open(self.LOCK_FILE, "w") as f:
                f.write(str(os.getpid()))
            return lock
        except Timeout:
            # Lock exists - check if stale
            if self._is_stale_lock(self.LOCK_FILE):
                logger.warning("Removing stale lock from crashed process")
                self.LOCK_FILE.unlink()
                return self._acquire_sync_lock()  # Retry
            else:
                raise RuntimeError("⚠️ Sync in progress. Retry in a few minutes.") from None

    def _is_stale_lock(self, lock_file: Path) -> bool:
        """Check if lock holder process is still alive (STORY-021e).

        Args:
            lock_file: Path to lock file containing PID

        Returns:
            True if lock is stale (process dead), False if process alive
        """
        if not lock_file.exists():
            return False

        try:
            with open(lock_file) as f:
                pid = int(f.read().strip())

            # Check if process exists
            return not psutil.pid_exists(pid)
        except (ValueError, FileNotFoundError):
            # Invalid PID or file deleted - treat as stale
            return True

    async def initialize(self) -> None:
        """Initialize SQLite database and schema.

        Creates database directory if missing, establishes connection,
        enables WAL mode for concurrent reads, and creates schema.
        Checks schema version compatibility and logs database statistics.

        Reference: STORY-021 AC8 (Database Maintenance)
        """
        from testio_mcp.database.schema import (
            check_schema_compatibility,
            configure_wal_mode,
            initialize_schema,
            vacuum_database,
        )

        # Create parent directory if missing
        self.db_path.parent.mkdir(parents=True, exist_ok=True)

        # Connect to database
        self.db = await aiosqlite.connect(str(self.db_path))
        self.db.row_factory = aiosqlite.Row

        # Configure WAL mode
        await configure_wal_mode(self._db)

        # Create schema (idempotent)
        await initialize_schema(self._db)

        # Check schema version compatibility (AC8)
        is_compatible, current_version, expected_version = await check_schema_compatibility(
            self._db
        )

        if not is_compatible:
            logger.warning(
                f"Schema version mismatch: current={current_version}, "
                f"expected={expected_version}. "
                f"Database may need migration. Consider running clear_cache() to recreate."
            )

        # Compact database on startup (AC8)
        await vacuum_database(self._db)

        # Initialize repository with database connection and API client (STORY-023c)
        self.repo = TestRepository(db=self._db, client=self.client, customer_id=self.customer_id)

        # Log database statistics on startup (AC8)
        db_size_mb = await self.get_db_size_mb()
        test_count = await self.count_tests()
        product_count = await self.count_products()

        logger.info(
            f"Initialized persistent cache at {self.db_path} for customer_id={self.customer_id} | "
            f"Schema v{expected_version} | "
            f"Size: {db_size_mb}MB | "
            f"Tests: {test_count} | "
            f"Products: {product_count}"
        )

    async def close(self) -> None:
        """Close database connection."""
        if self.db:
            await self.db.close()
            logger.info(f"Closed persistent cache at {self.db_path}")

    def _make_cache_key(self, *parts: Any) -> str:
        """Create cache key from parts (for compatibility with existing patterns).

        Args:
            *parts: Key components (e.g., "test", 123, "status")

        Returns:
            Cache key string (e.g., "test:123:status")
        """
        return ":".join(str(p) for p in parts)

    async def get_db_size_mb(self) -> float:
        """Get database file size in megabytes.

        Returns:
            Database file size in MB
        """
        if not self.db_path.exists():
            return 0.0
        size_bytes = self.db_path.stat().st_size
        return round(size_bytes / (1024 * 1024), 2)

    async def count_tests(self) -> int:
        """Count total tests in database for current customer.

        Returns:
            Total number of tests
        """
        if not self.repo:
            return 0
        return await self._repo.count_tests()

    async def count_products(self) -> int:
        """Count total products in database for current customer.

        Returns:
            Total number of products
        """
        if not self.repo:
            return 0
        return await self._repo.count_products()

    async def get_oldest_test_date(self) -> str | None:
        """Get oldest test end date for current customer.

        Returns:
            ISO 8601 timestamp of oldest test end date, or None if no tests
        """
        if not self.repo:
            return None
        return await self._repo.get_oldest_test_date()

    async def get_newest_test_date(self) -> str | None:
        """Get newest test end date for current customer.

        Returns:
            ISO 8601 timestamp of newest test end date, or None if no tests
        """
        if not self.repo:
            return None
        return await self._repo.get_newest_test_date()

    async def query_tests(
        self,
        product_id: int,
        statuses: list[str] | None = None,
        start_date: Any | None = None,
        end_date: Any | None = None,
        date_field: str = "start_at",
        page: int = 1,
        per_page: int = 100,
    ) -> list[dict[str, Any]]:
        """Query tests from local database with filtering and pagination (AC3).

        MVP: Uses self.customer_id internally for WHERE clause.
        Future (STORY-010): Will accept customer_id parameter.

        Args:
            product_id: Product identifier
            statuses: Optional list of status values to filter by
            start_date: Optional start date for date range filter (datetime or str)
            end_date: Optional end date for date range filter (datetime or str)
            date_field: Field to use for date filtering (start_at, end_at, created_at)
            page: Page number (1-indexed)
            per_page: Number of results per page

        Returns:
            List of test dictionaries (deserialized from JSON data column)
        """
        if not self.repo:
            return []
        return await self._repo.query_tests(
            product_id=product_id,
            statuses=statuses,
            start_date=start_date,
            end_date=end_date,
            date_field=date_field,
            page=page,
            per_page=per_page,
        )

    async def clear_database(self) -> None:
        """Clear all data from database (for current customer only).

        Deletes all tests and products for current customer, then vacuums to reclaim space.
        """
        from testio_mcp.database.schema import vacuum_database

        if not self.repo:
            return

        await self._repo.delete_all_tests()
        await self._repo.delete_all_products()
        await vacuum_database(self._db)

        logger.info(f"Cleared database for customer_id={self.customer_id}")

    async def delete_product_tests(self, product_id: int) -> None:
        """Delete all tests for a specific product.

        Args:
            product_id: Product identifier
        """
        if not self.repo:
            return

        await self._repo.delete_product_tests(product_id)

        logger.info(f"Deleted tests for product_id={product_id}, customer_id={self.customer_id}")

    async def refresh_all_tests(
        self,
        product_id: int,
        *,
        since: datetime | None = None,
        product_data: dict[str, Any] | None = None,
        sync_mode: str = "force",
        command_run_at: str | None = None,
        progress_callback: Any = None,
    ) -> RefreshResult:
        """Refresh all tests for product using upsert strategy (STORY-021d).

        Fetches ALL tests from API (no incremental logic) and updates database
        using INSERT OR REPLACE. Non-destructive: updates existing + adds new.

        This is the new --force behavior: refreshes data without deleting database.

        Args:
            product_id: Product identifier
            since: Only sync tests with end_at >= this date (optional)
            product_data: Optional product metadata to store (STORY-021a)
            sync_mode: Sync mode that triggered this ("force", "incremental", etc.)
            command_run_at: ISO timestamp when sync command started

        Returns:
            RefreshResult with tests_updated, tests_added, total_tests, errors
        """
        if not self.repo:
            return RefreshResult(tests_updated=0, tests_added=0, total_tests=0, errors=[])

        # Import timezone utilities for normalization
        from testio_mcp.utilities.timezone_utils import normalize_to_utc

        # Generate command_run_at if not provided
        if command_run_at is None:
            from datetime import UTC, datetime

            command_run_at = datetime.now(UTC).isoformat()

        logger.info(f"Refreshing all tests for product {product_id} (upsert strategy)")

        # Store product metadata if provided (STORY-021a)
        if product_data:
            product_json = json.dumps(product_data)
            now_utc = datetime.now(UTC).isoformat()
            await self._db.execute(
                """
                INSERT OR REPLACE INTO products (id, customer_id, data, last_synced)
                VALUES (?, ?, ?, ?)
                """,
                (product_id, self.customer_id, product_json, now_utc),
            )
            await self._db.commit()

        # Track counts for result
        tests_before = await self._repo.count_product_tests(product_id)
        tests_inserted = 0
        errors: list[str] = []
        page = 1
        page_size = 25
        total_recovery_attempts = 0
        pending_problematic_tests: list[dict[str, Any]] = []
        last_test_id: int | None = None
        last_test_end_at: str | None = None

        # Fetch ALL pages using recovery logic (matches incremental sync)
        while True:
            # Use same recovery logic as incremental sync
            page_data, recovery_attempts = await self._fetch_page_with_recovery(
                product_id=product_id,
                page=page,
                page_size=page_size,
                since=since,  # Respect --since date filter
                progress_callback=progress_callback,
                current_count=tests_inserted,
            )

            total_recovery_attempts += recovery_attempts

            # Extract tests from response (may be empty, partial, or complete)
            tests = page_data.get("exploratory_tests", [])
            recovery_failed = page_data.get("recovery_failed", False)

            # If recovery failed completely with no partial data, skip this page
            if recovery_failed and not tests:
                # Track problematic test for logging
                pending_test_info = {
                    "product_id": product_id,
                    "page": page,
                    "position_range": f"page_{page}",
                    "recovery_attempts": recovery_attempts,
                    "reason": "unrecoverable_500_error",
                    "boundary_before_id": last_test_id,
                    "boundary_before_end_at": last_test_end_at,
                    "sync_mode": sync_mode,
                    "command_run_at": command_run_at,
                }
                pending_problematic_tests.append(pending_test_info)
                errors.append(
                    f"Page {page}: Unrecoverable 500 error after {recovery_attempts} attempts"
                )
                page += 1
                continue

            if not tests:
                logger.info(f"No more tests found at page {page}, stopping refresh")
                break

            # Log date range for this page (matches incremental sync pattern)
            first_end_at = tests[0].get("end_at", "N/A")
            last_end_at = tests[-1].get("end_at", "N/A")
            logger.debug(
                f"Page {page} date range: "
                f"newest={first_end_at}, oldest={last_end_at}, "
                f"count={len(tests)}"
            )

            # If we have pending problematic tests, log them with boundary info
            if pending_problematic_tests:
                boundary_after_id = tests[0].get("id")
                # Normalize boundary_after to UTC for consistency
                boundary_after_end_at = normalize_to_utc(tests[0].get("end_at"))

                for pending in pending_problematic_tests:
                    pending["boundary_after_id"] = boundary_after_id
                    pending["boundary_after_end_at"] = boundary_after_end_at
                    await self.log_problematic_test(pending)

                pending_problematic_tests.clear()

            # Client-side date filtering (API doesn't support date filtering)
            # Since tests are ordered by end_at DESC (newest first), we can stop
            # as soon as we hit the first test older than cutoff
            if since:
                filtered_tests = []
                hit_cutoff = False

                for test in tests:
                    test_end_at_str = test.get("end_at")
                    if test_end_at_str:
                        from datetime import datetime

                        # Parse end_at and normalize to UTC for comparison
                        test_end_at = datetime.fromisoformat(test_end_at_str.replace("Z", "+00:00"))

                        # Stop processing as soon as we hit first test older than cutoff
                        # (all subsequent tests will also be older due to DESC ordering)
                        if test_end_at < since:
                            logger.info(
                                f"Hit cutoff at test {test.get('id')} with "
                                f"end_at={test_end_at_str} "
                                f"(older than --since {since.isoformat()}), stopping pagination"
                            )
                            hit_cutoff = True
                            break

                    filtered_tests.append(test)

                # If we filtered out some tests, log it
                if len(filtered_tests) < len(tests):
                    logger.info(
                        f"Page {page}: Kept {len(filtered_tests)} tests, "
                        f"stopped at cutoff (tests are ordered by end_at DESC)"
                    )

                tests = filtered_tests

                # Stop pagination if we hit the cutoff date
                if hit_cutoff:
                    break

            # Insert or update each test (same upsert pattern as incremental)
            for test in tests:
                await self._repo.insert_test(test, product_id)
                tests_inserted += 1

            # Report progress after each page (for real-time progress bar updates)
            if progress_callback:
                progress_callback(tests_inserted)

            # Track last test for boundary info (normalize to UTC for consistency)
            if tests:
                last_test_id = tests[-1].get("id")
                last_test_end_at = normalize_to_utc(tests[-1].get("end_at"))

            # Check if there are more pages (SAME LOGIC AS INCREMENTAL SYNC)
            # If recovery failed with partial data, MUST continue to next page
            # to capture boundary_after for complete gap identification
            if recovery_failed and tests:
                has_more = True  # Force continuation to get boundary_after
            else:
                # API doesn't return "next" field, so detect by page fullness
                has_more = len(tests) >= page_size

            # If recovery failed but we saved partial data, log as problematic
            if recovery_failed and tests:
                pending_test_info = {
                    "product_id": product_id,
                    "page": page,
                    "position_range": f"page_{page}",
                    "recovery_attempts": total_recovery_attempts,
                    "reason": "partial_recovery_incomplete",
                    "boundary_before_id": last_test_id,
                    "boundary_before_end_at": last_test_end_at,
                    "sync_mode": sync_mode,
                    "command_run_at": command_run_at,
                }
                pending_problematic_tests.append(pending_test_info)

            # Log page stats (matches incremental sync output)
            logger.debug(
                f"Page {page}: {len(tests)} tests, "
                f"has_more={has_more} (page_full={len(tests) >= page_size})"
            )

            # For force refresh, continue to next page if has_more
            # (same as incremental, but without stop-at-known-test logic)
            if not has_more:
                break

            page += 1

        tests_after = await self._repo.count_product_tests(product_id)
        tests_added = tests_after - tests_before

        logger.info(
            f"Refresh complete: {tests_inserted} tests processed, "
            f"{tests_added} new tests added, {tests_after} total tests"
        )

        return RefreshResult(
            tests_updated=tests_inserted - tests_added,
            tests_added=tests_added,
            total_tests=tests_after,
            errors=errors,
        )

    async def nuke_and_rebuild(
        self,
        product_id: int,
        *,
        product_data: dict[str, Any] | None = None,
    ) -> RebuildResult:
        """Delete all tests and resync from scratch (STORY-021d).

        This is the new --nuke behavior: destructive rebuild.
        Old --force behavior moved here.

        Use case: Database corruption, testing, complete reset.
        Requires explicit confirmation via --yes flag.

        Args:
            product_id: Product identifier
            product_data: Optional product metadata to store (STORY-021a)

        Returns:
            RebuildResult with tests_synced count and database_recreated flag
        """
        if not self.repo:
            return RebuildResult(tests_synced=0, database_recreated=False)

        logger.warning(f"DESTRUCTIVE: Deleting all tests for product {product_id}")

        # Delete all tests for product
        await self._repo.delete_product_tests(product_id)

        # Resync from scratch using incremental sync
        sync_result = await self.sync_product_tests(product_id, product_data=product_data)

        logger.info(f"Rebuild complete: {sync_result.new_tests_count} tests synced")

        return RebuildResult(
            tests_synced=sync_result.new_tests_count,
            database_recreated=False,  # Only deleted product tests, not entire DB
        )

    async def refresh_active_cycle(
        self,
        product_id: int,
        *,
        product_data: dict[str, Any] | None = None,
    ) -> dict[str, Any]:
        """Hybrid refresh: Discover new tests AND update mutable tests (STORY-021g).

        Part A: Incremental sync (discover new tests until known ID + 2 pages)
        Part B: Refresh mutable tests (customer_finalized, waiting, running, locked, initialized)

        Mutable tests are those that can still change. Immutable tests (archived, cancelled)
        are skipped as they never change.

        This is more efficient than full refresh and more comprehensive than
        background refresh. Perfect for active test cycles.

        Args:
            product_id: Product identifier
            product_data: Optional product metadata to store (STORY-021a)

        Returns:
            Dict with new_tests_discovered, mutable_tests_updated, errors
        """
        if not self.repo:
            return {"new_tests_discovered": 0, "mutable_tests_updated": 0, "errors": []}

        print(f"\n🔄 Hybrid refresh for product {product_id}...")

        # PART A: Discover new tests via incremental sync
        print("\n📥 Part A: Discovering new tests...")
        logger.info(f"Part A: Discovering new tests for product {product_id}...")
        sync_result = await self.sync_product_tests(product_id, product_data=product_data)
        new_tests = sync_result.new_tests_count
        print(f"   Found {new_tests} new tests")
        logger.info(f"Discovered {new_tests} new tests")

        # PART B: Update mutable tests via refresh_active_tests
        # (which now has summary reporting built-in)
        print("\n🔄 Part B: Refreshing mutable tests...")
        logger.info(f"Part B: Refreshing mutable tests for product {product_id}...")
        refresh_result = await self.refresh_active_tests(product_id)

        # Combine errors from both parts
        all_errors = []
        if sync_result.skipped_tests:
            all_errors.extend([f"Sync: {s}" for s in sync_result.skipped_tests])
        if refresh_result.get("errors"):
            all_errors.extend([f"Refresh: {e}" for e in refresh_result["errors"]])

        # Print overall summary
        print("\n✅ Hybrid refresh complete:")
        print(f"   New tests discovered: {new_tests}")
        print(f"   Mutable tests refreshed: {refresh_result['tests_updated']}")
        print(f"   Status changes: {refresh_result['status_changed']}")
        print(f"   Total errors: {len(all_errors)}")

        logger.info(
            f"Hybrid refresh complete: {new_tests} new, "
            f"{refresh_result['tests_updated']} refreshed, "
            f"{refresh_result['status_changed']} status changes"
        )

        return {
            "new_tests_discovered": new_tests,
            "mutable_tests_updated": refresh_result["tests_updated"],  # For backward compatibility
            "status_changed": refresh_result["status_changed"],
            "errors": all_errors,
            "updated_tests": refresh_result.get("updated_tests", []),
        }

    async def get_synced_products_info(self) -> list[dict[str, Any]]:
        """Get information about synced products for current customer.

        Returns:
            List of product info dictionaries with id, name, last_synced, test_count
        """
        if not self.repo:
            return []
        return await self._repo.get_synced_products_info()

    async def get_product_info(self, product_id: int) -> dict[str, Any] | None:
        """Get product information from database.

        Args:
            product_id: Product identifier

        Returns:
            Product info dictionary with id, name, type, or None if not found
        """
        if not self.repo:
            return None
        return await self._repo.get_product_info(product_id)

    async def should_run_initial_sync(self, refresh_interval_seconds: int) -> bool:
        """Check if initial sync should run based on last sync time.

        Decision logic:
        - Circuit breaker: If 3+ sync failures in last 5 minutes, skip (restart loop protection)
        - If database is empty (no products): Run sync (first run)
        - Check oldest product sync AND oldest mutable test sync
        - If either is > refresh_interval ago: Run sync (data stale)
        - Otherwise: Skip sync (data fresh)

        Only checks mutable test statuses (customer_finalized, waiting, running,
        locked, initialized). Archived and cancelled tests are immutable.

        Args:
            refresh_interval_seconds: Refresh interval from settings

        Returns:
            True if sync should run, False to skip
        """
        from datetime import datetime, timedelta

        # Circuit breaker: Check recent failures (restart loop protection)
        recent_failures = await self.count_sync_failures_since(
            datetime.now(UTC) - timedelta(minutes=5)
        )

        if recent_failures >= 3:
            logger.error(
                "🛑 Circuit breaker: 3+ sync failures in last 5 minutes. "
                "Server may be in restart loop. Skipping initial sync. "
                "Check logs and fix issues before retrying."
            )
            return False

        # Check if database has any products
        product_count = await self.count_products()
        if product_count == 0:
            logger.info("Database empty (first run), initial sync needed")
            return True

        # Get products with last_synced timestamps
        products_info = await self.get_synced_products_info()
        if not products_info:
            logger.info("No synced products found, initial sync needed")
            return True

        now = datetime.now(UTC)

        # Check 1: Find oldest product sync
        oldest_product_sync: datetime | None = None
        for product in products_info:
            last_synced_str = product.get("last_synced")
            if not last_synced_str:
                continue

            # Parse timestamp (ISO format with timezone)
            try:
                last_synced = datetime.fromisoformat(last_synced_str)

                if oldest_product_sync is None or last_synced < oldest_product_sync:
                    oldest_product_sync = last_synced
            except ValueError as e:
                logger.warning(f"Failed to parse product last_synced '{last_synced_str}': {e}")
                continue

        if oldest_product_sync is None:
            logger.info("No valid product last_synced timestamps found, initial sync needed")
            return True

        # Check 2: Find oldest mutable test sync
        oldest_test_sync_str = await self._repo.get_oldest_mutable_test_synced_at()
        oldest_test_sync: datetime | None = None

        if oldest_test_sync_str:
            try:
                oldest_test_sync = datetime.fromisoformat(oldest_test_sync_str)
            except ValueError as e:
                logger.warning(f"Failed to parse test synced_at '{oldest_test_sync_str}': {e}")

        # Determine overall oldest sync (either products or mutable tests)
        overall_oldest = oldest_product_sync
        if oldest_test_sync and oldest_test_sync < overall_oldest:
            overall_oldest = oldest_test_sync

        # Check if data is stale
        time_since_sync = (now - overall_oldest).total_seconds()

        if time_since_sync > refresh_interval_seconds:
            logger.info(
                f"Data stale (oldest: {time_since_sync:.0f}s ago, "
                f"threshold: {refresh_interval_seconds}s), initial sync needed"
            )
            return True

        logger.info(
            f"Data fresh (oldest: {time_since_sync:.0f}s ago, "
            f"threshold: {refresh_interval_seconds}s), skipping initial sync"
        )
        return False

    async def get_problematic_tests(self, product_id: int | None = None) -> list[dict[str, Any]]:
        """Get tests that failed to sync due to API 500 errors.

        Args:
            product_id: Optional filter for specific product

        Returns:
            List of problematic test records with boundary information
        """
        if not self.repo:
            return []
        return await self._repo.get_problematic_tests(product_id)

    async def log_problematic_test(self, test_info: dict[str, Any]) -> None:
        """Append problematic test info to sync_metadata.

        Args:
            test_info: Test information dict with boundary data
        """
        if not self.repo:
            return
        await self._repo.log_problematic_test(test_info)

        # Format boundary information for logging
        boundary_before = (
            f"boundary_before=(id={test_info.get('boundary_before_id')}, "
            f"end_at={test_info.get('boundary_before_end_at')})"
        )

        # Include boundary_after if available
        boundary_after_id = test_info.get("boundary_after_id")
        if boundary_after_id:
            boundary_after = (
                f", boundary_after=(id={boundary_after_id}, "
                f"end_at={test_info.get('boundary_after_end_at')})"
            )
        else:
            boundary_after = ""

        logger.warning(
            f"Logged problematic test: product_id={test_info.get('product_id')}, "
            f"position_range={test_info.get('position_range')}, "
            f"recovery_attempts={test_info.get('recovery_attempts')}, "
            f"{boundary_before}{boundary_after}"
        )

    async def get_problematic_events(self, product_id: int | None = None) -> list[dict[str, Any]]:
        """Get all failed sync events with their mapped test IDs.

        Args:
            product_id: Filter by product, or None for all products

        Returns:
            List of event dicts with:
            - event_id: UUID of failed sync event
            - product_id: Product identifier
            - position_range: Page positions that failed
            - recovery_attempts: Number of retry attempts
            - boundary_before_id: Test ID before failed range
            - boundary_before_end_at: End timestamp of boundary test
            - mapped_test_ids: List of test IDs mapped to this event
        """
        if not self.repo:
            return []

        # Get failed events from problematic_tests
        events = await self._repo.get_problematic_tests(product_id)

        # Get test ID mappings from sync_metadata
        if not self.repo.db:
            return events

        cursor = await self.repo.db.execute(
            "SELECT value FROM sync_metadata WHERE key = 'problematic_test_mappings'"
        )
        result = await cursor.fetchone()

        mappings: dict[str, list[int]] = {}
        if result and result[0]:
            mappings = json.loads(result[0])

        # Merge mapped test IDs into events
        for event in events:
            event_id = event.get("event_id", "")
            event["mapped_test_ids"] = mappings.get(event_id, [])

        return events

    async def map_test_ids_to_event(self, event_id: str, test_ids: list[int]) -> None:
        """Map test IDs to a specific failed sync event.

        Simple metadata write - no API calls, no validation.
        User decides which test IDs belong to which event.

        Args:
            event_id: UUID of failed sync event (from get_problematic_events)
            test_ids: One or more test IDs to map to this event

        Raises:
            ValueError: If event_id doesn't exist
        """
        if not self.repo or not self.repo.db:
            return

        # Validate event_id exists
        events = await self._repo.get_problematic_tests(product_id=None)
        event_ids = {e.get("event_id") for e in events}
        if event_id not in event_ids:
            raise ValueError(f"Event ID '{event_id}' not found in problematic tests")

        # Get existing mappings
        cursor = await self.repo.db.execute(
            "SELECT value FROM sync_metadata WHERE key = 'problematic_test_mappings'"
        )
        result = await cursor.fetchone()

        mappings: dict[str, list[int]] = {}
        if result and result[0]:
            mappings = json.loads(result[0])

        # Add new test IDs to event (append to existing if any)
        if event_id not in mappings:
            mappings[event_id] = []

        # Merge new test IDs (avoid duplicates)
        existing_ids = set(mappings[event_id])
        for test_id in test_ids:
            if test_id not in existing_ids:
                mappings[event_id].append(test_id)

        # Update metadata
        await self.repo.db.execute(
            """
            INSERT OR REPLACE INTO sync_metadata (key, value)
            VALUES ('problematic_test_mappings', ?)
            """,
            (json.dumps(mappings),),
        )
        await self.repo.db.commit()

        logger.info(f"Mapped {len(test_ids)} test IDs to event {event_id}: {test_ids}")

    async def retry_problematic_tests(self, product_id: int) -> dict[str, Any]:
        """Retry fetching all mapped test IDs for a product.

        Fetches each test individually via GET /tests/{id}.
        Removes from event mapping on success, keeps on failure.

        Args:
            product_id: Product ID to retry

        Returns:
            Dict with tests_retried, tests_succeeded, tests_failed, errors
        """
        if not self.repo or not self.repo.db:
            return {
                "tests_retried": 0,
                "tests_succeeded": 0,
                "tests_failed": 0,
                "errors": [],
            }

        # Get all events for this product with their mapped test IDs
        events = await self.get_problematic_events(product_id=product_id)

        # Collect all mapped test IDs for this product
        test_ids_to_retry: list[int] = []
        for event in events:
            test_ids_to_retry.extend(event.get("mapped_test_ids", []))

        results: dict[str, Any] = {
            "tests_retried": len(test_ids_to_retry),
            "tests_succeeded": 0,
            "tests_failed": 0,
            "errors": [],
        }

        # Retry each test ID individually
        for test_id in test_ids_to_retry:
            try:
                # Fetch individual test from API
                test_data = await self.client.get(f"exploratory_tests/{test_id}")

                # Insert to database (INSERT OR REPLACE handles duplicates)
                await self._repo.insert_test(test_data, product_id)

                # Remove from ALL event mappings on success
                await self._remove_test_id_from_mappings(test_id)
                results["tests_succeeded"] = results["tests_succeeded"] + 1

                logger.info(f"Successfully retried test_id={test_id}")

            except Exception as e:
                # Keep in mapping on failure
                results["tests_failed"] = results["tests_failed"] + 1
                error_msg = f"Test {test_id}: {type(e).__name__} - {str(e)}"
                results["errors"].append(error_msg)
                logger.warning(f"Failed to retry test_id={test_id}: {e}")

        return results

    async def _remove_test_id_from_mappings(self, test_id: int) -> None:
        """Remove a test ID from all event mappings after successful fetch.

        Args:
            test_id: Test ID to remove from mappings
        """
        if not self.repo or not self.repo.db:
            return

        # Get existing mappings
        cursor = await self.repo.db.execute(
            "SELECT value FROM sync_metadata WHERE key = 'problematic_test_mappings'"
        )
        result = await cursor.fetchone()

        if not result or not result[0]:
            return

        mappings: dict[str, list[int]] = json.loads(result[0])

        # Remove test_id from all events
        updated = False
        for _event_id, test_ids in mappings.items():
            if test_id in test_ids:
                test_ids.remove(test_id)
                updated = True

        # Remove events with empty test ID lists
        mappings = {k: v for k, v in mappings.items() if v}

        if updated:
            # Update metadata
            await self.repo.db.execute(
                """
                INSERT OR REPLACE INTO sync_metadata (key, value)
                VALUES ('problematic_test_mappings', ?)
                """,
                (json.dumps(mappings),),
            )
            await self.repo.db.commit()

    async def clear_problematic_tests(self) -> dict[str, Any]:
        """Clear all problematic records (position ranges AND tracked test IDs).

        Returns:
            Dict with position_ranges_cleared and test_ids_cleared counts
        """
        if not self.repo or not self.repo.db:
            return {"position_ranges_cleared": 0, "test_ids_cleared": 0}

        # Get counts before clearing
        events = await self._repo.get_problematic_tests(product_id=None)
        position_ranges_cleared = len(events)

        # Get test ID mappings count
        cursor = await self.repo.db.execute(
            "SELECT value FROM sync_metadata WHERE key = 'problematic_test_mappings'"
        )
        result = await cursor.fetchone()

        test_ids_cleared = 0
        if result and result[0]:
            mappings: dict[str, list[int]] = json.loads(result[0])
            test_ids_cleared = sum(len(ids) for ids in mappings.values())

        # Clear both metadata keys
        await self.repo.db.execute(
            """
            DELETE FROM sync_metadata
            WHERE key IN ('problematic_tests', 'problematic_test_mappings')
            """
        )
        await self.repo.db.commit()

        logger.info(
            f"Cleared {position_ranges_cleared} position ranges "
            f"and {test_ids_cleared} test ID mappings"
        )

        return {
            "position_ranges_cleared": position_ranges_cleared,
            "test_ids_cleared": test_ids_cleared,
        }

    # Sync event logging for observability and circuit breaker

    async def log_sync_event_start(self, event_type: str, trigger_source: str) -> int:
        """Create sync event record with status='running'.

        Args:
            event_type: Type of sync ('initial_sync' or 'background_refresh')
            trigger_source: What triggered the sync ('startup', 'background_task', 'cli_command')

        Returns:
            Event ID for updating later with complete/failed status
        """
        if not self.repo or not self.repo.db:
            return -1

        from datetime import UTC, datetime

        now = datetime.now(UTC).isoformat()

        cursor = await self.repo.db.execute(
            """
            INSERT INTO sync_events (
                event_type, started_at, status, trigger_source
            ) VALUES (?, ?, 'running', ?)
            """,
            (event_type, now, trigger_source),
        )
        await self.repo.db.commit()

        event_id = cursor.lastrowid
        logger.debug(
            f"Sync event started: id={event_id}, type={event_type}, source={trigger_source}"
        )
        return event_id if event_id else -1

    async def log_sync_event_complete(
        self,
        event_id: int,
        products_synced: int,
        tests_discovered: int,
        tests_refreshed: int,
        duration_seconds: float,
    ) -> None:
        """Update sync event with status='completed' and statistics.

        Args:
            event_id: Event ID from log_sync_event_start()
            products_synced: Number of products synced
            tests_discovered: Number of new tests discovered
            tests_refreshed: Number of tests refreshed
            duration_seconds: Time taken for sync
        """
        if not self.repo or not self.repo.db or event_id < 0:
            return

        from datetime import UTC, datetime

        now = datetime.now(UTC).isoformat()

        await self.repo.db.execute(
            """
            UPDATE sync_events
            SET completed_at = ?,
                status = 'completed',
                products_synced = ?,
                tests_discovered = ?,
                tests_refreshed = ?,
                duration_seconds = ?
            WHERE id = ?
            """,
            (now, products_synced, tests_discovered, tests_refreshed, duration_seconds, event_id),
        )
        await self.repo.db.commit()

        logger.debug(
            f"Sync event completed: id={event_id}, "
            f"products={products_synced}, discovered={tests_discovered}, "
            f"refreshed={tests_refreshed}, duration={duration_seconds:.2f}s"
        )

    async def log_sync_event_failed(
        self, event_id: int, error_message: str, duration_seconds: float
    ) -> None:
        """Update sync event with status='failed' and error message.

        Args:
            event_id: Event ID from log_sync_event_start()
            error_message: Error description
            duration_seconds: Time taken before failure
        """
        if not self.repo or not self.repo.db or event_id < 0:
            return

        from datetime import UTC, datetime

        now = datetime.now(UTC).isoformat()

        await self.repo.db.execute(
            """
            UPDATE sync_events
            SET completed_at = ?,
                status = 'failed',
                error_message = ?,
                duration_seconds = ?
            WHERE id = ?
            """,
            (now, error_message, duration_seconds, event_id),
        )
        await self.repo.db.commit()

        logger.debug(f"Sync event failed: id={event_id}, error={error_message}")

    async def log_sync_event_cancelled(self, event_id: int, duration_seconds: float) -> None:
        """Update sync event with status='cancelled' (server shutdown).

        Args:
            event_id: Event ID from log_sync_event_start()
            duration_seconds: Time elapsed before cancellation
        """
        if not self.repo or not self.repo.db or event_id < 0:
            return

        from datetime import UTC, datetime

        now = datetime.now(UTC).isoformat()

        await self.repo.db.execute(
            """
            UPDATE sync_events
            SET completed_at = ?,
                status = 'cancelled',
                error_message = 'Task cancelled (server shutdown)',
                duration_seconds = ?
            WHERE id = ?
            """,
            (now, duration_seconds, event_id),
        )
        await self.repo.db.commit()

        logger.debug(
            f"Sync event cancelled: id={event_id}, duration={duration_seconds:.2f}s "
            f"(server shutdown)"
        )

    async def get_sync_events(self, limit: int = 10) -> list[dict[str, Any]]:
        """Get recent sync events for observability.

        Args:
            limit: Maximum number of events to return (default: 10)

        Returns:
            List of sync event dicts ordered by started_at DESC
        """
        if not self.repo or not self.repo.db:
            return []

        cursor = await self.repo.db.execute(
            """
            SELECT
                id, event_type, started_at, completed_at, status,
                products_synced, tests_discovered, tests_refreshed,
                duration_seconds, error_message, trigger_source
            FROM sync_events
            ORDER BY started_at DESC
            LIMIT ?
            """,
            (limit,),
        )
        rows = await cursor.fetchall()

        events = []
        for row in rows:
            events.append(
                {
                    "id": row[0],
                    "event_type": row[1],
                    "started_at": row[2],
                    "completed_at": row[3],
                    "status": row[4],
                    "products_synced": row[5],
                    "tests_discovered": row[6],
                    "tests_refreshed": row[7],
                    "duration_seconds": row[8],
                    "error_message": row[9],
                    "trigger_source": row[10],
                }
            )

        return events

    async def count_sync_failures_since(self, since: datetime) -> int:
        """Count sync failures since given time for circuit breaker.

        Args:
            since: Count failures after this timestamp

        Returns:
            Number of failed sync events
        """
        if not self.repo or not self.repo.db:
            return 0

        since_iso = since.isoformat()

        cursor = await self.repo.db.execute(
            """
            SELECT COUNT(*)
            FROM sync_events
            WHERE status = 'failed' AND started_at >= ?
            """,
            (since_iso,),
        )
        result = await cursor.fetchone()

        return result[0] if result else 0

    async def get_last_destructive_timestamp(self) -> datetime | None:
        """Get timestamp of last destructive operation for rate limiting (STORY-022a).

        Returns:
            Datetime of last destructive operation, or None if never executed
        """
        if not self.repo or not self.repo.db:
            return None

        cursor = await self.repo.db.execute(
            "SELECT value FROM sync_metadata WHERE key = 'last_destructive_op'"
        )
        result = await cursor.fetchone()

        if result and result[0]:
            # Parse ISO 8601 timestamp
            return datetime.fromisoformat(result[0])

        return None

    async def set_last_destructive_timestamp(self, timestamp: datetime) -> None:
        """Set timestamp of last destructive operation for rate limiting (STORY-022a).

        Args:
            timestamp: Datetime of destructive operation (should be UTC)
        """
        if not self.repo or not self.repo.db:
            return

        # Store as ISO 8601 string
        await self.repo.db.execute(
            """
            INSERT OR REPLACE INTO sync_metadata (key, value)
            VALUES ('last_destructive_op', ?)
            """,
            (timestamp.isoformat(),),
        )
        await self.repo.db.commit()

    async def sync_product_tests(
        self,
        product_id: int,
        *,
        since: datetime | None = None,
        product_data: dict[str, Any] | None = None,
        sync_mode: str = "incremental",
        command_run_at: str | None = None,
        progress_callback: Any = None,
    ) -> SyncResult:
        """Sync tests for a product using incremental algorithm.

        Algorithm:
        1. Fetch tests sorted by -end_at (newest first from API)
        2. For each test, upsert to database (INSERT OR REPLACE)
        3. Track when we hit first known test (by existence check)
        4. Stop after hitting known test + 1 safety page (catch stragglers)
        5. Multi-pass recovery: reduce page size on 500 errors (25→10→5→2→1)

        Upsert Behavior:
        - ALL tests fetched from API are upserted (both new and existing)
        - This ensures any test we see has fresh data in the database
        - Only truly new tests (not previously in DB) increment new_tests_count
        - Known test detection determines when to stop fetching, not what to update

        Note: Tests are ordered by end_at DESC (not test ID) because IDs are
        globally sequential across ALL TestIO customers, so gaps are normal.

        Args:
            product_id: Product identifier
            since: Only sync tests with end_at >= this date (optional)
            product_data: Optional product metadata to store in database (STORY-021a)
            sync_mode: Sync mode that triggered this ("incremental", "force", etc.)
            command_run_at: ISO timestamp when sync command started
            progress_callback: Optional callback function called after each page with current count

        Returns:
            SyncResult with counts, skipped tests, and boundary info
        """
        date_filter_str = ""
        if since:
            date_filter_str += f", since={since.isoformat()}"

        # Generate command_run_at if not provided
        if command_run_at is None:
            from datetime import UTC, datetime

            command_run_at = datetime.now(UTC).isoformat()

        logger.info(
            f"Starting sync for product_id={product_id}, "
            f"customer_id={self.customer_id}{date_filter_str}"
        )

        # Import timezone utilities for normalization
        from testio_mcp.utilities.timezone_utils import normalize_to_utc

        new_tests_count = 0
        tests_checked = 0  # Track total tests processed from API
        skipped_tests: list[dict[str, Any]] = []
        page = 1
        pages_fetched = 0  # Track pages actually fetched
        safety_pages_remaining = 1  # Pages to fetch AFTER hitting known ID
        hit_known_test = False
        hit_known_test_this_page = False  # Track if we hit known test on current page
        total_recovery_attempts = 0
        page_size = 25  # Default page size for API requests

        # Boundary tracking for diagnostics (BLOCKER 4)
        last_test_id: int | None = None
        last_test_end_at: str | None = None
        pending_problematic_tests: list[dict[str, Any]] = []  # Tests awaiting boundary_after

        # Safety limit to prevent infinite loops
        MAX_PAGES = 50

        while page <= MAX_PAGES:
            hit_known_test_this_page = False  # Reset for each page

            # Fetch page with multi-pass recovery
            page_data, recovery_attempts = await self._fetch_page_with_recovery(
                product_id=product_id,
                page=page,
                page_size=page_size,
                since=since,
                progress_callback=progress_callback,
                current_count=new_tests_count,
            )

            total_recovery_attempts += recovery_attempts

            # Extract tests from response (may be empty, partial, or complete)
            tests = page_data.get("exploratory_tests", [])
            recovery_failed = page_data.get("recovery_failed", False)

            # If recovery failed completely with no partial data, skip this page
            if recovery_failed and not tests:
                # Add to pending list - will log with boundary_after when next page succeeds
                pending_test_info = {
                    "product_id": product_id,
                    "page": page,
                    "position_range": f"page_{page}",
                    "recovery_attempts": recovery_attempts,
                    "reason": "unrecoverable_500_error",
                    "boundary_before_id": last_test_id,
                    "boundary_before_end_at": last_test_end_at,
                    "sync_mode": sync_mode,
                    "command_run_at": command_run_at,
                }
                pending_problematic_tests.append(pending_test_info)
                skipped_tests.append(pending_test_info.copy())
                page += 1
                continue

            # Page fetched (either successfully or with partial recovery data)
            pages_fetched += 1

            if not tests:
                logger.info(f"No more tests found at page {page}, stopping sync")
                break

            # Log date range for this page (verify chronological ordering)
            if tests:
                first_end_at = tests[0].get("end_at", "N/A")
                last_end_at = tests[-1].get("end_at", "N/A")
                logger.debug(
                    f"Page {page} date range: "
                    f"newest={first_end_at}, oldest={last_end_at}, "
                    f"count={len(tests)}"
                )

            # If we have pending problematic tests, capture boundary_after and log them
            if pending_problematic_tests:
                boundary_after_id = tests[0].get("id")
                # Normalize boundary_after to UTC for consistency
                boundary_after_end_at = normalize_to_utc(tests[0].get("end_at"))

                for pending in pending_problematic_tests:
                    pending["boundary_after_id"] = boundary_after_id
                    pending["boundary_after_end_at"] = boundary_after_end_at
                    await self.log_problematic_test(pending)

                    # Update skipped_tests with boundary_after
                    for skipped in skipped_tests:
                        if skipped.get("page") == pending.get("page") and skipped.get(
                            "product_id"
                        ) == pending.get("product_id"):
                            skipped["boundary_after_id"] = boundary_after_id
                            skipped["boundary_after_end_at"] = boundary_after_end_at

                pending_problematic_tests.clear()

            # Client-side date filtering (API doesn't support date filtering)
            if since:
                filtered_tests = []
                for test in tests:
                    test_end_at_str = test.get("end_at")
                    if test_end_at_str:
                        from datetime import datetime

                        # Parse end_at and normalize to UTC for comparison
                        test_end_at = datetime.fromisoformat(test_end_at_str.replace("Z", "+00:00"))

                        # Skip tests older than cutoff
                        if test_end_at < since:
                            logger.info(
                                f"Skipping test {test.get('id')} with end_at={test_end_at_str} "
                                f"(older than --since {since.isoformat()})"
                            )
                            continue

                    filtered_tests.append(test)

                # If we filtered out some tests, check if we should stop fetching
                if len(filtered_tests) < len(tests):
                    logger.info(
                        f"Page {page}: Filtered {len(tests) - len(filtered_tests)} tests "
                        f"older than {since.isoformat()}, keeping {len(filtered_tests)}"
                    )

                tests = filtered_tests

            # Insert/update tests and check for known IDs
            for test in tests:
                test_id = test.get("id")
                tests_checked += 1  # Count every test we process from API

                # Check if we've hit a known test (for stop-early optimization)
                # Even if test exists, we still upsert it to ensure fresh data
                is_new_test = True
                if test_id:
                    exists = await self._repo.test_exists(test_id, product_id)
                    if exists:
                        is_new_test = False
                        if not hit_known_test:
                            logger.info(
                                f"Hit known test_id={test_id} at page {page}, "
                                f"will continue for {safety_pages_remaining} more pages"
                            )
                            hit_known_test = True
                            hit_known_test_this_page = True

                # Always upsert test (INSERT OR REPLACE handles both new and existing)
                # This ensures any test we fetch from API has fresh data in database
                await self._repo.insert_test(test, product_id)

                # Only count truly new tests (not updates of existing tests)
                if is_new_test:
                    new_tests_count += 1

                # Update boundary tracking for diagnostics (normalize to UTC)
                last_test_id = test_id
                last_test_end_at = normalize_to_utc(test.get("end_at"))

            # Report progress after each page (for real-time progress bar updates)
            if progress_callback:
                progress_callback(new_tests_count)

            # Stop fetching if all tests on this page were filtered out (older than cutoff)
            if since and len(tests) == 0 and not recovery_failed:
                logger.info(f"Page {page}: All tests older than {since.isoformat()}, stopping sync")
                break

            # Check if there are more pages
            # If recovery failed with partial data, we MUST continue to next page
            # to capture boundary_after for complete gap identification (STORY-021b)
            if recovery_failed and tests:
                has_more = True  # Force continuation to get boundary_after
            else:
                # API doesn't return "next" field, so detect by page fullness
                has_more = len(tests) >= page_size

            # If recovery failed but we saved partial data, log as problematic
            # (boundary_before is now correct because we updated last_test_id above)
            if recovery_failed and tests:
                pending_test_info = {
                    "product_id": product_id,
                    "page": page,
                    "position_range": f"page_{page}",
                    "recovery_attempts": recovery_attempts,
                    "reason": "partial_recovery_incomplete",
                    "boundary_before_id": last_test_id,
                    "boundary_before_end_at": last_test_end_at,
                    "sync_mode": sync_mode,
                    "command_run_at": command_run_at,
                }
                pending_problematic_tests.append(pending_test_info)
                skipped_tests.append(pending_test_info.copy())

            # DEBUG: Log page stats for debugging
            logger.debug(
                f"Page {page}: {len(tests)} tests, "
                f"has_more={has_more} (page_full={len(tests) >= page_size})"
            )

            logger.info(
                f"Page {page} complete: fetched {len(tests)} tests, "
                f"has_more={has_more}, next={page_data.get('next')}"
            )

            # Check stop condition AFTER processing current page
            # Don't decrement on the page where we first hit the known test
            if hit_known_test and not hit_known_test_this_page:
                safety_pages_remaining -= 1
                if safety_pages_remaining <= 0:
                    logger.info("Completed safety margin (1 page after known ID), stopping sync")
                    break

            # Stop if no more pages
            if not has_more:
                logger.info("No more pages available, sync complete")
                break

            page += 1

        # Check if we hit safety limit (AC2 requirement)
        if page > MAX_PAGES:
            logger.warning(
                f"Hit {MAX_PAGES}-page safety limit for product {product_id} "
                f"({new_tests_count} tests fetched). "
                f"If you need full history, run force_sync_product({product_id}) "
                f"or clear_cache() to re-sync."
            )

        # Log any remaining pending problematic tests (failures at end of sync)
        if pending_problematic_tests:
            logger.warning(
                f"Sync ended with {len(pending_problematic_tests)} unresolved failures "
                f"(no boundary_after available)"
            )
            for pending in pending_problematic_tests:
                # No boundary_after available - these were at the end
                await self.log_problematic_test(pending)

        # Update product last_synced timestamp and store product metadata (STORY-021a)
        now_utc = datetime.now(UTC).isoformat()
        if product_data is not None:
            # Full upsert: insert or replace with new metadata
            product_json = json.dumps(product_data)
            await self._db.execute(
                """
                INSERT OR REPLACE INTO products (id, customer_id, data, last_synced)
                VALUES (?, ?, ?, ?)
                """,
                (product_id, self.customer_id, product_json, now_utc),
            )
        else:
            # Partial update: only update last_synced, preserve existing data
            # If row doesn't exist, create with empty JSON (backward compatibility)
            await self._db.execute(
                """
                INSERT INTO products (id, customer_id, data, last_synced)
                VALUES (?, ?, '{}', ?)
                ON CONFLICT(id) DO UPDATE SET
                    last_synced = ?
                WHERE id = excluded.id AND customer_id = excluded.customer_id
                """,
                (product_id, self.customer_id, now_utc, now_utc),
            )
        await self._db.commit()

        # Build result
        completed = len(skipped_tests) == 0
        boundary_info = {
            "pages_fetched": pages_fetched,
            "hit_known_test": hit_known_test,
            "safety_pages_used": 1 - safety_pages_remaining if hit_known_test else 0,
        }

        logger.info(
            f"Sync complete: product_id={product_id}, checked={tests_checked}, "
            f"new={new_tests_count}, skipped={len(skipped_tests)}, "
            f"completed={completed}, recovery_attempts={total_recovery_attempts}"
        )

        return SyncResult(
            new_tests_count=new_tests_count,
            tests_checked=tests_checked,
            skipped_tests=skipped_tests,
            completed=completed,
            boundary_info=boundary_info,
            recovery_attempts=total_recovery_attempts,
        )

    async def refresh_active_tests(self, product_id: int) -> dict[str, Any]:
        """Refresh status for mutable tests (can still change).

        Implements AC4: Batch fetch fresh status from API with concurrency control.

        Mutable tests: customer_finalized, waiting, running, locked, initialized
        Immutable tests: archived, cancelled (skipped, never change)

        Args:
            product_id: Product identifier

        Returns:
            Dict with refresh statistics (tests_checked, tests_updated, errors)
        """
        if not self.repo:
            return {"tests_checked": 0, "tests_updated": 0, "status_changed": 0, "errors": []}

        logger.info(f"Starting mutable test refresh for product {product_id}")

        # Get mutable tests from database
        mutable_tests = await self._repo.get_mutable_tests(product_id)
        mutable_test_ids = [t["id"] for t in mutable_tests]

        if not mutable_test_ids:
            logger.info(f"No mutable tests found for product {product_id}")
            # Still update last_synced to mark that we checked (prevents staleness on restart)
            await self._repo.update_product_last_synced(product_id)
            return {"tests_checked": 0, "tests_updated": 0, "status_changed": 0, "errors": []}

        # Create status map (before refresh) and test title map
        status_before = {t["id"]: t["status"] for t in mutable_tests}

        # Query titles from database (stored in data JSON column)
        test_titles = {}
        for test_id in mutable_test_ids:
            cursor = await self._db.execute(
                "SELECT data FROM tests WHERE id = ? AND product_id = ? AND customer_id = ?",
                (test_id, product_id, self.customer_id),
            )
            row = await cursor.fetchone()
            if row and row[0]:
                import json

                test_data = json.loads(row[0])
                test_titles[test_id] = test_data.get("title", f"Test {test_id}")
            else:
                test_titles[test_id] = f"Test {test_id}"

        print(f"\n🔄 Refreshing {len(mutable_test_ids)} mutable tests for product {product_id}...")

        # Batch fetch from API with concurrency control
        tests_updated = 0  # Successfully refreshed from API
        status_changed = 0  # Status actually changed
        errors: list[dict[str, Any]] = []
        updated_tests: list[dict[str, Any]] = []

        async def fetch_and_update_test(test_id: int, index: int) -> None:
            """Fetch single test and update in database."""
            nonlocal tests_updated, status_changed
            try:
                # Get old status from status_before map
                old_status = status_before.get(test_id)

                # Use refresh_test which has built-in validation
                await self._repo.refresh_test(test_id)

                # Count as successfully updated (API fetch succeeded)
                tests_updated += 1

                # Query new status after refresh
                cursor = await self._db.execute(
                    "SELECT status FROM tests WHERE id = ? AND product_id = ? AND customer_id = ?",
                    (test_id, product_id, self.customer_id),
                )
                row = await cursor.fetchone()
                new_status = row[0] if row else None

                # Track if status actually changed
                if old_status != new_status:
                    status_changed += 1
                    updated_tests.append(
                        {
                            "id": test_id,
                            "title": test_titles[test_id],
                            "old_status": old_status,
                            "new_status": new_status,
                        }
                    )

                # Show progress every 10 tests
                if (index + 1) % 10 == 0 or (index + 1) == len(mutable_test_ids):
                    print(
                        f"  Progress: {index + 1}/{len(mutable_test_ids)} tests checked "
                        f"({status_changed} changed, "
                        f"{tests_updated - status_changed} unchanged, {len(errors)} errors)"
                    )

            except Exception as e:
                logger.warning(f"Failed to refresh test {test_id}: {e}")
                errors.append({"test_id": test_id, "title": test_titles[test_id], "error": str(e)})

        # Suppress individual HTTP request logs during bulk operation
        with suppress_httpx_logs():
            # Use asyncio.gather for concurrent fetching
            # Concurrency limit is handled by client's semaphore
            tasks = [fetch_and_update_test(tid, idx) for idx, tid in enumerate(mutable_test_ids)]
            await asyncio.gather(*tasks)

        # Print summary
        print("\n✅ Refresh complete:")
        print(f"   Total checked: {len(mutable_test_ids)}")
        print(f"   Successfully refreshed: {tests_updated}")
        print(f"   Status changed: {status_changed}")
        print(f"   Status unchanged: {tests_updated - status_changed}")
        print(f"   Errors: {len(errors)}")

        # Show updated test cycles (if any changed)
        if updated_tests:
            print("\n📊 Test cycles with status changes:")
            for test in updated_tests[:10]:  # Show first 10
                status_change = f"{test['old_status']} → {test['new_status']}"
                print(f"   • {test['title']} (ID: {test['id']}): {status_change}")
            if len(updated_tests) > 10:
                print(f"   ... and {len(updated_tests) - 10} more")

        # Show errors (if any)
        if errors:
            print(f"\n⚠️  Failed to refresh {len(errors)} tests:")
            for err in errors[:5]:  # Show first 5 errors
                print(f"   • {err['title']} (ID: {err['test_id']}): {err['error']}")
            if len(errors) > 5:
                print(f"   ... and {len(errors) - 5} more errors")

        logger.info(
            f"Mutable test refresh complete: product_id={product_id}, "
            f"checked={len(mutable_test_ids)}, updated={tests_updated}, "
            f"status_changed={status_changed}, errors={len(errors)}"
        )

        # Update product's last_synced timestamp (mark data as fresh)
        # This ensures staleness check knows we just refreshed the data
        await self._repo.update_product_last_synced(product_id)

        return {
            "tests_checked": len(mutable_test_ids),
            "tests_updated": tests_updated,  # Successfully refreshed from API
            "status_changed": status_changed,  # Status actually changed
            "errors": errors,
            "updated_tests": updated_tests,
        }

    async def initial_sync(self, since_filter: str | None = None) -> dict[str, Any]:
        """Perform initial sync of all products and their tests (AC6).

        This method is designed to run as a background task during server startup.
        It performs two operations:
        1. Discovers new tests using incremental sync algorithm
        2. Refreshes mutable tests to update synced_at timestamps (makes data fresh)

        The server remains available during initial sync (serves partial data).

        Uses file-based locking to prevent conflicts with manual CLI operations (STORY-021e).

        Args:
            since_filter: Optional date filter (ISO date or relative, e.g., "2023-01-01" or
                         "90 days ago"). Only syncs tests with end_at >= this date.
                         Reduces API load for customers with long test histories.

        Returns:
            Dict with sync statistics (products_synced, total_new_tests,
                                      tests_refreshed, duration_seconds)
        """
        import time

        # Parse date filter if provided
        since: datetime | None = None
        if since_filter:
            try:
                from testio_mcp.utilities.date_utils import parse_flexible_date

                iso_string = parse_flexible_date(since_filter, start_of_day=True)
                since = datetime.fromisoformat(iso_string.replace("Z", "+00:00"))
                logger.info(
                    f"Using date filter for initial sync: {since_filter} "
                    f"(parsed as {since.isoformat()})"
                )
            except Exception as e:
                logger.warning(
                    f"Failed to parse since_filter='{since_filter}': {e}. "
                    f"Syncing all tests without date filter."
                )
                since = None

        # Acquire lock before any writes (STORY-021e) - Fixed per Codex review
        try:
            lock = self._acquire_sync_lock()
        except RuntimeError as e:
            # Lock acquisition failed (another sync in progress)
            logger.warning(f"Initial sync skipped: {e}")
            return {
                "products_synced": 0,
                "total_products": 0,
                "total_new_tests": 0,
                "duration_seconds": 0,
                "skipped": True,
            }

        # Log sync event start
        event_id = await self.log_sync_event_start(
            event_type="initial_sync", trigger_source="startup"
        )

        try:
            logger.info("Starting initial sync of all products")
            start_time = time.time()

            # Check if database is empty (first run detection)
            product_count = await self.count_products()
            test_count = await self.count_tests()
            logger.info(f"Database state before sync: {product_count} products, {test_count} tests")
            # Fetch all products from API
            response = await self.client.get("products")
            products = response.get("products", [])
            total_products = len(products)

            if not products:
                logger.warning("No products found in API, initial sync complete (0 products)")
                return {
                    "products_synced": 0,
                    "total_new_tests": 0,
                    "duration_seconds": round(time.time() - start_time, 2),
                }

            logger.info(f"Found {total_products} products, starting sync")

            # Phase 1: Discover new tests for each product
            total_new_tests = 0
            products_synced = 0

            for idx, product in enumerate(products, start=1):
                product_id = product.get("id")
                product_name = product.get("name", "Unknown")

                logger.info(
                    f"Syncing product {idx}/{total_products}: id={product_id}, name={product_name}"
                )

                try:
                    # Sync tests for this product (pass product data for storage)
                    # Use TESTIO_SYNC_SINCE date filter if configured
                    result = await self.sync_product_tests(
                        product_id, product_data=product, since=since
                    )

                    total_new_tests += result.new_tests_count
                    products_synced += 1

                    logger.info(
                        f"Product {idx}/{total_products} synced: "
                        f"{result.new_tests_count} new tests, "
                        f"completed={result.completed}, "
                        f"skipped={len(result.skipped_tests)}"
                    )

                except Exception as e:
                    logger.error(
                        f"Failed to sync product {product_id} ({product_name}): {e}", exc_info=True
                    )
                    # Continue with next product (partial sync is acceptable)

            # Phase 2: Refresh mutable tests to make data truly fresh
            # This updates synced_at timestamps so data won't be considered stale
            logger.info("Phase 2: Refreshing mutable tests to update timestamps")
            total_tests_refreshed = 0

            for idx, product in enumerate(products, start=1):
                product_id = product.get("id")
                product_name = product.get("name", "Unknown")

                try:
                    logger.info(
                        f"Refreshing mutable tests for product {idx}/{total_products}: "
                        f"id={product_id}, name={product_name}"
                    )

                    # Suppress HTTP logs during bulk refresh
                    with suppress_httpx_logs():
                        refresh_result = await self.refresh_active_tests(product_id)

                    total_tests_refreshed += refresh_result["tests_updated"]

                    logger.info(
                        f"Product {idx}/{total_products} refreshed: "
                        f"{refresh_result['tests_updated']} tests updated, "
                        f"{refresh_result['status_changed']} status changes"
                    )

                except Exception as e:
                    logger.error(
                        f"Failed to refresh product {product_id} ({product_name}): {e}",
                        exc_info=True,
                    )
                    # Continue with next product (partial refresh is acceptable)

            duration = round(time.time() - start_time, 2)
            logger.info(
                f"Initial sync complete: {products_synced}/{total_products} products synced, "
                f"{total_new_tests} new tests discovered, {total_tests_refreshed} tests refreshed, "
                f"{duration} seconds"
            )

            # Log sync event completion
            await self.log_sync_event_complete(
                event_id=event_id,
                products_synced=products_synced,
                tests_discovered=total_new_tests,
                tests_refreshed=total_tests_refreshed,
                duration_seconds=duration,
            )

            return {
                "products_synced": products_synced,
                "total_products": total_products,
                "total_new_tests": total_new_tests,
                "tests_refreshed": total_tests_refreshed,
                "duration_seconds": duration,
            }

        except asyncio.CancelledError:
            # Task was cancelled (server shutdown) - mark as cancelled
            duration = round(time.time() - start_time, 2)
            logger.warning(f"Initial sync cancelled after {duration} seconds (server shutdown)")

            # Mark event as cancelled (not failed)
            await self.log_sync_event_cancelled(event_id=event_id, duration_seconds=duration)

            raise  # Re-raise to allow proper task cleanup

        except Exception as e:
            duration = round(time.time() - start_time, 2)
            logger.error(f"Initial sync failed after {duration} seconds: {e}", exc_info=True)

            # Log sync event failure
            await self.log_sync_event_failed(
                event_id=event_id, error_message=str(e), duration_seconds=duration
            )

            raise
        finally:
            # Always release lock
            lock.release()

    async def run_background_refresh(
        self, interval_seconds: int, since_filter: str | None = None
    ) -> None:
        """Run background refresh task at regular intervals (AC4, AC6).

        This method is designed to run as a long-running background task.
        It periodically performs two operations for all synced products:
        1. Discovers new tests (incremental sync)
        2. Refreshes existing mutable tests (status updates)

        Uses file-based locking to prevent conflicts with manual CLI operations (STORY-021e).

        Args:
            interval_seconds: Seconds between refresh cycles (e.g., 900 for 15 minutes)
            since_filter: Optional date filter (ISO date or relative, e.g., "2023-01-01" or
                         "90 days ago"). Only syncs tests with end_at >= this date.
                         Reduces API load for customers with long test histories.
        """
        logger.info(f"Starting background refresh task (interval: {interval_seconds}s)")

        # Parse date filter once at startup (static during server lifetime)
        since: datetime | None = None
        if since_filter:
            try:
                from testio_mcp.utilities.date_utils import parse_flexible_date

                iso_string = parse_flexible_date(since_filter, start_of_day=True)
                since = datetime.fromisoformat(iso_string.replace("Z", "+00:00"))
                logger.info(
                    f"Background refresh using date filter: {since_filter} "
                    f"(parsed as {since.isoformat()})"
                )
            except Exception as e:
                logger.warning(
                    f"Failed to parse since_filter='{since_filter}': {e}. "
                    f"Background refresh will sync all tests without date filter."
                )
                since = None

        while True:
            try:
                await asyncio.sleep(interval_seconds)

                logger.info("Starting background refresh cycle")

                # Acquire lock before writes (STORY-021e)
                try:
                    lock = self._acquire_sync_lock()
                except RuntimeError as e:
                    logger.warning(f"Skipping refresh cycle: {e}")
                    continue

                # Log sync event start
                import time

                event_id = await self.log_sync_event_start(
                    event_type="background_refresh", trigger_source="background_task"
                )
                start_time = time.time()

                try:
                    # Get all synced products
                    products_info = await self.get_synced_products_info()

                    if not products_info:
                        logger.info("No synced products found, skipping refresh cycle")
                        # Log as completed with 0 refreshes
                        duration = round(time.time() - start_time, 2)
                        await self.log_sync_event_complete(
                            event_id=event_id,
                            products_synced=0,
                            tests_discovered=0,
                            tests_refreshed=0,
                            duration_seconds=duration,
                        )
                        continue

                    logger.info(f"Background refresh for {len(products_info)} products")

                    # Phase 1: Discover new tests (incremental sync)
                    # Use TESTIO_SYNC_SINCE date filter if configured
                    total_new_tests = 0
                    for product in products_info:
                        product_id = product["id"]
                        product_name = product.get("name", "Unknown")
                        try:
                            sync_result = await self.sync_product_tests(
                                product_id, product_data=None, since=since
                            )
                            total_new_tests += sync_result.new_tests_count
                            logger.debug(
                                f"Product {product_id} ({product_name}): "
                                f"{sync_result.new_tests_count} new tests discovered"
                            )
                        except Exception as e:
                            logger.error(
                                f"Failed to sync product {product_id} ({product_name}): {e}"
                            )
                            # Continue with next product

                    # Phase 2: Refresh mutable tests
                    total_refreshed = 0
                    for product in products_info:
                        product_id = product["id"]
                        try:
                            refresh_result = await self.refresh_active_tests(product_id)
                            total_refreshed += refresh_result["tests_updated"]
                            logger.debug(
                                f"Refreshed product {product_id}: "
                                f"{refresh_result['tests_updated']} tests updated"
                            )
                        except Exception as e:
                            logger.error(f"Failed to refresh product {product_id}: {e}")
                            # Continue with next product

                    logger.info(
                        f"Background refresh cycle complete: "
                        f"{total_new_tests} new tests discovered, "
                        f"{total_refreshed} tests refreshed across "
                        f"{len(products_info)} products"
                    )

                    # Log sync event completion
                    duration = round(time.time() - start_time, 2)
                    await self.log_sync_event_complete(
                        event_id=event_id,
                        products_synced=len(products_info),
                        tests_discovered=total_new_tests,
                        tests_refreshed=total_refreshed,
                        duration_seconds=duration,
                    )

                except asyncio.CancelledError:
                    # Task was cancelled (server shutdown) - mark as cancelled
                    duration = round(time.time() - start_time, 2)
                    logger.warning(
                        f"Background refresh cycle cancelled after {duration} seconds "
                        f"(server shutdown)"
                    )
                    await self.log_sync_event_cancelled(
                        event_id=event_id, duration_seconds=duration
                    )
                    raise  # Re-raise to outer handler to break the loop

                except Exception as e:
                    # Log sync event failure
                    duration = round(time.time() - start_time, 2)
                    await self.log_sync_event_failed(
                        event_id=event_id, error_message=str(e), duration_seconds=duration
                    )
                    raise  # Re-raise to be caught by outer exception handler

                finally:
                    # Always release lock
                    lock.release()

            except asyncio.CancelledError:
                logger.info("Background refresh task cancelled, shutting down")
                break
            except Exception as e:
                logger.error(f"Error in background refresh cycle: {e}", exc_info=True)
                # Continue running despite errors

    async def _fetch_page_with_recovery(
        self,
        product_id: int,
        page: int,
        page_size: int,
        *,
        since: datetime | None = None,
        progress_callback: Any = None,
        current_count: int = 0,
    ) -> tuple[dict[str, Any], int]:
        """Fetch page with multi-pass recovery for 500 errors.

        Algorithm (BLOCKER 2 fix - full range coverage):
        - Try with page_size=25
        - If 500 error, retry with page_size=10 across ALL chunks needed
        - Example: page=2, size=25 (items 26-50) fails:
          → Retry with size=10: fetch 3 chunks (items 21-30, 31-40, 41-50)
          → If any chunk fails, move to size=5 and retry ALL sub-chunks
        - Continue reducing size (5 → 2 → 1) until successful or exhausted

        This ensures complete range coverage - no data is skipped due to
        partial chunk failures.

        Args:
            product_id: Product identifier
            page: Page number (1-indexed)
            page_size: Initial page size (25)
            since: Only fetch tests with end_at >= this date (optional)
            progress_callback: Optional callback for real-time progress updates
            current_count: Current test count (for progress updates)

        Returns:
            Tuple of (page_data, recovery_attempts)
            - page_data: Combined results from all chunks, or None if unrecoverable
            - recovery_attempts: Total number of 500 errors encountered
        """
        recovery_attempts = 0
        sizes_to_try = [25, 10, 5, 2, 1]

        # Calculate absolute offset and range size from original request
        original_offset = (page - 1) * page_size
        original_range_size = page_size

        # Preserve tests across recovery attempts (STORY-021b fix)
        # Use dict to deduplicate by test ID (smaller page sizes may re-fetch same tests)
        combined_results_by_id: dict[int, dict[str, Any]] = {}
        all_chunks_succeeded = True
        last_response_next = None

        for attempt_size in sizes_to_try:
            try:
                # Calculate how many chunks needed to cover original range
                # Example: size=25 failed, retry at size=10 → need 3 chunks (25/10 = 2.5 → ceil = 3)
                num_chunks = math.ceil(original_range_size / attempt_size)

                # DEBUG: Show recovery attempt
                if recovery_attempts > 0:
                    print(
                        f"[RECOVERY ATTEMPT] Trying page_size={attempt_size}, "
                        f"num_chunks={num_chunks}, "
                        f"offset={original_offset}, "
                        f"recovery_attempts={recovery_attempts}"
                    )

                # Reset success flag for this attempt size (but preserve combined_results!)
                all_chunks_succeeded = True

                # Calculate starting page that covers original_offset with new attempt_size
                # Example: original_offset=25, attempt_size=10 → start_page = ceil(26/10) = 3
                start_page = (original_offset // attempt_size) + 1

                # Fetch all chunks at this size level
                for chunk_idx in range(num_chunks):
                    chunk_offset = original_offset + (chunk_idx * attempt_size)
                    chunk_page = start_page + chunk_idx

                    try:
                        # Build query parameters
                        params: dict[str, Any] = {
                            "page": chunk_page,
                            "per_page": attempt_size,  # TestIO API uses "per_page" not "page_size"
                        }

                        # Note: API doesn't support ordering or date filtering
                        # Results come pre-sorted by end_at DESC (API default)

                        response = await self.client.get(
                            f"products/{product_id}/exploratory_tests",
                            params=params,
                        )

                        # Calculate what offset range this page actually covers
                        page_start_offset = (chunk_page - 1) * attempt_size
                        page_end_offset = page_start_offset + attempt_size - 1

                        # DEBUG: Log API response structure for recovery debugging
                        logger.debug(
                            f"API recovery chunk {chunk_idx + 1}/{num_chunks}: "
                            f"Requested per_page={attempt_size}, page={chunk_page}, "
                            f"offset_range=[{page_start_offset}-{page_end_offset}], "
                            f"Got {len(response.get('exploratory_tests', []))} tests "
                            f"(need offset [{original_offset}-"
                            f"{original_offset + original_range_size - 1}])"
                        )

                        # Debug: Log response structure
                        # Note: API does not provide pagination metadata (next/count fields)
                        # See docs/architecture/API_CAPABILITIES.md for details
                        logger.info(
                            f"API response keys: {list(response.keys())}, "
                            f"exploratory_tests count: "
                            f"{len(response.get('exploratory_tests', []))}"
                        )

                        # Collect results from this chunk (deduplicate by ID)
                        chunk_tests = response.get("exploratory_tests", [])
                        for test in chunk_tests:
                            test_id = test.get("id")
                            if test_id is not None:
                                combined_results_by_id[test_id] = test

                        # Track "next" from last successful response
                        last_response_next = response.get("next")

                        # Update progress after each successful chunk (real-time during recovery)
                        if progress_callback and recovery_attempts > 0:
                            progress_callback(current_count + len(combined_results_by_id))

                    except Exception as e:
                        error_str = str(e)
                        if "500" in error_str or "Internal Server Error" in error_str:
                            recovery_attempts += 1
                            logger.warning(
                                f"API 500 error at offset {chunk_offset} "
                                f"(page={chunk_page}, size={attempt_size}, "
                                f"chunk {chunk_idx + 1}/{num_chunks}), "
                                f"attempt {recovery_attempts}"
                            )
                            all_chunks_succeeded = False

                            # Special case: At page_size=1, we can't go smaller
                            # Just skip this single test and continue to next chunk
                            if attempt_size == 1:
                                print(
                                    f"[SKIP] Offset {chunk_offset} failed at page_size=1, "
                                    f"skipping and continuing to next offset"
                                )
                                continue  # Try next chunk instead of breaking

                            # Update original_offset to start from the actual
                            # range this failed page covers
                            # The page covers: (chunk_page - 1) * attempt_size
                            # to chunk_page * attempt_size - 1
                            failed_page_start = (chunk_page - 1) * attempt_size

                            # Calculate the end of the range we're currently trying to cover
                            current_range_end = original_offset + original_range_size - 1

                            # Next recovery should start from where the failed page starts
                            # and cover up to the end of our current range
                            original_offset = failed_page_start
                            original_range_size = current_range_end - failed_page_start + 1

                            print(
                                f"[RECOVERY UPDATE] Failed page covered "
                                f"[{failed_page_start}-{failed_page_start + attempt_size - 1}], "
                                f"combined_results={len(combined_results_by_id)} tests, "
                                f"next attempt will cover [{original_offset}-{current_range_end}]"
                            )

                            break  # Stop trying chunks at this size, move to smaller size
                        else:
                            # Non-500 error, re-raise
                            raise

                # If all chunks succeeded, return combined results
                if all_chunks_succeeded:
                    # Convert dict back to list (preserve API sort order: newest first by end_at)
                    combined_results = list(combined_results_by_id.values())

                    if recovery_attempts > 0:
                        logger.info(
                            f"Recovery successful: fetched {len(combined_results)} unique tests "
                            f"across {num_chunks} chunks at size={attempt_size}, "
                            f"attempts={recovery_attempts}"
                        )

                        # Always emit DATE RANGE logs for recovery results (even if empty)
                        print(
                            f"[RECOVERY] Page {page}: Recovered {len(combined_results)} tests "
                            f"using {num_chunks} chunks at page_size={attempt_size}"
                        )

                        if combined_results:
                            first_end_at = combined_results[0].get("end_at", "N/A")
                            last_end_at = combined_results[-1].get("end_at", "N/A")
                            logger.debug(
                                f"Recovery page {page} date range: "
                                f"newest={first_end_at}, oldest={last_end_at}, "
                                f"count={len(combined_results)}"
                            )
                        else:
                            # Log for empty recoveries (AC1, AC3)
                            logger.debug(
                                f"Recovery page {page}: 0 tests recovered (empty recovery)"
                            )
                            logger.info(f"Recovery for page {page}: 0 tests recovered")

                    # Build response matching API format
                    # Use "next" from last successful chunk (for pagination continuity)
                    return {
                        "exploratory_tests": combined_results,
                        "next": last_response_next,
                    }, recovery_attempts

                # Some chunk failed, continue to next smaller size

            except Exception as e:
                # Non-500 error during setup, re-raise
                error_str = str(e)
                if "500" not in error_str and "Internal Server Error" not in error_str:
                    raise

        # All recovery attempts failed
        # Convert dict to list for final logging
        combined_results = list(combined_results_by_id.values())

        logger.error(
            f"Failed to fetch offset {original_offset} "
            f"after {recovery_attempts} recovery attempts, giving up on this range"
        )

        # Log what we managed to recover before giving up (STORY-021b)
        if combined_results:
            first_test = combined_results[0]
            last_test = combined_results[-1]
            first_end_at = first_test.get("end_at", "N/A")
            last_end_at = last_test.get("end_at", "N/A")
            first_id = first_test.get("id", "N/A")
            last_id = last_test.get("id", "N/A")

            logger.debug(
                f"Recovery partial page {page} date range: "
                f"Recovered {len(combined_results)} unique tests before final failure, "
                f"newest={first_end_at} (id={first_id}), "
                f"oldest={last_end_at} (id={last_id})"
            )
            logger.warning(
                f"Page {page}: Recovered {len(combined_results)} unique tests "
                f"but failed to complete range [offset {original_offset}], "
                f"boundary tests: first_id={first_id}, last_id={last_id}"
            )
        else:
            # All recovery attempts failed with 0 tests (STORY-021b: log for consistency)
            logger.debug(f"Recovery page {page}: 0 tests recovered (unrecoverable)")
            logger.warning(
                f"Page {page}: 0 tests recovered after {recovery_attempts} recovery attempts"
            )

        # Return partial results even though recovery failed (STORY-021b critical fix)
        # This allows caller to save partial data and set correct boundary_before
        return {
            "exploratory_tests": combined_results,
            "next": None,
            "recovery_failed": True,  # Flag incomplete recovery
        }, recovery_attempts
