"""
Dead Letter Queue (DLQ) for AtomIngest.

Provides file-based persistent storage, retry orchestration, and export
capabilities for rows that fail during ingestion.

Usage::

    from atomingest.pipeline.dlq import DLQManager

    dlq = DLQManager(path="dlq/failures.ndjson", max_retries=3)

    # Capture a failure
    dlq.record_failure(row_data, error, target_table="orders", source="orders.csv", run_id="abc123")

    # Retry all pending records
    results = dlq.retry_all(processor=my_row_processor)
    print(results)  # {"resolved": 5, "exhausted": 1}

    # Export for manual review
    dlq.export_csv("dlq_export.csv")
    dlq.export_json("dlq_export.json", status=DLQStatus.EXHAUSTED)

    # Dashboard
    print(dlq.summary())  # {"pending": 2, "retrying": 0, "resolved": 5, "exhausted": 1}
"""

import csv
import json
import logging
import uuid
from dataclasses import asdict, dataclass
from datetime import datetime
from enum import Enum
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional

logger = logging.getLogger(__name__)


class DLQStatus(str, Enum):
    """Lifecycle state of a DLQ record."""

    PENDING = "pending"  # Failed, awaiting retry
    RETRYING = "retrying"  # Currently being retried
    RESOLVED = "resolved"  # Successfully reprocessed
    EXHAUSTED = "exhausted"  # Max retries reached


@dataclass
class DLQRecord:
    """
    A single failed row captured by the Dead Letter Queue.

    Tracks the original data, failure context, and full retry history.
    Serialises to/from plain dicts for NDJSON file storage.
    """

    id: str
    row_data: Dict[str, Any]
    error_message: str
    error_type: str
    target_table: str
    source: str
    run_id: str
    status: DLQStatus
    retry_count: int
    max_retries: int
    created_at: str  # ISO-8601 timestamp
    last_attempted_at: Optional[str]
    resolved_at: Optional[str]
    retry_errors: List[str]  # Ordered history of retry error messages

    @classmethod
    def create(
        cls,
        row_data: Dict[str, Any],
        error: Exception,
        target_table: str,
        source: str = "unknown",
        run_id: str = "unknown",
        max_retries: int = 3,
    ) -> "DLQRecord":
        """Factory: build a fresh PENDING record from a caught exception."""
        return cls(
            id=str(uuid.uuid4()),
            row_data=row_data,
            error_message=str(error),
            error_type=type(error).__name__,
            target_table=target_table,
            source=source,
            run_id=run_id,
            status=DLQStatus.PENDING,
            retry_count=0,
            max_retries=max_retries,
            created_at=datetime.now().isoformat(),
            last_attempted_at=None,
            resolved_at=None,
            retry_errors=[],
        )

    def to_dict(self) -> Dict[str, Any]:
        """Serialise to a plain dict (status as string for JSON compat)."""
        d = asdict(self)
        d["status"] = self.status.value
        return d

    @classmethod
    def from_dict(cls, data: Dict[str, Any]) -> "DLQRecord":
        """Deserialise from a plain dict loaded from JSON."""
        data = dict(data)
        data["status"] = DLQStatus(data["status"])
        return cls(**data)

    @property
    def is_retryable(self) -> bool:
        """True when the record can still be retried."""
        return self.status == DLQStatus.PENDING and self.retry_count < self.max_retries


class DLQStore:
    """
    File-based persistent store for DLQ records.

    Stores records as newline-delimited JSON (NDJSON) for efficient appending.
    Updates and deletes rewrite the file atomically via a temp-swap pattern.

    Args:
        path: Path to the ``.ndjson`` store file. Parent directories are
              created automatically.
    """

    def __init__(self, path: Path):
        self.path = Path(path)
        self.path.parent.mkdir(parents=True, exist_ok=True)
        if not self.path.exists():
            self.path.touch()

    # ------------------------------------------------------------------
    # Write operations
    # ------------------------------------------------------------------

    def append(self, record: DLQRecord) -> None:
        """Append a new record to the store (O(1) write)."""
        with self.path.open("a", encoding="utf-8") as f:
            f.write(json.dumps(record.to_dict()) + "\n")
        logger.debug(
            f"DLQ: persisted record {record.id} for table '{record.target_table}'"
        )

    def update(self, record: DLQRecord) -> None:
        """Update an existing record in place (rewrites the file)."""
        records = self.all()
        updated = [record if r.id == record.id else r for r in records]
        self._rewrite(updated)

    def _rewrite(self, records: List[DLQRecord]) -> None:
        """Overwrite the store file with the provided records."""
        tmp = self.path.with_suffix(".tmp")
        with tmp.open("w", encoding="utf-8") as f:
            for r in records:
                f.write(json.dumps(r.to_dict()) + "\n")
        tmp.replace(self.path)

    # ------------------------------------------------------------------
    # Read operations
    # ------------------------------------------------------------------

    def all(self) -> List[DLQRecord]:
        """Load and return every record from the store."""
        records: List[DLQRecord] = []
        with self.path.open("r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if line:
                    records.append(DLQRecord.from_dict(json.loads(line)))
        return records

    def filter(
        self,
        status: Optional[DLQStatus] = None,
        table: Optional[str] = None,
    ) -> List[DLQRecord]:
        """Return records matching the given filters."""
        records = self.all()
        if status is not None:
            records = [r for r in records if r.status == status]
        if table is not None:
            records = [r for r in records if r.target_table == table]
        return records

    def get(self, record_id: str) -> Optional[DLQRecord]:
        """Look up a single record by ID. Returns None if not found."""
        for record in self.all():
            if record.id == record_id:
                return record
        return None

    def count(self, status: Optional[DLQStatus] = None) -> int:
        """Return the number of records, optionally filtered by status."""
        return len(self.filter(status=status))

    def summary(self) -> Dict[str, int]:
        """Return record counts grouped by status."""
        counts: Dict[str, int] = {s.value: 0 for s in DLQStatus}
        for r in self.all():
            counts[r.status.value] += 1
        return counts


class DLQRetryRunner:
    """
    Replays PENDING DLQ records through a caller-supplied row processor.

    The *processor* is any callable that accepts a row dict and either
    succeeds silently or raises an exception on failure.  Typically this
    is a thin wrapper around your ``Ingester`` that processes one row.

    Args:
        store: The :class:`DLQStore` to read from and update.
    """

    def __init__(self, store: DLQStore):
        self.store = store

    def retry_pending(
        self,
        processor: Callable[[Dict[str, Any]], None],
        table: Optional[str] = None,
    ) -> Dict[str, int]:
        """
        Attempt to reprocess all PENDING records.

        Args:
            processor: Callable ``(row_data: dict) -> None`` that raises on
                       failure.
            table: When provided, only retry records for that target table.

        Returns:
            ``{"resolved": int, "exhausted": int}`` counts for the run.
        """
        pending = self.store.filter(status=DLQStatus.PENDING, table=table)
        resolved = 0
        exhausted = 0

        for record in pending:
            if not record.is_retryable:
                continue

            # Mark as in-flight before attempting
            record.status = DLQStatus.RETRYING
            record.last_attempted_at = datetime.now().isoformat()
            self.store.update(record)

            try:
                processor(record.row_data)
                record.status = DLQStatus.RESOLVED
                record.resolved_at = datetime.now().isoformat()
                resolved += 1
                logger.info(f"DLQ: resolved record {record.id}")
            except Exception as exc:
                record.retry_count += 1
                record.retry_errors.append(str(exc))
                if record.retry_count >= record.max_retries:
                    record.status = DLQStatus.EXHAUSTED
                    exhausted += 1
                    logger.error(
                        f"DLQ: record {record.id} exhausted after "
                        f"{record.retry_count} retries"
                    )
                else:
                    record.status = DLQStatus.PENDING
                    logger.warning(
                        f"DLQ: record {record.id} retry "
                        f"{record.retry_count}/{record.max_retries} failed: {exc}"
                    )

            self.store.update(record)

        logger.info(
            f"DLQ retry run complete — resolved: {resolved}, exhausted: {exhausted}"
        )
        return {"resolved": resolved, "exhausted": exhausted}


class DLQExporter:
    """
    Exports DLQ records to CSV or JSON for manual review or downstream tooling.

    The ``row_data`` and ``retry_errors`` fields are JSON-encoded when writing
    to CSV so that every cell remains a flat string.

    Args:
        store: The :class:`DLQStore` to read from.
    """

    def __init__(self, store: DLQStore):
        self.store = store

    def to_json(
        self,
        output_path: Path,
        status: Optional[DLQStatus] = None,
        table: Optional[str] = None,
        pretty: bool = True,
    ) -> int:
        """
        Export records to a JSON file.

        Args:
            output_path: Destination file path.
            status: Filter by status (``None`` exports all).
            table: Filter by target table (``None`` exports all).
            pretty: Indent the JSON for readability.

        Returns:
            Number of records exported.
        """
        records = self.store.filter(status=status, table=table)
        output_path = Path(output_path)
        indent = 2 if pretty else None
        with output_path.open("w", encoding="utf-8") as f:
            json.dump([r.to_dict() for r in records], f, indent=indent)
        logger.info(f"DLQ: exported {len(records)} records to {output_path}")
        return len(records)

    def to_csv(
        self,
        output_path: Path,
        status: Optional[DLQStatus] = None,
        table: Optional[str] = None,
    ) -> int:
        """
        Export records to a CSV file.

        Args:
            output_path: Destination file path.
            status: Filter by status (``None`` exports all).
            table: Filter by target table (``None`` exports all).

        Returns:
            Number of records exported.
        """
        records = self.store.filter(status=status, table=table)
        if not records:
            logger.info("DLQ: no records to export")
            return 0

        output_path = Path(output_path)
        fieldnames = list(records[0].to_dict().keys())

        with output_path.open("w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(f, fieldnames=fieldnames)
            writer.writeheader()
            for r in records:
                row = r.to_dict()
                row["row_data"] = json.dumps(row["row_data"])
                row["retry_errors"] = json.dumps(row["retry_errors"])
                writer.writerow(row)

        logger.info(f"DLQ: exported {len(records)} records to {output_path}")
        return len(records)


class DLQManager:
    """
    Facade for the full Dead Letter Queue system.

    Provides a single, stable entry point for capturing failures, retrying
    records, and exporting data.  Compose it with the pipeline by passing
    the instance into ``HookContext.metadata["dlq_manager"]`` so that
    :class:`~atomingest.pipeline.hooks.DeadLetterQueueHook` can pick it up.

    Args:
        path: Path to the NDJSON store file.  Defaults to
              ``dlq/failures.ndjson`` relative to the working directory.
        max_retries: How many retry attempts before a record is marked
                     ``EXHAUSTED``.

    Example::

        dlq = DLQManager(path="dlq/failures.ndjson", max_retries=5)

        # Record a failure
        dlq.record_failure(row_data, error, target_table="orders",
                           source="orders.csv", run_id="run-42")

        # Retry all pending
        results = dlq.retry_all(processor=my_row_processor)

        # Inspect
        print(dlq.summary())

        # Export
        dlq.export_csv("review.csv", status=DLQStatus.EXHAUSTED)
    """

    def __init__(
        self,
        path: Path = Path("dlq/failures.ndjson"),
        max_retries: int = 3,
    ):
        self.store = DLQStore(Path(path))
        self.retry_runner = DLQRetryRunner(self.store)
        self.exporter = DLQExporter(self.store)
        self.max_retries = max_retries

    # ------------------------------------------------------------------
    # Core API
    # ------------------------------------------------------------------

    def record_failure(
        self,
        row_data: Dict[str, Any],
        error: Exception,
        target_table: str,
        source: str = "unknown",
        run_id: str = "unknown",
    ) -> DLQRecord:
        """
        Capture a failed row and persist it to the DLQ store.

        Returns:
            The persisted :class:`DLQRecord`.
        """
        record = DLQRecord.create(
            row_data=row_data,
            error=error,
            target_table=target_table,
            source=source,
            run_id=run_id,
            max_retries=self.max_retries,
        )
        self.store.append(record)
        return record

    def retry_all(
        self,
        processor: Callable[[Dict[str, Any]], None],
        table: Optional[str] = None,
    ) -> Dict[str, int]:
        """
        Retry all PENDING records.

        Args:
            processor: ``(row_data: dict) -> None``; raises on failure.
            table: Limit retries to a specific target table.

        Returns:
            ``{"resolved": int, "exhausted": int}``
        """
        return self.retry_runner.retry_pending(processor=processor, table=table)

    # ------------------------------------------------------------------
    # Export
    # ------------------------------------------------------------------

    def export_json(
        self,
        output_path: Path,
        status: Optional[DLQStatus] = None,
        table: Optional[str] = None,
    ) -> int:
        """Export records to JSON. Returns number of records written."""
        return self.exporter.to_json(output_path, status=status, table=table)

    def export_csv(
        self,
        output_path: Path,
        status: Optional[DLQStatus] = None,
        table: Optional[str] = None,
    ) -> int:
        """Export records to CSV. Returns number of records written."""
        return self.exporter.to_csv(output_path, status=status, table=table)

    # ------------------------------------------------------------------
    # Inspection
    # ------------------------------------------------------------------

    def summary(self) -> Dict[str, int]:
        """Return record counts grouped by status."""
        return self.store.summary()

    def pending_count(self) -> int:
        """Number of records awaiting retry."""
        return self.store.count(status=DLQStatus.PENDING)

    def exhausted_count(self) -> int:
        """Number of records that have exhausted all retry attempts."""
        return self.store.count(status=DLQStatus.EXHAUSTED)
