"""Custom DataFrame wrapper for local dataframe workflows."""

from __future__ import annotations

import os
import json
import re
import logging
import webbrowser
import unittest
import copy
import tempfile
import sqlite3
from html import escape as html_escape
import random
import uuid
from pathlib import Path
from datetime import datetime, timezone
from typing import Any, Iterable, Mapping, Optional, Sequence, Tuple, Union

import pandas as pd

try:
    from sqlalchemy import create_engine
except ImportError:  # pragma: no cover - optional dependency in some environments
    create_engine = None

__all__ = [
    "DataTable",
    "datatable",
    "DatabaseConnector",
    "db_connector",
    "from_sql",
    "from_mysql_db",
    "from_maria_db",
    "from_oracle_db",
    "from_postgres_db",
    "from_sqlite_db",
    "from_google_sheet",
    "to_google_sheet",
    "from_faker",
    "from_factory",
    "df",
]


def _default_sqlite_connection_string() -> str:
    for key in ("SOIGIA_DB_PATH", "SOIGIA_SQLITE_PATH", "SOIGIA_SQLITE_URL", "SOIGIA_DB_URL"):
        value = os.getenv(key)
        if not value:
            continue
        if value.startswith("sqlite:///"):
            path = Path(value[len("sqlite:///") :]).expanduser().resolve()
        elif value.startswith("sqlite://"):
            path = Path(value[len("sqlite://") :]).expanduser().resolve()
        elif value == "sqlite:///:memory:":
            return value
        else:
            path = Path(value).expanduser().resolve()
        path.parent.mkdir(parents=True, exist_ok=True)
        return f"sqlite:///{path.as_posix()}"

    path = (Path.cwd() / "data" / "soigia.sqlite3").resolve()
    path.parent.mkdir(parents=True, exist_ok=True)
    return f"sqlite:///{path.as_posix()}"


class _ValuesResult(list):
    """List-like container that exposes Django-style helpers."""

    def first(self):
        return self[0] if self else None

    def last(self):
        return self[-1] if self else None

    def filter(self, predicate=None, **conditions):
        """Return filtered records by predicate or exact field matches."""
        if predicate is not None and not callable(predicate):
            raise TypeError("predicate must be callable")
        result = _ValuesResult()
        for row in self:
            if predicate is not None:
                if predicate(row):
                    result.append(row)
                continue
            match = True
            for key, expected in conditions.items():
                if row.get(key) != expected:
                    match = False
                    break
            if match:
                result.append(row)
        return result

    def exclude(self, predicate=None, **conditions):
        """Return records excluded by predicate or exact field matches."""
        if predicate is not None and not callable(predicate):
            raise TypeError("predicate must be callable")
        result = _ValuesResult()
        for row in self:
            if predicate is not None:
                if not predicate(row):
                    result.append(row)
                continue
            match = True
            for key, expected in conditions.items():
                if row.get(key) != expected:
                    match = False
                    break
            if not match:
                result.append(row)
        return result

    def order_by(self, key, reverse: bool = False):
        """Return records ordered by a key or callable."""
        if not callable(key) and not isinstance(key, str):
            raise TypeError("key must be a field name or callable")
        return _ValuesResult(sorted(self, key=(key if callable(key) else lambda row: row.get(key)), reverse=reverse))

    def distinct(self, *fields: str):
        """Return unique records, optionally keyed by selected fields."""
        result = _ValuesResult()
        seen = set()
        for row in self:
            if fields and isinstance(row, dict):
                signature = tuple(row.get(field) for field in fields)
            elif isinstance(row, dict):
                signature = tuple(sorted(row.items()))
            elif isinstance(row, (tuple, list)):
                signature = tuple(row)
            else:
                signature = row
            if signature in seen:
                continue
            seen.add(signature)
            result.append(row)
        return result


class _NoLogsContext:
    """Fallback context manager for assertNoLogs on older Python versions."""

    def __init__(self, logger=None, level=logging.INFO):
        self.logger = logging.getLogger(logger) if isinstance(logger, str) else (logger or logging.getLogger())
        self.level = logging._nameToLevel.get(level, level) if isinstance(level, str) else level
        self.records = []
        self._handler = logging.Handler(level=self.level)
        self._handler.emit = self._emit  # type: ignore[assignment]
        self._old_level = None

    def _emit(self, record):
        if record.levelno >= self.level:
            self.records.append(record)

    def __enter__(self):
        self._old_level = self.logger.level
        self.logger.addHandler(self._handler)
        return self

    def __exit__(self, exc_type, exc, tb):
        self.logger.removeHandler(self._handler)
        if self._old_level is not None:
            self.logger.setLevel(self._old_level)
        if exc_type is None and self.records:
            raise AssertionError("Unexpected logs were emitted")
        return False


class _MiniFaker:
    """Small fallback faker used when the optional faker package is unavailable."""

    _first_names = ["Alice", "Bob", "Carol", "Dave", "Eve", "Frank", "Grace", "Huy", "Linh", "Minh"]
    _last_names = ["Nguyen", "Tran", "Le", "Pham", "Hoang", "Vo", "Huynh", "Pham"]
    _cities = ["Hanoi", "Saigon", "Danang", "Hue", "Can Tho", "Nha Trang"]
    _companies = ["Acme", "Nimbus", "SoiGia", "Atlas", "Vertex", "Orchid"]
    _words = ["alpha", "beta", "gamma", "delta", "omega", "signal", "market", "trade", "data", "frame"]

    def __init__(self, seed: Optional[int] = None):
        self._rng = random.Random(seed)

    def seed_instance(self, seed: int) -> None:
        self._rng.seed(seed)

    def _choice(self, values: Sequence[str]) -> str:
        return self._rng.choice(list(values))

    def name(self) -> str:
        return f"{self._choice(self._first_names)} {self._choice(self._last_names)}"

    def first_name(self) -> str:
        return self._choice(self._first_names)

    def last_name(self) -> str:
        return self._choice(self._last_names)

    def city(self) -> str:
        return self._choice(self._cities)

    def company(self) -> str:
        return self._choice(self._companies)

    def word(self) -> str:
        return self._choice(self._words)

    def words(self, nb: int = 3) -> list[str]:
        return [self.word() for _ in range(max(1, nb))]

    def sentence(self, nb_words: int = 6) -> str:
        words = self.words(nb_words)
        sentence = " ".join(words).capitalize()
        return f"{sentence}."

    def text(self, max_nb_chars: int = 200) -> str:
        text = self.sentence(max(3, min(12, max_nb_chars // 12 or 3)))
        return text[:max_nb_chars]

    def email(self) -> str:
        user = f"{self.first_name().lower()}.{self.last_name().lower()}".replace(" ", "")
        domain = self._choice(["example.com", "mail.com", "soigia.dev"])
        return f"{user}@{domain}"

    def phone_number(self) -> str:
        return f"+84-{self._rng.randint(100, 999)}-{self._rng.randint(1000, 9999)}"

    def uuid4(self) -> str:
        return str(uuid.UUID(int=self._rng.getrandbits(128)))

    def random_int(self, min: int = 0, max: int = 9999) -> int:  # noqa: A002
        return self._rng.randint(min, max)

    def pyfloat(self, left_digits: int = 2, right_digits: int = 2, positive: bool = True) -> float:
        left = self._rng.randint(0 if positive else -(10**left_digits), 10**left_digits)
        right = self._rng.randint(0, 10**right_digits - 1)
        value = float(f"{abs(left)}.{right:0{right_digits}d}")
        return abs(value) if positive else value

    def date(self) -> datetime:
        return datetime.fromtimestamp(self._rng.randint(1_600_000_000, 1_800_000_000), tz=timezone.utc)

    def datetime(self) -> datetime:
        return self.date()


class _FallbackRichTable:
    """Small fallback object that mimics the minimal rich.Table API used in tests."""

    def __init__(self, title: Optional[str] = None):
        self.title = title
        self.columns: list[str] = []
        self.rows: list[tuple[str, ...]] = []

    def add_column(self, name: str) -> None:
        self.columns.append(name)

    def add_row(self, *values: str) -> None:
        self.rows.append(tuple(values))


def _build_faker(locale: str = "en_US", seed: Optional[int] = None):
    try:
        from faker import Faker  # type: ignore
    except ImportError:
        return _MiniFaker(seed=seed)

    fake = Faker(locale)
    if seed is not None:
        fake.seed_instance(seed)
    return fake


def _resolve_fake_value(generator: Any, spec: Any) -> Any:
    if callable(spec):
        return spec(generator)
    if isinstance(spec, tuple) and len(spec) == 2 and isinstance(spec[0], str) and isinstance(spec[1], Mapping):
        provider = getattr(generator, spec[0], None)
        if callable(provider):
            return provider(**dict(spec[1]))
    if isinstance(spec, Mapping) and "provider" in spec:
        provider_name = spec["provider"]
        provider = getattr(generator, provider_name, None)
        if callable(provider):
            kwargs = dict(spec.get("kwargs", {}))
            return provider(**kwargs)
    if isinstance(spec, str):
        provider = getattr(generator, spec, None)
        if callable(provider):
            return provider()
    return spec


def _factory_item_to_record(item: Any) -> dict[str, Any]:
    if isinstance(item, Mapping):
        return dict(item)
    if hasattr(item, "_asdict"):
        return dict(item._asdict())
    if hasattr(item, "dict") and callable(item.dict):
        value = item.dict()
        if isinstance(value, Mapping):
            return dict(value)
    if hasattr(item, "__dict__"):
        return {key: value for key, value in vars(item).items() if not key.startswith("_") and not callable(value)}
    raise TypeError("factory item cannot be converted into a record")


def _build_factory_rows(factory_class: Any, count: int, **build_kwargs: Any) -> list[Any]:
    if hasattr(factory_class, "build_batch") and callable(factory_class.build_batch):
        return list(factory_class.build_batch(size=count, **build_kwargs))
    if callable(factory_class):
        return [factory_class(**build_kwargs) for _ in range(count)]
    raise TypeError("factory_class must provide build_batch() or be callable")


class DatabaseConnector:
    """Hold one generic connection string and optional backend-specific URLs."""

    def __init__(
        self,
        connection_string: Optional[str] = None,
        mysql: Optional[str] = None,
        maria: Optional[str] = None,
        mariadb: Optional[str] = None,
        oracle: Optional[str] = None,
        postgres: Optional[str] = None,
        sqlite: Optional[str] = None,
    ):
        self.connection_string = connection_string
        self.mysql = mysql
        self.maria = maria
        self.mariadb = mariadb
        self.oracle = oracle
        self.postgres = postgres
        self.sqlite = sqlite

    def _engine_url(self, backend: str, connection_string: Optional[str] = None) -> str:
        url = (
            connection_string
            or getattr(self, backend, None)
            or self.connection_string
            or os.getenv(f"SOIGIA_{backend.upper()}_URL")
        )
        if not url and backend == "sqlite":
            return _default_sqlite_connection_string()
        if not url:
            raise ValueError(f"Missing connection string for backend: {backend}")
        return url

    def _engine(self, backend: str, connection_string: Optional[str] = None):
        if create_engine is None:
            raise ImportError("sqlalchemy is required to use database helpers")
        return create_engine(self._engine_url(backend, connection_string))

    def _sqlite_connect(self, connection_string: Optional[str] = None) -> sqlite3.Connection:
        url = self._engine_url("sqlite", connection_string)
        if url == "sqlite:///:memory:":
            return sqlite3.connect(":memory:")
        if url.startswith("sqlite:///"):
            return sqlite3.connect(url[len("sqlite:///") :])
        if url.startswith("sqlite://"):
            return sqlite3.connect(url[len("sqlite://") :])
        return sqlite3.connect(url)

    def query(self, sql: str, backend: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        """Run SQL and return a DataTable."""
        self._engine_url(backend, connection_string)
        if backend == "sqlite" and create_engine is None:
            connection = self._sqlite_connect(connection_string)
            try:
                frame = pd.read_sql_query(sql, connection, **kwargs)
            finally:
                connection.close()
            return DataTable(frame)
        engine = self._engine(backend, connection_string)
        try:
            frame = pd.read_sql_query(sql, engine, **kwargs)
        finally:
            dispose = getattr(engine, "dispose", None)
            if callable(dispose):
                dispose()
        return DataTable(frame)

    def from_sql(
        self,
        sql: str,
        backend: str = "sqlite",
        connection_string: Optional[str] = None,
        **kwargs: Any,
    ) -> "DataTable":
        """Run SQL against the selected backend and return a DataTable."""
        return self.query(sql, backend, connection_string=connection_string, **kwargs)

    def from_mysql_db(self, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        return self.query(sql, "mysql", connection_string=connection_string, **kwargs)

    def from_maria_db(self, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        return self.query(sql, "maria", connection_string=connection_string, **kwargs)

    def from_oracle_db(self, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        return self.query(sql, "oracle", connection_string=connection_string, **kwargs)

    def from_postgres_db(self, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        return self.query(sql, "postgres", connection_string=connection_string, **kwargs)

    def from_sqlite_db(self, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        return self.query(sql, "sqlite", connection_string=connection_string, **kwargs)


_DEFAULT_CONNECTOR: Optional[DatabaseConnector] = None


def db_connector(connection_string: Optional[str] = None, **connections: Any) -> DatabaseConnector:
    """Configure and return a reusable database connector."""
    global _DEFAULT_CONNECTOR
    _DEFAULT_CONNECTOR = DatabaseConnector(connection_string=connection_string, **connections)
    return _DEFAULT_CONNECTOR


def _connector() -> DatabaseConnector:
    global _DEFAULT_CONNECTOR
    if _DEFAULT_CONNECTOR is None:
        _DEFAULT_CONNECTOR = DatabaseConnector()
    return _DEFAULT_CONNECTOR


def from_sql(sql: str, backend: str = "sqlite", connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
    return _connector().from_sql(sql, backend=backend, connection_string=connection_string, **kwargs)


def from_mysql_db(sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
    return _connector().from_mysql_db(sql, connection_string=connection_string, **kwargs)


def from_maria_db(sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
    return _connector().from_maria_db(sql, connection_string=connection_string, **kwargs)


def from_oracle_db(sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
    return _connector().from_oracle_db(sql, connection_string=connection_string, **kwargs)


def from_postgres_db(sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
    return _connector().from_postgres_db(sql, connection_string=connection_string, **kwargs)


def from_sqlite_db(sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
    return _connector().from_sqlite_db(sql, connection_string=connection_string, **kwargs)


def _is_google_worksheet(candidate: Any) -> bool:
    return (
        hasattr(candidate, "get_all_records")
        or hasattr(candidate, "get_all_values")
        or hasattr(candidate, "update")
        or hasattr(candidate, "append_rows")
    )


def _build_google_client(
    client: Any = None,
    credentials_json: Optional[str] = None,
    service_account_file: Optional[Union[str, Path]] = None,
    scopes: Optional[Sequence[str]] = None,
):
    if client is not None:
        return client
    try:
        import gspread  # type: ignore
    except ImportError as exc:  # pragma: no cover - optional dependency
        raise ImportError("gspread is required to use Google Sheet helpers") from exc

    if credentials_json or service_account_file:
        try:
            from google.oauth2.service_account import Credentials  # type: ignore
        except ImportError as exc:  # pragma: no cover - optional dependency
            raise ImportError("google-auth is required to use credential-based Google Sheet helpers") from exc

        credential_scopes = list(
            scopes
            or [
                "https://www.googleapis.com/auth/spreadsheets",
                "https://www.googleapis.com/auth/drive",
            ]
        )
        if credentials_json:
            credentials = Credentials.from_service_account_info(json.loads(credentials_json), scopes=credential_scopes)
        else:
            credentials = Credentials.from_service_account_file(str(service_account_file), scopes=credential_scopes)
        return gspread.authorize(credentials)

    if hasattr(gspread, "service_account"):
        return gspread.service_account()
    raise ImportError("gspread is required to use Google Sheet helpers")


def _resolve_google_worksheet(
    spreadsheet: Any,
    worksheet: Any = 0,
    *,
    client: Any = None,
    credentials_json: Optional[str] = None,
    service_account_file: Optional[Union[str, Path]] = None,
    scopes: Optional[Sequence[str]] = None,
):
    if _is_google_worksheet(spreadsheet):
        return spreadsheet
    if _is_google_worksheet(worksheet):
        return worksheet

    resolved_client = _build_google_client(
        client=client,
        credentials_json=credentials_json,
        service_account_file=service_account_file,
        scopes=scopes,
    )

    workbook = spreadsheet
    if isinstance(spreadsheet, (str, Path)):
        if hasattr(resolved_client, "open_by_url") and str(spreadsheet).startswith(("http://", "https://")):
            workbook = resolved_client.open_by_url(str(spreadsheet))
        elif hasattr(resolved_client, "open_by_key"):
            workbook = resolved_client.open_by_key(str(spreadsheet))
        elif hasattr(resolved_client, "open"):
            workbook = resolved_client.open(str(spreadsheet))
        else:  # pragma: no cover - defensive
            workbook = resolved_client
    else:
        workbook = spreadsheet

    if isinstance(worksheet, int):
        if hasattr(workbook, "get_worksheet"):
            resolved = workbook.get_worksheet(worksheet)
            if resolved is not None:
                return resolved
        if hasattr(workbook, "worksheet"):
            return workbook.worksheet(worksheet)
    else:
        if hasattr(workbook, "worksheet"):
            return workbook.worksheet(str(worksheet))

    raise TypeError("Could not resolve a Google Sheet worksheet from the provided inputs")


def _google_worksheet_to_frame(worksheet: Any, header: int = 0, range_name: Optional[str] = None) -> "DataTable":
    if range_name is not None and hasattr(worksheet, "get"):
        values = worksheet.get(range_name)
        if not values:
            return DataTable()
        headers = values[header]
        rows = values[header + 1 :]
        return DataTable(rows, columns=headers)

    if hasattr(worksheet, "get_all_records"):
        records = worksheet.get_all_records()
        return DataTable(records)

    if hasattr(worksheet, "get_all_values"):
        values = worksheet.get_all_values()
        if not values:
            return DataTable()
        headers = values[header]
        rows = values[header + 1 :]
        return DataTable(rows, columns=headers)

    raise TypeError("worksheet object must provide get_all_records() or get_all_values()")


def _frame_to_google_values(frame: pd.DataFrame, include_index: bool = False) -> list[list[Any]]:
    data = frame.reset_index() if include_index else frame
    clean = data.astype(object).where(pd.notna(data), None)
    return [list(clean.columns)] + clean.values.tolist()


def from_google_sheet(
    spreadsheet: Any,
    worksheet: Any = 0,
    header: int = 0,
    range_name: Optional[str] = None,
    client: Any = None,
    credentials_json: Optional[str] = None,
    service_account_file: Optional[Union[str, Path]] = None,
    scopes: Optional[Sequence[str]] = None,
) -> "DataTable":
    return DataTable.from_google_sheet(
        spreadsheet,
        worksheet=worksheet,
        header=header,
        range_name=range_name,
        client=client,
        credentials_json=credentials_json,
        service_account_file=service_account_file,
        scopes=scopes,
    )


def to_google_sheet(
    table: "DataTable",
    spreadsheet: Any,
    worksheet: Any = 0,
    clear: bool = True,
    include_index: bool = False,
    start_cell: str = "A1",
    value_input_option: str = "RAW",
    client: Any = None,
    credentials_json: Optional[str] = None,
    service_account_file: Optional[Union[str, Path]] = None,
    scopes: Optional[Sequence[str]] = None,
) -> Any:
    return table.to_google_sheet(
        spreadsheet,
        worksheet=worksheet,
        clear=clear,
        include_index=include_index,
        start_cell=start_cell,
        value_input_option=value_input_option,
        client=client,
        credentials_json=credentials_json,
        service_account_file=service_account_file,
        scopes=scopes,
    )


def from_faker(
    schema: Mapping[str, Any],
    count: int = 10,
    locale: str = "en_US",
    seed: Optional[int] = None,
) -> "DataTable":
    return DataTable.from_faker(schema=schema, count=count, locale=locale, seed=seed)


def from_factory(factory_class: Any, count: int = 10, **build_kwargs: Any) -> "DataTable":
    return DataTable.from_factory(factory_class, count=count, **build_kwargs)


class _HelperTestCase(unittest.TestCase):
    """Concrete unittest case used only as an assertion delegate."""

    def runTest(self):
        return None


class _TestCaseMixin:
    """Mixin that exposes unittest.TestCase-style helpers on a dataframe."""

    _class_cleanups: list[tuple[Any, tuple[Any, ...], dict[str, Any]]] = []

    def _testcase(self) -> unittest.TestCase:
        helper = self.__dict__.get("_unittest_helper")
        if helper is None:
            helper = _HelperTestCase("runTest")
            object.__setattr__(self, "_unittest_helper", helper)
        return helper

    @classmethod
    def _class_cleanup_stack(cls) -> list[tuple[Any, tuple[Any, ...], dict[str, Any]]]:
        stack = cls.__dict__.get("_class_cleanups")
        if stack is None:
            stack = []
            setattr(cls, "_class_cleanups", stack)
        return stack

    @classmethod
    def setUpClass(cls):
        """Hook kept for Django/TestCase-like symmetry."""
        return None

    @classmethod
    def tearDownClass(cls):
        """Run any class cleanups that were registered."""
        cls.doClassCleanups()

    @classmethod
    def addClassCleanup(cls, function, /, *args, **kwargs):
        cls._class_cleanup_stack().append((function, args, kwargs))

    @classmethod
    def doClassCleanups(cls):
        errors = []
        stack = cls._class_cleanup_stack()
        while stack:
            function, args, kwargs = stack.pop()
            try:
                function(*args, **kwargs)
            except Exception as exc:  # pragma: no cover - defensive
                errors.append(exc)
        if errors:
            raise errors[0]

    @classmethod
    def enterClassContext(cls, cm):
        result = cm.__enter__()
        cls.addClassCleanup(cm.__exit__, None, None, None)
        return result

    def setUp(self):
        """Hook kept for Django/TestCase-like symmetry."""
        return None

    def tearDown(self):
        """Hook kept for Django/TestCase-like symmetry."""
        return None

    def assertIsSubclass(self, cls, class_or_tuple, msg=None):
        helper = self._testcase()
        if not isinstance(cls, type):
            helper.fail(helper._formatMessage(msg, f"{cls!r} is not a class"))
        if not issubclass(cls, class_or_tuple):
            helper.fail(helper._formatMessage(msg, f"{cls!r} is not a subclass of {class_or_tuple!r}"))

    def assertNotIsSubclass(self, cls, class_or_tuple, msg=None):
        helper = self._testcase()
        if not isinstance(cls, type):
            helper.fail(helper._formatMessage(msg, f"{cls!r} is not a class"))
        if issubclass(cls, class_or_tuple):
            helper.fail(helper._formatMessage(msg, f"{cls!r} is a subclass of {class_or_tuple!r}"))

    def assertNoLogs(self, logger=None, level=logging.INFO):
        helper = self._testcase()
        if hasattr(helper, "assertNoLogs"):
            return helper.assertNoLogs(logger, level=level)
        return _NoLogsContext(logger, level)


def _delegate_testcase_method(name: str):
    def method(self, *args, **kwargs):
        return getattr(self._testcase(), name)(*args, **kwargs)

    method.__name__ = name
    method.__qualname__ = f"_TestCaseMixin.{name}"
    return method


for _method_name in [
    "assertEqual",
    "assertNotEqual",
    "assertTrue",
    "assertFalse",
    "assertIs",
    "assertIsNot",
    "assertIsNone",
    "assertIsNotNone",
    "assertIn",
    "assertNotIn",
    "assertIsInstance",
    "assertNotIsInstance",
    "assertAlmostEqual",
    "assertNotAlmostEqual",
    "assertGreater",
    "assertGreaterEqual",
    "assertLess",
    "assertLessEqual",
    "assertRaises",
    "assertRaisesRegex",
    "assertWarns",
    "assertWarnsRegex",
    "assertLogs",
    "assertMultiLineEqual",
    "assertSequenceEqual",
    "assertListEqual",
    "assertTupleEqual",
    "assertSetEqual",
    "assertDictEqual",
    "assertCountEqual",
    "assertRegex",
    "assertNotRegex",
    "fail",
    "addTypeEqualityFunc",
    "addCleanup",
    "doCleanups",
    "enterContext",
    "run",
    "debug",
    "defaultTestResult",
    "countTestCases",
    "id",
    "shortDescription",
    "subTest",
]:
    setattr(_TestCaseMixin, _method_name, _delegate_testcase_method(_method_name))


class _DjangoStyleManager:
    """A minimal manager facade that reads like Django."""

    def __init__(self, queryset_factory):
        self._queryset_factory = queryset_factory

    def all(self):
        return self._queryset_factory().all()

    def none(self):
        return self._queryset_factory().none()

    def filter(self, *expressions: str, **conditions: Any):
        return self._queryset_factory().filter(*expressions, **conditions)

    def exclude(self, *expressions: str, **conditions: Any):
        return self._queryset_factory().exclude(*expressions, **conditions)

    def order_by(self, *fields: str, **kwargs: Any):
        return self._queryset_factory().order_by(*fields, **kwargs)

    def random_order(self):
        return self._queryset_factory().random_order()

    def values(self, *fields: str):
        return self._queryset_factory().values(*fields)

    def values_list(self, *fields: str, flat: bool = False):
        return self._queryset_factory().values_list(*fields, flat=flat)

    def select(self, *fields: str):
        return self._queryset_factory().select(*fields)

    def exclude_fields(self, *fields: str):
        return self._queryset_factory().exclude_fields(*fields)

    def get(self, *expressions: str, **conditions: Any):
        return self._queryset_factory().get(*expressions, **conditions)

    def count(self) -> int:
        return self._queryset_factory().count()

    def exists(self) -> bool:
        return self._queryset_factory().exists()

    def first(self):
        return self._queryset_factory().first()

    def last(self):
        return self._queryset_factory().last()

    def earliest(self, *fields: str):
        return self._queryset_factory().earliest(*fields)

    def latest(self, *fields: str):
        return self._queryset_factory().latest(*fields)

    def paginate(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().paginate(*args, **kwargs)

    def group_by(self, *fields: str):
        return self._queryset_factory().group_by(*fields)

    def aggregate(self, **aggregations: Any):
        return self._queryset_factory().aggregate(**aggregations)

    def sum(self, *fields: str):
        return self._queryset_factory().sum(*fields)

    def avg(self, *fields: str):
        return self._queryset_factory().avg(*fields)

    def min(self, *fields: str):
        return self._queryset_factory().min(*fields)

    def max(self, *fields: str):
        return self._queryset_factory().max(*fields)

    def diff_columns_against(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_columns_against(*args, **kwargs)

    def diff_added_rows(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_added_rows(*args, **kwargs)

    def diff_removed_rows(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_removed_rows(*args, **kwargs)

    def diff_unchanged_rows(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_unchanged_rows(*args, **kwargs)

    def diff_changed_rows(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_changed_rows(*args, **kwargs)

    def diff_changed_cells(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_changed_cells(*args, **kwargs)

    def diff_report(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_report(*args, **kwargs)

    def create(self, **fields: Any):
        return self._queryset_factory().create(**fields)

    def append_row(self, **fields: Any):
        return self._queryset_factory().append_row(**fields)

    def bulk_create(self, rows: Sequence[Mapping[str, Any]]):
        return self._queryset_factory().bulk_create(rows)

    def update(self, **fields: Any):
        return self._queryset_factory().update(**fields)

    def delete(self):
        return self._queryset_factory().delete()

    def bulk_update(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().bulk_update(*args, **kwargs)

    def save(self):
        return self._queryset_factory().save()

    def get_or_create(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().get_or_create(*args, **kwargs)

    def update_or_create(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().update_or_create(*args, **kwargs)

    def __getattr__(self, name: str):
        return getattr(self._queryset_factory(), name)


class DataTable(_TestCaseMixin, pd.DataFrame):
    """A thin pandas.DataFrame subclass for personal dataframe work."""

    _metadata = ["_soigia_version_history", "_soigia_auto_version", "_soigia_auto_version_label"]

    def __init__(self, *args: Any, **kwargs: Any):
        super().__init__(*args, **kwargs)
        self.columns = [column.lower() if isinstance(column, str) else column for column in self.columns]

    @property
    def _constructor(self):
        return DataTable

    def __finalize__(self, other, method=None, **kwargs):
        result = super().__finalize__(other, method=method, **kwargs)
        if isinstance(other, DataTable) and hasattr(other, "_soigia_version_history"):
            object.__setattr__(
                self, "_soigia_version_history", copy.deepcopy(getattr(other, "_soigia_version_history"))
            )
        if isinstance(other, DataTable):
            if hasattr(other, "_soigia_auto_version"):
                object.__setattr__(self, "_soigia_auto_version", getattr(other, "_soigia_auto_version"))
            if hasattr(other, "_soigia_auto_version_label"):
                object.__setattr__(self, "_soigia_auto_version_label", getattr(other, "_soigia_auto_version_label"))
        return result

    def _inherit_versions(self, frame: pd.DataFrame) -> "DataTable":
        result = DataTable(frame)
        if hasattr(self, "_soigia_version_history"):
            object.__setattr__(result, "_soigia_version_history", copy.deepcopy(self._soigia_version_history))
        if hasattr(self, "_soigia_auto_version"):
            object.__setattr__(result, "_soigia_auto_version", getattr(self, "_soigia_auto_version"))
        if hasattr(self, "_soigia_auto_version_label"):
            object.__setattr__(result, "_soigia_auto_version_label", getattr(self, "_soigia_auto_version_label"))
        if getattr(self, "_soigia_auto_version", False):
            result.mark_version(getattr(self, "_soigia_auto_version_label", "auto"))
        return result

    def _snapshot_payload(self, label: Optional[str] = None) -> dict[str, Any]:
        return {
            "version": self.version_count(),
            "label": label,
            "created_at": datetime.now(timezone.utc).isoformat(),
            "shape": tuple(self.shape),
            "columns": list(self.columns),
            "records": self.to_dict(orient="records"),
        }

    def _resolve_version_index(self, version: int | str) -> int:
        history = self.version_history(include_data=True)
        if isinstance(version, int):
            if version < 0:
                version = len(history) + version
            if version < 0 or version >= len(history):
                raise IndexError("Version index out of range")
            return version
        for index, item in enumerate(history):
            if item.get("label") == version:
                return index
        raise KeyError(f"Unknown version label: {version}")

    @property
    def orm(self) -> "_ORMManager":
        """Return a Django-like query helper bound to this dataframe."""
        return _ORMManager(self)

    @property
    def qs(self) -> "_ORMManager":
        """Alias for orm."""
        return self.orm

    @property
    def objects(self) -> "_ORMManager":
        """Django-style alias for orm."""
        return _DjangoStyleManager(lambda: _ORMManager(self))

    @property
    def manager(self) -> "_ORMManager":
        """Alias for objects."""
        return self.objects

    def filter(self, *expressions: str, **conditions: Any) -> "DataTable":
        """Return a filtered DataTable using row expressions and field lookups."""
        return self.objects.filter(*expressions, **conditions).to_df()

    def exclude(self, *expressions: str, **conditions: Any) -> "DataTable":
        """Return rows excluded by expressions and field lookups."""
        return self.objects.exclude(*expressions, **conditions).to_df()

    def order_by(
        self,
        *fields: str,
        nulls_first: bool = False,
        nulls_last: bool = True,
        nulls: Optional[Mapping[str, str]] = None,
    ) -> "DataTable":
        """Return rows ordered by the given fields."""
        return self.objects.order_by(
            *fields,
            nulls_first=nulls_first,
            nulls_last=nulls_last,
            nulls=nulls,
        ).to_df()

    def random_order(self) -> "DataTable":
        """Return rows in random order."""
        return self.objects.random_order().to_df()

    def get(self, *expressions: str, **conditions: Any) -> "DataTable":
        """Return exactly one matching row as a one-row DataTable."""
        return self.objects.get(*expressions, **conditions)

    def first(self) -> "DataTable":
        """Return the first row as a one-row DataTable."""
        return self.objects.first()

    def last(self) -> "DataTable":
        """Return the last row as a one-row DataTable."""
        return self.objects.last()

    def earliest(self, *fields: str) -> "DataTable":
        """Return the earliest row according to the given ordering fields."""
        return self.objects.earliest(*fields)

    def latest(self, *fields: str) -> "DataTable":
        """Return the latest row according to the given ordering fields."""
        return self.objects.latest(*fields)

    def paginate(self, limit: int, offset: int = 0) -> "DataTable":
        """Return a page of rows as a DataTable."""
        return self.objects.paginate(limit, offset).to_df()

    def count(self) -> int:
        """Return the number of rows."""
        return self.objects.count()

    def exists(self) -> bool:
        """Return whether the table has at least one row."""
        return self.objects.exists()

    @classmethod
    def from_csv(cls, path: Union[str, Path], **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from a CSV file."""
        return cls(pd.read_csv(path, **kwargs))

    @classmethod
    def load_from_csv(cls, path: Union[str, Path], **kwargs: Any) -> "DataTable":
        """Alias for from_csv()."""
        return cls.from_csv(path, **kwargs)

    @classmethod
    def from_dict(cls, data: Optional[dict] = None, **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from a dictionary."""
        return cls(pd.DataFrame.from_dict(data or {}, **kwargs))

    @classmethod
    def from_records(cls, records: Sequence[Mapping[str, Any]], **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from records."""
        return cls(pd.DataFrame.from_records(records, **kwargs))

    @classmethod
    def from_json(cls, path_or_buf: Any, **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from JSON input."""
        return cls(pd.read_json(path_or_buf, **kwargs))

    @classmethod
    def from_excel(cls, io: Any, **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from an Excel workbook or sheet."""
        return cls(pd.read_excel(io, **kwargs))

    @classmethod
    def from_parquet(cls, path: Any, **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from a Parquet file."""
        return cls(pd.read_parquet(path, **kwargs))

    @classmethod
    def load_from_parquet(cls, path: Any, **kwargs: Any) -> "DataTable":
        """Alias for from_parquet()."""
        return cls.from_parquet(path, **kwargs)

    @classmethod
    def from_feather(cls, path: Any, **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from a Feather file."""
        return cls(pd.read_feather(path, **kwargs))

    @classmethod
    def from_pickle(cls, filepath_or_buffer: Any, **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from a pickle file or buffer."""
        return cls(pd.read_pickle(filepath_or_buffer, **kwargs))

    @classmethod
    def from_faker(
        cls,
        schema: Mapping[str, Any],
        count: int = 10,
        locale: str = "en_US",
        seed: Optional[int] = None,
    ) -> "DataTable":
        """Create a DataTable instance from a fake-data schema."""
        generator = _build_faker(locale=locale, seed=seed)
        records = []
        for _ in range(max(0, count)):
            row = {field: _resolve_fake_value(generator, spec) for field, spec in schema.items()}
            records.append(row)
        return cls(records)

    @classmethod
    def from_factory(cls, factory_class: Any, count: int = 10, **build_kwargs: Any) -> "DataTable":
        """Create a DataTable instance from a factory_boy-style factory or callable."""
        rows = _build_factory_rows(factory_class, count, **build_kwargs)
        records = [_factory_item_to_record(item) for item in rows]
        return cls(records)

    @classmethod
    def from_sql(
        cls,
        sql: str,
        backend: str = "sqlite",
        connection_string: Optional[str] = None,
        **kwargs: Any,
    ) -> "DataTable":
        """Run SQL against the selected backend and return a DataTable."""
        return cls(from_sql(sql, backend=backend, connection_string=connection_string, **kwargs))

    @classmethod
    def load_from_db(
        cls,
        sql: str,
        backend: str = "sqlite",
        connection_string: Optional[str] = None,
        **kwargs: Any,
    ) -> "DataTable":
        """Alias for from_sql()."""
        return cls.from_sql(sql, backend=backend, connection_string=connection_string, **kwargs)

    def init_versions(self, label: str = "initial") -> "DataTable":
        """Initialize version history with the current dataframe snapshot."""
        object.__setattr__(self, "_soigia_version_history", [self._snapshot_payload(label)])
        return self

    def mark_version(self, label: Optional[str] = None) -> "DataTable":
        """Append the current dataframe snapshot to version history."""
        history = list(getattr(self, "_soigia_version_history", []))
        history.append(self._snapshot_payload(label))
        object.__setattr__(self, "_soigia_version_history", history)
        return self

    def version_history(self, include_data: bool = False) -> list[dict[str, Any]]:
        """Return the stored version history."""
        history = copy.deepcopy(getattr(self, "_soigia_version_history", []) or [])
        if include_data:
            return history
        for item in history:
            item.pop("records", None)
        return history

    def version_count(self) -> int:
        """Return the number of stored versions."""
        history = getattr(self, "_soigia_version_history", None) or []
        return len(history)

    def current_version(self) -> Optional[dict[str, Any]]:
        """Return the latest version metadata if available."""
        history = self.version_history(include_data=True)
        return history[-1] if history else None

    def get_version(self, version: int | str = -1) -> "DataTable":
        """Return a stored version as a DataTable."""
        index = self._resolve_version_index(version)
        history = self.version_history(include_data=True)
        payload = history[index]
        frame = DataTable(payload.get("records", []))
        if payload.get("columns"):
            frame = frame.reindex(columns=payload["columns"])
        return frame

    def restore_version(self, version: int | str = -1) -> "DataTable":
        """Alias for get_version()."""
        return self.get_version(version)

    def snapshot(self, label: Optional[str] = None) -> "DataTable":
        """Alias for mark_version()."""
        return self.mark_version(label)

    def auto_version(self, label: str = "auto") -> "DataTable":
        """Enable automatic version snapshots for helper-driven transforms."""
        if self.version_count() == 0:
            self.init_versions("initial")
        object.__setattr__(self, "_soigia_auto_version", True)
        object.__setattr__(self, "_soigia_auto_version_label", label)
        return self

    def disable_auto_version(self) -> "DataTable":
        """Disable automatic version snapshots."""
        object.__setattr__(self, "_soigia_auto_version", False)
        return self

    def is_auto_versioning(self) -> bool:
        """Return whether automatic version snapshots are enabled."""
        return bool(getattr(self, "_soigia_auto_version", False))

    def diff_versions(self, old: int | str = 0, new: int | str = -1) -> dict[str, Any]:
        """Compare two versions and return row-level differences."""
        old_frame = self.get_version(old)
        new_frame = self.get_version(new)

        def _signature(row: Mapping[str, Any]) -> str:
            return repr(tuple(sorted((key, repr(value)) for key, value in row.items())))

        old_rows = old_frame.to_dict(orient="records")
        new_rows = new_frame.to_dict(orient="records")

        old_map = {_signature(row): row for row in old_rows}
        new_map = {_signature(row): row for row in new_rows}

        old_set = set(old_map)
        new_set = set(new_map)

        added = [new_map[key] for key in new_set - old_set]
        removed = [old_map[key] for key in old_set - new_set]

        return {
            "old_version": old,
            "new_version": new,
            "added_rows": added,
            "removed_rows": removed,
            "old_count": len(old_rows),
            "new_count": len(new_rows),
            "row_delta": len(new_rows) - len(old_rows),
        }

    def diff_records(self, old: int | str = 0, new: int | str = -1) -> dict[str, Any]:
        """Alias for diff_versions()."""
        return self.diff_versions(old, new)

    def diff_columns(self, old: int | str = 0, new: int | str = -1) -> dict[str, Any]:
        """Compare column layouts between two versions."""
        old_frame = self.get_version(old)
        new_frame = self.get_version(new)
        old_columns = list(old_frame.columns)
        new_columns = list(new_frame.columns)
        return {
            "old_version": old,
            "new_version": new,
            "old_columns": old_columns,
            "new_columns": new_columns,
            "added_columns": [column for column in new_columns if column not in old_columns],
            "removed_columns": [column for column in old_columns if column not in new_columns],
            "reordered_columns": [column for column in new_columns if column in old_columns],
            "column_order_changed": old_columns != new_columns,
        }

    @staticmethod
    def _cell_equal(left: Any, right: Any) -> bool:
        """Compare two cell values while handling NaN and unhashable types."""
        if left is right:
            return True
        try:
            if pd.isna(left) and pd.isna(right):
                return True
        except Exception:
            pass
        try:
            comparison = left == right
            if isinstance(comparison, bool):
                return comparison
            return bool(comparison)
        except Exception:
            return repr(left) == repr(right)

    def diff_cells(
        self,
        old: int | str = 0,
        new: int | str = -1,
        key_columns: Optional[Sequence[str]] = None,
    ) -> dict[str, Any]:
        """Compare two versions at cell level."""
        old_frame = self.get_version(old)
        new_frame = self.get_version(new)
        old_rows = old_frame.to_dict(orient="records")
        new_rows = new_frame.to_dict(orient="records")

        if key_columns:
            keys = list(key_columns)
            for column in keys:
                if column not in old_frame.columns or column not in new_frame.columns:
                    raise KeyError(f"Unknown key column: {column}")

            def make_key(row: Mapping[str, Any]) -> tuple[Any, ...]:
                return tuple(row.get(column) for column in keys)

            old_map = {make_key(row): row for row in old_rows}
            new_map = {make_key(row): row for row in new_rows}
            common_keys = old_map.keys() & new_map.keys()
            added_rows = [new_map[key] for key in new_map.keys() - old_map.keys()]
            removed_rows = [old_map[key] for key in old_map.keys() - new_map.keys()]
            changed_cells = []
            changed_rows = set()
            columns = [
                column for column in sorted(set(old_frame.columns) | set(new_frame.columns)) if column not in keys
            ]
            for key in common_keys:
                old_row = old_map[key]
                new_row = new_map[key]
                for column in columns:
                    if not self._cell_equal(old_row.get(column), new_row.get(column)):
                        changed_rows.add(key)
                        changed_cells.append(
                            {
                                "key": key,
                                "column": column,
                                "old": old_row.get(column),
                                "new": new_row.get(column),
                            }
                        )
        else:
            max_len = max(len(old_rows), len(new_rows))
            added_rows = []
            removed_rows = []
            changed_cells = []
            changed_rows = set()
            columns = sorted(set(old_frame.columns) | set(new_frame.columns))
            for index in range(max_len):
                if index >= len(old_rows):
                    added_rows.append({"index": index, "row": new_rows[index]})
                    continue
                if index >= len(new_rows):
                    removed_rows.append({"index": index, "row": old_rows[index]})
                    continue
                old_row = old_rows[index]
                new_row = new_rows[index]
                for column in columns:
                    if not self._cell_equal(old_row.get(column), new_row.get(column)):
                        changed_rows.add(index)
                        changed_cells.append(
                            {
                                "index": index,
                                "column": column,
                                "old": old_row.get(column),
                                "new": new_row.get(column),
                            }
                        )

        return {
            "old_version": old,
            "new_version": new,
            "added_rows": added_rows,
            "removed_rows": removed_rows,
            "changed_cells": changed_cells,
            "changed_rows": sorted(changed_rows),
        }

    def diff_summary(
        self,
        old: int | str = 0,
        new: int | str = -1,
        key_columns: Optional[Sequence[str]] = None,
    ) -> dict[str, Any]:
        """Return a compact summary of column, row, and cell differences."""
        records = self.diff_versions(old, new)
        columns = self.diff_columns(old, new)
        cells = self.diff_cells(old, new, key_columns=key_columns)
        return {
            "old_version": old,
            "new_version": new,
            "row_delta": records["row_delta"],
            "added_rows": len(records["added_rows"]),
            "removed_rows": len(records["removed_rows"]),
            "added_columns": columns["added_columns"],
            "removed_columns": columns["removed_columns"],
            "column_order_changed": columns["column_order_changed"],
            "changed_cells": len(cells["changed_cells"]),
            "changed_rows": len(cells["changed_rows"]),
        }

    @classmethod
    def db_connector(cls, connection_string: Optional[str] = None, **connections: Any):
        """Configure the default SQL connector used by the DB helpers."""
        return db_connector(connection_string=connection_string, **connections)

    @classmethod
    def from_mysql_db(cls, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        """Run SQL against a MySQL database and return a DataTable."""
        return cls(from_mysql_db(sql, connection_string=connection_string, **kwargs))

    @classmethod
    def from_maria_db(cls, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        """Run SQL against a MariaDB database and return a DataTable."""
        return cls(from_maria_db(sql, connection_string=connection_string, **kwargs))

    @classmethod
    def from_oracle_db(cls, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        """Run SQL against an Oracle database and return a DataTable."""
        return cls(from_oracle_db(sql, connection_string=connection_string, **kwargs))

    @classmethod
    def from_postgres_db(cls, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        """Run SQL against a PostgreSQL database and return a DataTable."""
        return cls(from_postgres_db(sql, connection_string=connection_string, **kwargs))

    @classmethod
    def from_sqlite_db(cls, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        """Run SQL against a SQLite database and return a DataTable."""
        return cls(from_sqlite_db(sql, connection_string=connection_string, **kwargs))

    @classmethod
    def from_google_sheet(
        cls,
        spreadsheet: Any,
        worksheet: Any = 0,
        header: int = 0,
        range_name: Optional[str] = None,
        client: Any = None,
        credentials_json: Optional[str] = None,
        service_account_file: Optional[Union[str, Path]] = None,
        scopes: Optional[Sequence[str]] = None,
    ) -> "DataTable":
        """Load data from a Google Sheet worksheet into a DataTable."""
        resolved = _resolve_google_worksheet(
            spreadsheet,
            worksheet,
            client=client,
            credentials_json=credentials_json,
            service_account_file=service_account_file,
            scopes=scopes,
        )
        return _google_worksheet_to_frame(resolved, header=header, range_name=range_name)

    def to_google_sheet(
        self,
        spreadsheet: Any,
        worksheet: Any = 0,
        clear: bool = True,
        include_index: bool = False,
        start_cell: str = "A1",
        value_input_option: str = "RAW",
        client: Any = None,
        credentials_json: Optional[str] = None,
        service_account_file: Optional[Union[str, Path]] = None,
        scopes: Optional[Sequence[str]] = None,
    ) -> Any:
        """Write the dataframe to a Google Sheet worksheet."""
        resolved = _resolve_google_worksheet(
            spreadsheet,
            worksheet,
            client=client,
            credentials_json=credentials_json,
            service_account_file=service_account_file,
            scopes=scopes,
        )
        values = _frame_to_google_values(self, include_index=include_index)
        if clear and hasattr(resolved, "clear"):
            resolved.clear()
        if hasattr(resolved, "update"):
            resolved.update(start_cell, values, value_input_option=value_input_option)
            return resolved
        if hasattr(resolved, "append_rows"):
            resolved.append_rows(values, value_input_option=value_input_option)
            return resolved
        raise TypeError("worksheet object must provide update() or append_rows()")

    def to_rows(self) -> list[dict[str, Any]]:
        """Return the dataframe as a list of record dictionaries."""
        return self.to_dict(orient="records")

    def to_markdown_table(self, index: bool = False, **kwargs: Any) -> str:
        """Return a Markdown table string."""
        return self.to_markdown(index=index, **kwargs)

    def to_html_table(self, index: bool = False, **kwargs: Any) -> str:
        """Return an HTML table string."""
        return self.to_html(index=index, **kwargs)

    def to_csv_table(self, index: bool = False, **kwargs: Any) -> str:
        """Return a CSV string."""
        return self.to_csv(index=index, **kwargs)

    def save_to_csv(self, path: Union[str, Path], index: bool = False, **kwargs: Any) -> Path:
        """Write the dataframe to CSV and return the output path."""
        output = Path(path)
        output.parent.mkdir(parents=True, exist_ok=True)
        self.to_csv(output, index=index, **kwargs)
        return output

    def save_to_parquet(self, path: Union[str, Path], index: bool = False, **kwargs: Any) -> Path:
        """Write the dataframe to Parquet and return the output path."""
        output = Path(path)
        output.parent.mkdir(parents=True, exist_ok=True)
        self.to_parquet(output, index=index, **kwargs)
        return output

    def save_to_db(
        self,
        table_name: str,
        connection_string: Optional[str] = None,
        *,
        backend: str = "sqlite",
        if_exists: str = "replace",
        index: bool = False,
        **kwargs: Any,
    ) -> "DataTable":
        """Write the dataframe to a database table and return self."""
        frame = pd.DataFrame(self)
        if backend == "sqlite":
            url = _connector()._engine_url("sqlite", connection_string)
            if url == "sqlite:///:memory:":
                connection = sqlite3.connect(":memory:")
            elif url.startswith("sqlite:///"):
                connection = sqlite3.connect(url[len("sqlite:///") :])
            elif url.startswith("sqlite://"):
                connection = sqlite3.connect(url[len("sqlite://") :])
            else:
                connection = sqlite3.connect(url)
            try:
                frame.to_sql(table_name, connection, if_exists=if_exists, index=index, **kwargs)
            finally:
                connection.close()
            return self

        if not connection_string:
            raise ValueError("connection_string is required for save_to_db() on non-sqlite backends")

        if create_engine is None:
            raise ImportError("sqlalchemy is required to write to non-sqlite databases")
        engine = create_engine(connection_string)
        try:
            frame.to_sql(table_name, engine, if_exists=if_exists, index=index, **kwargs)
        finally:
            dispose = getattr(engine, "dispose", None)
            if callable(dispose):
                dispose()
        return self

    def to_rich_table(self, title: Optional[str] = None, show_index: bool = False):
        """Return a rich Table object for terminal display."""
        try:
            from rich.table import Table
        except ImportError:
            table = _FallbackRichTable(title=title)
            if show_index:
                table.add_column("index")
            for column in self.columns:
                table.add_column(str(column))
            for index, row in self.iterrows():
                values = [str(index)] if show_index else []
                values.extend("" if pd.isna(value) else str(value) for value in row.tolist())
                table.add_row(*values)
            return table

        table = Table(title=title)
        if show_index:
            table.add_column("index")
        for column in self.columns:
            table.add_column(str(column))
        for index, row in self.iterrows():
            values = [str(index)] if show_index else []
            values.extend("" if pd.isna(value) else str(value) for value in row.tolist())
            table.add_row(*values)
        return table

    def to_excel_table(
        self,
        path: Union[str, Path],
        sheet_name: str = "Sheet1",
        index: bool = False,
        style_header: bool = True,
        autofit: bool = True,
        freeze_header: bool = True,
        **kwargs: Any,
    ) -> Path:
        """Write a styled Excel file and return the path."""
        output = Path(path)
        self.to_excel(output, sheet_name=sheet_name, index=index, **kwargs)
        if not (style_header or autofit or freeze_header):
            return output
        try:
            from openpyxl import load_workbook
            from openpyxl.styles import Font, PatternFill
        except ImportError:
            return output

        workbook = load_workbook(output)
        worksheet = workbook[sheet_name]
        if freeze_header:
            worksheet.freeze_panes = "A2"
        if style_header:
            fill = PatternFill("solid", fgColor="1F2937")
            for cell in worksheet[1]:
                cell.font = Font(bold=True, color="FFFFFF")
                cell.fill = fill
        if autofit:
            for column_cells in worksheet.columns:
                column_letter = column_cells[0].column_letter
                max_length = 0
                for cell in column_cells:
                    value = "" if cell.value is None else str(cell.value)
                    if len(value) > max_length:
                        max_length = len(value)
                worksheet.column_dimensions[column_letter].width = min(max_length + 2, 60)
        workbook.save(output)
        return output

    def render_table(self, format: str = "markdown", **kwargs: Any):
        """Render the dataframe in a selected format."""
        fmt = format.lower()
        if fmt == "markdown":
            return self.to_markdown_table(**kwargs)
        if fmt == "html":
            return self.to_html_table(**kwargs)
        if fmt == "csv":
            return self.to_csv_table(**kwargs)
        if fmt == "rich":
            return self.to_rich_table(**kwargs)
        if fmt == "excel":
            path = kwargs.pop("path", None)
            if path is None:
                raise ValueError("render_table(format='excel') requires path=")
            return self.to_excel_table(path, **kwargs)
        raise ValueError(f"Unsupported format: {format}")

    def to_display_text(self, format: str = "markdown", **kwargs: Any) -> str:
        """Return printable text for a selected display format."""
        fmt = format.lower()
        display_kwargs = dict(kwargs)
        width = display_kwargs.pop("width", 120)
        rendered = self.render_table(fmt, **display_kwargs)
        if fmt != "rich":
            return rendered
        try:
            from rich.console import Console
        except ImportError:
            return str(rendered)

        console = Console(record=True, width=width)
        console.print(rendered)
        return console.export_text()

    def print_table(self, format: str = "markdown", file=None, end: str = "\n", **kwargs: Any) -> str:
        """Print a rendered table and return the printable text."""
        text = self.to_display_text(format=format, **kwargs)
        print(text, file=file, end=end)
        return text

    def log_table(
        self,
        format: str = "markdown",
        logger: Any = None,
        level: int = logging.INFO,
        **kwargs: Any,
    ) -> str:
        """Log a rendered table and return the printable text."""
        text = self.to_display_text(format=format, **kwargs)
        target_logger = (
            logging.getLogger(logger) if isinstance(logger, str) else (logger or logging.getLogger(__name__))
        )
        target_logger.log(level, "%s", text)
        return text

    def _row_signature(self, row: Mapping[str, Any], columns: Sequence[str]) -> str:
        return repr(tuple((column, repr(row.get(column))) for column in columns))

    def _records_frame(self, records: Sequence[Mapping[str, Any]], columns: Sequence[str]) -> "DataTable":
        frame = DataTable(list(records))
        if columns:
            frame = frame.reindex(columns=list(columns))
        return frame

    def compare_frames(
        self,
        other: "DataTable",
        key_columns: Optional[Sequence[str]] = None,
    ) -> dict[str, Any]:
        """Compare two dataframes and return matched, changed, and unmatched rows."""
        other_frame = DataTable(other)
        columns = list(dict.fromkeys(list(self.columns) + list(other_frame.columns)))
        left_rows = self.to_dict(orient="records")
        right_rows = other_frame.to_dict(orient="records")

        if key_columns:
            keys = list(key_columns)
            for column in keys:
                if column not in self.columns or column not in other_frame.columns:
                    raise KeyError(f"Unknown key column: {column}")

            def make_key(row: Mapping[str, Any]) -> tuple[Any, ...]:
                return tuple(row.get(column) for column in keys)

            left_map = {make_key(row): row for row in left_rows}
            right_map = {make_key(row): row for row in right_rows}
            common_keys = left_map.keys() & right_map.keys()
            same_rows = []
            changed_rows = []
            left_only_rows = [left_map[key] for key in left_map.keys() - right_map.keys()]
            right_only_rows = [right_map[key] for key in right_map.keys() - left_map.keys()]

            for key in common_keys:
                left_row = left_map[key]
                right_row = right_map[key]
                row_equal = all(self._cell_equal(left_row.get(column), right_row.get(column)) for column in columns)
                if row_equal:
                    same_rows.append(left_row)
                    continue
                changed_columns = [
                    column for column in columns if not self._cell_equal(left_row.get(column), right_row.get(column))
                ]
                changed_rows.append(
                    {
                        "key": key,
                        "left": left_row,
                        "right": right_row,
                        "changed_columns": changed_columns,
                    }
                )
        else:
            left_groups: dict[str, list[Mapping[str, Any]]] = {}
            right_groups: dict[str, list[Mapping[str, Any]]] = {}
            for row in left_rows:
                left_groups.setdefault(self._row_signature(row, columns), []).append(row)
            for row in right_rows:
                right_groups.setdefault(self._row_signature(row, columns), []).append(row)

            same_rows = []
            left_only_rows = []
            right_only_rows = []
            changed_rows = []
            for signature in set(left_groups) | set(right_groups):
                left_list = left_groups.get(signature, [])
                right_list = right_groups.get(signature, [])
                common = min(len(left_list), len(right_list))
                same_rows.extend(left_list[:common])
                left_only_rows.extend(left_list[common:])
                right_only_rows.extend(right_list[common:])

        same_frame = self._records_frame(same_rows, columns)
        left_only_frame = self._records_frame(left_only_rows, columns)
        right_only_frame = self._records_frame(right_only_rows, columns)
        return {
            "same_rows": same_frame,
            "left_only_rows": left_only_frame,
            "right_only_rows": right_only_frame,
            "changed_rows": changed_rows,
            "same_count": len(same_rows),
            "left_only_count": len(left_only_rows),
            "right_only_count": len(right_only_rows),
            "changed_count": len(changed_rows),
        }

    def _normalize_join_inputs(
        self,
        other: Any,
        on: Optional[Union[str, Sequence[str]]] = None,
        left_on: Optional[Union[str, Sequence[str]]] = None,
        right_on: Optional[Union[str, Sequence[str]]] = None,
    ) -> tuple["DataTable", list[str], list[str]]:
        other_frame = DataTable(other)
        if on is not None:
            keys = [on] if isinstance(on, str) else list(on)
            if not keys:
                raise ValueError("join key list cannot be empty")
            return other_frame, keys, keys
        if left_on is not None or right_on is not None:
            if left_on is None or right_on is None:
                raise ValueError("left_on and right_on must be provided together")
            left_keys = [left_on] if isinstance(left_on, str) else list(left_on)
            right_keys = [right_on] if isinstance(right_on, str) else list(right_on)
            if len(left_keys) != len(right_keys):
                raise ValueError("left_on and right_on must have the same number of keys")
            return other_frame, left_keys, right_keys
        common = [column for column in self.columns if column in other_frame.columns]
        if not common:
            raise ValueError("join requires 'on' or at least one common column")
        return other_frame, common, common

    def join(
        self,
        other: Any,
        on: Optional[Union[str, Sequence[str]]] = None,
        how: str = "inner",
        left_on: Optional[Union[str, Sequence[str]]] = None,
        right_on: Optional[Union[str, Sequence[str]]] = None,
        suffixes: tuple[str, str] = ("_x", "_y"),
        sort: bool = False,
    ) -> "DataTable":
        """Join with another dataframe using pandas merge semantics."""
        other_frame, left_keys, right_keys = self._normalize_join_inputs(
            other, on=on, left_on=left_on, right_on=right_on
        )
        merged = pd.merge(
            self,
            other_frame,
            how=how,
            left_on=left_keys,
            right_on=right_keys,
            suffixes=suffixes,
            sort=sort,
        )
        return self._inherit_versions(merged)

    def left_join(
        self,
        other: Any,
        on: Optional[Union[str, Sequence[str]]] = None,
        left_on: Optional[Union[str, Sequence[str]]] = None,
        right_on: Optional[Union[str, Sequence[str]]] = None,
        suffixes: tuple[str, str] = ("_x", "_y"),
        sort: bool = False,
    ) -> "DataTable":
        """Left join helper."""
        return self.join(other, on=on, how="left", left_on=left_on, right_on=right_on, suffixes=suffixes, sort=sort)

    def inner_join(
        self,
        other: Any,
        on: Optional[Union[str, Sequence[str]]] = None,
        left_on: Optional[Union[str, Sequence[str]]] = None,
        right_on: Optional[Union[str, Sequence[str]]] = None,
        suffixes: tuple[str, str] = ("_x", "_y"),
        sort: bool = False,
    ) -> "DataTable":
        """Inner join helper."""
        return self.join(other, on=on, how="inner", left_on=left_on, right_on=right_on, suffixes=suffixes, sort=sort)

    def anti_join(
        self,
        other: Any,
        on: Optional[Union[str, Sequence[str]]] = None,
        left_on: Optional[Union[str, Sequence[str]]] = None,
        right_on: Optional[Union[str, Sequence[str]]] = None,
        suffixes: tuple[str, str] = ("_x", "_y"),
        sort: bool = False,
    ) -> "DataTable":
        """Return rows from the left dataframe that have no match on the right."""
        other_frame, left_keys, right_keys = self._normalize_join_inputs(
            other, on=on, left_on=left_on, right_on=right_on
        )
        merged = pd.merge(
            self,
            other_frame,
            how="left",
            left_on=left_keys,
            right_on=right_keys,
            suffixes=suffixes,
            sort=sort,
            indicator=True,
        )
        anti = merged.loc[merged["_merge"] == "left_only", list(self.columns)].copy()
        return self._inherit_versions(anti)

    def ensure_columns(self, *columns: str, fill_value: Any = pd.NA) -> "DataTable":
        """Ensure columns exist, creating missing ones with a default value."""
        result = self.copy()
        for column in columns:
            if column not in result.columns:
                result[column] = fill_value
        return self._inherit_versions(result)

    def coerce_types(self, mapping: Mapping[str, Any], errors: str = "raise") -> "DataTable":
        """Coerce selected columns to the requested types."""
        result = self.copy()
        for column, dtype in mapping.items():
            if column not in result.columns:
                raise KeyError(f"Unknown field: {column}")
            if dtype in {"datetime", "datetime64", "datetime64[ns]", datetime}:
                result[column] = pd.to_datetime(result[column], errors=errors)
            elif dtype in {"date"}:
                result[column] = pd.to_datetime(result[column], errors=errors).dt.date
            else:
                result[column] = (
                    result[column].astype(dtype, errors=errors) if hasattr(result[column], "astype") else result[column]
                )
        return self._inherit_versions(result)

    def validate_schema(self, schema: Mapping[str, Any], strict: bool = False) -> "DataTable":
        """Validate column presence and basic column constraints."""
        if strict:
            extra_columns = [column for column in self.columns if column not in schema]
            if extra_columns:
                raise ValueError(f"Unexpected columns: {extra_columns}")

        for column, spec in schema.items():
            if column not in self.columns:
                required = True
                if isinstance(spec, Mapping):
                    required = bool(spec.get("required", True))
                if required:
                    raise KeyError(f"Missing required column: {column}")
                continue

            series = self[column]
            nullable = True
            expected_type = None
            choices = None
            minimum = maximum = None
            regex_pattern = None
            length_spec = None

            if isinstance(spec, Mapping):
                nullable = bool(spec.get("nullable", True))
                expected_type = spec.get("type", spec.get("dtype"))
                choices = spec.get("choices")
                minimum = spec.get("min", spec.get("ge"))
                maximum = spec.get("max", spec.get("le"))
                regex_pattern = spec.get("regex")
                length_spec = spec.get("length")
            else:
                expected_type = spec

            if not nullable and series.isna().any():
                raise ValueError(f"Column {column!r} contains missing values")

            non_null = series.dropna()
            if expected_type is not None:
                if isinstance(expected_type, tuple):
                    valid_types = expected_type
                elif isinstance(expected_type, type):
                    valid_types = (expected_type,)
                else:
                    valid_types = ()
                if valid_types:
                    invalid = [value for value in non_null if not isinstance(value, valid_types)]
                    if invalid:
                        raise TypeError(f"Column {column!r} does not match expected type {expected_type!r}")

            if choices is not None:
                invalid = [value for value in non_null if value not in choices]
                if invalid:
                    raise ValueError(f"Column {column!r} contains values outside the allowed choices")

            if minimum is not None and any(value < minimum for value in non_null):
                raise ValueError(f"Column {column!r} contains values below minimum {minimum!r}")
            if maximum is not None and any(value > maximum for value in non_null):
                raise ValueError(f"Column {column!r} contains values above maximum {maximum!r}")

            if regex_pattern is not None:
                pattern = re.compile(regex_pattern)
                invalid = [value for value in non_null if not pattern.search(str(value))]
                if invalid:
                    raise ValueError(f"Column {column!r} does not match regex {regex_pattern!r}")

            if length_spec is not None:
                if isinstance(length_spec, int):
                    invalid = [value for value in non_null if len(str(value)) != length_spec]
                elif isinstance(length_spec, Mapping):
                    min_length = length_spec.get("min")
                    max_length = length_spec.get("max")
                    invalid = [
                        value
                        for value in non_null
                        if (min_length is not None and len(str(value)) < min_length)
                        or (max_length is not None and len(str(value)) > max_length)
                    ]
                else:
                    invalid = []
                if invalid:
                    raise ValueError(f"Column {column!r} fails length validation")

        return self

    def same_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Return rows that are equal in both dataframes."""
        return self.compare_frames(other, key_columns=key_columns)["same_rows"]

    def same(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Alias for same_rows()."""
        return self.same_rows(other, key_columns=key_columns)

    def different_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> dict[str, Any]:
        """Return rows that differ between two dataframes."""
        comparison = self.compare_frames(other, key_columns=key_columns)
        return {
            "left_only_rows": comparison["left_only_rows"],
            "right_only_rows": comparison["right_only_rows"],
            "changed_rows": comparison["changed_rows"],
            "left_only_count": comparison["left_only_count"],
            "right_only_count": comparison["right_only_count"],
            "changed_count": comparison["changed_count"],
        }

    def diff_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> dict[str, Any]:
        """Alias for different_rows()."""
        return self.different_rows(other, key_columns=key_columns)

    def compare_with_status(self, other: "DataTable", status_column: str = "status") -> "DataTable":
        """Return a combined dataframe with exact-row membership status."""
        comparison = self.compare_frames(other)
        other_frame = DataTable(other)
        columns = list(dict.fromkeys(list(self.columns) + list(other_frame.columns) + [status_column]))
        rows: list[dict[str, Any]] = []

        for status, frame in (
            ("same", comparison["same_rows"]),
            ("left_only", comparison["left_only_rows"]),
            ("right_only", comparison["right_only_rows"]),
        ):
            for record in frame.to_rows():
                row = dict(record)
                row[status_column] = status
                rows.append(row)

        result = DataTable(rows)
        if columns:
            result = result.reindex(columns=columns)
        return result

    def compare_status_summary(self, other: "DataTable", status_column: str = "status") -> "DataTable":
        """Return one summary row per status with grouped records."""
        status_table = self.compare_with_status(other, status_column=status_column)
        rows: list[dict[str, Any]] = []
        for status in ("same", "left_only", "right_only"):
            subset = status_table[status_table[status_column] == status].drop(columns=[status_column], errors="ignore")
            rows.append(
                {
                    status_column: status,
                    "count": int(len(subset)),
                    "rows": subset.to_rows(),
                }
            )
        return DataTable(rows)

    def compare_status_rows(self, other: "DataTable", status_column: str = "status") -> "DataTable":
        """Return one representative row for each status."""
        status_table = self.compare_with_status(other, status_column=status_column)
        columns = list(status_table.columns)
        rows: list[dict[str, Any]] = []

        for status in ("same", "left_only", "right_only"):
            subset = status_table[status_table[status_column] == status]
            if len(subset) > 0:
                row = subset.iloc[0].to_dict()
            else:
                row = {column: pd.NA for column in columns}
                row[status_column] = status
            rows.append(row)

        result = DataTable(rows)
        if columns:
            result = result.reindex(columns=columns)
        return result

    def compare_sample_rows(self, other: "DataTable", status_column: str = "status") -> "DataTable":
        """Alias for compare_status_rows()."""
        return self.compare_status_rows(other, status_column=status_column)

    def diff_columns_against(self, other: "DataTable") -> dict[str, Any]:
        """Compare column layouts against another dataframe."""
        other_frame = DataTable(other)
        self_columns = list(self.columns)
        other_columns = list(other_frame.columns)
        return {
            "left_columns": self_columns,
            "right_columns": other_columns,
            "added_columns": [column for column in other_columns if column not in self_columns],
            "removed_columns": [column for column in self_columns if column not in other_columns],
            "shared_columns": [column for column in self_columns if column in other_columns],
            "column_order_changed": self_columns != other_columns,
        }

    def diff_added_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Return rows that exist only in the other dataframe."""
        return self.compare_frames(other, key_columns=key_columns)["right_only_rows"]

    def right_only(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Alias for diff_added_rows()."""
        return self.diff_added_rows(other, key_columns=key_columns)

    def diff_removed_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Return rows that exist only in the current dataframe."""
        return self.compare_frames(other, key_columns=key_columns)["left_only_rows"]

    def left_only(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Alias for diff_removed_rows()."""
        return self.diff_removed_rows(other, key_columns=key_columns)

    def diff_unchanged_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Return rows that are identical in both dataframes."""
        return self.compare_frames(other, key_columns=key_columns)["same_rows"]

    def diff_changed_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Return the row-level changes between two dataframes."""
        comparison = self.compare_frames(other, key_columns=key_columns)
        return DataTable(comparison["changed_rows"])

    def diff_changed_cells(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Return a cell-level diff between two dataframes."""
        other_frame = DataTable(other)
        columns = list(dict.fromkeys(list(self.columns) + list(other_frame.columns)))
        left_rows = self.to_dict(orient="records")
        right_rows = other_frame.to_dict(orient="records")
        changes: list[dict[str, Any]] = []

        if key_columns:
            keys = list(key_columns)
            for column in keys:
                if column not in self.columns or column not in other_frame.columns:
                    raise KeyError(f"Unknown key column: {column}")

            def make_key(row: Mapping[str, Any]) -> tuple[Any, ...]:
                return tuple(row.get(column) for column in keys)

            left_map = {make_key(row): row for row in left_rows}
            right_map = {make_key(row): row for row in right_rows}
            for key in left_map.keys() & right_map.keys():
                left_row = left_map[key]
                right_row = right_map[key]
                for column in columns:
                    if column in keys:
                        continue
                    if not self._cell_equal(left_row.get(column), right_row.get(column)):
                        changes.append(
                            {
                                "key": key,
                                "column": column,
                                "left": left_row.get(column),
                                "right": right_row.get(column),
                            }
                        )
        else:
            max_len = max(len(left_rows), len(right_rows))
            for index in range(max_len):
                if index >= len(left_rows) or index >= len(right_rows):
                    continue
                left_row = left_rows[index]
                right_row = right_rows[index]
                for column in columns:
                    if not self._cell_equal(left_row.get(column), right_row.get(column)):
                        changes.append(
                            {
                                "index": index,
                                "column": column,
                                "left": left_row.get(column),
                                "right": right_row.get(column),
                            }
                        )

        return DataTable(changes)

    def diff_report(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> dict[str, Any]:
        """Return a deep diff report with rows, cells, and columns."""
        comparison = self.compare_frames(other, key_columns=key_columns)
        added_rows = comparison["right_only_rows"]
        removed_rows = comparison["left_only_rows"]
        same_rows = comparison["same_rows"]
        changed_rows = DataTable(comparison["changed_rows"])
        changed_cells = self.diff_changed_cells(other, key_columns=key_columns)
        column_diff = self.diff_columns_against(other)
        return {
            "same_rows": same_rows,
            "added_rows": added_rows,
            "removed_rows": removed_rows,
            "changed_rows": changed_rows,
            "changed_cells": changed_cells,
            "column_diff": column_diff,
            "same_count": comparison["same_count"],
            "added_count": comparison["right_only_count"],
            "removed_count": comparison["left_only_count"],
            "changed_count": comparison["changed_count"],
            "changed_cell_count": len(changed_cells),
            "has_changes": bool(
                comparison["left_only_count"]
                or comparison["right_only_count"]
                or comparison["changed_count"]
                or len(changed_cells)
                or column_diff["added_columns"]
                or column_diff["removed_columns"]
                or column_diff["column_order_changed"]
            ),
        }

    def only(self, *fields: str) -> "DataTable":
        """Return a frame with only the selected columns."""
        if not fields:
            return self._inherit_versions(self.copy())
        return self._inherit_versions(self.loc[:, list(fields)].copy())

    def select(self, *fields: str) -> "DataTable":
        """Alias for only()."""
        return self.only(*fields)

    def exclude_fields(self, *fields: str) -> "DataTable":
        """Return a frame without the selected columns."""
        if not fields:
            return self._inherit_versions(self.copy())
        drop_fields = [field for field in fields if field in self.columns]
        return self._inherit_versions(self.drop(columns=drop_fields).copy())

    def clean_columns(self, lower: bool = True, replace_spaces: str = "_") -> "DataTable":
        """Normalize column names for consistent downstream use."""
        columns = []
        for column in self.columns:
            name = str(column).strip()
            if lower:
                name = name.lower()
            if replace_spaces is not None:
                name = re.sub(r"\s+", replace_spaces, name)
            columns.append(name)
        result = self.copy()
        result.columns = columns
        return self._inherit_versions(result)

    def drop_empty_columns(self) -> "DataTable":
        """Drop columns that contain only missing values."""
        return self._inherit_versions(super().dropna(axis=1, how="all"))

    def fill_missing(self, value: Any = 0, columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Fill missing values in selected columns or in the whole frame."""
        result = self.copy()
        if columns is None:
            return self._inherit_versions(result.fillna(value))
        result.loc[:, list(columns)] = result.loc[:, list(columns)].fillna(value)
        return self._inherit_versions(result)

    def to_numeric(self, columns: Optional[Iterable[str]] = None, errors: str = "coerce") -> "DataTable":
        """Convert selected columns to numeric values."""
        result = self.copy()
        target_columns = list(columns) if columns is not None else list(result.columns)
        for column in target_columns:
            result[column] = pd.to_numeric(result[column], errors=errors)
        return self._inherit_versions(result)

    @staticmethod
    def _normalize_compare_value(
        value: Any,
        strip_text: bool = True,
        lower_text: bool = False,
        collapse_whitespace: bool = False,
    ) -> Any:
        """Normalize a single scalar value for comparison."""
        if isinstance(value, str):
            text = value
            if strip_text:
                text = text.strip()
            if collapse_whitespace:
                text = re.sub(r"\s+", " ", text)
            if lower_text:
                text = text.lower()
            return text
        return value

    def normalize(
        self,
        rename: Optional[Mapping[str, str]] = None,
        columns: Optional[Sequence[str]] = None,
        numeric_columns: Optional[Sequence[str]] = None,
        datetime_columns: Optional[Sequence[str]] = None,
        strip_text: bool = True,
        lower_text: bool = False,
        collapse_whitespace: bool = False,
        drop_empty_columns: bool = False,
        sort_columns: bool = False,
    ) -> "DataTable":
        """Return a normalized copy that is easier to compare across sources."""
        result = self.copy()

        if rename:
            result = result.rename(columns=dict(rename))

        if columns is not None:
            missing = [column for column in columns if column not in result.columns]
            if missing:
                raise KeyError(f"Unknown field: {missing[0]}")
            result = result.loc[:, list(columns)].copy()

        numeric_set = set(numeric_columns or [])
        datetime_set = set(datetime_columns or [])

        for column in numeric_set:
            if column not in result.columns:
                raise KeyError(f"Unknown field: {column}")
            result[column] = pd.to_numeric(result[column], errors="coerce")

        for column in datetime_set:
            if column not in result.columns:
                raise KeyError(f"Unknown field: {column}")
            result[column] = pd.to_datetime(result[column], errors="coerce")

        for column in result.columns:
            if column in numeric_set or column in datetime_set:
                continue
            series = result[column]
            if series.dtype == object or pd.api.types.is_string_dtype(series):
                result[column] = series.map(
                    lambda value: self._normalize_compare_value(
                        value,
                        strip_text=strip_text,
                        lower_text=lower_text,
                        collapse_whitespace=collapse_whitespace,
                    )
                )

        if drop_empty_columns:
            result = result.dropna(axis=1, how="all")

        if sort_columns:
            result = result.reindex(columns=sorted(result.columns))

        return self._inherit_versions(result)

    def reconcile(
        self,
        other: "DataTable",
        key_columns: Optional[Sequence[str]] = None,
        column_map: Optional[Mapping[str, str]] = None,
        ignore_columns: Optional[Sequence[str]] = None,
        strip_text: bool = True,
        lower_text: bool = False,
        collapse_whitespace: bool = False,
        numeric_tolerance: Optional[float] = None,
        datetime_tolerance: Optional[Any] = None,
        duplicate_strategy: str = "error",
    ) -> dict[str, Any]:
        """Compare two dataframes with optional normalization and tolerance rules."""

        other_frame = DataTable(other)
        left_frame = self.copy()
        right_frame = other_frame.copy()

        if column_map:
            right_frame = right_frame.rename(columns=dict(column_map))

        ignored = set(ignore_columns or [])
        if ignored:
            left_frame = left_frame.drop(
                columns=[column for column in left_frame.columns if column in ignored], errors="ignore"
            )
            right_frame = right_frame.drop(
                columns=[column for column in right_frame.columns if column in ignored], errors="ignore"
            )

        if key_columns is None:
            key_columns = [column for column in left_frame.columns if column in right_frame.columns]
        keys = list(key_columns)
        if not keys:
            raise ValueError("reconcile requires at least one key column")

        missing_left = [column for column in keys if column not in left_frame.columns]
        missing_right = [column for column in keys if column not in right_frame.columns]
        if missing_left or missing_right:
            missing = missing_left or missing_right
            raise KeyError(f"Unknown key column: {missing[0]}")

        left_frame = left_frame.normalize(
            columns=list(left_frame.columns),
            strip_text=strip_text,
            lower_text=lower_text,
            collapse_whitespace=collapse_whitespace,
        )
        right_frame = right_frame.normalize(
            columns=list(right_frame.columns),
            strip_text=strip_text,
            lower_text=lower_text,
            collapse_whitespace=collapse_whitespace,
        )

        compare_columns = [
            column
            for column in dict.fromkeys(list(left_frame.columns) + list(right_frame.columns))
            if column not in keys
        ]
        common_columns = [
            column for column in compare_columns if column in left_frame.columns and column in right_frame.columns
        ]

        def _row_key(row: Mapping[str, Any]) -> tuple[Any, ...]:
            return tuple(row.get(column) for column in keys)

        def _group_rows(frame: DataTable) -> dict[tuple[Any, ...], list[dict[str, Any]]]:
            grouped: dict[tuple[Any, ...], list[dict[str, Any]]] = {}
            for row in frame.to_dict(orient="records"):
                grouped.setdefault(_row_key(row), []).append(row)
            return grouped

        def _choose_rows(rows: list[dict[str, Any]]) -> dict[str, Any]:
            if duplicate_strategy == "first":
                return rows[0]
            if duplicate_strategy == "last":
                return rows[-1]
            raise ValueError("duplicate key rows found; set duplicate_strategy='first' or 'last' to continue")

        left_groups = _group_rows(left_frame)
        right_groups = _group_rows(right_frame)
        left_duplicates = [row for rows in left_groups.values() if len(rows) > 1 for row in rows]
        right_duplicates = [row for rows in right_groups.values() if len(rows) > 1 for row in rows]

        if duplicate_strategy not in {"error", "first", "last"}:
            raise ValueError("duplicate_strategy must be 'error', 'first', or 'last'")

        if duplicate_strategy == "error" and (left_duplicates or right_duplicates):
            raise ValueError("duplicate key rows found; set duplicate_strategy='first' or 'last' to continue")

        left_map = {key: _choose_rows(rows) if len(rows) > 1 else rows[0] for key, rows in left_groups.items()}
        right_map = {key: _choose_rows(rows) if len(rows) > 1 else rows[0] for key, rows in right_groups.items()}

        same_rows: list[dict[str, Any]] = []
        left_only_rows: list[dict[str, Any]] = []
        right_only_rows: list[dict[str, Any]] = []
        changed_rows: list[dict[str, Any]] = []
        changed_cells: list[dict[str, Any]] = []

        common_keys = left_map.keys() & right_map.keys()
        for key in common_keys:
            left_row = left_map[key]
            right_row = right_map[key]
            row_changed_columns: list[str] = []

            for column in common_columns:
                left_value = left_row.get(column)
                right_value = right_row.get(column)

                if numeric_tolerance is not None:
                    left_numeric = pd.to_numeric(pd.Series([left_value]), errors="coerce").iloc[0]
                    right_numeric = pd.to_numeric(pd.Series([right_value]), errors="coerce").iloc[0]
                    if pd.notna(left_numeric) and pd.notna(right_numeric):
                        if abs(float(left_numeric) - float(right_numeric)) <= float(numeric_tolerance):
                            continue

                if datetime_tolerance is not None:
                    left_dt = pd.to_datetime(pd.Series([left_value]), errors="coerce").iloc[0]
                    right_dt = pd.to_datetime(pd.Series([right_value]), errors="coerce").iloc[0]
                    if pd.notna(left_dt) and pd.notna(right_dt):
                        if abs(left_dt - right_dt) <= pd.Timedelta(datetime_tolerance):
                            continue

                if self._cell_equal(left_value, right_value):
                    continue

                row_changed_columns.append(column)
                changed_cells.append(
                    {
                        "key": key,
                        "column": column,
                        "left": left_value,
                        "right": right_value,
                    }
                )

            if row_changed_columns:
                changed_rows.append(
                    {
                        "key": key,
                        "left": left_row,
                        "right": right_row,
                        "changed_columns": row_changed_columns,
                    }
                )
            else:
                same_rows.append(left_row)

        for key in left_map.keys() - right_map.keys():
            left_only_rows.append(left_map[key])
        for key in right_map.keys() - left_map.keys():
            right_only_rows.append(right_map[key])

        all_columns = list(dict.fromkeys(list(left_frame.columns) + list(right_frame.columns)))

        return {
            "key_columns": keys,
            "compare_columns": common_columns,
            "left_only_rows": self._records_frame(left_only_rows, all_columns),
            "right_only_rows": self._records_frame(right_only_rows, all_columns),
            "same_rows": self._records_frame(same_rows, all_columns),
            "changed_rows": DataTable(changed_rows),
            "changed_cells": DataTable(changed_cells),
            "duplicate_left_rows": self._records_frame(left_duplicates, all_columns),
            "duplicate_right_rows": self._records_frame(right_duplicates, all_columns),
            "same_count": len(same_rows),
            "left_only_count": len(left_only_rows),
            "right_only_count": len(right_only_rows),
            "changed_count": len(changed_rows),
            "changed_cell_count": len(changed_cells),
            "duplicate_left_count": len(left_duplicates),
            "duplicate_right_count": len(right_duplicates),
            "has_changes": bool(
                left_only_rows
                or right_only_rows
                or changed_rows
                or changed_cells
                or left_duplicates
                or right_duplicates
            ),
        }

    def reconcile_html_report(
        self,
        other: "DataTable",
        key_columns: Optional[Sequence[str]] = None,
        column_map: Optional[Mapping[str, str]] = None,
        ignore_columns: Optional[Sequence[str]] = None,
        strip_text: bool = True,
        lower_text: bool = False,
        collapse_whitespace: bool = False,
        numeric_tolerance: Optional[float] = None,
        datetime_tolerance: Optional[Any] = None,
        duplicate_strategy: str = "error",
        title: Optional[str] = None,
        max_preview_rows: int = 25,
        top_n_mismatched_keys: int = 10,
        include_raw_json: bool = True,
        output_path: Optional[Union[str, Path]] = None,
        download_json_path: Optional[Union[str, Path]] = None,
        open_report_in_browser: bool = False,
        return_payload: bool = False,
    ) -> str | tuple[str, dict[str, Any]]:
        """Return a standalone HTML report for dataframe reconciliation."""

        report = self.reconcile(
            other,
            key_columns=key_columns,
            column_map=column_map,
            ignore_columns=ignore_columns,
            strip_text=strip_text,
            lower_text=lower_text,
            collapse_whitespace=collapse_whitespace,
            numeric_tolerance=numeric_tolerance,
            datetime_tolerance=datetime_tolerance,
            duplicate_strategy=duplicate_strategy,
        )

        other_frame = DataTable(other)
        page_title = title or "DataTable Reconciliation Report"
        key_labels = ", ".join(html_escape(str(column)) for column in report["key_columns"])
        compare_labels = ", ".join(html_escape(str(column)) for column in report["compare_columns"])
        left_name = html_escape(getattr(self, "name", "left"))
        right_name = html_escape(getattr(other_frame, "name", "right"))

        def _card_kind(kind: str) -> str:
            return kind if kind in {"good", "warn", "bad", "info"} else "info"

        def _preview(frame: DataTable, limit: int = max_preview_rows) -> DataTable:
            if limit is None or limit <= 0:
                return frame
            return DataTable(frame.head(limit).copy())

        def _table_html(frame: DataTable, *, index: bool = False, empty_message: str = "No rows") -> str:
            if frame is None or len(frame) == 0:
                return f'<div class="empty-state">{html_escape(empty_message)}</div>'
            return frame.to_html(index=index, border=0, classes=["datatable", "report-table"], escape=True)

        def _summary_card(label: str, value: Any, hint: str = "", kind: str = "info") -> str:
            hint_html = f'<div class="card-hint">{html_escape(str(hint))}</div>' if hint else ""
            return (
                f'<section class="card {_card_kind(kind)}">'
                f'<div class="card-label">{html_escape(label)}</div>'
                f'<div class="card-value">{html_escape(str(value))}</div>'
                f"{hint_html}"
                "</section>"
            )

        column_diff = self.diff_columns_against(other_frame)
        changed_rows_preview = _preview(report["changed_rows"], max_preview_rows)
        changed_cells_preview = _preview(report["changed_cells"], max_preview_rows)
        left_only_preview = _preview(report["left_only_rows"], max_preview_rows)
        right_only_preview = _preview(report["right_only_rows"], max_preview_rows)
        same_preview = _preview(report["same_rows"], max_preview_rows)
        duplicates_left_preview = _preview(report["duplicate_left_rows"], max_preview_rows)
        duplicates_right_preview = _preview(report["duplicate_right_rows"], max_preview_rows)
        mismatch_rows = sorted(
            (
                {
                    "key": row.get("key"),
                    "changed_columns": ", ".join(str(column) for column in row.get("changed_columns", [])),
                    "changed_count": len(row.get("changed_columns", [])),
                    "left": ", ".join(
                        f"{column}={row.get('left', {}).get(column)!r}" for column in row.get("changed_columns", [])
                    ),
                    "right": ", ".join(
                        f"{column}={row.get('right', {}).get(column)!r}" for column in row.get("changed_columns", [])
                    ),
                }
                for row in report["changed_rows"].to_dict(orient="records")
            ),
            key=lambda item: (-item["changed_count"], repr(item["key"])),
        )
        mismatch_limit = max(0, int(top_n_mismatched_keys))
        mismatch_preview = DataTable(
            mismatch_rows[:mismatch_limit], columns=["key", "changed_count", "changed_columns", "left", "right"]
        )

        def _json_default(value: Any) -> Any:
            if isinstance(value, (datetime, Path)):
                return str(value)
            if hasattr(value, "item"):
                try:
                    return value.item()
                except Exception:
                    pass
            return str(value)

        raw_payload = {
            "title": page_title,
            "left_name": getattr(self, "name", "left"),
            "right_name": getattr(other_frame, "name", "right"),
            "summary": {
                "same_count": report["same_count"],
                "left_only_count": report["left_only_count"],
                "right_only_count": report["right_only_count"],
                "changed_count": report["changed_count"],
                "changed_cell_count": report["changed_cell_count"],
                "duplicate_left_count": report["duplicate_left_count"],
                "duplicate_right_count": report["duplicate_right_count"],
                "has_changes": report["has_changes"],
            },
            "key_columns": report["key_columns"],
            "compare_columns": report["compare_columns"],
            "column_diff": column_diff,
            "changed_rows": report["changed_rows"].to_dict(orient="records"),
            "changed_cells": report["changed_cells"].to_dict(orient="records"),
            "left_only_rows": report["left_only_rows"].to_dict(orient="records"),
            "right_only_rows": report["right_only_rows"].to_dict(orient="records"),
            "same_rows": report["same_rows"].to_dict(orient="records"),
            "duplicate_left_rows": report["duplicate_left_rows"].to_dict(orient="records"),
            "duplicate_right_rows": report["duplicate_right_rows"].to_dict(orient="records"),
            "top_mismatched_keys": mismatch_rows[:mismatch_limit],
            "inputs": {
                "column_map": dict(column_map or {}),
                "ignore_columns": list(ignore_columns or []),
                "strip_text": strip_text,
                "lower_text": lower_text,
                "collapse_whitespace": collapse_whitespace,
                "numeric_tolerance": numeric_tolerance,
                "datetime_tolerance": datetime_tolerance,
                "duplicate_strategy": duplicate_strategy,
                "max_preview_rows": max_preview_rows,
                "top_n_mismatched_keys": mismatch_limit,
            },
        }
        raw_json = json.dumps(raw_payload, indent=2, ensure_ascii=False, default=_json_default)

        status_class = "status-clean" if not report["has_changes"] else "status-warning"
        status_text = "Clean match" if not report["has_changes"] else "Differences detected"
        summary_rows = [
            ["Same rows", report["same_count"]],
            ["Left only rows", report["left_only_count"]],
            ["Right only rows", report["right_only_count"]],
            ["Changed rows", report["changed_count"]],
            ["Changed cells", report["changed_cell_count"]],
            ["Duplicate rows on left", report["duplicate_left_count"]],
            ["Duplicate rows on right", report["duplicate_right_count"]],
        ]
        summary_table = DataTable(summary_rows, columns=["metric", "value"])

        html_parts = [
            "<!doctype html>",
            '<html lang="en">',
            "<head>",
            '<meta charset="utf-8">',
            '<meta name="viewport" content="width=device-width, initial-scale=1">',
            f"<title>{html_escape(page_title)}</title>",
            "<style>",
            ":root{--bg:#0f172a;--panel:#111827;--panel-soft:#1f2937;--text:#e5e7eb;--muted:#9ca3af;--line:#374151;--good:#10b981;--warn:#f59e0b;--bad:#ef4444;--accent:#60a5fa;}",
            "body{margin:0;background:linear-gradient(180deg,#0b1120 0%,#111827 100%);color:var(--text);font-family:Arial,Helvetica,sans-serif;}",
            ".wrap{max-width:1440px;margin:0 auto;padding:24px;}",
            ".hero{display:flex;flex-wrap:wrap;justify-content:space-between;gap:16px;align-items:flex-end;margin-bottom:20px;}",
            ".hero h1{margin:0;font-size:28px;line-height:1.2;}",
            ".hero .subtitle{color:var(--muted);margin-top:6px;}",
            ".badge{display:inline-flex;align-items:center;gap:8px;padding:8px 12px;border-radius:999px;font-weight:700;}",
            ".status-clean{background:rgba(16,185,129,.15);color:#86efac;border:1px solid rgba(16,185,129,.35);}",
            ".status-warning{background:rgba(245,158,11,.15);color:#fde68a;border:1px solid rgba(245,158,11,.35);}",
            ".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:12px;margin:16px 0 24px;}",
            ".card{background:rgba(17,24,39,.96);border:1px solid var(--line);border-radius:14px;padding:16px;box-shadow:0 12px 28px rgba(0,0,0,.18);}",
            ".card.good{border-color:rgba(16,185,129,.45);background:linear-gradient(180deg,rgba(6,78,59,.45),rgba(17,24,39,.96));}",
            ".card.warn{border-color:rgba(245,158,11,.45);background:linear-gradient(180deg,rgba(120,53,15,.45),rgba(17,24,39,.96));}",
            ".card.bad{border-color:rgba(239,68,68,.5);background:linear-gradient(180deg,rgba(127,29,29,.45),rgba(17,24,39,.96));}",
            ".card.info{border-color:rgba(96,165,250,.4);background:linear-gradient(180deg,rgba(30,41,59,.6),rgba(17,24,39,.96));}",
            ".card-label{color:var(--muted);font-size:12px;text-transform:uppercase;letter-spacing:.08em;}",
            ".card-value{font-size:24px;font-weight:800;margin-top:8px;}",
            ".card-hint{color:var(--muted);font-size:12px;margin-top:8px;}",
            ".status-row{display:flex;flex-wrap:wrap;gap:10px;margin-top:10px;}",
            ".mini-pill{display:inline-flex;align-items:center;border-radius:999px;padding:5px 10px;font-size:12px;font-weight:700;border:1px solid transparent;}",
            ".mini-pill.good{background:rgba(16,185,129,.12);color:#86efac;border-color:rgba(16,185,129,.3);}",
            ".mini-pill.warn{background:rgba(245,158,11,.12);color:#fde68a;border-color:rgba(245,158,11,.3);}",
            ".mini-pill.bad{background:rgba(239,68,68,.12);color:#fca5a5;border-color:rgba(239,68,68,.3);}",
            ".section{background:rgba(17,24,39,.88);border:1px solid var(--line);border-radius:16px;padding:18px;margin:18px 0;}",
            ".section h2{margin:0 0 12px;font-size:18px;}",
            ".section h3{margin:14px 0 10px;font-size:15px;color:#dbeafe;}",
            ".meta{display:grid;grid-template-columns:repeat(auto-fit,minmax(240px,1fr));gap:12px;}",
            ".meta .card-value{font-size:16px;word-break:break-word;}",
            ".empty-state{padding:14px;border:1px dashed var(--line);border-radius:12px;color:var(--muted);background:rgba(31,41,55,.5);}",
            "table.datatable{width:100%;border-collapse:collapse;background:rgba(15,23,42,.7);}",
            "table.datatable th,table.datatable td{border:1px solid var(--line);padding:8px 10px;vertical-align:top;font-size:13px;}",
            "table.datatable th{background:#1f2937;color:#f9fafb;position:sticky;top:0;z-index:1;}",
            "table.datatable tr:nth-child(even) td{background:rgba(31,41,55,.45);}",
            ".table-wrap{overflow:auto;border-radius:12px;}",
            ".pill{display:inline-block;padding:4px 10px;border-radius:999px;background:rgba(96,165,250,.15);color:#bfdbfe;border:1px solid rgba(96,165,250,.25);margin-right:6px;margin-bottom:6px;}",
            ".small{color:var(--muted);font-size:12px;}",
            "details{margin-top:10px;}",
            "summary{cursor:pointer;color:#bfdbfe;font-weight:700;}",
            ".footer{color:var(--muted);font-size:12px;margin-top:20px;padding-top:12px;border-top:1px solid var(--line);}",
            "</style>",
            "</head>",
            "<body>",
            '<div class="wrap">',
            '<header class="hero">',
            "<div>",
            f"<h1>{html_escape(page_title)}</h1>",
            f'<div class="subtitle">Left: {left_name} | Right: {right_name}</div>',
            '<div class="status-row">',
            f'<span class="mini-pill {"good" if not report["has_changes"] else "warn"}">{html_escape(status_text)}</span>',
            f'<span class="mini-pill {"good" if report["same_count"] else "warn"}">Same: {report["same_count"]}</span>',
            f'<span class="mini-pill {"bad" if report["left_only_count"] else "good"}">Left only: {report["left_only_count"]}</span>',
            f'<span class="mini-pill {"bad" if report["right_only_count"] else "good"}">Right only: {report["right_only_count"]}</span>',
            f'<span class="mini-pill {"warn" if report["changed_count"] else "good"}">Changed: {report["changed_count"]}</span>',
            f'<span class="mini-pill {"bad" if report["duplicate_left_count"] or report["duplicate_right_count"] else "good"}">Duplicates: {report["duplicate_left_count"] + report["duplicate_right_count"]}</span>',
            "</div>",
            "</div>",
            f'<div class="badge {status_class}">{status_text}</div>',
            "</header>",
            '<section class="section">',
            "<h2>Summary</h2>",
            '<div class="grid">',
            _summary_card("Match status", status_text, kind="good" if not report["has_changes"] else "warn"),
            _summary_card(
                "Key columns", len(report["key_columns"]), key_labels or "No explicit key columns", kind="info"
            ),
            _summary_card(
                "Compare columns",
                len(report["compare_columns"]),
                compare_labels or "All shared non-key columns",
                kind="info",
            ),
            _summary_card("Changed rows", report["changed_count"], kind="warn" if report["changed_count"] else "good"),
            _summary_card(
                "Changed cells", report["changed_cell_count"], kind="warn" if report["changed_cell_count"] else "good"
            ),
            _summary_card("Left only", report["left_only_count"], kind="bad" if report["left_only_count"] else "good"),
            _summary_card(
                "Right only", report["right_only_count"], kind="bad" if report["right_only_count"] else "good"
            ),
            _summary_card(
                "Duplicate rows",
                f'{report["duplicate_left_count"]} / {report["duplicate_right_count"]}',
                kind="bad" if report["duplicate_left_count"] or report["duplicate_right_count"] else "good",
            ),
            "</div>",
            '<div class="table-wrap">',
            _table_html(summary_table, index=False, empty_message="No summary available"),
            "</div>",
            "</section>",
            '<section class="section">',
            "<h2>Inputs</h2>",
            '<div class="meta">',
            _summary_card("Left columns", len(self.columns), ", ".join(html_escape(str(col)) for col in self.columns)),
            _summary_card(
                "Right columns",
                len(other_frame.columns),
                ", ".join(html_escape(str(col)) for col in other_frame.columns),
            ),
            _summary_card(
                "Column map",
                len(column_map or {}),
                ", ".join(f"{html_escape(str(k))} → {html_escape(str(v))}" for k, v in (column_map or {}).items())
                or "No column map",
            ),
            _summary_card(
                "Ignore columns",
                len(ignore_columns or []),
                ", ".join(html_escape(str(col)) for col in (ignore_columns or [])) or "No ignored columns",
            ),
            _summary_card("Strip text", strip_text),
            _summary_card("Lower text", lower_text),
            _summary_card("Collapse whitespace", collapse_whitespace),
            _summary_card("Numeric tolerance", numeric_tolerance if numeric_tolerance is not None else "none"),
            _summary_card("Datetime tolerance", datetime_tolerance if datetime_tolerance is not None else "none"),
            "</div>",
            "</section>",
            '<section class="section">',
            "<h2>Column Differences</h2>",
            '<div class="table-wrap">',
            _table_html(DataTable([column_diff]), index=False, empty_message="No column diff available"),
            "</div>",
            "</section>",
            '<section class="section">',
            "<h2>Matched Rows</h2>",
            '<div class="small">Rows that matched after normalization and tolerance checks.</div>',
            '<div class="table-wrap">',
            _table_html(same_preview, index=False, empty_message="No matched rows"),
            "</div>",
            "</section>",
            '<section class="section">',
            "<h2>Changed Rows</h2>",
            '<div class="small">Rows with the same key but different values.</div>',
            '<div class="table-wrap">',
            _table_html(changed_rows_preview, index=False, empty_message="No changed rows"),
            "</div>",
            "<h3>Top Mismatched Keys</h3>",
            f'<div class="small">Keys with the most changed columns, sorted descending. Showing top {mismatch_limit}.</div>',
            '<div class="table-wrap">',
            _table_html(mismatch_preview, index=False, empty_message="No mismatched keys"),
            "</div>",
            "<details>",
            "<summary>All changed row details</summary>",
            '<div class="table-wrap" style="margin-top:10px;">',
            _table_html(report["changed_rows"], index=False, empty_message="No changed rows"),
            "</div>",
            "</details>",
            "<details>",
            "<summary>Changed cell details</summary>",
            '<div class="table-wrap" style="margin-top:10px;">',
            _table_html(changed_cells_preview, index=False, empty_message="No changed cells"),
            "</div>",
            "</details>",
            "</section>",
            '<section class="section">',
            "<h2>Only on Left</h2>",
            '<div class="table-wrap">',
            _table_html(left_only_preview, index=False, empty_message="No rows only on left"),
            "</div>",
            "<details>",
            "<summary>All left-only rows</summary>",
            '<div class="table-wrap" style="margin-top:10px;">',
            _table_html(report["left_only_rows"], index=False, empty_message="No rows only on left"),
            "</div>",
            "</details>",
            "</section>",
            '<section class="section">',
            "<h2>Only on Right</h2>",
            '<div class="table-wrap">',
            _table_html(right_only_preview, index=False, empty_message="No rows only on right"),
            "</div>",
            "<details>",
            "<summary>All right-only rows</summary>",
            '<div class="table-wrap" style="margin-top:10px;">',
            _table_html(report["right_only_rows"], index=False, empty_message="No rows only on right"),
            "</div>",
            "</details>",
            "</section>",
            '<section class="section">',
            "<h2>Duplicates</h2>",
            '<div class="small">Duplicate key rows can hide reconciliation issues. Review them first.</div>',
            "<h3>Left duplicates</h3>",
            '<div class="table-wrap">',
            _table_html(duplicates_left_preview, index=False, empty_message="No duplicate rows on left"),
            "</div>",
            "<h3>Right duplicates</h3>",
            '<div class="table-wrap">',
            _table_html(duplicates_right_preview, index=False, empty_message="No duplicate rows on right"),
            "</div>",
            "</section>",
            '<section class="section">',
            "<h2>Raw Report</h2>",
            '<div class="small">Compact summary of the reconciliation result.</div>',
            '<div class="table-wrap">',
            _table_html(
                DataTable([report]).exclude_fields(
                    "same_rows",
                    "left_only_rows",
                    "right_only_rows",
                    "changed_rows",
                    "changed_cells",
                    "duplicate_left_rows",
                    "duplicate_right_rows",
                ),
                index=False,
            ),
            "</div>",
            "</section>",
            '<section class="section">',
            "<h2>Raw JSON</h2>",
            '<div class="small">Machine-readable payload for downstream automation.</div>',
            "<details open>",
            "<summary>Show raw JSON</summary>",
            f'<script type="application/json" id="reconcile-report-json">{html_escape(raw_json)}</script>',
            f'<pre class="table-wrap" style="white-space:pre-wrap;word-break:break-word;margin-top:10px;">{html_escape(raw_json)}</pre>',
            "</details>",
            "</section>",
            '<div class="footer">',
            f"Generated from {html_escape(left_name)} vs {html_escape(right_name)} using DataTable.reconcile_html_report().",
            "</div>",
            "</div>",
            "</body>",
            "</html>",
        ]
        html = "\n".join(html_parts)
        report_path: Optional[Path] = Path(output_path) if output_path is not None else None
        if output_path is not None:
            report_path = Path(output_path)
            report_path.write_text(html, encoding="utf-8")
        if download_json_path is not None:
            json_output = Path(download_json_path)
            json_output.write_text(raw_json, encoding="utf-8")
        if open_report_in_browser:
            if report_path is None:
                temp_file = tempfile.NamedTemporaryFile(
                    "w", suffix=".html", prefix="reconcile-report-", delete=False, encoding="utf-8"
                )
                try:
                    temp_file.write(html)
                    temp_file.flush()
                finally:
                    temp_file.close()
                report_path = Path(temp_file.name)
            webbrowser.open(report_path.resolve().as_uri())
        if return_payload:
            return html, raw_payload
        return html

    def to_html_report(self, other: "DataTable", **kwargs: Any) -> str:
        """Alias for reconcile_html_report()."""
        return self.reconcile_html_report(other, **kwargs)

    def reconcile_report_path(
        self,
        other: "DataTable",
        path: Optional[Union[str, Path]] = None,
        **kwargs: Any,
    ) -> Path:
        """Write a reconciliation HTML report and return the output path."""
        if path is None:
            fd, temp_path = tempfile.mkstemp(prefix="reconcile-report-", suffix=".html")
            os.close(fd)
            output = Path(temp_path)
        else:
            output = Path(path)
        self.reconcile_html_report(other, output_path=output, **kwargs)
        return output

    def reconcile_report_json_path(
        self,
        other: "DataTable",
        path: Optional[Union[str, Path]] = None,
        **kwargs: Any,
    ) -> Path:
        """Write the reconciliation payload JSON and return the output path."""
        if path is None:
            fd, temp_path = tempfile.mkstemp(prefix="reconcile-report-", suffix=".json")
            os.close(fd)
            output = Path(temp_path)
        else:
            output = Path(path)
        self.reconcile_html_report(other, download_json_path=output, **kwargs)
        return output

    def reconcile_bundle(
        self,
        other: "DataTable",
        key_columns: Optional[Sequence[str]] = None,
        column_map: Optional[Mapping[str, str]] = None,
        ignore_columns: Optional[Sequence[str]] = None,
        strip_text: bool = True,
        lower_text: bool = False,
        collapse_whitespace: bool = False,
        numeric_tolerance: Optional[float] = None,
        datetime_tolerance: Optional[Any] = None,
        duplicate_strategy: str = "error",
        title: Optional[str] = None,
        max_preview_rows: int = 25,
        top_n_mismatched_keys: int = 10,
        include_raw_json: bool = True,
        html_path: Optional[Union[str, Path]] = None,
        json_path: Optional[Union[str, Path]] = None,
        open_report_in_browser: bool = False,
    ) -> dict[str, Any]:
        """Return the HTML, JSON payload, and file paths for a reconciliation report."""
        if html_path is None:
            fd, temp_path = tempfile.mkstemp(prefix="reconcile-report-", suffix=".html")
            os.close(fd)
            resolved_html_path = Path(temp_path)
        else:
            resolved_html_path = Path(html_path)

        if json_path is None:
            fd, temp_path = tempfile.mkstemp(prefix="reconcile-report-", suffix=".json")
            os.close(fd)
            resolved_json_path = Path(temp_path)
        else:
            resolved_json_path = Path(json_path)

        html_result = self.reconcile_html_report(
            other,
            key_columns=key_columns,
            column_map=column_map,
            ignore_columns=ignore_columns,
            strip_text=strip_text,
            lower_text=lower_text,
            collapse_whitespace=collapse_whitespace,
            numeric_tolerance=numeric_tolerance,
            datetime_tolerance=datetime_tolerance,
            duplicate_strategy=duplicate_strategy,
            title=title,
            max_preview_rows=max_preview_rows,
            top_n_mismatched_keys=top_n_mismatched_keys,
            include_raw_json=include_raw_json,
            output_path=resolved_html_path,
            download_json_path=resolved_json_path,
            open_report_in_browser=False,
            return_payload=True,
        )
        if not isinstance(html_result, tuple):
            raise TypeError("reconcile_html_report() did not return the expected payload tuple")
        html, payload = html_result

        if open_report_in_browser:
            webbrowser.open(resolved_html_path.resolve().as_uri())

        return {
            "html": html,
            "payload": payload,
            "html_path": resolved_html_path,
            "json_path": resolved_json_path,
            "opened_in_browser": bool(open_report_in_browser),
        }

    def write_reconcile_html_report(
        self,
        path: Union[str, Path],
        other: "DataTable",
        **kwargs: Any,
    ) -> Path:
        """Write a reconciliation HTML report to disk."""
        return self.reconcile_report_path(other, path=path, **kwargs)

    def open_reconcile_html_report(self, *args: Any, **kwargs: Any) -> Path:
        """Write a reconciliation HTML report and open it in the default browser."""
        path: Optional[Union[str, Path]] = kwargs.pop("path", None)
        other: Optional["DataTable"] = kwargs.pop("other", None)

        if len(args) == 1:
            if isinstance(args[0], DataTable):
                other = args[0]
            else:
                path = args[0]
        elif len(args) >= 2:
            path, other = args[0], args[1]

        if other is None:
            raise TypeError("open_reconcile_html_report() requires a comparison dataframe")

        output = self.reconcile_report_path(other, path=path, open_report_in_browser=True, **kwargs)
        return output

    def reconcile_open(self, other: "DataTable", **kwargs: Any) -> dict[str, Any]:
        """Return a bundle and open the HTML report in the default browser."""
        return self.reconcile_bundle(other, open_report_in_browser=True, **kwargs)


class _ORMQuerySet:
    """Lightweight Django-like queryset helper for DataTable."""

    def __init__(self, frame: DataTable):
        frame_copy = frame.copy()
        self._frame = frame_copy if isinstance(frame_copy, DataTable) else DataTable(frame_copy)

    def _spawn(self, frame: DataTable) -> "_ORMQuerySet":
        """Create a new queryset of the same logical kind."""
        return _ORMQuerySet(frame)

    def __getitem__(self, key: Any) -> Any:
        """Proxy column/subset access to the underlying dataframe."""
        result = self._frame.__getitem__(key)
        if isinstance(result, pd.DataFrame):
            return DataTable(result)
        return result

    def all(self) -> "_ORMQuerySet":
        """Return a clone of the queryset."""
        return self._spawn(self._frame)

    def none(self) -> "_ORMQuerySet":
        """Return an empty queryset with the same columns."""
        return _ORMQuerySet(self._frame.iloc[0:0].copy())

    def filter(self, *expressions: str, **conditions: Any) -> "_ORMQuerySet":
        """Filter rows using query expressions and Django-style field lookups."""
        result = self._frame
        for expression in expressions:
            result = DataTable(result.query(expression))
        if not conditions:
            return self._spawn(result.copy())
        mask = self._build_mask(result, conditions)
        return self._spawn(result.loc[mask].copy())

    def exclude(self, *expressions: str, **conditions: Any) -> "_ORMQuerySet":
        """Exclude rows using query expressions and Django-style field lookups."""
        result = self._frame
        for expression in expressions:
            result = DataTable(result.query(expression))
        if not conditions:
            return self._spawn(self._frame.loc[~self._frame.index.isin(result.index)].copy())
        mask = self._build_mask(result, conditions)
        return self._spawn(result.loc[~mask].copy())

    def order_by(
        self,
        *fields: str,
        nulls_first: bool = False,
        nulls_last: bool = True,
        nulls: Optional[Mapping[str, str]] = None,
    ) -> "_ORMQuerySet":
        """Sort rows by one or more fields, supporting Django's '-' prefix."""
        if not fields:
            return self.all()
        if fields == ("?",):
            shuffled = self._frame.sample(frac=1).copy()
            shuffled.index = self._frame.index
            return self._spawn(shuffled)
        columns = []
        ascending = []
        for field in fields:
            descending = field.startswith("-")
            columns.append(field[1:] if descending else field)
            ascending.append(not descending)
        if nulls:
            ordered = self._frame.copy()
            for index in range(len(columns) - 1, -1, -1):
                column = columns[index]
                na_position = nulls.get(column, "first" if nulls_first or not nulls_last else "last")
                ordered = ordered.sort_values(
                    by=column,
                    ascending=ascending[index],
                    kind="mergesort",
                    na_position=na_position,
                )
            return self._spawn(ordered.copy())
        return self._spawn(
            self._frame.sort_values(
                by=columns,
                ascending=ascending,
                kind="mergesort",
                na_position="first" if nulls_first or not nulls_last else "last",
            ).copy()
        )

    def random_order(self) -> "_ORMQuerySet":
        """Return rows in random order."""
        return self.order_by("?")

    def distinct(self, *fields: str) -> "_ORMQuerySet":
        """Return unique rows or unique combinations of selected fields."""
        if fields:
            return self._spawn(self._frame.loc[:, list(fields)].drop_duplicates().copy())
        return self._spawn(self._frame.drop_duplicates().copy())

    def only(self, *fields: str) -> "_ORMQuerySet":
        """Return a queryset with only the selected columns."""
        if not fields:
            return self.all()
        return self._spawn(self._frame.loc[:, list(fields)].copy())

    def select(self, *fields: str) -> "_ORMQuerySet":
        """Alias for only()."""
        return self.only(*fields)

    def exclude_fields(self, *fields: str) -> "_ORMQuerySet":
        """Return a queryset without the selected columns."""
        if not fields:
            return self.all()
        drop_fields = [field for field in fields if field in self._frame.columns]
        return self._spawn(self._frame.drop(columns=drop_fields).copy())

    def values(self, *fields: str) -> _ValuesResult:
        """Return queryset rows as dictionaries."""
        frame = self._select_fields(*fields)
        return _ValuesResult(frame.to_dict(orient="records"))

    def values_list(self, *fields: str, flat: bool = False) -> _ValuesResult:
        """Return queryset rows as tuples or a flat list for one field."""
        frame = self._select_fields(*fields)
        if flat:
            if len(fields) != 1:
                raise ValueError("flat=True requires exactly one field")
            return _ValuesResult(frame.iloc[:, 0].tolist())
        return _ValuesResult([tuple(row) for row in frame.itertuples(index=False, name=None)])

    def first(self) -> DataTable:
        """Return the first row as a one-row dataframe."""
        return DataTable(self._frame.head(1).copy())

    def last(self) -> DataTable:
        """Return the last row as a one-row dataframe."""
        return DataTable(self._frame.tail(1).copy())

    def earliest(self, *fields: str) -> DataTable:
        """Return the earliest row according to the given ordering fields."""
        ordered = self.order_by(*fields) if fields else self.all()
        return ordered.first()

    def latest(self, *fields: str) -> DataTable:
        """Return the latest row according to the given ordering fields."""
        ordered = self.order_by(*fields) if fields else self.all()
        return ordered.last()

    def get(self, *expressions: str, **conditions: Any) -> DataTable:
        """Return exactly one matching row as a one-row dataframe."""
        result = self.filter(*expressions, **conditions)._frame
        if len(result) == 1:
            return DataTable(result.copy())
        if len(result) == 0:
            raise KeyError("No rows matched the given query")
        raise ValueError("Multiple rows matched the given query")

    def count(self) -> int:
        """Return the number of rows."""
        return int(len(self._frame))

    def exists(self) -> bool:
        """Return whether the queryset has at least one row."""
        return not self._frame.empty

    def paginate(self, limit: int, offset: int = 0) -> "_ORMQuerySet":
        """Return a page of rows using limit/offset semantics."""
        if limit < 0 or offset < 0:
            raise ValueError("limit and offset must be non-negative")
        return self._spawn(self._frame.iloc[offset : offset + limit].copy())

    def group_by(self, *fields: str) -> "_GroupedQuery":
        """Return a grouped query helper for Django-like aggregation."""
        if not fields:
            raise ValueError("group_by() requires at least one field")
        return _GroupedQuery(self._frame, fields)

    def sum(self, *fields: str) -> dict[str, Any]:
        """Return column sums for the requested fields."""
        return self._aggregate_columns("sum", *fields)

    def avg(self, *fields: str) -> dict[str, Any]:
        """Return column means for the requested fields."""
        return self._aggregate_columns("mean", *fields)

    def min(self, *fields: str) -> dict[str, Any]:
        """Return column minimums for the requested fields."""
        return self._aggregate_columns("min", *fields)

    def max(self, *fields: str) -> dict[str, Any]:
        """Return column maximums for the requested fields."""
        return self._aggregate_columns("max", *fields)

    def diff_columns_against(self, other: "DataTable") -> dict[str, Any]:
        """Compare column layouts against another dataframe."""
        return self._frame.diff_columns_against(other)

    def same(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Alias for same_rows()."""
        return self._frame.same(other, key_columns=key_columns)

    def compare_with_status(self, other: "DataTable", status_column: str = "status") -> DataTable:
        """Return a combined dataframe with exact-row membership status."""
        return self._frame.compare_with_status(other, status_column=status_column)

    def compare_status_summary(self, other: "DataTable", status_column: str = "status") -> DataTable:
        """Return one summary row per status with grouped records."""
        return self._frame.compare_status_summary(other, status_column=status_column)

    def compare_status_rows(self, other: "DataTable", status_column: str = "status") -> DataTable:
        """Return one representative row for each status."""
        return self._frame.compare_status_rows(other, status_column=status_column)

    def compare_sample_rows(self, other: "DataTable", status_column: str = "status") -> DataTable:
        """Alias for compare_status_rows()."""
        return self._frame.compare_sample_rows(other, status_column=status_column)

    def diff_added_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Return rows that exist only in the other dataframe."""
        return self._frame.diff_added_rows(other, key_columns=key_columns)

    def right_only(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Alias for diff_added_rows()."""
        return self._frame.right_only(other, key_columns=key_columns)

    def diff_removed_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Return rows that exist only in the current dataframe."""
        return self._frame.diff_removed_rows(other, key_columns=key_columns)

    def left_only(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Alias for diff_removed_rows()."""
        return self._frame.left_only(other, key_columns=key_columns)

    def diff_unchanged_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Return rows that are identical in both dataframes."""
        return self._frame.diff_unchanged_rows(other, key_columns=key_columns)

    def diff_changed_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Return row-level diff records."""
        return self._frame.diff_changed_rows(other, key_columns=key_columns)

    def diff_changed_cells(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Return cell-level diff records."""
        return self._frame.diff_changed_cells(other, key_columns=key_columns)

    def diff_report(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> dict[str, Any]:
        """Return a deep diff report."""
        return self._frame.diff_report(other, key_columns=key_columns)

    def aggregate(self, **aggregations: Any) -> dict[str, Any]:
        """Return named aggregates for the whole queryset."""
        if not aggregations:
            return {}
        return self._resolve_aggregations(self._frame, aggregations)

    def to_df(self) -> DataTable:
        """Materialize the queryset as a DataTable instance."""
        return DataTable(self._frame.copy())

    def to_markdown_table(self, *args: Any, **kwargs: Any) -> str:
        return self.to_df().to_markdown_table(*args, **kwargs)

    def to_html_table(self, *args: Any, **kwargs: Any) -> str:
        return self.to_df().to_html_table(*args, **kwargs)

    def to_csv_table(self, *args: Any, **kwargs: Any) -> str:
        return self.to_df().to_csv_table(*args, **kwargs)

    def to_rich_table(self, *args: Any, **kwargs: Any):
        return self.to_df().to_rich_table(*args, **kwargs)

    def to_excel_table(self, *args: Any, **kwargs: Any) -> Path:
        return self.to_df().to_excel_table(*args, **kwargs)

    def render_table(self, *args: Any, **kwargs: Any):
        return self.to_df().render_table(*args, **kwargs)

    def to_display_text(self, *args: Any, **kwargs: Any) -> str:
        return self.to_df().to_display_text(*args, **kwargs)

    def print_table(self, *args: Any, **kwargs: Any) -> str:
        return self.to_df().print_table(*args, **kwargs)

    def log_table(self, *args: Any, **kwargs: Any) -> str:
        return self.to_df().log_table(*args, **kwargs)

    def join(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.join(*args, **kwargs))

    def left_join(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.left_join(*args, **kwargs))

    def inner_join(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.inner_join(*args, **kwargs))

    def anti_join(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.anti_join(*args, **kwargs))

    def ensure_columns(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.ensure_columns(*args, **kwargs))

    def coerce_types(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.coerce_types(*args, **kwargs))

    def validate_schema(self, *args: Any, **kwargs: Any) -> DataTable:
        return self._frame.validate_schema(*args, **kwargs)

    def auto_version(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.auto_version(*args, **kwargs))

    def disable_auto_version(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.disable_auto_version(*args, **kwargs))

    def snapshot(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.snapshot(*args, **kwargs))

    def _select_fields(self, *fields: str) -> pd.DataFrame:
        if not fields:
            return self._frame
        return self._frame.loc[:, list(fields)]

    def _build_mask(self, frame: DataTable, conditions: Mapping[str, Any]) -> pd.Series:
        mask = pd.Series(True, index=frame.index)
        for key, expected in conditions.items():
            field, lookup = self._split_lookup(key)
            if field not in frame.columns:
                raise KeyError(f"Unknown field: {field}")
            series = frame[field]
            mask &= self._apply_lookup(series, lookup, expected)
        return mask

    def _aggregate_columns(self, func: str, *fields: str) -> dict[str, Any]:
        selected = list(fields) if fields else list(self._frame.columns)
        result: dict[str, Any] = {}
        for field in selected:
            if field not in self._frame.columns:
                raise KeyError(f"Unknown field: {field}")
            result[field] = getattr(self._frame[field], func)()
        return result

    def _resolve_aggregations(self, frame: pd.DataFrame, aggregations: Mapping[str, Any]) -> dict[str, Any]:
        result: dict[str, Any] = {}
        for alias, spec in aggregations.items():
            source, func = self._normalize_spec(alias, spec)
            if source not in frame.columns:
                raise KeyError(f"Unknown field: {source}")
            result[alias] = getattr(frame[source], func)()
        return result

    @staticmethod
    def _split_lookup(key: str) -> Tuple[str, str]:
        if "__" not in key:
            return key, "exact"
        field, lookup = key.rsplit("__", 1)
        return field, lookup

    @staticmethod
    def _apply_lookup(series: pd.Series, lookup: str, expected: Any) -> pd.Series:
        if lookup == "exact":
            return series.eq(expected)
        if lookup == "ne":
            return series.ne(expected)
        if lookup == "gt":
            return series.gt(expected)
        if lookup == "gte":
            return series.ge(expected)
        if lookup == "lt":
            return series.lt(expected)
        if lookup == "lte":
            return series.le(expected)
        if lookup == "in":
            return series.isin(expected)
        if lookup == "contains":
            return series.astype(str).str.contains(str(expected), regex=False, na=False)
        if lookup == "icontains":
            return series.astype(str).str.contains(str(expected), case=False, regex=False, na=False)
        if lookup == "startswith":
            return series.astype(str).str.startswith(str(expected), na=False)
        if lookup == "istartswith":
            return series.astype(str).str.lower().str.startswith(str(expected).lower(), na=False)
        if lookup == "endswith":
            return series.astype(str).str.endswith(str(expected), na=False)
        if lookup == "iendswith":
            return series.astype(str).str.lower().str.endswith(str(expected).lower(), na=False)
        if lookup == "isnull":
            return series.isna() if expected else series.notna()
        if lookup in {"range", "between"}:
            start, end = expected
            return series.between(start, end, inclusive="both")
        if lookup in {"year", "month", "day", "hour", "minute", "second", "date"}:
            dt = pd.to_datetime(series, errors="coerce")
            if lookup == "date":
                return dt.dt.date.eq(expected)
            return getattr(dt.dt, lookup).eq(expected)
        raise ValueError(f"Unsupported lookup: {lookup}")

    @staticmethod
    def _normalize_spec(alias: str, spec: Any) -> Tuple[str, Any]:
        if isinstance(spec, tuple) and len(spec) == 2:
            return spec[0], spec[1]
        if isinstance(spec, str):
            if spec in {"sum", "mean", "min", "max", "count", "first", "last", "median"}:
                return alias, spec
            return spec, "first"
        if callable(spec):
            return alias, spec
        raise TypeError("Annotation must be a string, callable, or (field, func) tuple")


class _ORMManager(_ORMQuerySet):
    """QuerySet with Django-like write helpers."""

    def _spawn(self, frame: DataTable) -> "_ORMQuerySet":
        return _ORMManager(frame)

    def create(self, **fields: Any) -> DataTable:
        """Create a one-row dataframe from the provided fields."""
        return DataTable([fields])

    def append_row(self, **fields: Any) -> DataTable:
        """Return a new dataframe with one appended row."""
        created = self.create(**fields)
        return DataTable(pd.concat([self._frame, created], ignore_index=True))

    def bulk_create(self, rows: Sequence[Mapping[str, Any]]) -> DataTable:
        """Return a new dataframe with multiple appended rows."""
        if not rows:
            return self.to_df()
        created = DataTable(list(rows))
        return DataTable(pd.concat([self._frame, created], ignore_index=True))

    def update(self, **fields: Any) -> int:
        """Return the number of matched rows that would be updated."""
        return self.count()

    def delete(self) -> int:
        """Return the number of matched rows that would be deleted."""
        return self.count()

    def bulk_update(
        self,
        values: Mapping[str, Any],
        return_count: bool = False,
        **conditions: Any,
    ) -> DataTable | tuple[DataTable, int]:
        """Return an updated dataframe and optionally the number of affected rows."""
        mask = self._build_mask(self._frame, conditions) if conditions else pd.Series(True, index=self._frame.index)
        updated = self._frame.copy()
        for field, value in values.items():
            if field not in updated.columns:
                updated[field] = pd.NA
            updated.loc[mask, field] = value
        updated_frame = DataTable(updated)
        if return_count:
            return updated_frame, int(mask.sum())
        return updated_frame

    def save(self) -> DataTable:
        """Return the current dataframe snapshot."""
        return self.to_df()

    def get_or_create(
        self,
        defaults: Optional[Mapping[str, Any]] = None,
        **lookup: Any,
    ) -> tuple[DataTable, bool]:
        """Return a matching row or create a one-row dataframe."""
        try:
            return self.get(**lookup), False
        except KeyError:
            data = dict(defaults or {})
            data.update(lookup)
            return self.create(**data), True

    def update_or_create(
        self,
        defaults: Optional[Mapping[str, Any]] = None,
        **lookup: Any,
    ) -> tuple[DataTable, bool]:
        """Return an updated row or create a new one-row dataframe."""
        try:
            row = self.get(**lookup)
        except KeyError:
            data = dict(defaults or {})
            data.update(lookup)
            return self.create(**data), True

        data = row.iloc[0].to_dict()
        data.update(defaults or {})
        data.update(lookup)
        return DataTable([data]), False


class _GroupedQuery(_ORMQuerySet):
    """Grouped queryset helper returned by group_by()."""

    def __init__(self, frame: DataTable, fields: Sequence[str]):
        super().__init__(frame)
        self._fields = list(fields)

    def annotate(self, **aggregations: Any) -> _ORMQuerySet:
        """Aggregate grouped rows into a queryset with named results."""
        grouped = self._frame.groupby(self._fields, dropna=False, sort=False)
        if not aggregations:
            result = grouped.size().reset_index(name="count")
            return _ORMQuerySet(DataTable(result))

        result = grouped.size().reset_index(name="_count").drop(columns=["_count"])
        for alias, spec in aggregations.items():
            source, func = self._normalize_spec(alias, spec)
            if source not in self._frame.columns:
                raise KeyError(f"Unknown field: {source}")
            aggregate_frame = getattr(grouped[source], func)().reset_index(name=alias)
            result = result.merge(aggregate_frame, on=self._fields, how="left", sort=False)
        return _ORMQuerySet(DataTable(result))

    def count(self) -> _ORMQuerySet:
        """Return group counts as a queryset."""
        result = self._frame.groupby(self._fields, dropna=False, sort=False).size().reset_index(name="count")
        return _ORMQuerySet(DataTable(result))

    def sum(self, *fields: str) -> dict[str, Any]:
        """Return grouped column sums for the requested fields."""
        return self._aggregate_columns("sum", *fields)

    def avg(self, *fields: str) -> dict[str, Any]:
        """Return grouped column means for the requested fields."""
        return self._aggregate_columns("mean", *fields)

    def min(self, *fields: str) -> dict[str, Any]:
        """Return grouped column minimums for the requested fields."""
        return self._aggregate_columns("min", *fields)

    def max(self, *fields: str) -> dict[str, Any]:
        """Return grouped column maximums for the requested fields."""
        return self._aggregate_columns("max", *fields)

    def aggregate(self, **aggregations: Any) -> dict[str, Any]:
        """Return named aggregates for the grouped queryset."""
        grouped = self._frame.groupby(self._fields, dropna=False, sort=False)
        result: dict[str, Any] = {}
        for alias, spec in aggregations.items():
            source, func = self._normalize_spec(alias, spec)
            if source not in self._frame.columns:
                raise KeyError(f"Unknown field: {source}")
            aggregate_frame = getattr(grouped[source], func)().reset_index(name=alias)
            result[alias] = aggregate_frame.to_dict(orient="records")
        return result

    def _aggregate_columns(self, func: str, *fields: str) -> dict[str, Any]:
        grouped = self._frame.groupby(self._fields, dropna=False, sort=False)
        selected = list(fields) if fields else [field for field in self._frame.columns if field not in self._fields]
        result: dict[str, Any] = {}
        for field in selected:
            if field not in self._frame.columns:
                raise KeyError(f"Unknown field: {field}")
            result[field] = getattr(grouped[field], func)().reset_index().to_dict(orient="records")
        return result


df = DataTable
datatable = DataTable
BaseDataFrame = DataTable
