"""Custom DataFrame wrapper for local dataframe workflows."""

from __future__ import annotations

import os
import json
import re
import logging
import unittest
import copy
import random
import uuid
from pathlib import Path
from datetime import datetime, timezone
from typing import Any, Iterable, Mapping, Optional, Sequence, Tuple, Union

import pandas as pd

try:
    from sqlalchemy import create_engine
except ImportError:  # pragma: no cover - optional dependency in some environments
    create_engine = None

__all__ = [
    "DataTable",
    "datatable",
    "DatabaseConnector",
    "db_connector",
    "from_sql",
    "from_mysql_db",
    "from_maria_db",
    "from_oracle_db",
    "from_postgres_db",
    "from_sqlite_db",
    "from_google_sheet",
    "to_google_sheet",
    "from_faker",
    "from_factory",
    "df",
]


class _ValuesResult(list):
    """List-like container that exposes Django-style helpers."""

    def first(self):
        return self[0] if self else None

    def last(self):
        return self[-1] if self else None

    def filter(self, predicate=None, **conditions):
        """Return filtered records by predicate or exact field matches."""
        if predicate is not None and not callable(predicate):
            raise TypeError("predicate must be callable")
        result = _ValuesResult()
        for row in self:
            if predicate is not None:
                if predicate(row):
                    result.append(row)
                continue
            match = True
            for key, expected in conditions.items():
                if row.get(key) != expected:
                    match = False
                    break
            if match:
                result.append(row)
        return result

    def exclude(self, predicate=None, **conditions):
        """Return records excluded by predicate or exact field matches."""
        if predicate is not None and not callable(predicate):
            raise TypeError("predicate must be callable")
        result = _ValuesResult()
        for row in self:
            if predicate is not None:
                if not predicate(row):
                    result.append(row)
                continue
            match = True
            for key, expected in conditions.items():
                if row.get(key) != expected:
                    match = False
                    break
            if not match:
                result.append(row)
        return result

    def order_by(self, key, reverse: bool = False):
        """Return records ordered by a key or callable."""
        if not callable(key) and not isinstance(key, str):
            raise TypeError("key must be a field name or callable")
        return _ValuesResult(sorted(self, key=(key if callable(key) else lambda row: row.get(key)), reverse=reverse))

    def distinct(self, *fields: str):
        """Return unique records, optionally keyed by selected fields."""
        result = _ValuesResult()
        seen = set()
        for row in self:
            if fields and isinstance(row, dict):
                signature = tuple(row.get(field) for field in fields)
            elif isinstance(row, dict):
                signature = tuple(sorted(row.items()))
            elif isinstance(row, (tuple, list)):
                signature = tuple(row)
            else:
                signature = row
            if signature in seen:
                continue
            seen.add(signature)
            result.append(row)
        return result


class _NoLogsContext:
    """Fallback context manager for assertNoLogs on older Python versions."""

    def __init__(self, logger=None, level=logging.INFO):
        self.logger = logging.getLogger(logger) if isinstance(logger, str) else (logger or logging.getLogger())
        self.level = logging._nameToLevel.get(level, level) if isinstance(level, str) else level
        self.records = []
        self._handler = logging.Handler(level=self.level)
        self._handler.emit = self._emit  # type: ignore[assignment]
        self._old_level = None

    def _emit(self, record):
        if record.levelno >= self.level:
            self.records.append(record)

    def __enter__(self):
        self._old_level = self.logger.level
        self.logger.addHandler(self._handler)
        return self

    def __exit__(self, exc_type, exc, tb):
        self.logger.removeHandler(self._handler)
        if self._old_level is not None:
            self.logger.setLevel(self._old_level)
        if exc_type is None and self.records:
            raise AssertionError("Unexpected logs were emitted")
        return False


class _MiniFaker:
    """Small fallback faker used when the optional faker package is unavailable."""

    _first_names = ["Alice", "Bob", "Carol", "Dave", "Eve", "Frank", "Grace", "Huy", "Linh", "Minh"]
    _last_names = ["Nguyen", "Tran", "Le", "Pham", "Hoang", "Vo", "Huynh", "Pham"]
    _cities = ["Hanoi", "Saigon", "Danang", "Hue", "Can Tho", "Nha Trang"]
    _companies = ["Acme", "Nimbus", "SoiGia", "Atlas", "Vertex", "Orchid"]
    _words = ["alpha", "beta", "gamma", "delta", "omega", "signal", "market", "trade", "data", "frame"]

    def __init__(self, seed: Optional[int] = None):
        self._rng = random.Random(seed)

    def seed_instance(self, seed: int) -> None:
        self._rng.seed(seed)

    def _choice(self, values: Sequence[str]) -> str:
        return self._rng.choice(list(values))

    def name(self) -> str:
        return f"{self._choice(self._first_names)} {self._choice(self._last_names)}"

    def first_name(self) -> str:
        return self._choice(self._first_names)

    def last_name(self) -> str:
        return self._choice(self._last_names)

    def city(self) -> str:
        return self._choice(self._cities)

    def company(self) -> str:
        return self._choice(self._companies)

    def word(self) -> str:
        return self._choice(self._words)

    def words(self, nb: int = 3) -> list[str]:
        return [self.word() for _ in range(max(1, nb))]

    def sentence(self, nb_words: int = 6) -> str:
        words = self.words(nb_words)
        sentence = " ".join(words).capitalize()
        return f"{sentence}."

    def text(self, max_nb_chars: int = 200) -> str:
        text = self.sentence(max(3, min(12, max_nb_chars // 12 or 3)))
        return text[:max_nb_chars]

    def email(self) -> str:
        user = f"{self.first_name().lower()}.{self.last_name().lower()}".replace(" ", "")
        domain = self._choice(["example.com", "mail.com", "soigia.dev"])
        return f"{user}@{domain}"

    def phone_number(self) -> str:
        return f"+84-{self._rng.randint(100, 999)}-{self._rng.randint(1000, 9999)}"

    def uuid4(self) -> str:
        return str(uuid.UUID(int=self._rng.getrandbits(128)))

    def random_int(self, min: int = 0, max: int = 9999) -> int:  # noqa: A002
        return self._rng.randint(min, max)

    def pyfloat(self, left_digits: int = 2, right_digits: int = 2, positive: bool = True) -> float:
        left = self._rng.randint(0 if positive else -10 ** left_digits, 10**left_digits)
        right = self._rng.randint(0, 10**right_digits - 1)
        value = float(f"{abs(left)}.{right:0{right_digits}d}")
        return abs(value) if positive else value

    def date(self) -> datetime:
        return datetime.fromtimestamp(self._rng.randint(1_600_000_000, 1_800_000_000), tz=timezone.utc)

    def datetime(self) -> datetime:
        return self.date()


def _build_faker(locale: str = "en_US", seed: Optional[int] = None):
    try:
        from faker import Faker  # type: ignore
    except ImportError:
        return _MiniFaker(seed=seed)

    fake = Faker(locale)
    if seed is not None:
        fake.seed_instance(seed)
    return fake


def _resolve_fake_value(generator: Any, spec: Any) -> Any:
    if callable(spec):
        return spec(generator)
    if isinstance(spec, tuple) and len(spec) == 2 and isinstance(spec[0], str) and isinstance(spec[1], Mapping):
        provider = getattr(generator, spec[0], None)
        if callable(provider):
            return provider(**dict(spec[1]))
    if isinstance(spec, Mapping) and "provider" in spec:
        provider_name = spec["provider"]
        provider = getattr(generator, provider_name, None)
        if callable(provider):
            kwargs = dict(spec.get("kwargs", {}))
            return provider(**kwargs)
    if isinstance(spec, str):
        provider = getattr(generator, spec, None)
        if callable(provider):
            return provider()
    return spec


def _factory_item_to_record(item: Any) -> dict[str, Any]:
    if isinstance(item, Mapping):
        return dict(item)
    if hasattr(item, "_asdict"):
        return dict(item._asdict())
    if hasattr(item, "dict") and callable(item.dict):
        value = item.dict()
        if isinstance(value, Mapping):
            return dict(value)
    if hasattr(item, "__dict__"):
        return {key: value for key, value in vars(item).items() if not key.startswith("_") and not callable(value)}
    raise TypeError("factory item cannot be converted into a record")


def _build_factory_rows(factory_class: Any, count: int, **build_kwargs: Any) -> list[Any]:
    if hasattr(factory_class, "build_batch") and callable(factory_class.build_batch):
        return list(factory_class.build_batch(size=count, **build_kwargs))
    if callable(factory_class):
        return [factory_class(**build_kwargs) for _ in range(count)]
    raise TypeError("factory_class must provide build_batch() or be callable")


class DatabaseConnector:
    """Hold one generic connection string and optional backend-specific URLs."""

    def __init__(
        self,
        connection_string: Optional[str] = None,
        mysql: Optional[str] = None,
        maria: Optional[str] = None,
        mariadb: Optional[str] = None,
        oracle: Optional[str] = None,
        postgres: Optional[str] = None,
        sqlite: Optional[str] = None,
    ):
        self.connection_string = connection_string
        self.mysql = mysql
        self.maria = maria
        self.mariadb = mariadb
        self.oracle = oracle
        self.postgres = postgres
        self.sqlite = sqlite

    def _engine_url(self, backend: str, connection_string: Optional[str] = None) -> str:
        url = (
            connection_string
            or getattr(self, backend, None)
            or self.connection_string
            or os.getenv(f"SOIGIA_{backend.upper()}_URL")
        )
        if not url:
            raise ValueError(f"Missing connection string for backend: {backend}")
        return url

    def _engine(self, backend: str, connection_string: Optional[str] = None):
        if create_engine is None:
            raise ImportError("sqlalchemy is required to use database helpers")
        return create_engine(self._engine_url(backend, connection_string))

    def query(self, sql: str, backend: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        """Run SQL and return a DataTable."""
        engine = self._engine(backend, connection_string)
        try:
            frame = pd.read_sql_query(sql, engine, **kwargs)
        finally:
            dispose = getattr(engine, "dispose", None)
            if callable(dispose):
                dispose()
        return DataTable(frame)

    def from_sql(
        self,
        sql: str,
        backend: str = "sqlite",
        connection_string: Optional[str] = None,
        **kwargs: Any,
    ) -> "DataTable":
        """Run SQL against the selected backend and return a DataTable."""
        return self.query(sql, backend, connection_string=connection_string, **kwargs)

    def from_mysql_db(self, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        return self.query(sql, "mysql", connection_string=connection_string, **kwargs)

    def from_maria_db(self, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        return self.query(sql, "maria", connection_string=connection_string, **kwargs)

    def from_oracle_db(self, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        return self.query(sql, "oracle", connection_string=connection_string, **kwargs)

    def from_postgres_db(self, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        return self.query(sql, "postgres", connection_string=connection_string, **kwargs)

    def from_sqlite_db(self, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        return self.query(sql, "sqlite", connection_string=connection_string, **kwargs)


_DEFAULT_CONNECTOR: Optional[DatabaseConnector] = None


def db_connector(connection_string: Optional[str] = None, **connections: Any) -> DatabaseConnector:
    """Configure and return a reusable database connector."""
    global _DEFAULT_CONNECTOR
    _DEFAULT_CONNECTOR = DatabaseConnector(connection_string=connection_string, **connections)
    return _DEFAULT_CONNECTOR


def _connector() -> DatabaseConnector:
    global _DEFAULT_CONNECTOR
    if _DEFAULT_CONNECTOR is None:
        _DEFAULT_CONNECTOR = DatabaseConnector()
    return _DEFAULT_CONNECTOR


def from_sql(sql: str, backend: str = "sqlite", connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
    return _connector().from_sql(sql, backend=backend, connection_string=connection_string, **kwargs)


def from_mysql_db(sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
    return _connector().from_mysql_db(sql, connection_string=connection_string, **kwargs)


def from_maria_db(sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
    return _connector().from_maria_db(sql, connection_string=connection_string, **kwargs)


def from_oracle_db(sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
    return _connector().from_oracle_db(sql, connection_string=connection_string, **kwargs)


def from_postgres_db(sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
    return _connector().from_postgres_db(sql, connection_string=connection_string, **kwargs)


def from_sqlite_db(sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
    return _connector().from_sqlite_db(sql, connection_string=connection_string, **kwargs)


def _is_google_worksheet(candidate: Any) -> bool:
    return (
        hasattr(candidate, "get_all_records")
        or hasattr(candidate, "get_all_values")
        or hasattr(candidate, "update")
        or hasattr(candidate, "append_rows")
    )


def _build_google_client(
    client: Any = None,
    credentials_json: Optional[str] = None,
    service_account_file: Optional[Union[str, Path]] = None,
    scopes: Optional[Sequence[str]] = None,
):
    if client is not None:
        return client
    try:
        import gspread  # type: ignore
    except ImportError as exc:  # pragma: no cover - optional dependency
        raise ImportError("gspread is required to use Google Sheet helpers") from exc

    if credentials_json or service_account_file:
        try:
            from google.oauth2.service_account import Credentials  # type: ignore
        except ImportError as exc:  # pragma: no cover - optional dependency
            raise ImportError("google-auth is required to use credential-based Google Sheet helpers") from exc

        credential_scopes = list(
            scopes
            or [
                "https://www.googleapis.com/auth/spreadsheets",
                "https://www.googleapis.com/auth/drive",
            ]
        )
        if credentials_json:
            credentials = Credentials.from_service_account_info(json.loads(credentials_json), scopes=credential_scopes)
        else:
            credentials = Credentials.from_service_account_file(str(service_account_file), scopes=credential_scopes)
        return gspread.authorize(credentials)

    if hasattr(gspread, "service_account"):
        return gspread.service_account()
    raise ImportError("gspread is required to use Google Sheet helpers")


def _resolve_google_worksheet(
    spreadsheet: Any,
    worksheet: Any = 0,
    *,
    client: Any = None,
    credentials_json: Optional[str] = None,
    service_account_file: Optional[Union[str, Path]] = None,
    scopes: Optional[Sequence[str]] = None,
):
    if _is_google_worksheet(spreadsheet):
        return spreadsheet
    if _is_google_worksheet(worksheet):
        return worksheet

    resolved_client = _build_google_client(
        client=client,
        credentials_json=credentials_json,
        service_account_file=service_account_file,
        scopes=scopes,
    )

    workbook = spreadsheet
    if isinstance(spreadsheet, (str, Path)):
        if hasattr(resolved_client, "open_by_url") and str(spreadsheet).startswith(("http://", "https://")):
            workbook = resolved_client.open_by_url(str(spreadsheet))
        elif hasattr(resolved_client, "open_by_key"):
            workbook = resolved_client.open_by_key(str(spreadsheet))
        elif hasattr(resolved_client, "open"):
            workbook = resolved_client.open(str(spreadsheet))
        else:  # pragma: no cover - defensive
            workbook = resolved_client
    else:
        workbook = spreadsheet

    if isinstance(worksheet, int):
        if hasattr(workbook, "get_worksheet"):
            resolved = workbook.get_worksheet(worksheet)
            if resolved is not None:
                return resolved
        if hasattr(workbook, "worksheet"):
            return workbook.worksheet(worksheet)
    else:
        if hasattr(workbook, "worksheet"):
            return workbook.worksheet(str(worksheet))

    raise TypeError("Could not resolve a Google Sheet worksheet from the provided inputs")


def _google_worksheet_to_frame(worksheet: Any, header: int = 0, range_name: Optional[str] = None) -> "DataTable":
    if range_name is not None and hasattr(worksheet, "get"):
        values = worksheet.get(range_name)
        if not values:
            return DataTable()
        headers = values[header]
        rows = values[header + 1 :]
        return DataTable(rows, columns=headers)

    if hasattr(worksheet, "get_all_records"):
        records = worksheet.get_all_records()
        return DataTable(records)

    if hasattr(worksheet, "get_all_values"):
        values = worksheet.get_all_values()
        if not values:
            return DataTable()
        headers = values[header]
        rows = values[header + 1 :]
        return DataTable(rows, columns=headers)

    raise TypeError("worksheet object must provide get_all_records() or get_all_values()")


def _frame_to_google_values(frame: pd.DataFrame, include_index: bool = False) -> list[list[Any]]:
    data = frame.reset_index() if include_index else frame
    clean = data.astype(object).where(pd.notna(data), None)
    return [list(clean.columns)] + clean.values.tolist()


def from_google_sheet(
    spreadsheet: Any,
    worksheet: Any = 0,
    header: int = 0,
    range_name: Optional[str] = None,
    client: Any = None,
    credentials_json: Optional[str] = None,
    service_account_file: Optional[Union[str, Path]] = None,
    scopes: Optional[Sequence[str]] = None,
) -> "DataTable":
    return DataTable.from_google_sheet(
        spreadsheet,
        worksheet=worksheet,
        header=header,
        range_name=range_name,
        client=client,
        credentials_json=credentials_json,
        service_account_file=service_account_file,
        scopes=scopes,
    )


def to_google_sheet(
    table: "DataTable",
    spreadsheet: Any,
    worksheet: Any = 0,
    clear: bool = True,
    include_index: bool = False,
    start_cell: str = "A1",
    value_input_option: str = "RAW",
    client: Any = None,
    credentials_json: Optional[str] = None,
    service_account_file: Optional[Union[str, Path]] = None,
    scopes: Optional[Sequence[str]] = None,
) -> Any:
    return table.to_google_sheet(
        spreadsheet,
        worksheet=worksheet,
        clear=clear,
        include_index=include_index,
        start_cell=start_cell,
        value_input_option=value_input_option,
        client=client,
        credentials_json=credentials_json,
        service_account_file=service_account_file,
        scopes=scopes,
    )


def from_faker(
    schema: Mapping[str, Any],
    count: int = 10,
    locale: str = "en_US",
    seed: Optional[int] = None,
) -> "DataTable":
    return DataTable.from_faker(schema=schema, count=count, locale=locale, seed=seed)


def from_factory(factory_class: Any, count: int = 10, **build_kwargs: Any) -> "DataTable":
    return DataTable.from_factory(factory_class, count=count, **build_kwargs)


class _HelperTestCase(unittest.TestCase):
    """Concrete unittest case used only as an assertion delegate."""

    def runTest(self):
        return None


class _TestCaseMixin:
    """Mixin that exposes unittest.TestCase-style helpers on a dataframe."""

    _class_cleanups: list[tuple[Any, tuple[Any, ...], dict[str, Any]]] = []

    def _testcase(self) -> unittest.TestCase:
        helper = self.__dict__.get("_unittest_helper")
        if helper is None:
            helper = _HelperTestCase("runTest")
            object.__setattr__(self, "_unittest_helper", helper)
        return helper

    @classmethod
    def _class_cleanup_stack(cls) -> list[tuple[Any, tuple[Any, ...], dict[str, Any]]]:
        stack = cls.__dict__.get("_class_cleanups")
        if stack is None:
            stack = []
            setattr(cls, "_class_cleanups", stack)
        return stack

    @classmethod
    def setUpClass(cls):
        """Hook kept for Django/TestCase-like symmetry."""
        return None

    @classmethod
    def tearDownClass(cls):
        """Run any class cleanups that were registered."""
        cls.doClassCleanups()

    @classmethod
    def addClassCleanup(cls, function, /, *args, **kwargs):
        cls._class_cleanup_stack().append((function, args, kwargs))

    @classmethod
    def doClassCleanups(cls):
        errors = []
        stack = cls._class_cleanup_stack()
        while stack:
            function, args, kwargs = stack.pop()
            try:
                function(*args, **kwargs)
            except Exception as exc:  # pragma: no cover - defensive
                errors.append(exc)
        if errors:
            raise errors[0]

    @classmethod
    def enterClassContext(cls, cm):
        result = cm.__enter__()
        cls.addClassCleanup(cm.__exit__, None, None, None)
        return result

    def setUp(self):
        """Hook kept for Django/TestCase-like symmetry."""
        return None

    def tearDown(self):
        """Hook kept for Django/TestCase-like symmetry."""
        return None

    def assertIsSubclass(self, cls, class_or_tuple, msg=None):
        helper = self._testcase()
        if not isinstance(cls, type):
            helper.fail(helper._formatMessage(msg, f"{cls!r} is not a class"))
        if not issubclass(cls, class_or_tuple):
            helper.fail(helper._formatMessage(msg, f"{cls!r} is not a subclass of {class_or_tuple!r}"))

    def assertNotIsSubclass(self, cls, class_or_tuple, msg=None):
        helper = self._testcase()
        if not isinstance(cls, type):
            helper.fail(helper._formatMessage(msg, f"{cls!r} is not a class"))
        if issubclass(cls, class_or_tuple):
            helper.fail(helper._formatMessage(msg, f"{cls!r} is a subclass of {class_or_tuple!r}"))

    def assertNoLogs(self, logger=None, level=logging.INFO):
        helper = self._testcase()
        if hasattr(helper, "assertNoLogs"):
            return helper.assertNoLogs(logger, level=level)
        return _NoLogsContext(logger, level)


def _delegate_testcase_method(name: str):
    def method(self, *args, **kwargs):
        return getattr(self._testcase(), name)(*args, **kwargs)

    method.__name__ = name
    method.__qualname__ = f"_TestCaseMixin.{name}"
    return method


for _method_name in [
    "assertEqual",
    "assertNotEqual",
    "assertTrue",
    "assertFalse",
    "assertIs",
    "assertIsNot",
    "assertIsNone",
    "assertIsNotNone",
    "assertIn",
    "assertNotIn",
    "assertIsInstance",
    "assertNotIsInstance",
    "assertAlmostEqual",
    "assertNotAlmostEqual",
    "assertGreater",
    "assertGreaterEqual",
    "assertLess",
    "assertLessEqual",
    "assertRaises",
    "assertRaisesRegex",
    "assertWarns",
    "assertWarnsRegex",
    "assertLogs",
    "assertMultiLineEqual",
    "assertSequenceEqual",
    "assertListEqual",
    "assertTupleEqual",
    "assertSetEqual",
    "assertDictEqual",
    "assertCountEqual",
    "assertRegex",
    "assertNotRegex",
    "fail",
    "addTypeEqualityFunc",
    "addCleanup",
    "doCleanups",
    "enterContext",
    "run",
    "debug",
    "defaultTestResult",
    "countTestCases",
    "id",
    "shortDescription",
    "subTest",
]:
    setattr(_TestCaseMixin, _method_name, _delegate_testcase_method(_method_name))


class _DjangoStyleManager:
    """A minimal manager facade that reads like Django."""

    def __init__(self, queryset_factory):
        self._queryset_factory = queryset_factory

    def all(self):
        return self._queryset_factory().all()

    def none(self):
        return self._queryset_factory().none()

    def filter(self, *expressions: str, **conditions: Any):
        return self._queryset_factory().filter(*expressions, **conditions)

    def exclude(self, *expressions: str, **conditions: Any):
        return self._queryset_factory().exclude(*expressions, **conditions)

    def order_by(self, *fields: str, **kwargs: Any):
        return self._queryset_factory().order_by(*fields, **kwargs)

    def random_order(self):
        return self._queryset_factory().random_order()

    def values(self, *fields: str):
        return self._queryset_factory().values(*fields)

    def values_list(self, *fields: str, flat: bool = False):
        return self._queryset_factory().values_list(*fields, flat=flat)

    def select(self, *fields: str):
        return self._queryset_factory().select(*fields)

    def exclude_fields(self, *fields: str):
        return self._queryset_factory().exclude_fields(*fields)

    def get(self, *expressions: str, **conditions: Any):
        return self._queryset_factory().get(*expressions, **conditions)

    def count(self) -> int:
        return self._queryset_factory().count()

    def exists(self) -> bool:
        return self._queryset_factory().exists()

    def first(self):
        return self._queryset_factory().first()

    def last(self):
        return self._queryset_factory().last()

    def earliest(self, *fields: str):
        return self._queryset_factory().earliest(*fields)

    def latest(self, *fields: str):
        return self._queryset_factory().latest(*fields)

    def paginate(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().paginate(*args, **kwargs)

    def group_by(self, *fields: str):
        return self._queryset_factory().group_by(*fields)

    def aggregate(self, **aggregations: Any):
        return self._queryset_factory().aggregate(**aggregations)

    def sum(self, *fields: str):
        return self._queryset_factory().sum(*fields)

    def avg(self, *fields: str):
        return self._queryset_factory().avg(*fields)

    def min(self, *fields: str):
        return self._queryset_factory().min(*fields)

    def max(self, *fields: str):
        return self._queryset_factory().max(*fields)

    def diff_columns_against(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_columns_against(*args, **kwargs)

    def diff_added_rows(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_added_rows(*args, **kwargs)

    def diff_removed_rows(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_removed_rows(*args, **kwargs)

    def diff_unchanged_rows(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_unchanged_rows(*args, **kwargs)

    def diff_changed_rows(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_changed_rows(*args, **kwargs)

    def diff_changed_cells(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_changed_cells(*args, **kwargs)

    def diff_report(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().diff_report(*args, **kwargs)

    def create(self, **fields: Any):
        return self._queryset_factory().create(**fields)

    def append_row(self, **fields: Any):
        return self._queryset_factory().append_row(**fields)

    def bulk_create(self, rows: Sequence[Mapping[str, Any]]):
        return self._queryset_factory().bulk_create(rows)

    def update(self, **fields: Any):
        return self._queryset_factory().update(**fields)

    def delete(self):
        return self._queryset_factory().delete()

    def bulk_update(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().bulk_update(*args, **kwargs)

    def save(self):
        return self._queryset_factory().save()

    def get_or_create(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().get_or_create(*args, **kwargs)

    def update_or_create(self, *args: Any, **kwargs: Any):
        return self._queryset_factory().update_or_create(*args, **kwargs)

    def __getattr__(self, name: str):
        return getattr(self._queryset_factory(), name)


class DataTable(_TestCaseMixin, pd.DataFrame):
    """A thin pandas.DataFrame subclass for personal dataframe work."""

    _metadata = ["_soigia_version_history", "_soigia_auto_version", "_soigia_auto_version_label"]

    @property
    def _constructor(self):
        return DataTable

    def __finalize__(self, other, method=None, **kwargs):
        result = super().__finalize__(other, method=method, **kwargs)
        if isinstance(other, DataTable) and hasattr(other, "_soigia_version_history"):
            object.__setattr__(self, "_soigia_version_history", copy.deepcopy(getattr(other, "_soigia_version_history")))
        if isinstance(other, DataTable):
            if hasattr(other, "_soigia_auto_version"):
                object.__setattr__(self, "_soigia_auto_version", getattr(other, "_soigia_auto_version"))
            if hasattr(other, "_soigia_auto_version_label"):
                object.__setattr__(self, "_soigia_auto_version_label", getattr(other, "_soigia_auto_version_label"))
        return result

    def _inherit_versions(self, frame: pd.DataFrame) -> "DataTable":
        result = DataTable(frame)
        if hasattr(self, "_soigia_version_history"):
            object.__setattr__(result, "_soigia_version_history", copy.deepcopy(self._soigia_version_history))
        if hasattr(self, "_soigia_auto_version"):
            object.__setattr__(result, "_soigia_auto_version", getattr(self, "_soigia_auto_version"))
        if hasattr(self, "_soigia_auto_version_label"):
            object.__setattr__(result, "_soigia_auto_version_label", getattr(self, "_soigia_auto_version_label"))
        if getattr(self, "_soigia_auto_version", False):
            result.mark_version(getattr(self, "_soigia_auto_version_label", "auto"))
        return result

    def _snapshot_payload(self, label: Optional[str] = None) -> dict[str, Any]:
        return {
            "version": self.version_count(),
            "label": label,
            "created_at": datetime.now(timezone.utc).isoformat(),
            "shape": tuple(self.shape),
            "columns": list(self.columns),
            "records": self.to_dict(orient="records"),
        }

    def _resolve_version_index(self, version: int | str) -> int:
        history = self.version_history(include_data=True)
        if isinstance(version, int):
            if version < 0:
                version = len(history) + version
            if version < 0 or version >= len(history):
                raise IndexError("Version index out of range")
            return version
        for index, item in enumerate(history):
            if item.get("label") == version:
                return index
        raise KeyError(f"Unknown version label: {version}")

    @property
    def orm(self) -> "_ORMManager":
        """Return a Django-like query helper bound to this dataframe."""
        return _ORMManager(self)

    @property
    def qs(self) -> "_ORMManager":
        """Alias for orm."""
        return self.orm

    @property
    def objects(self) -> "_ORMManager":
        """Django-style alias for orm."""
        return _DjangoStyleManager(lambda: _ORMManager(self))

    @property
    def manager(self) -> "_ORMManager":
        """Alias for objects."""
        return self.objects

    @classmethod
    def from_csv(cls, path: Union[str, Path], **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from a CSV file."""
        return cls(pd.read_csv(path, **kwargs))

    @classmethod
    def from_dict(cls, data: Optional[dict] = None, **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from a dictionary."""
        return cls(pd.DataFrame.from_dict(data or {}, **kwargs))

    @classmethod
    def from_records(cls, records: Sequence[Mapping[str, Any]], **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from records."""
        return cls(pd.DataFrame.from_records(records, **kwargs))

    @classmethod
    def from_json(cls, path_or_buf: Any, **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from JSON input."""
        return cls(pd.read_json(path_or_buf, **kwargs))

    @classmethod
    def from_excel(cls, io: Any, **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from an Excel workbook or sheet."""
        return cls(pd.read_excel(io, **kwargs))

    @classmethod
    def from_parquet(cls, path: Any, **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from a Parquet file."""
        return cls(pd.read_parquet(path, **kwargs))

    @classmethod
    def from_feather(cls, path: Any, **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from a Feather file."""
        return cls(pd.read_feather(path, **kwargs))

    @classmethod
    def from_pickle(cls, filepath_or_buffer: Any, **kwargs: Any) -> "DataTable":
        """Create a DataTable instance from a pickle file or buffer."""
        return cls(pd.read_pickle(filepath_or_buffer, **kwargs))

    @classmethod
    def from_faker(
        cls,
        schema: Mapping[str, Any],
        count: int = 10,
        locale: str = "en_US",
        seed: Optional[int] = None,
    ) -> "DataTable":
        """Create a DataTable instance from a fake-data schema."""
        generator = _build_faker(locale=locale, seed=seed)
        records = []
        for _ in range(max(0, count)):
            row = {field: _resolve_fake_value(generator, spec) for field, spec in schema.items()}
            records.append(row)
        return cls(records)

    @classmethod
    def from_factory(cls, factory_class: Any, count: int = 10, **build_kwargs: Any) -> "DataTable":
        """Create a DataTable instance from a factory_boy-style factory or callable."""
        rows = _build_factory_rows(factory_class, count, **build_kwargs)
        records = [_factory_item_to_record(item) for item in rows]
        return cls(records)

    @classmethod
    def from_sql(
        cls,
        sql: str,
        backend: str = "sqlite",
        connection_string: Optional[str] = None,
        **kwargs: Any,
    ) -> "DataTable":
        """Run SQL against the selected backend and return a DataTable."""
        return cls(from_sql(sql, backend=backend, connection_string=connection_string, **kwargs))

    def init_versions(self, label: str = "initial") -> "DataTable":
        """Initialize version history with the current dataframe snapshot."""
        object.__setattr__(self, "_soigia_version_history", [self._snapshot_payload(label)])
        return self

    def mark_version(self, label: Optional[str] = None) -> "DataTable":
        """Append the current dataframe snapshot to version history."""
        history = list(getattr(self, "_soigia_version_history", []))
        history.append(self._snapshot_payload(label))
        object.__setattr__(self, "_soigia_version_history", history)
        return self

    def version_history(self, include_data: bool = False) -> list[dict[str, Any]]:
        """Return the stored version history."""
        history = copy.deepcopy(getattr(self, "_soigia_version_history", []) or [])
        if include_data:
            return history
        for item in history:
            item.pop("records", None)
        return history

    def version_count(self) -> int:
        """Return the number of stored versions."""
        history = getattr(self, "_soigia_version_history", None) or []
        return len(history)

    def current_version(self) -> Optional[dict[str, Any]]:
        """Return the latest version metadata if available."""
        history = self.version_history(include_data=True)
        return history[-1] if history else None

    def get_version(self, version: int | str = -1) -> "DataTable":
        """Return a stored version as a DataTable."""
        index = self._resolve_version_index(version)
        history = self.version_history(include_data=True)
        payload = history[index]
        frame = DataTable(payload.get("records", []))
        if payload.get("columns"):
            frame = frame.reindex(columns=payload["columns"])
        return frame

    def restore_version(self, version: int | str = -1) -> "DataTable":
        """Alias for get_version()."""
        return self.get_version(version)

    def snapshot(self, label: Optional[str] = None) -> "DataTable":
        """Alias for mark_version()."""
        return self.mark_version(label)

    def auto_version(self, label: str = "auto") -> "DataTable":
        """Enable automatic version snapshots for helper-driven transforms."""
        if self.version_count() == 0:
            self.init_versions("initial")
        object.__setattr__(self, "_soigia_auto_version", True)
        object.__setattr__(self, "_soigia_auto_version_label", label)
        return self

    def disable_auto_version(self) -> "DataTable":
        """Disable automatic version snapshots."""
        object.__setattr__(self, "_soigia_auto_version", False)
        return self

    def is_auto_versioning(self) -> bool:
        """Return whether automatic version snapshots are enabled."""
        return bool(getattr(self, "_soigia_auto_version", False))

    def diff_versions(self, old: int | str = 0, new: int | str = -1) -> dict[str, Any]:
        """Compare two versions and return row-level differences."""
        old_frame = self.get_version(old)
        new_frame = self.get_version(new)

        def _signature(row: Mapping[str, Any]) -> str:
            return repr(tuple(sorted((key, repr(value)) for key, value in row.items())))

        old_rows = old_frame.to_dict(orient="records")
        new_rows = new_frame.to_dict(orient="records")

        old_map = {_signature(row): row for row in old_rows}
        new_map = {_signature(row): row for row in new_rows}

        old_set = set(old_map)
        new_set = set(new_map)

        added = [new_map[key] for key in new_set - old_set]
        removed = [old_map[key] for key in old_set - new_set]

        return {
            "old_version": old,
            "new_version": new,
            "added_rows": added,
            "removed_rows": removed,
            "old_count": len(old_rows),
            "new_count": len(new_rows),
            "row_delta": len(new_rows) - len(old_rows),
        }

    def diff_records(self, old: int | str = 0, new: int | str = -1) -> dict[str, Any]:
        """Alias for diff_versions()."""
        return self.diff_versions(old, new)

    def diff_columns(self, old: int | str = 0, new: int | str = -1) -> dict[str, Any]:
        """Compare column layouts between two versions."""
        old_frame = self.get_version(old)
        new_frame = self.get_version(new)
        old_columns = list(old_frame.columns)
        new_columns = list(new_frame.columns)
        return {
            "old_version": old,
            "new_version": new,
            "old_columns": old_columns,
            "new_columns": new_columns,
            "added_columns": [column for column in new_columns if column not in old_columns],
            "removed_columns": [column for column in old_columns if column not in new_columns],
            "reordered_columns": [column for column in new_columns if column in old_columns],
            "column_order_changed": old_columns != new_columns,
        }

    @staticmethod
    def _cell_equal(left: Any, right: Any) -> bool:
        """Compare two cell values while handling NaN and unhashable types."""
        if left is right:
            return True
        try:
            if pd.isna(left) and pd.isna(right):
                return True
        except Exception:
            pass
        try:
            comparison = left == right
            if isinstance(comparison, bool):
                return comparison
            return bool(comparison)
        except Exception:
            return repr(left) == repr(right)

    def diff_cells(
        self,
        old: int | str = 0,
        new: int | str = -1,
        key_columns: Optional[Sequence[str]] = None,
    ) -> dict[str, Any]:
        """Compare two versions at cell level."""
        old_frame = self.get_version(old)
        new_frame = self.get_version(new)
        old_rows = old_frame.to_dict(orient="records")
        new_rows = new_frame.to_dict(orient="records")

        if key_columns:
            keys = list(key_columns)
            for column in keys:
                if column not in old_frame.columns or column not in new_frame.columns:
                    raise KeyError(f"Unknown key column: {column}")

            def make_key(row: Mapping[str, Any]) -> tuple[Any, ...]:
                return tuple(row.get(column) for column in keys)

            old_map = {make_key(row): row for row in old_rows}
            new_map = {make_key(row): row for row in new_rows}
            common_keys = old_map.keys() & new_map.keys()
            added_rows = [new_map[key] for key in new_map.keys() - old_map.keys()]
            removed_rows = [old_map[key] for key in old_map.keys() - new_map.keys()]
            changed_cells = []
            changed_rows = set()
            columns = [column for column in sorted(set(old_frame.columns) | set(new_frame.columns)) if column not in keys]
            for key in common_keys:
                old_row = old_map[key]
                new_row = new_map[key]
                for column in columns:
                    if not self._cell_equal(old_row.get(column), new_row.get(column)):
                        changed_rows.add(key)
                        changed_cells.append(
                            {
                                "key": key,
                                "column": column,
                                "old": old_row.get(column),
                                "new": new_row.get(column),
                            }
                        )
        else:
            max_len = max(len(old_rows), len(new_rows))
            added_rows = []
            removed_rows = []
            changed_cells = []
            changed_rows = set()
            columns = sorted(set(old_frame.columns) | set(new_frame.columns))
            for index in range(max_len):
                if index >= len(old_rows):
                    added_rows.append({"index": index, "row": new_rows[index]})
                    continue
                if index >= len(new_rows):
                    removed_rows.append({"index": index, "row": old_rows[index]})
                    continue
                old_row = old_rows[index]
                new_row = new_rows[index]
                for column in columns:
                    if not self._cell_equal(old_row.get(column), new_row.get(column)):
                        changed_rows.add(index)
                        changed_cells.append(
                            {
                                "index": index,
                                "column": column,
                                "old": old_row.get(column),
                                "new": new_row.get(column),
                            }
                        )

        return {
            "old_version": old,
            "new_version": new,
            "added_rows": added_rows,
            "removed_rows": removed_rows,
            "changed_cells": changed_cells,
            "changed_rows": sorted(changed_rows),
        }

    def diff_summary(
        self,
        old: int | str = 0,
        new: int | str = -1,
        key_columns: Optional[Sequence[str]] = None,
    ) -> dict[str, Any]:
        """Return a compact summary of column, row, and cell differences."""
        records = self.diff_versions(old, new)
        columns = self.diff_columns(old, new)
        cells = self.diff_cells(old, new, key_columns=key_columns)
        return {
            "old_version": old,
            "new_version": new,
            "row_delta": records["row_delta"],
            "added_rows": len(records["added_rows"]),
            "removed_rows": len(records["removed_rows"]),
            "added_columns": columns["added_columns"],
            "removed_columns": columns["removed_columns"],
            "column_order_changed": columns["column_order_changed"],
            "changed_cells": len(cells["changed_cells"]),
            "changed_rows": len(cells["changed_rows"]),
        }

    @classmethod
    def db_connector(cls, connection_string: Optional[str] = None, **connections: Any):
        """Configure the default SQL connector used by the DB helpers."""
        return db_connector(connection_string=connection_string, **connections)

    @classmethod
    def from_mysql_db(cls, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        """Run SQL against a MySQL database and return a DataTable."""
        return cls(from_mysql_db(sql, connection_string=connection_string, **kwargs))

    @classmethod
    def from_maria_db(cls, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        """Run SQL against a MariaDB database and return a DataTable."""
        return cls(from_maria_db(sql, connection_string=connection_string, **kwargs))

    @classmethod
    def from_oracle_db(cls, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        """Run SQL against an Oracle database and return a DataTable."""
        return cls(from_oracle_db(sql, connection_string=connection_string, **kwargs))

    @classmethod
    def from_postgres_db(cls, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        """Run SQL against a PostgreSQL database and return a DataTable."""
        return cls(from_postgres_db(sql, connection_string=connection_string, **kwargs))

    @classmethod
    def from_sqlite_db(cls, sql: str, connection_string: Optional[str] = None, **kwargs: Any) -> "DataTable":
        """Run SQL against a SQLite database and return a DataTable."""
        return cls(from_sqlite_db(sql, connection_string=connection_string, **kwargs))

    @classmethod
    def from_google_sheet(
        cls,
        spreadsheet: Any,
        worksheet: Any = 0,
        header: int = 0,
        range_name: Optional[str] = None,
        client: Any = None,
        credentials_json: Optional[str] = None,
        service_account_file: Optional[Union[str, Path]] = None,
        scopes: Optional[Sequence[str]] = None,
    ) -> "DataTable":
        """Load data from a Google Sheet worksheet into a DataTable."""
        resolved = _resolve_google_worksheet(
            spreadsheet,
            worksheet,
            client=client,
            credentials_json=credentials_json,
            service_account_file=service_account_file,
            scopes=scopes,
        )
        return _google_worksheet_to_frame(resolved, header=header, range_name=range_name)

    def to_google_sheet(
        self,
        spreadsheet: Any,
        worksheet: Any = 0,
        clear: bool = True,
        include_index: bool = False,
        start_cell: str = "A1",
        value_input_option: str = "RAW",
        client: Any = None,
        credentials_json: Optional[str] = None,
        service_account_file: Optional[Union[str, Path]] = None,
        scopes: Optional[Sequence[str]] = None,
    ) -> Any:
        """Write the dataframe to a Google Sheet worksheet."""
        resolved = _resolve_google_worksheet(
            spreadsheet,
            worksheet,
            client=client,
            credentials_json=credentials_json,
            service_account_file=service_account_file,
            scopes=scopes,
        )
        values = _frame_to_google_values(self, include_index=include_index)
        if clear and hasattr(resolved, "clear"):
            resolved.clear()
        if hasattr(resolved, "update"):
            resolved.update(start_cell, values, value_input_option=value_input_option)
            return resolved
        if hasattr(resolved, "append_rows"):
            resolved.append_rows(values, value_input_option=value_input_option)
            return resolved
        raise TypeError("worksheet object must provide update() or append_rows()")

    def to_rows(self) -> list[dict[str, Any]]:
        """Return the dataframe as a list of record dictionaries."""
        return self.to_dict(orient="records")

    def to_markdown_table(self, index: bool = False, **kwargs: Any) -> str:
        """Return a Markdown table string."""
        return self.to_markdown(index=index, **kwargs)

    def to_html_table(self, index: bool = False, **kwargs: Any) -> str:
        """Return an HTML table string."""
        return self.to_html(index=index, **kwargs)

    def to_csv_table(self, index: bool = False, **kwargs: Any) -> str:
        """Return a CSV string."""
        return self.to_csv(index=index, **kwargs)

    def to_rich_table(self, title: Optional[str] = None, show_index: bool = False):
        """Return a rich Table object for terminal display."""
        from rich.table import Table

        table = Table(title=title)
        if show_index:
            table.add_column("index")
        for column in self.columns:
            table.add_column(str(column))
        for index, row in self.iterrows():
            values = [str(index)] if show_index else []
            values.extend("" if pd.isna(value) else str(value) for value in row.tolist())
            table.add_row(*values)
        return table

    def to_excel_table(
        self,
        path: Union[str, Path],
        sheet_name: str = "Sheet1",
        index: bool = False,
        style_header: bool = True,
        autofit: bool = True,
        freeze_header: bool = True,
        **kwargs: Any,
    ) -> Path:
        """Write a styled Excel file and return the path."""
        output = Path(path)
        self.to_excel(output, sheet_name=sheet_name, index=index, **kwargs)
        if not (style_header or autofit or freeze_header):
            return output
        try:
            from openpyxl import load_workbook
            from openpyxl.styles import Font, PatternFill
        except ImportError:
            return output

        workbook = load_workbook(output)
        worksheet = workbook[sheet_name]
        if freeze_header:
            worksheet.freeze_panes = "A2"
        if style_header:
            fill = PatternFill("solid", fgColor="1F2937")
            for cell in worksheet[1]:
                cell.font = Font(bold=True, color="FFFFFF")
                cell.fill = fill
        if autofit:
            for column_cells in worksheet.columns:
                column_letter = column_cells[0].column_letter
                max_length = 0
                for cell in column_cells:
                    value = "" if cell.value is None else str(cell.value)
                    if len(value) > max_length:
                        max_length = len(value)
                worksheet.column_dimensions[column_letter].width = min(max_length + 2, 60)
        workbook.save(output)
        return output

    def render_table(self, format: str = "markdown", **kwargs: Any):
        """Render the dataframe in a selected format."""
        fmt = format.lower()
        if fmt == "markdown":
            return self.to_markdown_table(**kwargs)
        if fmt == "html":
            return self.to_html_table(**kwargs)
        if fmt == "csv":
            return self.to_csv_table(**kwargs)
        if fmt == "rich":
            return self.to_rich_table(**kwargs)
        if fmt == "excel":
            path = kwargs.pop("path", None)
            if path is None:
                raise ValueError("render_table(format='excel') requires path=")
            return self.to_excel_table(path, **kwargs)
        raise ValueError(f"Unsupported format: {format}")

    def to_display_text(self, format: str = "markdown", **kwargs: Any) -> str:
        """Return printable text for a selected display format."""
        fmt = format.lower()
        display_kwargs = dict(kwargs)
        width = display_kwargs.pop("width", 120)
        rendered = self.render_table(fmt, **display_kwargs)
        if fmt != "rich":
            return rendered
        try:
            from rich.console import Console
        except ImportError:
            return str(rendered)

        console = Console(record=True, width=width)
        console.print(rendered)
        return console.export_text()

    def print_table(self, format: str = "markdown", file=None, end: str = "\n", **kwargs: Any) -> str:
        """Print a rendered table and return the printable text."""
        text = self.to_display_text(format=format, **kwargs)
        print(text, file=file, end=end)
        return text

    def log_table(
        self,
        format: str = "markdown",
        logger: Any = None,
        level: int = logging.INFO,
        **kwargs: Any,
    ) -> str:
        """Log a rendered table and return the printable text."""
        text = self.to_display_text(format=format, **kwargs)
        target_logger = logging.getLogger(logger) if isinstance(logger, str) else (logger or logging.getLogger(__name__))
        target_logger.log(level, "%s", text)
        return text

    def _row_signature(self, row: Mapping[str, Any], columns: Sequence[str]) -> str:
        return repr(tuple((column, repr(row.get(column))) for column in columns))

    def _records_frame(self, records: Sequence[Mapping[str, Any]], columns: Sequence[str]) -> "DataTable":
        frame = DataTable(list(records))
        if columns:
            frame = frame.reindex(columns=list(columns))
        return frame

    def compare_frames(
        self,
        other: "DataTable",
        key_columns: Optional[Sequence[str]] = None,
    ) -> dict[str, Any]:
        """Compare two dataframes and return matched, changed, and unmatched rows."""
        other_frame = DataTable(other)
        columns = list(dict.fromkeys(list(self.columns) + list(other_frame.columns)))
        left_rows = self.to_dict(orient="records")
        right_rows = other_frame.to_dict(orient="records")

        if key_columns:
            keys = list(key_columns)
            for column in keys:
                if column not in self.columns or column not in other_frame.columns:
                    raise KeyError(f"Unknown key column: {column}")

            def make_key(row: Mapping[str, Any]) -> tuple[Any, ...]:
                return tuple(row.get(column) for column in keys)

            left_map = {make_key(row): row for row in left_rows}
            right_map = {make_key(row): row for row in right_rows}
            common_keys = left_map.keys() & right_map.keys()
            same_rows = []
            changed_rows = []
            left_only_rows = [left_map[key] for key in left_map.keys() - right_map.keys()]
            right_only_rows = [right_map[key] for key in right_map.keys() - left_map.keys()]

            for key in common_keys:
                left_row = left_map[key]
                right_row = right_map[key]
                row_equal = all(
                    self._cell_equal(left_row.get(column), right_row.get(column))
                    for column in columns
                )
                if row_equal:
                    same_rows.append(left_row)
                    continue
                changed_columns = [
                    column
                    for column in columns
                    if not self._cell_equal(left_row.get(column), right_row.get(column))
                ]
                changed_rows.append(
                    {
                        "key": key,
                        "left": left_row,
                        "right": right_row,
                        "changed_columns": changed_columns,
                    }
                )
        else:
            left_groups: dict[str, list[Mapping[str, Any]]] = {}
            right_groups: dict[str, list[Mapping[str, Any]]] = {}
            for row in left_rows:
                left_groups.setdefault(self._row_signature(row, columns), []).append(row)
            for row in right_rows:
                right_groups.setdefault(self._row_signature(row, columns), []).append(row)

            same_rows = []
            left_only_rows = []
            right_only_rows = []
            changed_rows = []
            for signature in set(left_groups) | set(right_groups):
                left_list = left_groups.get(signature, [])
                right_list = right_groups.get(signature, [])
                common = min(len(left_list), len(right_list))
                same_rows.extend(left_list[:common])
                left_only_rows.extend(left_list[common:])
                right_only_rows.extend(right_list[common:])

        same_frame = self._records_frame(same_rows, columns)
        left_only_frame = self._records_frame(left_only_rows, columns)
        right_only_frame = self._records_frame(right_only_rows, columns)
        return {
            "same_rows": same_frame,
            "left_only_rows": left_only_frame,
            "right_only_rows": right_only_frame,
            "changed_rows": changed_rows,
            "same_count": len(same_rows),
            "left_only_count": len(left_only_rows),
            "right_only_count": len(right_only_rows),
            "changed_count": len(changed_rows),
        }

    def _normalize_join_inputs(
        self,
        other: Any,
        on: Optional[Union[str, Sequence[str]]] = None,
        left_on: Optional[Union[str, Sequence[str]]] = None,
        right_on: Optional[Union[str, Sequence[str]]] = None,
    ) -> tuple["DataTable", list[str], list[str]]:
        other_frame = DataTable(other)
        if on is not None:
            keys = [on] if isinstance(on, str) else list(on)
            if not keys:
                raise ValueError("join key list cannot be empty")
            return other_frame, keys, keys
        if left_on is not None or right_on is not None:
            if left_on is None or right_on is None:
                raise ValueError("left_on and right_on must be provided together")
            left_keys = [left_on] if isinstance(left_on, str) else list(left_on)
            right_keys = [right_on] if isinstance(right_on, str) else list(right_on)
            if len(left_keys) != len(right_keys):
                raise ValueError("left_on and right_on must have the same number of keys")
            return other_frame, left_keys, right_keys
        common = [column for column in self.columns if column in other_frame.columns]
        if not common:
            raise ValueError("join requires 'on' or at least one common column")
        return other_frame, common, common

    def join(
        self,
        other: Any,
        on: Optional[Union[str, Sequence[str]]] = None,
        how: str = "inner",
        left_on: Optional[Union[str, Sequence[str]]] = None,
        right_on: Optional[Union[str, Sequence[str]]] = None,
        suffixes: tuple[str, str] = ("_x", "_y"),
        sort: bool = False,
    ) -> "DataTable":
        """Join with another dataframe using pandas merge semantics."""
        other_frame, left_keys, right_keys = self._normalize_join_inputs(other, on=on, left_on=left_on, right_on=right_on)
        merged = pd.merge(
            self,
            other_frame,
            how=how,
            left_on=left_keys,
            right_on=right_keys,
            suffixes=suffixes,
            sort=sort,
        )
        return self._inherit_versions(merged)

    def left_join(
        self,
        other: Any,
        on: Optional[Union[str, Sequence[str]]] = None,
        left_on: Optional[Union[str, Sequence[str]]] = None,
        right_on: Optional[Union[str, Sequence[str]]] = None,
        suffixes: tuple[str, str] = ("_x", "_y"),
        sort: bool = False,
    ) -> "DataTable":
        """Left join helper."""
        return self.join(other, on=on, how="left", left_on=left_on, right_on=right_on, suffixes=suffixes, sort=sort)

    def inner_join(
        self,
        other: Any,
        on: Optional[Union[str, Sequence[str]]] = None,
        left_on: Optional[Union[str, Sequence[str]]] = None,
        right_on: Optional[Union[str, Sequence[str]]] = None,
        suffixes: tuple[str, str] = ("_x", "_y"),
        sort: bool = False,
    ) -> "DataTable":
        """Inner join helper."""
        return self.join(other, on=on, how="inner", left_on=left_on, right_on=right_on, suffixes=suffixes, sort=sort)

    def anti_join(
        self,
        other: Any,
        on: Optional[Union[str, Sequence[str]]] = None,
        left_on: Optional[Union[str, Sequence[str]]] = None,
        right_on: Optional[Union[str, Sequence[str]]] = None,
        suffixes: tuple[str, str] = ("_x", "_y"),
        sort: bool = False,
    ) -> "DataTable":
        """Return rows from the left dataframe that have no match on the right."""
        other_frame, left_keys, right_keys = self._normalize_join_inputs(other, on=on, left_on=left_on, right_on=right_on)
        merged = pd.merge(
            self,
            other_frame,
            how="left",
            left_on=left_keys,
            right_on=right_keys,
            suffixes=suffixes,
            sort=sort,
            indicator=True,
        )
        anti = merged.loc[merged["_merge"] == "left_only", list(self.columns)].copy()
        return self._inherit_versions(anti)

    def ensure_columns(self, *columns: str, fill_value: Any = pd.NA) -> "DataTable":
        """Ensure columns exist, creating missing ones with a default value."""
        result = self.copy()
        for column in columns:
            if column not in result.columns:
                result[column] = fill_value
        return self._inherit_versions(result)

    def coerce_types(self, mapping: Mapping[str, Any], errors: str = "raise") -> "DataTable":
        """Coerce selected columns to the requested types."""
        result = self.copy()
        for column, dtype in mapping.items():
            if column not in result.columns:
                raise KeyError(f"Unknown field: {column}")
            if dtype in {"datetime", "datetime64", "datetime64[ns]", datetime}:
                result[column] = pd.to_datetime(result[column], errors=errors)
            elif dtype in {"date"}:
                result[column] = pd.to_datetime(result[column], errors=errors).dt.date
            else:
                result[column] = result[column].astype(dtype, errors=errors) if hasattr(result[column], "astype") else result[column]
        return self._inherit_versions(result)

    def validate_schema(self, schema: Mapping[str, Any], strict: bool = False) -> "DataTable":
        """Validate column presence and basic column constraints."""
        if strict:
            extra_columns = [column for column in self.columns if column not in schema]
            if extra_columns:
                raise ValueError(f"Unexpected columns: {extra_columns}")

        for column, spec in schema.items():
            if column not in self.columns:
                required = True
                if isinstance(spec, Mapping):
                    required = bool(spec.get("required", True))
                if required:
                    raise KeyError(f"Missing required column: {column}")
                continue

            series = self[column]
            nullable = True
            expected_type = None
            choices = None
            minimum = maximum = None
            regex_pattern = None
            length_spec = None

            if isinstance(spec, Mapping):
                nullable = bool(spec.get("nullable", True))
                expected_type = spec.get("type", spec.get("dtype"))
                choices = spec.get("choices")
                minimum = spec.get("min", spec.get("ge"))
                maximum = spec.get("max", spec.get("le"))
                regex_pattern = spec.get("regex")
                length_spec = spec.get("length")
            else:
                expected_type = spec

            if not nullable and series.isna().any():
                raise ValueError(f"Column {column!r} contains missing values")

            non_null = series.dropna()
            if expected_type is not None:
                if isinstance(expected_type, tuple):
                    valid_types = expected_type
                elif isinstance(expected_type, type):
                    valid_types = (expected_type,)
                else:
                    valid_types = ()
                if valid_types:
                    invalid = [value for value in non_null if not isinstance(value, valid_types)]
                    if invalid:
                        raise TypeError(f"Column {column!r} does not match expected type {expected_type!r}")

            if choices is not None:
                invalid = [value for value in non_null if value not in choices]
                if invalid:
                    raise ValueError(f"Column {column!r} contains values outside the allowed choices")

            if minimum is not None and any(value < minimum for value in non_null):
                raise ValueError(f"Column {column!r} contains values below minimum {minimum!r}")
            if maximum is not None and any(value > maximum for value in non_null):
                raise ValueError(f"Column {column!r} contains values above maximum {maximum!r}")

            if regex_pattern is not None:
                pattern = re.compile(regex_pattern)
                invalid = [value for value in non_null if not pattern.search(str(value))]
                if invalid:
                    raise ValueError(f"Column {column!r} does not match regex {regex_pattern!r}")

            if length_spec is not None:
                if isinstance(length_spec, int):
                    invalid = [value for value in non_null if len(str(value)) != length_spec]
                elif isinstance(length_spec, Mapping):
                    min_length = length_spec.get("min")
                    max_length = length_spec.get("max")
                    invalid = [
                        value
                        for value in non_null
                        if (min_length is not None and len(str(value)) < min_length)
                        or (max_length is not None and len(str(value)) > max_length)
                    ]
                else:
                    invalid = []
                if invalid:
                    raise ValueError(f"Column {column!r} fails length validation")

        return self

    def same_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Return rows that are equal in both dataframes."""
        return self.compare_frames(other, key_columns=key_columns)["same_rows"]

    def same(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Alias for same_rows()."""
        return self.same_rows(other, key_columns=key_columns)

    def different_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> dict[str, Any]:
        """Return rows that differ between two dataframes."""
        comparison = self.compare_frames(other, key_columns=key_columns)
        return {
            "left_only_rows": comparison["left_only_rows"],
            "right_only_rows": comparison["right_only_rows"],
            "changed_rows": comparison["changed_rows"],
            "left_only_count": comparison["left_only_count"],
            "right_only_count": comparison["right_only_count"],
            "changed_count": comparison["changed_count"],
        }

    def diff_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> dict[str, Any]:
        """Alias for different_rows()."""
        return self.different_rows(other, key_columns=key_columns)

    def compare_with_status(self, other: "DataTable", status_column: str = "status") -> "DataTable":
        """Return a combined dataframe with exact-row membership status."""
        comparison = self.compare_frames(other)
        other_frame = DataTable(other)
        columns = list(dict.fromkeys(list(self.columns) + list(other_frame.columns) + [status_column]))
        rows: list[dict[str, Any]] = []

        for status, frame in (
            ("same", comparison["same_rows"]),
            ("left_only", comparison["left_only_rows"]),
            ("right_only", comparison["right_only_rows"]),
        ):
            for record in frame.to_rows():
                row = dict(record)
                row[status_column] = status
                rows.append(row)

        result = DataTable(rows)
        if columns:
            result = result.reindex(columns=columns)
        return result

    def compare_status_summary(self, other: "DataTable", status_column: str = "status") -> "DataTable":
        """Return one summary row per status with grouped records."""
        status_table = self.compare_with_status(other, status_column=status_column)
        rows: list[dict[str, Any]] = []
        for status in ("same", "left_only", "right_only"):
            subset = status_table[status_table[status_column] == status].drop(columns=[status_column], errors="ignore")
            rows.append(
                {
                    status_column: status,
                    "count": int(len(subset)),
                    "rows": subset.to_rows(),
                }
            )
        return DataTable(rows)

    def compare_status_rows(self, other: "DataTable", status_column: str = "status") -> "DataTable":
        """Return one representative row for each status."""
        status_table = self.compare_with_status(other, status_column=status_column)
        columns = list(status_table.columns)
        rows: list[dict[str, Any]] = []

        for status in ("same", "left_only", "right_only"):
            subset = status_table[status_table[status_column] == status]
            if len(subset) > 0:
                row = subset.iloc[0].to_dict()
            else:
                row = {column: pd.NA for column in columns}
                row[status_column] = status
            rows.append(row)

        result = DataTable(rows)
        if columns:
            result = result.reindex(columns=columns)
        return result

    def compare_sample_rows(self, other: "DataTable", status_column: str = "status") -> "DataTable":
        """Alias for compare_status_rows()."""
        return self.compare_status_rows(other, status_column=status_column)

    def diff_columns_against(self, other: "DataTable") -> dict[str, Any]:
        """Compare column layouts against another dataframe."""
        other_frame = DataTable(other)
        self_columns = list(self.columns)
        other_columns = list(other_frame.columns)
        return {
            "left_columns": self_columns,
            "right_columns": other_columns,
            "added_columns": [column for column in other_columns if column not in self_columns],
            "removed_columns": [column for column in self_columns if column not in other_columns],
            "shared_columns": [column for column in self_columns if column in other_columns],
            "column_order_changed": self_columns != other_columns,
        }

    def diff_added_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Return rows that exist only in the other dataframe."""
        return self.compare_frames(other, key_columns=key_columns)["right_only_rows"]

    def right_only(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Alias for diff_added_rows()."""
        return self.diff_added_rows(other, key_columns=key_columns)

    def diff_removed_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Return rows that exist only in the current dataframe."""
        return self.compare_frames(other, key_columns=key_columns)["left_only_rows"]

    def left_only(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Alias for diff_removed_rows()."""
        return self.diff_removed_rows(other, key_columns=key_columns)

    def diff_unchanged_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Return rows that are identical in both dataframes."""
        return self.compare_frames(other, key_columns=key_columns)["same_rows"]

    def diff_changed_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Return the row-level changes between two dataframes."""
        comparison = self.compare_frames(other, key_columns=key_columns)
        return DataTable(comparison["changed_rows"])

    def diff_changed_cells(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Return a cell-level diff between two dataframes."""
        other_frame = DataTable(other)
        columns = list(dict.fromkeys(list(self.columns) + list(other_frame.columns)))
        left_rows = self.to_dict(orient="records")
        right_rows = other_frame.to_dict(orient="records")
        changes: list[dict[str, Any]] = []

        if key_columns:
            keys = list(key_columns)
            for column in keys:
                if column not in self.columns or column not in other_frame.columns:
                    raise KeyError(f"Unknown key column: {column}")

            def make_key(row: Mapping[str, Any]) -> tuple[Any, ...]:
                return tuple(row.get(column) for column in keys)

            left_map = {make_key(row): row for row in left_rows}
            right_map = {make_key(row): row for row in right_rows}
            for key in left_map.keys() & right_map.keys():
                left_row = left_map[key]
                right_row = right_map[key]
                for column in columns:
                    if column in keys:
                        continue
                    if not self._cell_equal(left_row.get(column), right_row.get(column)):
                        changes.append(
                            {
                                "key": key,
                                "column": column,
                                "left": left_row.get(column),
                                "right": right_row.get(column),
                            }
                        )
        else:
            max_len = max(len(left_rows), len(right_rows))
            for index in range(max_len):
                if index >= len(left_rows) or index >= len(right_rows):
                    continue
                left_row = left_rows[index]
                right_row = right_rows[index]
                for column in columns:
                    if not self._cell_equal(left_row.get(column), right_row.get(column)):
                        changes.append(
                            {
                                "index": index,
                                "column": column,
                                "left": left_row.get(column),
                                "right": right_row.get(column),
                            }
                        )

        return DataTable(changes)

    def diff_report(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> dict[str, Any]:
        """Return a deep diff report with rows, cells, and columns."""
        comparison = self.compare_frames(other, key_columns=key_columns)
        added_rows = comparison["right_only_rows"]
        removed_rows = comparison["left_only_rows"]
        same_rows = comparison["same_rows"]
        changed_rows = DataTable(comparison["changed_rows"])
        changed_cells = self.diff_changed_cells(other, key_columns=key_columns)
        column_diff = self.diff_columns_against(other)
        return {
            "same_rows": same_rows,
            "added_rows": added_rows,
            "removed_rows": removed_rows,
            "changed_rows": changed_rows,
            "changed_cells": changed_cells,
            "column_diff": column_diff,
            "same_count": comparison["same_count"],
            "added_count": comparison["right_only_count"],
            "removed_count": comparison["left_only_count"],
            "changed_count": comparison["changed_count"],
            "changed_cell_count": len(changed_cells),
            "has_changes": bool(
                comparison["left_only_count"]
                or comparison["right_only_count"]
                or comparison["changed_count"]
                or len(changed_cells)
                or column_diff["added_columns"]
                or column_diff["removed_columns"]
                or column_diff["column_order_changed"]
            ),
        }

    def only(self, *fields: str) -> "DataTable":
        """Return a frame with only the selected columns."""
        if not fields:
            return self._inherit_versions(self.copy())
        return self._inherit_versions(self.loc[:, list(fields)].copy())

    def select(self, *fields: str) -> "DataTable":
        """Alias for only()."""
        return self.only(*fields)

    def exclude_fields(self, *fields: str) -> "DataTable":
        """Return a frame without the selected columns."""
        if not fields:
            return self._inherit_versions(self.copy())
        drop_fields = [field for field in fields if field in self.columns]
        return self._inherit_versions(self.drop(columns=drop_fields).copy())

    def clean_columns(self, lower: bool = True, replace_spaces: str = "_") -> "DataTable":
        """Normalize column names for consistent downstream use."""
        columns = []
        for column in self.columns:
            name = str(column).strip()
            if lower:
                name = name.lower()
            if replace_spaces is not None:
                name = re.sub(r"\s+", replace_spaces, name)
            columns.append(name)
        result = self.copy()
        result.columns = columns
        return self._inherit_versions(result)

    def drop_empty_columns(self) -> "DataTable":
        """Drop columns that contain only missing values."""
        return self._inherit_versions(super().dropna(axis=1, how="all"))

    def fill_missing(self, value: Any = 0, columns: Optional[Sequence[str]] = None) -> "DataTable":
        """Fill missing values in selected columns or in the whole frame."""
        result = self.copy()
        if columns is None:
            return self._inherit_versions(result.fillna(value))
        result.loc[:, list(columns)] = result.loc[:, list(columns)].fillna(value)
        return self._inherit_versions(result)

    def to_numeric(self, columns: Optional[Iterable[str]] = None, errors: str = "coerce") -> "DataTable":
        """Convert selected columns to numeric values."""
        result = self.copy()
        target_columns = list(columns) if columns is not None else list(result.columns)
        for column in target_columns:
            result[column] = pd.to_numeric(result[column], errors=errors)
        return self._inherit_versions(result)


class _ORMQuerySet:
    """Lightweight Django-like queryset helper for DataTable."""

    def __init__(self, frame: DataTable):
        frame_copy = frame.copy()
        self._frame = frame_copy if isinstance(frame_copy, DataTable) else DataTable(frame_copy)

    def _spawn(self, frame: DataTable) -> "_ORMQuerySet":
        """Create a new queryset of the same logical kind."""
        return _ORMQuerySet(frame)

    def __getitem__(self, key: Any) -> Any:
        """Proxy column/subset access to the underlying dataframe."""
        result = self._frame.__getitem__(key)
        if isinstance(result, pd.DataFrame):
            return DataTable(result)
        return result

    def all(self) -> "_ORMQuerySet":
        """Return a clone of the queryset."""
        return self._spawn(self._frame)

    def none(self) -> "_ORMQuerySet":
        """Return an empty queryset with the same columns."""
        return _ORMQuerySet(self._frame.iloc[0:0].copy())

    def filter(self, *expressions: str, **conditions: Any) -> "_ORMQuerySet":
        """Filter rows using query expressions and Django-style field lookups."""
        result = self._frame
        for expression in expressions:
            result = DataTable(result.query(expression))
        if not conditions:
            return self._spawn(result.copy())
        mask = self._build_mask(result, conditions)
        return self._spawn(result.loc[mask].copy())

    def exclude(self, *expressions: str, **conditions: Any) -> "_ORMQuerySet":
        """Exclude rows using query expressions and Django-style field lookups."""
        result = self._frame
        for expression in expressions:
            result = DataTable(result.query(expression))
        if not conditions:
            return self._spawn(self._frame.loc[~self._frame.index.isin(result.index)].copy())
        mask = self._build_mask(result, conditions)
        return self._spawn(result.loc[~mask].copy())

    def order_by(
        self,
        *fields: str,
        nulls_first: bool = False,
        nulls_last: bool = True,
        nulls: Optional[Mapping[str, str]] = None,
    ) -> "_ORMQuerySet":
        """Sort rows by one or more fields, supporting Django's '-' prefix."""
        if not fields:
            return self.all()
        if fields == ("?",):
            shuffled = self._frame.sample(frac=1).copy()
            shuffled.index = self._frame.index
            return self._spawn(shuffled)
        columns = []
        ascending = []
        for field in fields:
            descending = field.startswith("-")
            columns.append(field[1:] if descending else field)
            ascending.append(not descending)
        if nulls:
            ordered = self._frame.copy()
            for index in range(len(columns) - 1, -1, -1):
                column = columns[index]
                na_position = nulls.get(column, "first" if nulls_first or not nulls_last else "last")
                ordered = ordered.sort_values(
                    by=column,
                    ascending=ascending[index],
                    kind="mergesort",
                    na_position=na_position,
                )
            return self._spawn(ordered.copy())
        return self._spawn(
            self._frame.sort_values(
                by=columns,
                ascending=ascending,
                kind="mergesort",
                na_position="first" if nulls_first or not nulls_last else "last",
            ).copy()
        )

    def random_order(self) -> "_ORMQuerySet":
        """Return rows in random order."""
        return self.order_by("?")

    def distinct(self, *fields: str) -> "_ORMQuerySet":
        """Return unique rows or unique combinations of selected fields."""
        if fields:
            return self._spawn(self._frame.loc[:, list(fields)].drop_duplicates().copy())
        return self._spawn(self._frame.drop_duplicates().copy())

    def only(self, *fields: str) -> "_ORMQuerySet":
        """Return a queryset with only the selected columns."""
        if not fields:
            return self.all()
        return self._spawn(self._frame.loc[:, list(fields)].copy())

    def select(self, *fields: str) -> "_ORMQuerySet":
        """Alias for only()."""
        return self.only(*fields)

    def exclude_fields(self, *fields: str) -> "_ORMQuerySet":
        """Return a queryset without the selected columns."""
        if not fields:
            return self.all()
        drop_fields = [field for field in fields if field in self._frame.columns]
        return self._spawn(self._frame.drop(columns=drop_fields).copy())

    def values(self, *fields: str) -> _ValuesResult:
        """Return queryset rows as dictionaries."""
        frame = self._select_fields(*fields)
        return _ValuesResult(frame.to_dict(orient="records"))

    def values_list(self, *fields: str, flat: bool = False) -> _ValuesResult:
        """Return queryset rows as tuples or a flat list for one field."""
        frame = self._select_fields(*fields)
        if flat:
            if len(fields) != 1:
                raise ValueError("flat=True requires exactly one field")
            return _ValuesResult(frame.iloc[:, 0].tolist())
        return _ValuesResult([tuple(row) for row in frame.itertuples(index=False, name=None)])

    def first(self) -> DataTable:
        """Return the first row as a one-row dataframe."""
        return DataTable(self._frame.head(1).copy())

    def last(self) -> DataTable:
        """Return the last row as a one-row dataframe."""
        return DataTable(self._frame.tail(1).copy())

    def earliest(self, *fields: str) -> DataTable:
        """Return the earliest row according to the given ordering fields."""
        ordered = self.order_by(*fields) if fields else self.all()
        return ordered.first()

    def latest(self, *fields: str) -> DataTable:
        """Return the latest row according to the given ordering fields."""
        ordered = self.order_by(*fields) if fields else self.all()
        return ordered.last()

    def get(self, *expressions: str, **conditions: Any) -> DataTable:
        """Return exactly one matching row as a one-row dataframe."""
        result = self.filter(*expressions, **conditions)._frame
        if len(result) == 1:
            return DataTable(result.copy())
        if len(result) == 0:
            raise KeyError("No rows matched the given query")
        raise ValueError("Multiple rows matched the given query")

    def count(self) -> int:
        """Return the number of rows."""
        return int(len(self._frame))

    def exists(self) -> bool:
        """Return whether the queryset has at least one row."""
        return not self._frame.empty

    def paginate(self, limit: int, offset: int = 0) -> "_ORMQuerySet":
        """Return a page of rows using limit/offset semantics."""
        if limit < 0 or offset < 0:
            raise ValueError("limit and offset must be non-negative")
        return self._spawn(self._frame.iloc[offset : offset + limit].copy())

    def group_by(self, *fields: str) -> "_GroupedQuery":
        """Return a grouped query helper for Django-like aggregation."""
        if not fields:
            raise ValueError("group_by() requires at least one field")
        return _GroupedQuery(self._frame, fields)

    def sum(self, *fields: str) -> dict[str, Any]:
        """Return column sums for the requested fields."""
        return self._aggregate_columns("sum", *fields)

    def avg(self, *fields: str) -> dict[str, Any]:
        """Return column means for the requested fields."""
        return self._aggregate_columns("mean", *fields)

    def min(self, *fields: str) -> dict[str, Any]:
        """Return column minimums for the requested fields."""
        return self._aggregate_columns("min", *fields)

    def max(self, *fields: str) -> dict[str, Any]:
        """Return column maximums for the requested fields."""
        return self._aggregate_columns("max", *fields)

    def diff_columns_against(self, other: "DataTable") -> dict[str, Any]:
        """Compare column layouts against another dataframe."""
        return self._frame.diff_columns_against(other)

    def same(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Alias for same_rows()."""
        return self._frame.same(other, key_columns=key_columns)

    def compare_with_status(self, other: "DataTable", status_column: str = "status") -> DataTable:
        """Return a combined dataframe with exact-row membership status."""
        return self._frame.compare_with_status(other, status_column=status_column)

    def compare_status_summary(self, other: "DataTable", status_column: str = "status") -> DataTable:
        """Return one summary row per status with grouped records."""
        return self._frame.compare_status_summary(other, status_column=status_column)

    def compare_status_rows(self, other: "DataTable", status_column: str = "status") -> DataTable:
        """Return one representative row for each status."""
        return self._frame.compare_status_rows(other, status_column=status_column)

    def compare_sample_rows(self, other: "DataTable", status_column: str = "status") -> DataTable:
        """Alias for compare_status_rows()."""
        return self._frame.compare_sample_rows(other, status_column=status_column)

    def diff_added_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Return rows that exist only in the other dataframe."""
        return self._frame.diff_added_rows(other, key_columns=key_columns)

    def right_only(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Alias for diff_added_rows()."""
        return self._frame.right_only(other, key_columns=key_columns)

    def diff_removed_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Return rows that exist only in the current dataframe."""
        return self._frame.diff_removed_rows(other, key_columns=key_columns)

    def left_only(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Alias for diff_removed_rows()."""
        return self._frame.left_only(other, key_columns=key_columns)

    def diff_unchanged_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Return rows that are identical in both dataframes."""
        return self._frame.diff_unchanged_rows(other, key_columns=key_columns)

    def diff_changed_rows(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Return row-level diff records."""
        return self._frame.diff_changed_rows(other, key_columns=key_columns)

    def diff_changed_cells(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> DataTable:
        """Return cell-level diff records."""
        return self._frame.diff_changed_cells(other, key_columns=key_columns)

    def diff_report(self, other: "DataTable", key_columns: Optional[Sequence[str]] = None) -> dict[str, Any]:
        """Return a deep diff report."""
        return self._frame.diff_report(other, key_columns=key_columns)

    def aggregate(self, **aggregations: Any) -> dict[str, Any]:
        """Return named aggregates for the whole queryset."""
        if not aggregations:
            return {}
        return self._resolve_aggregations(self._frame, aggregations)

    def to_df(self) -> DataTable:
        """Materialize the queryset as a DataTable instance."""
        return DataTable(self._frame.copy())

    def to_markdown_table(self, *args: Any, **kwargs: Any) -> str:
        return self.to_df().to_markdown_table(*args, **kwargs)

    def to_html_table(self, *args: Any, **kwargs: Any) -> str:
        return self.to_df().to_html_table(*args, **kwargs)

    def to_csv_table(self, *args: Any, **kwargs: Any) -> str:
        return self.to_df().to_csv_table(*args, **kwargs)

    def to_rich_table(self, *args: Any, **kwargs: Any):
        return self.to_df().to_rich_table(*args, **kwargs)

    def to_excel_table(self, *args: Any, **kwargs: Any) -> Path:
        return self.to_df().to_excel_table(*args, **kwargs)

    def render_table(self, *args: Any, **kwargs: Any):
        return self.to_df().render_table(*args, **kwargs)

    def to_display_text(self, *args: Any, **kwargs: Any) -> str:
        return self.to_df().to_display_text(*args, **kwargs)

    def print_table(self, *args: Any, **kwargs: Any) -> str:
        return self.to_df().print_table(*args, **kwargs)

    def log_table(self, *args: Any, **kwargs: Any) -> str:
        return self.to_df().log_table(*args, **kwargs)

    def join(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.join(*args, **kwargs))

    def left_join(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.left_join(*args, **kwargs))

    def inner_join(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.inner_join(*args, **kwargs))

    def anti_join(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.anti_join(*args, **kwargs))

    def ensure_columns(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.ensure_columns(*args, **kwargs))

    def coerce_types(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.coerce_types(*args, **kwargs))

    def validate_schema(self, *args: Any, **kwargs: Any) -> DataTable:
        return self._frame.validate_schema(*args, **kwargs)

    def auto_version(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.auto_version(*args, **kwargs))

    def disable_auto_version(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.disable_auto_version(*args, **kwargs))

    def snapshot(self, *args: Any, **kwargs: Any) -> "_ORMQuerySet":
        return self._spawn(self._frame.snapshot(*args, **kwargs))

    def _select_fields(self, *fields: str) -> pd.DataFrame:
        if not fields:
            return self._frame
        return self._frame.loc[:, list(fields)]

    def _build_mask(self, frame: DataTable, conditions: Mapping[str, Any]) -> pd.Series:
        mask = pd.Series(True, index=frame.index)
        for key, expected in conditions.items():
            field, lookup = self._split_lookup(key)
            if field not in frame.columns:
                raise KeyError(f"Unknown field: {field}")
            series = frame[field]
            mask &= self._apply_lookup(series, lookup, expected)
        return mask

    def _aggregate_columns(self, func: str, *fields: str) -> dict[str, Any]:
        selected = list(fields) if fields else list(self._frame.columns)
        result: dict[str, Any] = {}
        for field in selected:
            if field not in self._frame.columns:
                raise KeyError(f"Unknown field: {field}")
            result[field] = getattr(self._frame[field], func)()
        return result

    def _resolve_aggregations(self, frame: pd.DataFrame, aggregations: Mapping[str, Any]) -> dict[str, Any]:
        result: dict[str, Any] = {}
        for alias, spec in aggregations.items():
            source, func = self._normalize_spec(alias, spec)
            if source not in frame.columns:
                raise KeyError(f"Unknown field: {source}")
            result[alias] = getattr(frame[source], func)()
        return result

    @staticmethod
    def _split_lookup(key: str) -> Tuple[str, str]:
        if "__" not in key:
            return key, "exact"
        field, lookup = key.rsplit("__", 1)
        return field, lookup

    @staticmethod
    def _apply_lookup(series: pd.Series, lookup: str, expected: Any) -> pd.Series:
        if lookup == "exact":
            return series.eq(expected)
        if lookup == "ne":
            return series.ne(expected)
        if lookup == "gt":
            return series.gt(expected)
        if lookup == "gte":
            return series.ge(expected)
        if lookup == "lt":
            return series.lt(expected)
        if lookup == "lte":
            return series.le(expected)
        if lookup == "in":
            return series.isin(expected)
        if lookup == "contains":
            return series.astype(str).str.contains(str(expected), regex=False, na=False)
        if lookup == "icontains":
            return series.astype(str).str.contains(str(expected), case=False, regex=False, na=False)
        if lookup == "startswith":
            return series.astype(str).str.startswith(str(expected), na=False)
        if lookup == "istartswith":
            return series.astype(str).str.lower().str.startswith(str(expected).lower(), na=False)
        if lookup == "endswith":
            return series.astype(str).str.endswith(str(expected), na=False)
        if lookup == "iendswith":
            return series.astype(str).str.lower().str.endswith(str(expected).lower(), na=False)
        if lookup == "isnull":
            return series.isna() if expected else series.notna()
        if lookup in {"range", "between"}:
            start, end = expected
            return series.between(start, end, inclusive="both")
        if lookup in {"year", "month", "day", "hour", "minute", "second", "date"}:
            dt = pd.to_datetime(series, errors="coerce")
            if lookup == "date":
                return dt.dt.date.eq(expected)
            return getattr(dt.dt, lookup).eq(expected)
        raise ValueError(f"Unsupported lookup: {lookup}")

    @staticmethod
    def _normalize_spec(alias: str, spec: Any) -> Tuple[str, Any]:
        if isinstance(spec, tuple) and len(spec) == 2:
            return spec[0], spec[1]
        if isinstance(spec, str):
            if spec in {"sum", "mean", "min", "max", "count", "first", "last", "median"}:
                return alias, spec
            return spec, "first"
        if callable(spec):
            return alias, spec
        raise TypeError("Annotation must be a string, callable, or (field, func) tuple")


class _ORMManager(_ORMQuerySet):
    """QuerySet with Django-like write helpers."""

    def _spawn(self, frame: DataTable) -> "_ORMQuerySet":
        return _ORMManager(frame)

    def create(self, **fields: Any) -> DataTable:
        """Create a one-row dataframe from the provided fields."""
        return DataTable([fields])

    def append_row(self, **fields: Any) -> DataTable:
        """Return a new dataframe with one appended row."""
        created = self.create(**fields)
        return DataTable(pd.concat([self._frame, created], ignore_index=True))

    def bulk_create(self, rows: Sequence[Mapping[str, Any]]) -> DataTable:
        """Return a new dataframe with multiple appended rows."""
        if not rows:
            return self.to_df()
        created = DataTable(list(rows))
        return DataTable(pd.concat([self._frame, created], ignore_index=True))

    def update(self, **fields: Any) -> int:
        """Return the number of matched rows that would be updated."""
        return self.count()

    def delete(self) -> int:
        """Return the number of matched rows that would be deleted."""
        return self.count()

    def bulk_update(
        self,
        values: Mapping[str, Any],
        return_count: bool = False,
        **conditions: Any,
    ) -> DataTable | tuple[DataTable, int]:
        """Return an updated dataframe and optionally the number of affected rows."""
        mask = self._build_mask(self._frame, conditions) if conditions else pd.Series(True, index=self._frame.index)
        updated = self._frame.copy()
        for field, value in values.items():
            if field not in updated.columns:
                updated[field] = pd.NA
            updated.loc[mask, field] = value
        updated_frame = DataTable(updated)
        if return_count:
            return updated_frame, int(mask.sum())
        return updated_frame

    def save(self) -> DataTable:
        """Return the current dataframe snapshot."""
        return self.to_df()

    def get_or_create(
        self,
        defaults: Optional[Mapping[str, Any]] = None,
        **lookup: Any,
    ) -> tuple[DataTable, bool]:
        """Return a matching row or create a one-row dataframe."""
        try:
            return self.get(**lookup), False
        except KeyError:
            data = dict(defaults or {})
            data.update(lookup)
            return self.create(**data), True

    def update_or_create(
        self,
        defaults: Optional[Mapping[str, Any]] = None,
        **lookup: Any,
    ) -> tuple[DataTable, bool]:
        """Return an updated row or create a new one-row dataframe."""
        try:
            row = self.get(**lookup)
        except KeyError:
            data = dict(defaults or {})
            data.update(lookup)
            return self.create(**data), True

        data = row.iloc[0].to_dict()
        data.update(defaults or {})
        data.update(lookup)
        return DataTable([data]), False


class _GroupedQuery(_ORMQuerySet):
    """Grouped queryset helper returned by group_by()."""

    def __init__(self, frame: DataTable, fields: Sequence[str]):
        super().__init__(frame)
        self._fields = list(fields)

    def annotate(self, **aggregations: Any) -> _ORMQuerySet:
        """Aggregate grouped rows into a queryset with named results."""
        grouped = self._frame.groupby(self._fields, dropna=False, sort=False)
        if not aggregations:
            result = grouped.size().reset_index(name="count")
            return _ORMQuerySet(DataTable(result))

        result = grouped.size().reset_index(name="_count").drop(columns=["_count"])
        for alias, spec in aggregations.items():
            source, func = self._normalize_spec(alias, spec)
            if source not in self._frame.columns:
                raise KeyError(f"Unknown field: {source}")
            aggregate_frame = getattr(grouped[source], func)().reset_index(name=alias)
            result = result.merge(aggregate_frame, on=self._fields, how="left", sort=False)
        return _ORMQuerySet(DataTable(result))

    def count(self) -> _ORMQuerySet:
        """Return group counts as a queryset."""
        result = self._frame.groupby(self._fields, dropna=False, sort=False).size().reset_index(name="count")
        return _ORMQuerySet(DataTable(result))

    def sum(self, *fields: str) -> dict[str, Any]:
        """Return grouped column sums for the requested fields."""
        return self._aggregate_columns("sum", *fields)

    def avg(self, *fields: str) -> dict[str, Any]:
        """Return grouped column means for the requested fields."""
        return self._aggregate_columns("mean", *fields)

    def min(self, *fields: str) -> dict[str, Any]:
        """Return grouped column minimums for the requested fields."""
        return self._aggregate_columns("min", *fields)

    def max(self, *fields: str) -> dict[str, Any]:
        """Return grouped column maximums for the requested fields."""
        return self._aggregate_columns("max", *fields)

    def aggregate(self, **aggregations: Any) -> dict[str, Any]:
        """Return named aggregates for the grouped queryset."""
        grouped = self._frame.groupby(self._fields, dropna=False, sort=False)
        result: dict[str, Any] = {}
        for alias, spec in aggregations.items():
            source, func = self._normalize_spec(alias, spec)
            if source not in self._frame.columns:
                raise KeyError(f"Unknown field: {source}")
            aggregate_frame = getattr(grouped[source], func)().reset_index(name=alias)
            result[alias] = aggregate_frame.to_dict(orient="records")
        return result

    def _aggregate_columns(self, func: str, *fields: str) -> dict[str, Any]:
        grouped = self._frame.groupby(self._fields, dropna=False, sort=False)
        selected = list(fields) if fields else [field for field in self._frame.columns if field not in self._fields]
        result: dict[str, Any] = {}
        for field in selected:
            if field not in self._frame.columns:
                raise KeyError(f"Unknown field: {field}")
            result[field] = getattr(grouped[field], func)().reset_index().to_dict(orient="records")
        return result


df = DataTable
datatable = DataTable
BaseDataFrame = DataTable

