"""Base pipeline skeleton for business workflows and dataframe pipelines.

This module intentionally stays small and extensible:
- `BaseStep` encapsulates one business step.
- `BasePipeline` orchestrates execution and lifecycle hooks.
- `@pipeline.step(...)` provides a decorator-friendly API.

The implementation is conservative on purpose so we can iterate on the
execution model, retry/rollback semantics, and dataframe-specific helpers
without forcing a large framework design too early.
"""

from __future__ import annotations

import argparse
import inspect
import copy
import os
import sqlite3
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
import uuid
from dataclasses import dataclass, field
from enum import Enum
from types import SimpleNamespace
from typing import TYPE_CHECKING, Any, Callable, Optional, Sequence

from loguru import logger as loguru_logger

from .datatable import DataTable

if TYPE_CHECKING:
    pass

try:
    import yaml
except Exception:  # pragma: no cover - dependency should be installed
    yaml = None  # type: ignore[assignment]

__all__ = [
    "PipelineStatus",
    "StepOutcome",
    "PipelineContext",
    "BaseStep",
    "FunctionStep",
    "ConfigNamespace",
    "BasePipeline",
    "Pipeline",
]


_LOG_SINKS: dict[str, int] = {}


def _pipeline_log_path(class_name: str) -> Path:
    return (Path.cwd() / "logs" / f"{class_name}.log").resolve()


def _pipeline_data_dir(class_name: str) -> Path:
    return (Path.cwd() / "data" / class_name).resolve()


def _df_rows(data: Any) -> Optional[int]:
    try:
        return len(data)
    except Exception:
        return None


def _df_columns(data: Any) -> list[str]:
    columns = getattr(data, "columns", None)
    if columns is None:
        return []
    return [str(column) for column in columns]


def _model_items(model: Any) -> list[tuple[str, Any]]:
    if model is None:
        return []
    try:
        items = vars(model).items()
    except TypeError:
        return []
    return [(str(key), value) for key, value in items if not str(key).startswith("_")]


def _config_to_dict(config: Any) -> dict[str, Any]:
    if config is None:
        return {}
    if isinstance(config, ConfigNamespace):
        return config.as_dict()
    if isinstance(config, dict):
        return dict(config)
    if hasattr(config, "items"):
        try:
            return dict(config.items())
        except Exception:
            pass
    try:
        return dict(vars(config))
    except Exception:
        return {}


def _coerce_datatable(value: Any) -> Any:
    try:
        from .datatable import DataTable
    except Exception:
        return value
    if value is None:
        return DataTable()
    if isinstance(value, DataTable):
        return value
    if hasattr(value, "to_dict") and hasattr(value, "columns"):
        try:
            return DataTable(value)
        except Exception:
            return value
    if isinstance(value, (list, tuple, dict)):
        try:
            return DataTable(value)
        except Exception:
            return value
    try:
        return DataTable({"value": [value]})
    except Exception:
        return DataTable([value])


class ConfigNamespace(SimpleNamespace):
    """Namespace that also behaves like a lightweight mapping."""

    def get(self, key: str, default: Any = None) -> Any:
        return getattr(self, key, default)

    def setdefault(self, key: str, default: Any = None) -> Any:
        if not hasattr(self, key):
            setattr(self, key, default)
        return getattr(self, key)

    def update(self, other: Optional[dict[str, Any]] = None, **kwargs: Any) -> None:
        payload = dict(other or {})
        payload.update(kwargs)
        for key, value in payload.items():
            setattr(self, key, value)

    def as_dict(self) -> dict[str, Any]:
        return {key: value for key, value in vars(self).items() if not key.startswith("_")}

    def __contains__(self, key: object) -> bool:
        return isinstance(key, str) and hasattr(self, key)

    def __getitem__(self, key: str) -> Any:
        if not hasattr(self, key):
            raise KeyError(key)
        return getattr(self, key)

    def __setitem__(self, key: str, value: Any) -> None:
        setattr(self, key, value)


def _md_escape(value: Any) -> str:
    text = "-" if value is None else str(value)
    return text.replace("|", "\\|").replace("\n", " ")


def _pipeline_logger(class_name: str, pipeline_name: str) -> Any:
    log_path = _pipeline_log_path(class_name)
    sink_key = str(log_path)
    if sink_key not in _LOG_SINKS:
        log_path.parent.mkdir(parents=True, exist_ok=True)
        _LOG_SINKS[sink_key] = loguru_logger.add(
            log_path,
            level="INFO",
            rotation="10 MB",
            retention=14,
            encoding="utf-8",
            backtrace=False,
            diagnose=False,
            format=(
                "{time:YYYY-MM-DD HH:mm:ss} | {level:<8} | "
                "{extra[pipeline_class]}::{extra[pipeline_name]} | {message}"
            ),
        )
    return loguru_logger.bind(pipeline_class=class_name, pipeline_name=pipeline_name)


class PipelineSQLiteDB:
    """Small SQLite convenience wrapper attached to every Pipeline instance."""

    def __init__(self, pipeline: "Pipeline") -> None:
        self.pipeline = pipeline
        self.ensure()

    @property
    def path(self) -> Path:
        path = self.pipeline._resolve_db_path()
        path.parent.mkdir(parents=True, exist_ok=True)
        return path

    def ensure(self) -> Path:
        path = self.path
        if not path.exists():
            sqlite3.connect(path).close()
        with sqlite3.connect(path) as conn:
            conn.execute("""
                CREATE TABLE IF NOT EXISTS soigia_meta (
                    key TEXT PRIMARY KEY,
                    value TEXT NOT NULL
                )
                """)
            conn.executemany(
                "INSERT OR REPLACE INTO soigia_meta(key, value) VALUES (?, ?)",
                [
                    ("pipeline_class", self.pipeline.__class__.__name__),
                    ("pipeline_name", self.pipeline.name),
                    ("db_path", str(path)),
                    ("updated_at", datetime.now(timezone.utc).replace(microsecond=0).isoformat()),
                ],
            )
        return path

    def execute(self, sql: str, params: Any = None) -> sqlite3.Cursor:
        self.ensure()
        with sqlite3.connect(self.path) as conn:
            if params is None:
                cursor = conn.execute(sql)
            else:
                cursor = conn.execute(sql, params)
            conn.commit()
            return cursor

    def executemany(self, sql: str, params: Any) -> sqlite3.Cursor:
        self.ensure()
        with sqlite3.connect(self.path) as conn:
            cursor = conn.executemany(sql, params)
            conn.commit()
            return cursor

    def query(self, sql: str, params: Any = None, **kwargs: Any):
        self.ensure()
        import pandas as pd

        with sqlite3.connect(self.path) as conn:
            if params is None:
                frame = pd.read_sql_query(sql, conn, **kwargs)
            else:
                frame = pd.read_sql_query(sql, conn, params=params, **kwargs)
        return frame

    def list_tables(self) -> list[str]:
        frame = self.query("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")
        return [str(name) for name in frame["name"].tolist()]

    def table_exists(self, table_name: str) -> bool:
        frame = self.query(
            "SELECT 1 FROM sqlite_master WHERE type='table' AND name = ? LIMIT 1",
            params=(table_name,),
        )
        return not frame.empty

    def read_df(self, table_name: str, **kwargs: Any):
        return self.query(f'SELECT * FROM "{table_name}"', **kwargs)

    def read_table(self, table_name: str, **kwargs: Any):
        return self.read_df(table_name, **kwargs)

    def to_df(self, table_name: str, **kwargs: Any):
        return self.read_df(table_name, **kwargs)

    def get(self, table_name: str, **kwargs: Any):
        return self.read_table(table_name, **kwargs)

    def write_df(
        self,
        table_name: str,
        data: Any,
        *,
        if_exists: str = "replace",
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        self.ensure()
        import pandas as pd

        frame = data if isinstance(data, pd.DataFrame) else pd.DataFrame(data)
        with sqlite3.connect(self.path) as conn:
            frame.to_sql(table_name, conn, if_exists=if_exists, index=index, **kwargs)
        return self.path

    def write_table(
        self,
        table_name: str,
        data: Any,
        *,
        if_exists: str = "replace",
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        return self.write_df(table_name, data, if_exists=if_exists, index=index, **kwargs)

    def from_df(
        self,
        table_name: str,
        data: Any,
        *,
        if_exists: str = "replace",
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        return self.write_table(table_name, data, if_exists=if_exists, index=index, **kwargs)

    def set(
        self,
        table_name: str,
        data: Any,
        *,
        if_exists: str = "replace",
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        return self.write_table(table_name, data, if_exists=if_exists, index=index, **kwargs)

    def upsert_df(
        self,
        table_name: str,
        data: Any,
        *,
        mode: str = "upsert",
        key_columns: Sequence[str] | str | None = None,
        update_columns: Sequence[str] | str | None = None,
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        """Insert or update rows in a SQLite table using a conflict key."""

        self.ensure()
        import pandas as pd

        frame = data if isinstance(data, pd.DataFrame) else pd.DataFrame(data)
        if frame.empty:
            return self.path

        mode_normalized = str(mode).strip().lower().replace("_", "-")
        if mode_normalized in {"deleteinsert", "delete insert"}:
            mode_normalized = "delete-insert"
        if mode_normalized not in {"upsert", "delete-insert", "replace", "insert"}:
            raise ValueError(f"unsupported db write mode: {mode}")

        if mode_normalized == "replace":
            return self.write_df(table_name, frame, if_exists="replace", index=index, **kwargs)
        if mode_normalized == "insert":
            return self.write_df(table_name, frame, if_exists="append", index=index, **kwargs)

        if key_columns is None:
            key_columns = self._resolve_default_upsert_key_columns(table_name)
        if key_columns is None:
            if "id" in frame.columns:
                key_columns = ["id"]
            else:
                raise ValueError("key_columns is required for upsert_df when no id column is present")
        if isinstance(key_columns, str):
            key_columns = [key_columns]
        key_columns = [str(column) for column in key_columns]

        if update_columns is None:
            update_columns = [str(column) for column in frame.columns if str(column) not in key_columns]
        elif isinstance(update_columns, str):
            update_columns = [update_columns]
        else:
            update_columns = [str(column) for column in update_columns]

        missing_keys = [column for column in key_columns if column not in frame.columns]
        if missing_keys:
            raise KeyError(f"upsert key columns are missing from data: {', '.join(missing_keys)}")

        missing_updates = [column for column in update_columns if column not in frame.columns]
        if missing_updates:
            raise KeyError(f"upsert update columns are missing from data: {', '.join(missing_updates)}")

        quoted_table = f'"{table_name}"'
        quoted_columns = ", ".join(f'"{column}"' for column in frame.columns)
        placeholders = ", ".join("?" for _ in frame.columns)
        quoted_keys = ", ".join(f'"{column}"' for column in key_columns)

        with sqlite3.connect(self.path) as conn:
            if not self.table_exists(table_name):
                if mode_normalized == "delete-insert":
                    frame.head(0).to_sql(table_name, conn, if_exists="replace", index=index, **kwargs)
                else:
                    frame.to_sql(table_name, conn, if_exists="replace", index=index, **kwargs)
            if mode_normalized == "delete-insert":
                delete_sql = f'DELETE FROM "{table_name}" WHERE ' + " AND ".join(
                    f'"{column}" = ?' for column in key_columns
                )
                for row in frame.itertuples(index=False, name=None):
                    key_values = tuple(row[frame.columns.get_loc(column)] for column in key_columns)
                    conn.execute(delete_sql, key_values)
                conn.executemany(
                    f"INSERT INTO {quoted_table} ({quoted_columns}) VALUES ({placeholders})",
                    frame.itertuples(index=False, name=None),
                )
                conn.commit()
                return self.path
            index_name = f"ux_{table_name}_{'_'.join(key_columns)}"
            conn.execute(f'CREATE UNIQUE INDEX IF NOT EXISTS "{index_name}" ON {quoted_table} ({quoted_keys})')
            if update_columns:
                assignments = ", ".join(f'"{column}" = excluded."{column}"' for column in update_columns)
                sql = (
                    f"INSERT INTO {quoted_table} ({quoted_columns}) VALUES ({placeholders}) "
                    f"ON CONFLICT({quoted_keys}) DO UPDATE SET {assignments}"
                )
            else:
                sql = (
                    f"INSERT INTO {quoted_table} ({quoted_columns}) VALUES ({placeholders}) "
                    f"ON CONFLICT({quoted_keys}) DO NOTHING"
                )
            conn.executemany(sql, frame.itertuples(index=False, name=None))
            conn.commit()
        return self.path

    def upsert(
        self,
        table_name: str,
        data: Any,
        *,
        mode: str = "upsert",
        key_columns: Sequence[str] | str | None = None,
        update_columns: Sequence[str] | str | None = None,
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        return self.upsert_df(
            table_name,
            data,
            mode=mode,
            key_columns=key_columns,
            update_columns=update_columns,
            index=index,
            **kwargs,
        )

    def upsert_rows(
        self,
        table_name: str,
        rows: Sequence[dict[str, Any]] | Sequence[tuple[Any, ...]] | list[dict[str, Any]],
        *,
        mode: str = "upsert",
        key_columns: Sequence[str] | str | None = None,
        update_columns: Sequence[str] | str | None = None,
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        import pandas as pd

        frame = pd.DataFrame(rows)
        return self.upsert_df(
            table_name,
            frame,
            mode=mode,
            key_columns=key_columns,
            update_columns=update_columns,
            index=index,
            **kwargs,
        )

    def sync_data_df(
        self,
        table_name: str,
        *,
        mode: str = "replace",
        key_columns: Sequence[str] | str | None = None,
        update_columns: Sequence[str] | str | None = None,
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        if mode in {"upsert", "delete-insert"}:
            return self.upsert_df(
                table_name,
                self.pipeline.datatable,
                mode=mode,
                key_columns=key_columns,
                update_columns=update_columns,
                index=index,
                **kwargs,
            )
        return self.write_table(
            table_name,
            self.pipeline.datatable,
            if_exists=mode if mode in {"replace", "insert"} else "replace",
            index=index,
            **kwargs,
        )

    def load_data_df(self, table_name: str, **kwargs: Any):
        frame = self.read_df(table_name, **kwargs)
        self.pipeline.datatable = frame
        self.pipeline.set_data(table_name, frame)
        return frame

    def pull(self, table_name: str, **kwargs: Any):
        return self.load_data_df(table_name, **kwargs)

    def push(
        self,
        table_name: str,
        data: Any,
        *,
        mode: str = "replace",
        key_columns: Sequence[str] | str | None = None,
        update_columns: Sequence[str] | str | None = None,
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        if mode in {"upsert", "delete-insert"}:
            return self.upsert_df(
                table_name,
                data,
                mode=mode,
                key_columns=key_columns,
                update_columns=update_columns,
                index=index,
                **kwargs,
            )
        return self.write_table(
            table_name, data, if_exists=mode if mode in {"replace", "insert"} else "replace", index=index, **kwargs
        )

    def _resolve_default_upsert_key_columns(self, table_name: str) -> Sequence[str] | str | None:
        config = getattr(self.pipeline, "config", None)
        if config is None:
            return None
        candidates: list[Any] = []
        if hasattr(config, "get"):
            candidates.append(config.get("db_upsert_keys"))
            candidates.append(config.get("db_upsert_key_columns"))
        else:
            candidates.append(getattr(config, "db_upsert_keys", None))
            candidates.append(getattr(config, "db_upsert_key_columns", None))

        for candidate in candidates:
            if candidate is None:
                continue
            if isinstance(candidate, dict):
                if table_name in candidate:
                    return candidate[table_name]
                if "*" in candidate:
                    return candidate["*"]
                continue
            return candidate
        return None


@dataclass(slots=True)
class SQLiteQueryCondition:
    sql: str
    params: tuple[Any, ...] = ()


class SQLiteColumnRef:
    def __init__(self, table: "SQLiteTableQuery", name: str) -> None:
        self.table = table
        self.name = name

    def _condition(self, operator: str, value: Any) -> SQLiteQueryCondition:
        return SQLiteQueryCondition(sql=f'"{self.name}" {operator} ?', params=(value,))

    def __eq__(self, value: Any) -> SQLiteQueryCondition:  # type: ignore[override]
        return self._condition("=", value)

    def __ne__(self, value: Any) -> SQLiteQueryCondition:  # type: ignore[override]
        return self._condition("!=", value)

    def __gt__(self, value: Any) -> SQLiteQueryCondition:
        return self._condition(">", value)

    def __ge__(self, value: Any) -> SQLiteQueryCondition:
        return self._condition(">=", value)

    def __lt__(self, value: Any) -> SQLiteQueryCondition:
        return self._condition("<", value)

    def __le__(self, value: Any) -> SQLiteQueryCondition:
        return self._condition("<=", value)


class SQLiteTableQuery:
    def __init__(self, db: PipelineSQLiteDB, table_name: str) -> None:
        self._db = db
        self.table_name = table_name

    def __getattr__(self, name: str) -> SQLiteColumnRef:
        return SQLiteColumnRef(self, name)

    def _sqlize_condition(self, condition: Any) -> SQLiteQueryCondition:
        if isinstance(condition, SQLiteQueryCondition):
            return condition
        if isinstance(condition, tuple) and len(condition) == 3:
            column, operator, value = condition
            operator_text = {
                "=": "=",
                "==": "=",
                "eq": "=",
                "!=": "!=",
                "<>": "!=",
                "ne": "!=",
                ">": ">",
                "gt": ">",
                ">=": ">=",
                "gte": ">=",
                "<": "<",
                "lt": "<",
                "<=": "<=",
                "lte": "<=",
            }.get(str(operator), str(operator))
            return SQLiteQueryCondition(sql=f'"{column}" {operator_text} ?', params=(value,))
        if isinstance(condition, str):
            return SQLiteQueryCondition(sql=condition, params=())
        raise TypeError(f"unsupported condition: {condition!r}")

    def filter(self, *conditions: Any, **lookups: Any) -> "SQLiteTableQueryResult":
        normalized: list[SQLiteQueryCondition] = [self._sqlize_condition(condition) for condition in conditions]
        for key, value in lookups.items():
            if key.endswith("__gt"):
                column = key[:-4]
                normalized.append(SQLiteQueryCondition(sql=f'"{column}" > ?', params=(value,)))
            elif key.endswith("__gte"):
                column = key[:-5]
                normalized.append(SQLiteQueryCondition(sql=f'"{column}" >= ?', params=(value,)))
            elif key.endswith("__lt"):
                column = key[:-4]
                normalized.append(SQLiteQueryCondition(sql=f'"{column}" < ?', params=(value,)))
            elif key.endswith("__lte"):
                column = key[:-5]
                normalized.append(SQLiteQueryCondition(sql=f'"{column}" <= ?', params=(value,)))
            else:
                normalized.append(SQLiteQueryCondition(sql=f'"{key}" = ?', params=(value,)))
        return SQLiteTableQueryResult(self._db, self.table_name, normalized)

    def all(self) -> Any:
        return self._db.read_df(self.table_name)

    def first(self) -> Any:
        frame = self.all()
        return frame.head(1)

    def to_df(self) -> Any:
        return self.all()

    def __repr__(self) -> str:
        return f"<SQLiteTableQuery {self.table_name}>"


class SQLiteTableQueryResult(SQLiteTableQuery):
    def __init__(self, db: PipelineSQLiteDB, table_name: str, conditions: list[SQLiteQueryCondition]) -> None:
        super().__init__(db, table_name)
        self._conditions = conditions

    def _where_clause(self) -> tuple[str, tuple[Any, ...]]:
        if not self._conditions:
            return "", ()
        clauses = [condition.sql for condition in self._conditions]
        params: list[Any] = []
        for condition in self._conditions:
            params.extend(condition.params)
        return " WHERE " + " AND ".join(clauses), tuple(params)

    def all(self) -> Any:
        where_clause, params = self._where_clause()
        return self._db.query(f'SELECT * FROM "{self.table_name}"{where_clause}', params=params)

    def first(self) -> Any:
        frame = self.all()
        return frame.head(1)

    def to_df(self) -> Any:
        return self.all()


class PipelineEnvs:
    def __init__(self, db: PipelineSQLiteDB) -> None:
        self._db = db

    def __getattr__(self, table_name: str) -> SQLiteTableQuery:
        return SQLiteTableQuery(self._db, table_name)

    def table(self, table_name: str) -> SQLiteTableQuery:
        return SQLiteTableQuery(self._db, table_name)


class PipelineStatus(str, Enum):
    SUCCESS = "success"
    FAILED = "failed"
    SKIPPED = "skipped"
    ROLLED_BACK = "rolled_back"


@dataclass(slots=True)
class StepOutcome:
    """Normalized result returned by step execution."""

    status: PipelineStatus = PipelineStatus.SUCCESS
    data: Any = None
    message: str = ""
    metadata: dict[str, Any] = field(default_factory=dict)
    error: Optional[BaseException] = None
    attempts: int = 1
    skipped_reason: Optional[str] = None

    @property
    def success(self) -> bool:
        return self.status == PipelineStatus.SUCCESS

    @property
    def skipped(self) -> bool:
        return self.status == PipelineStatus.SKIPPED

    @property
    def failed(self) -> bool:
        return self.status == PipelineStatus.FAILED

    @property
    def summary(self) -> dict[str, Any]:
        return self.metadata

    @property
    def rows(self) -> Optional[int]:
        value = self.metadata.get("final_rows")
        return int(value) if value is not None else None

    @property
    def columns(self) -> list[str]:
        value = self.metadata.get("final_columns")
        if isinstance(value, list):
            return [str(column) for column in value]
        if isinstance(value, tuple):
            return [str(column) for column in value]
        if value is None:
            return []
        return [str(value)]

    @property
    def db_path(self) -> Optional[str]:
        value = self.metadata.get("db_output_path")
        return str(value) if value is not None else None

    @property
    def parquet_path(self) -> Optional[str]:
        value = self.metadata.get("parquet_output_path")
        return str(value) if value is not None else None

    @property
    def summary_path(self) -> Optional[str]:
        value = self.metadata.get("summary_path")
        return str(value) if value is not None else None

    @property
    def csv_path(self) -> Optional[str]:
        value = self.metadata.get("csv_output_path")
        return str(value) if value is not None else None

    @property
    def duration_seconds(self) -> Optional[float]:
        value = self.metadata.get("duration_seconds")
        return float(value) if value is not None else None


@dataclass(slots=True)
class PipelineContext:
    """Runtime state shared across steps."""

    pipeline: "BasePipeline"
    run_id: str = field(default_factory=lambda: uuid.uuid4().hex)
    config: dict[str, Any] = field(default_factory=dict)
    metadata: dict[str, Any] = field(default_factory=dict)
    current_step: Optional[str] = None
    current_attempt: int = 0
    input_data: Any = None
    output_data: Any = None
    step_outcomes: dict[str, StepOutcome] = field(default_factory=dict)
    artifacts: dict[str, Any] = field(default_factory=dict)
    errors: list[BaseException] = field(default_factory=list)
    rollback_actions: list[tuple[str, Callable[[], Any]]] = field(default_factory=list)

    def put(self, key: str, value: Any) -> Any:
        self.artifacts[key] = value
        return value

    def get(self, key: str, default: Any = None) -> Any:
        return self.artifacts.get(key, default)

    def add_rollback_action(self, action: Callable[[], Any], label: Optional[str] = None) -> str:
        action_label = label or getattr(action, "__name__", "rollback_action")
        self.rollback_actions.append((action_label, action))
        return action_label


def _call_with_context(func: Callable[..., Any], context: PipelineContext, data: Any) -> Any:
    try:
        sig = inspect.signature(func)
    except (TypeError, ValueError):
        return func(data, context)

    params = list(sig.parameters.values())
    if not params:
        return func()
    if len(params) == 1:
        return func(data)
    return func(data, context)


def _call_context_only(func: Callable[..., Any], context: PipelineContext) -> Any:
    try:
        sig = inspect.signature(func)
    except (TypeError, ValueError):
        return func(context)

    params = list(sig.parameters.values())
    if not params:
        return func()
    return func(context)


@dataclass(slots=True)
class BaseStep:
    """Base class for one business step."""

    name: str
    enabled: bool = True
    retries: int = 0

    def condition(self, context: PipelineContext, data: Any) -> bool:
        return True

    def should_skip(self, context: PipelineContext, data: Any) -> bool:
        return not self.enabled or not self.condition(context, data)

    def before(self, context: PipelineContext, data: Any) -> Any:
        return data

    def validate_input(self, context: PipelineContext, data: Any) -> None:
        return None

    def run(self, context: PipelineContext, data: Any) -> Any:
        raise NotImplementedError

    def validate_output(self, context: PipelineContext, output: Any) -> None:
        return None

    def after(self, context: PipelineContext, output: Any) -> Any:
        return output

    def rollback(self, context: PipelineContext, data: Any, error: BaseException) -> Any:
        return data

    def execute(self, context: PipelineContext, data: Any) -> StepOutcome:
        if self.should_skip(context, data):
            return StepOutcome(status=PipelineStatus.SKIPPED, data=data, skipped_reason="condition-disabled")

        attempts = 0
        last_error: Optional[BaseException] = None
        current_data = data
        while attempts <= self.retries:
            attempts += 1
            context.current_step = self.name
            context.current_attempt = attempts
            try:
                prepared = self.before(context, current_data)
                self.validate_input(context, prepared)
                output = self.run(context, prepared)
                self.validate_output(context, output)
                output = self.after(context, output)
                return StepOutcome(
                    status=PipelineStatus.SUCCESS,
                    data=output,
                    attempts=attempts,
                )
            except BaseException as exc:  # pragma: no cover - execution path depends on user steps
                last_error = exc
                context.errors.append(exc)
                if attempts > self.retries:
                    break
        return StepOutcome(
            status=PipelineStatus.FAILED,
            data=current_data,
            error=last_error,
            attempts=attempts,
        )


@dataclass(slots=True)
class FunctionStep(BaseStep):
    """Function-backed step used by the decorator API."""

    fn: Callable[..., Any] = field(default=lambda data, context=None: data, repr=False)
    pass_data: bool = True
    input_validator: Optional[Callable[..., Any]] = field(default=None, repr=False)
    output_validator: Optional[Callable[..., Any]] = field(default=None, repr=False)
    before_hook: Optional[Callable[..., Any]] = field(default=None, repr=False)
    after_hook: Optional[Callable[..., Any]] = field(default=None, repr=False)
    rollback_hook: Optional[Callable[..., Any]] = field(default=None, repr=False)
    condition_hook: Optional[Callable[..., bool]] = field(default=None, repr=False)
    dump_csv: bool = True

    def condition(self, context: PipelineContext, data: Any) -> bool:
        if self.condition_hook is None:
            return super().condition(context, data)
        if self.pass_data:
            return bool(_call_with_context(self.condition_hook, context, data))
        return bool(_call_context_only(self.condition_hook, context))

    def before(self, context: PipelineContext, data: Any) -> Any:
        if self.before_hook is None:
            return super().before(context, data)
        if self.pass_data:
            return _call_with_context(self.before_hook, context, data)
        return _call_context_only(self.before_hook, context)

    def validate_input(self, context: PipelineContext, data: Any) -> None:
        if self.input_validator is None:
            return super().validate_input(context, data)
        if self.pass_data:
            _call_with_context(self.input_validator, context, data)
        else:
            _call_context_only(self.input_validator, context)

    def run(self, context: PipelineContext, data: Any) -> Any:
        if self.pass_data:
            return _call_with_context(self.fn, context, data)
        return _call_context_only(self.fn, context)

    def validate_output(self, context: PipelineContext, output: Any) -> None:
        if self.output_validator is None:
            return super().validate_output(context, output)
        if self.pass_data:
            _call_with_context(self.output_validator, context, output)
        else:
            _call_context_only(self.output_validator, context)

    def after(self, context: PipelineContext, output: Any) -> Any:
        if self.after_hook is None:
            return super().after(context, output)
        if self.pass_data:
            return _call_with_context(self.after_hook, context, output)
        return _call_context_only(self.after_hook, context)

    def rollback(self, context: PipelineContext, data: Any, error: BaseException) -> Any:
        if self.rollback_hook is None:
            return super().rollback(context, data, error)
        try:
            if self.pass_data:
                return _call_with_context(self.rollback_hook, context, data)
            return _call_context_only(self.rollback_hook, context)
        except TypeError:
            return _call_context_only(self.rollback_hook, context)


@dataclass(slots=True)
class BasePipeline:
    """Orchestrates a sequence of steps."""

    name: str = "pipeline"
    config: dict[str, Any] = field(default_factory=dict)
    logger: Any = field(default=None)
    steps: list[BaseStep] = field(default_factory=list)

    def __post_init__(self) -> None:
        self.config = ConfigNamespace(**_config_to_dict(self.config))
        if self.logger is None:
            self.logger = _pipeline_logger(self.__class__.__name__, self.name)

    def add_step(self, step: BaseStep) -> BaseStep:
        self.steps.append(step)
        return step

    def step(
        self,
        name: Optional[str] = None,
        *,
        retries: int = 0,
        enabled: bool = True,
        condition: Optional[Callable[..., bool]] = None,
        before: Optional[Callable[..., Any]] = None,
        after: Optional[Callable[..., Any]] = None,
        validate_input: Optional[Callable[..., Any]] = None,
        validate_output: Optional[Callable[..., Any]] = None,
        rollback: Optional[Callable[..., Any]] = None,
        dump_csv: bool = True,
    ) -> Callable[[Callable[..., Any]], FunctionStep]:
        """Decorator API for registering a function-backed step."""

        def decorator(fn: Callable[..., Any]) -> FunctionStep:
            step = FunctionStep(
                name=name or fn.__name__,
                fn=fn,
                retries=retries,
                enabled=enabled,
                pass_data=True,
                condition_hook=condition,
                before_hook=before,
                after_hook=after,
                input_validator=validate_input,
                output_validator=validate_output,
                rollback_hook=rollback,
                dump_csv=dump_csv,
            )
            self.add_step(step)
            return step

        return decorator

    def before_run(self, func: Callable[..., Any]) -> Callable[..., Any]:
        self.config.setdefault("_before_run", []).append(func)
        return func

    def after_run(self, func: Callable[..., Any]) -> Callable[..., Any]:
        self.config.setdefault("_after_run", []).append(func)
        return func

    def run(self, data: Any = None, *, config: Optional[dict[str, Any]] = None) -> StepOutcome:
        context = PipelineContext(
            pipeline=self, config={**self.config.as_dict(), **_config_to_dict(config)}, input_data=data
        )
        self._run_hooks("_before_run", context)
        current = data
        for step_index, step in enumerate(self.steps, start=1):
            outcome = step.execute(context, current)
            context.step_outcomes[step.name] = outcome
            if outcome.skipped:
                self.logger.info("step skipped: {}", step.name)
                continue
            if outcome.failed:
                if outcome.error is not None:
                    self.logger.opt(exception=outcome.error).error("step failed: {}", step.name)
                else:
                    self.logger.error("step failed: {}", step.name)
                self._rollback(context, current, step)
                self._run_hooks("_after_run", context)
                return outcome
            current = outcome.data
            context.output_data = current
            self._dump_step_csv(step_index, step.name, current, context, step)
            self.logger.info("step success: {}", step.name)
        self._run_hooks("_after_run", context)
        return StepOutcome(status=PipelineStatus.SUCCESS, data=current, metadata={"run_id": context.run_id})

    def _rollback(self, context: PipelineContext, current: Any, failed_step: BaseStep) -> None:
        for step in reversed(self.steps):
            if step is failed_step:
                break
            try:
                step.rollback(
                    context, current, context.errors[-1] if context.errors else RuntimeError("pipeline failed")
                )
            except Exception as exc:  # pragma: no cover - rollback depends on user code
                self.logger.exception("rollback failed for step {}", step.name)
                context.errors.append(exc)

    def _restore_stage_snapshot(self, step_index: int, step_name: str) -> Any:
        snapshot_name = f"stage_{step_index:02d}_{step_name}"
        try:
            self.restore_df(snapshot_name)
            self.logger.info("stage snapshot restored: {}", snapshot_name)
        except KeyError:
            self.logger.warning("stage snapshot missing: {}", snapshot_name)
        return self.datatable

    def _run_rollback_actions(self, context: PipelineContext) -> None:
        for label, action in reversed(context.rollback_actions):
            try:
                action()
                self.logger.info("rollback action executed: {}", label)
            except Exception as exc:  # pragma: no cover - rollback depends on user code
                self.logger.exception("rollback action failed: {}", label)
                context.errors.append(exc)

    def _run_hooks(self, key: str, context: PipelineContext) -> None:
        for hook in self.config.get(key, []):
            try:
                _call_context_only(hook, context)
            except Exception:  # pragma: no cover - hook failures depend on user code
                self.logger.exception("hook failed: {}", key)

    def _dump_step_csv(
        self, step_index: int, step_name: str, data: Any, context: PipelineContext, step: BaseStep
    ) -> None:
        if not getattr(step, "dump_csv", True):
            return
        to_csv = getattr(data, "to_csv", None)
        if to_csv is None:
            return
        dump_dir = _pipeline_data_dir(self.__class__.__name__)
        dump_dir.mkdir(parents=True, exist_ok=True)
        dump_path = dump_dir / f"{step_index:02d}_{step_name}.csv"
        try:
            to_csv(dump_path, index=False)
            self.logger.info("step csv dumped: {}", dump_path)
        except Exception as exc:  # pragma: no cover - depends on user data
            self.logger.exception("step csv dump failed: {}", step_name)
            context.errors.append(exc)


class Pipeline(BasePipeline):
    """Public base class for business workflows.

    Subclasses declare step order through the `stages` class attribute and
    implement each step as a normal instance method.
    """

    datatable: DataTable
    df: DataTable
    path: Path
    stages: list[str] = []
    pipelines: list[str] = []
    cli_show_datatable_default: bool = False

    def __init_subclass__(cls, **kwargs: Any) -> None:
        super().__init_subclass__(**kwargs)
        protected = {"preprocess_data", "postprocess_data", "save_outputs", "save_data_to_db"}
        overridden = protected.intersection(cls.__dict__)
        if overridden:
            raise TypeError(f"{cls.__name__} cannot override protected pipeline hooks: {', '.join(sorted(overridden))}")

    @classmethod
    def step(
        cls,
        name: Optional[str] = None,
        *,
        retries: int = 0,
        enabled: bool = True,
        condition: Optional[Callable[..., bool]] = None,
        before: Optional[Callable[..., Any]] = None,
        after: Optional[Callable[..., Any]] = None,
        validate_input: Optional[Callable[..., Any]] = None,
        validate_output: Optional[Callable[..., Any]] = None,
        rollback: Optional[Callable[..., Any]] = None,
        dump_csv: bool = True,
    ) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
        """Optional metadata decorator for advanced step behavior."""

        def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
            fn._pipeline_step_spec = {  # type: ignore[attr-defined]
                "name": name or fn.__name__,
                "retries": retries,
                "enabled": enabled,
                "condition": condition,
                "before": before,
                "after": after,
                "validate_input": validate_input,
                "validate_output": validate_output,
                "rollback": rollback,
                "dump_csv": dump_csv,
            }
            return fn

        return decorator

    @staticmethod
    def _parse_cli_value(value: str) -> Any:
        if yaml is not None:
            try:
                parsed = yaml.safe_load(value)
            except Exception:
                return value
            return value if parsed is None else parsed
        return value

    @classmethod
    def _build_cli_parser(cls) -> argparse.ArgumentParser:
        parser = argparse.ArgumentParser(
            prog=cls.__name__,
            description=f"Run the {cls.__name__} pipeline.",
        )
        parser.add_argument("--name", default=cls.__name__, help="Pipeline instance name.")
        parser.add_argument(
            "--config",
            default=None,
            help="Optional config path or mapping-like value used to initialize the pipeline.",
        )
        parser.add_argument(
            "--set",
            action="append",
            default=[],
            metavar="KEY=VALUE",
            help="Override a config value. Can be used multiple times.",
        )
        parser.add_argument(
            "--step",
            action="append",
            default=[],
            metavar="STEP",
            help="Run only the selected step. Can be used multiple times.",
        )
        parser.add_argument(
            "--show-datatable",
            action="store_true",
            default=None,
            help="Print the current datatable after the pipeline finishes.",
        )
        parser.add_argument(
            "--no-show-datatable",
            action="store_false",
            dest="show_datatable",
            default=None,
            help="Disable datatable display even if the pipeline defaults to showing it.",
        )
        parser.add_argument(
            "--format",
            default="markdown",
            choices=("markdown", "html", "csv", "rich"),
            help="Display format used with --show-datatable.",
        )
        return parser

    @classmethod
    def cli(cls, argv: Optional[Sequence[str]] = None) -> StepOutcome:
        parser = cls._build_cli_parser()
        raw_argv = list(argv) if argv is not None else sys.argv[1:]
        args = parser.parse_args(raw_argv)
        show_datatable = getattr(cls, "cli_show_datatable_default", False)
        if args.show_datatable is True:
            show_datatable = True
        elif args.show_datatable is False:
            show_datatable = False

        pipeline = cls(name=args.name, config=args.config)
        for item in args.set:
            if "=" not in item:
                raise ValueError(f"Invalid --set value: {item!r}. Expected KEY=VALUE.")
            key, raw_value = item.split("=", 1)
            key = key.strip()
            if not key:
                raise ValueError(f"Invalid --set key in value: {item!r}")
            pipeline.config[key] = cls._parse_cli_value(raw_value)

        if args.step:
            selected = []
            step_map = {step.name: step for step in pipeline.steps}
            for step_name in args.step:
                if step_name not in step_map:
                    raise ValueError(f"Unknown step: {step_name!r}")
                selected.append(step_map[step_name])
            pipeline.steps = selected

        outcome = pipeline.run()
        if show_datatable:
            print(pipeline.display_datatable(format=args.format))
        return outcome

    def __init__(
        self,
        name: str = "pipeline",
        *,
        config: Any = None,
        logger: Any = None,
    ) -> None:
        super().__init__(name=name, config=self._load_config_payload(config), logger=logger)
        self.path = self._resolve_pipeline_path()
        self.db = PipelineSQLiteDB(self)
        self.envs = PipelineEnvs(self.db)
        self.config.setdefault("db_path", str(self.db_path))
        self._datatable: DataTable = _coerce_datatable(None)
        self._data: dict[str, Any] = {}
        self.model = SimpleNamespace()
        self.artifacts: dict[str, Any] = {}
        self._df_snapshots: dict[str, Any] = {}
        self._runtime_context: Optional[PipelineContext] = None
        self.steps = []
        self._register_decorated_members()

    @property
    def db_path(self) -> Path:
        return self._resolve_db_path()

    def _resolve_pipeline_path(self) -> Path:
        pipeline_module = inspect.getmodule(self.__class__)
        module_file = getattr(pipeline_module, "__file__", None)
        if module_file:
            return Path(module_file).resolve()
        return Path(__file__).resolve()

    def _resolve_db_path(self) -> Path:
        value = self.config.get("db_path")
        if value is None:
            env_value = os.getenv("SOIGIA_DB_PATH") or os.getenv("SOIGIA_SQLITE_PATH") or os.getenv("SOIGIA_DB_URL")
            if env_value:
                if env_value.startswith("sqlite:///"):
                    db_path = Path(env_value[len("sqlite:///") :]).expanduser().resolve()
                elif env_value.startswith("sqlite://"):
                    db_path = Path(env_value[len("sqlite://") :]).expanduser().resolve()
                elif env_value == "sqlite:///:memory:":
                    db_path = _pipeline_data_dir(self.__class__.__name__) / f"{self.__class__.__name__}.sqlite3"
                else:
                    db_path = Path(env_value).expanduser().resolve()
            else:
                db_path = _pipeline_data_dir(self.__class__.__name__) / f"{self.__class__.__name__}.sqlite3"
        else:
            db_path = Path(value).expanduser().resolve()
        db_path.parent.mkdir(parents=True, exist_ok=True)
        return db_path

    def _load_config_payload(self, config: Any) -> dict[str, Any]:
        if config is None:
            pipeline_module = inspect.getmodule(self.__class__)
            module_file = getattr(pipeline_module, "__file__", None)
            if module_file:
                module_dir = Path(module_file).resolve().parent
                default_path = module_dir / "config.yaml"
                if default_path.exists():
                    config = default_path
                else:
                    return {}
            else:
                return {}
        if isinstance(config, (str, Path)):
            if yaml is None:  # pragma: no cover - dependency should be installed
                raise RuntimeError("pyyaml is required to load pipeline config files")
            config_path = Path(config).expanduser()
            if not config_path.is_absolute():
                pipeline_module = inspect.getmodule(self.__class__)
                module_file = getattr(pipeline_module, "__file__", None)
                if module_file:
                    module_dir = Path(module_file).resolve().parent
                    candidate = (module_dir / config_path).resolve()
                    if candidate.exists():
                        config_path = candidate
            config_path = config_path.resolve()
            with config_path.open("r", encoding="utf-8") as handle:
                payload = yaml.safe_load(handle) or {}
            if not isinstance(payload, dict):
                raise TypeError(f"pipeline config file must contain a mapping: {config_path}")
            payload.setdefault("_config_path", str(config_path))
            return payload
        if isinstance(config, ConfigNamespace):
            return config.as_dict()
        if isinstance(config, dict):
            return dict(config)
        if hasattr(config, "items"):
            return dict(config.items())
        return dict(vars(config))

    def before_run(self, ctx: PipelineContext) -> Any:
        return None

    def after_run(self, ctx: PipelineContext) -> Any:
        return None

    def load_data(self) -> Any:
        """Load the initial dataframe for `run()`.

        Subclasses override this hook to fetch data from CSV, YAML, DB, API,
        or any other source. The returned value is assigned to `self.datatable`.
        If a subclass does not override it, the pipeline starts from an empty
        `DataTable`.
        """

        try:
            import pandas as pd
        except Exception:  # pragma: no cover - pandas should be installed
            self.datatable = []
            return self.datatable

        self.datatable = pd.DataFrame()
        return self.datatable

    def preprocess_data(self) -> Any:
        """Fixed preprocessing hook that runs after `load_data()`."""

        return self.datatable

    def postprocess_data(self) -> Any:
        """Fixed postprocessing hook that runs after all stages."""

        return self.datatable

    def save_outputs(self) -> Any:
        """Persist the final `self.datatable` snapshot to CSV, SQLite, and Parquet.

        Subclasses do not override this hook. They can change the target via
        config keys:
        - `csv_path`
        - `db_path`
        - `db_table`
        - `db_if_exists`
        - `parquet_path`
        """

        df = self.datatable
        if df is None:
            return self.datatable

        data_dir = _pipeline_data_dir(self.__class__.__name__)
        data_dir.mkdir(parents=True, exist_ok=True)

        csv_path = self.config.get("csv_path")
        if csv_path is None:
            csv_path = data_dir / f"{self.__class__.__name__}.csv"
        else:
            csv_path = Path(csv_path).expanduser().resolve()
            csv_path.parent.mkdir(parents=True, exist_ok=True)

        try:
            df.to_csv(csv_path, index=False, sep="\t")
            self.config["csv_output_path"] = str(csv_path)
            self.logger.info("step csv saved: {}", csv_path)
        except Exception as exc:  # pragma: no cover - depends on user data
            self.logger.exception("step csv save failed: {}", csv_path)
            context = getattr(self, "_runtime_context", None)
            if context is not None:
                context.errors.append(exc)

        columns = _df_columns(df)
        if not columns:
            self.config["db_output_path"] = None
            self.config["parquet_output_path"] = None
            return self.datatable

        table_name = str(self.config.get("db_table") or "final_data")
        if_exists = str(self.config.get("db_if_exists") or "replace")
        db_path = self.db.write_df(table_name, df, if_exists=if_exists, index=False)
        self.config["db_output_path"] = str(db_path)
        self.config["db_output_table"] = table_name
        self.logger.info("step db saved: {} -> {}", table_name, db_path)

        parquet_path = self.config.get("parquet_path")
        if parquet_path is None:
            parquet_path = db_path.with_suffix(".parquet")
        else:
            parquet_path = Path(parquet_path).expanduser().resolve()
            parquet_path.parent.mkdir(parents=True, exist_ok=True)

        try:
            df.to_parquet(parquet_path, index=False)
            self.config["parquet_output_path"] = str(parquet_path)
            self.logger.info("step parquet saved: {}", parquet_path)
        except Exception as exc:  # pragma: no cover - depends on optional parquet engine
            self.logger.exception("step parquet save failed: {}", parquet_path)
            context = getattr(self, "_runtime_context", None)
            if context is not None:
                context.errors.append(exc)
        return self.datatable

    def save_data_to_db(self) -> Any:
        """Backward-compatible alias for `save_outputs()`."""

        return self.save_outputs()

    def read_table(self, table_name: str, **kwargs: Any):
        return self.db.read_table(table_name, **kwargs)

    def to_df(self, table_name: str, **kwargs: Any):
        return self.db.to_df(table_name, **kwargs)

    def write_table(
        self,
        table_name: str,
        data: Any,
        *,
        if_exists: str = "replace",
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        return self.db.write_table(table_name, data, if_exists=if_exists, index=index, **kwargs)

    def from_df(
        self,
        table_name: str,
        data: Any,
        *,
        if_exists: str = "replace",
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        return self.db.from_df(table_name, data, if_exists=if_exists, index=index, **kwargs)

    def upsert_table(
        self,
        table_name: str,
        data: Any,
        *,
        mode: str = "upsert",
        key_columns: Sequence[str] | str | None = None,
        update_columns: Sequence[str] | str | None = None,
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        return self.db.upsert(
            table_name,
            data,
            mode=mode,
            key_columns=key_columns,
            update_columns=update_columns,
            index=index,
            **kwargs,
        )

    def upsert_rows(
        self,
        table_name: str,
        rows: Sequence[dict[str, Any]] | Sequence[tuple[Any, ...]] | list[dict[str, Any]],
        *,
        mode: str = "upsert",
        key_columns: Sequence[str] | str | None = None,
        update_columns: Sequence[str] | str | None = None,
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        return self.db.upsert_rows(
            table_name,
            rows,
            mode=mode,
            key_columns=key_columns,
            update_columns=update_columns,
            index=index,
            **kwargs,
        )

    def sync_data_df(
        self,
        table_name: str,
        *,
        mode: str = "replace",
        key_columns: Sequence[str] | str | None = None,
        update_columns: Sequence[str] | str | None = None,
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        return self.db.sync_data_df(
            table_name,
            mode=mode,
            key_columns=key_columns,
            update_columns=update_columns,
            index=index,
            **kwargs,
        )

    def load_data_df(self, table_name: str, **kwargs: Any):
        return self.db.load_data_df(table_name, **kwargs)

    def sync_datatable(
        self,
        table_name: str,
        *,
        mode: str = "replace",
        key_columns: Sequence[str] | str | None = None,
        update_columns: Sequence[str] | str | None = None,
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        return self.sync_data_df(
            table_name,
            mode=mode,
            key_columns=key_columns,
            update_columns=update_columns,
            index=index,
            **kwargs,
        )

    def load_datatable(self, table_name: str, **kwargs: Any):
        return self.load_data_df(table_name, **kwargs)

    def pull(
        self,
        table_name: str,
        **kwargs: Any,
    ):
        return self.load_data_df(table_name, **kwargs)

    def push(
        self,
        table_name: str,
        data: Any,
        *,
        mode: str = "replace",
        key_columns: Sequence[str] | str | None = None,
        update_columns: Sequence[str] | str | None = None,
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        return self.db.push(
            table_name,
            data,
            mode=mode,
            key_columns=key_columns,
            update_columns=update_columns,
            index=index,
            **kwargs,
        )

    def pull_datatable(self, table_name: str, **kwargs: Any):
        return self.load_datatable(table_name, **kwargs)

    def push_datatable(
        self,
        table_name: str,
        data: Any,
        *,
        mode: str = "replace",
        key_columns: Sequence[str] | str | None = None,
        update_columns: Sequence[str] | str | None = None,
        index: bool = False,
        **kwargs: Any,
    ) -> Path:
        return self.db.push(
            table_name,
            data,
            mode=mode,
            key_columns=key_columns,
            update_columns=update_columns,
            index=index,
            **kwargs,
        )

    def table_exists(self, table_name: str) -> bool:
        return self.db.table_exists(table_name)

    def list_tables(self) -> list[str]:
        return self.db.list_tables()

    def _summary_log_path(self) -> Path:
        summary_dir = _pipeline_data_dir(self.__class__.__name__)
        summary_dir.mkdir(parents=True, exist_ok=True)
        return summary_dir / "pipeline_summary.md"

    def _append_summary_log(self, markdown: str) -> None:
        summary_path = self._summary_log_path()
        if summary_path.exists() and summary_path.stat().st_size > 0:
            markdown = "\n\n---\n\n" + markdown.lstrip()
        with summary_path.open("a", encoding="utf-8") as handle:
            handle.write(markdown)

    def _build_summary_markdown(
        self,
        *,
        context: PipelineContext,
        input_data: Any,
        started_at: float,
        started_at_dt: datetime,
        status: str,
        step_records: list[dict[str, Any]],
        final_data: Any,
    ) -> str:
        duration_seconds = time.perf_counter() - started_at
        input_rows = _df_rows(input_data)
        input_columns = _df_columns(input_data)
        final_rows = _df_rows(final_data)
        final_columns = _df_columns(final_data)
        model_items = _model_items(self.model)

        lines = [
            "# Pipeline Run Summary",
            "",
            f"- **Pipeline:** `{_md_escape(self.__class__.__name__)}`",
            f"- **Name:** `{_md_escape(self.name)}`",
            f"- **Run ID:** `{_md_escape(context.run_id)}`",
            f"- **Status:** `{_md_escape(status)}`",
            f"- **Duration:** `{duration_seconds:.4f}s`",
            f"- **Input Rows:** `{_md_escape(input_rows)}`",
            f"- **Input Columns:** `{_md_escape(', '.join(input_columns) or '-')}`",
            f"- **Final Rows:** `{_md_escape(final_rows)}`",
            f"- **Final Columns:** `{_md_escape(', '.join(final_columns) or '-')}`",
            f"- **CSV Path:** `{_md_escape(self.config.get('csv_output_path', '-'))}`",
            f"- **DB Path:** `{_md_escape(self.config.get('db_output_path', '-'))}`",
            f"- **Parquet Path:** `{_md_escape(self.config.get('parquet_output_path', '-'))}`",
            "",
            "## Steps",
            "",
            "| # | Step | Status | Rows | Added Columns | Removed Columns |",
            "| --- | --- | --- | --- | --- | --- |",
        ]

        for record in step_records:
            lines.append(
                "| {index:02d} | {step} | {status} | {rows} | {added} | {removed} |".format(
                    index=record["index"],
                    step=_md_escape(record["step"]),
                    status=_md_escape(record["status"]),
                    rows=_md_escape(record["rows"]),
                    added=_md_escape(record["added"]),
                    removed=_md_escape(record["removed"]),
                )
            )

        lines.extend(
            [
                "",
                "## Notes",
                "",
                f"- **Started At:** `{_md_escape(started_at_dt.isoformat(timespec='seconds'))}`",
            ]
        )
        if model_items:
            lines.extend(["", "## Model", "", "| Key | Value |", "| --- | --- |"])
            for key, value in model_items:
                lines.append(f"| {_md_escape(key)} | {_md_escape(value)} |")
        return "\n".join(lines) + "\n"

    def load_csv(self, path: Any, **kwargs: Any) -> Any:
        """Load a CSV file into `self.datatable`."""

        import pandas as pd

        self.datatable = pd.read_csv(path, **kwargs)
        return self.datatable

    def load_data_from_csv(self, path: Any, **kwargs: Any) -> Any:
        """Backward-compatible alias for `load_csv()`."""

        return self.load_csv(path, **kwargs)

    def load_data_from_parquet(self, path: Any, **kwargs: Any) -> Any:
        """Load a Parquet file into `self.datatable`."""

        import pandas as pd

        self.datatable = pd.read_parquet(path, **kwargs)
        return self.datatable

    def load_yaml(self, path: Any, *, key: Optional[str] = None) -> Any:
        """Load a YAML file into `self.datatable`."""

        if yaml is None:  # pragma: no cover - dependency should be installed
            raise RuntimeError("pyyaml is required to load YAML files")
        with Path(path).expanduser().resolve().open("r", encoding="utf-8") as handle:
            payload = yaml.safe_load(handle)
        if key is not None and isinstance(payload, dict):
            payload = payload.get(key)
        self.datatable = payload
        return self.datatable

    def save_data_to_csv(self, path: Any = None, **kwargs: Any) -> Any:
        """Save `self.datatable` to CSV and remember the output path."""

        df = self.datatable
        if df is None:
            return self.datatable

        if path is None:
            data_dir = _pipeline_data_dir(self.__class__.__name__)
            data_dir.mkdir(parents=True, exist_ok=True)
            path = data_dir / f"{self.__class__.__name__}.csv"
        else:
            path = Path(path).expanduser().resolve()
            path.parent.mkdir(parents=True, exist_ok=True)

        df.to_csv(path, index=False, **kwargs)
        self.config["csv_output_path"] = str(path)
        return self.datatable

    def save_data_to_parquet(self, path: Any = None, **kwargs: Any) -> Any:
        """Save `self.datatable` to Parquet and remember the output path."""

        df = self.datatable
        if df is None:
            return self.datatable

        if path is None:
            data_dir = _pipeline_data_dir(self.__class__.__name__)
            data_dir.mkdir(parents=True, exist_ok=True)
            path = data_dir / f"{self.__class__.__name__}.parquet"
        else:
            path = Path(path).expanduser().resolve()
            path.parent.mkdir(parents=True, exist_ok=True)

        df.to_parquet(path, index=False, **kwargs)
        self.config["parquet_output_path"] = str(path)
        return self.datatable

    def logging(self, message: Any, *args: Any, level: str = "info", **kwargs: Any) -> None:
        """Write a log message through the pipeline logger.

        The default sink writes to `logs/<ClassName>.log`, so subclasses can
        simply call `self.logging("message")` without setting up logging.
        """

        log_fn = getattr(self.logger, level.lower(), self.logger.info)
        log_fn(message, *args, **kwargs)

    def display_datatable(self, format: str = "markdown", **kwargs: Any) -> str:
        """Render the current datatable for display and return the rendered text."""

        return self.datatable.to_display_text(format=format, **kwargs)

    def logging_datatable(self, format: str = "markdown", level: str = "info", **kwargs: Any) -> str:
        """Write the current datatable snapshot into the pipeline log and return the rendered text."""

        text = self.datatable.to_display_text(format=format, **kwargs)
        self.logging(text, level=level)
        return text

    def on_rollback(self, action: Callable[[], Any], *, label: Optional[str] = None) -> str:
        """Register a rollback action for external side effects."""

        context = self._runtime_context
        if context is None:
            raise RuntimeError("rollback actions can only be registered while the pipeline is running")
        return context.add_rollback_action(action, label=label)

    @property
    def datatable(self) -> "DataTable":
        return self._datatable

    @datatable.setter
    def datatable(self, value: Any) -> None:
        self._datatable = _coerce_datatable(value)

    @property
    def df(self) -> "DataTable":
        return self._datatable

    @df.setter
    def df(self, value: Any) -> None:
        self._datatable = _coerce_datatable(value)

    def set_df(self, value: Any) -> "DataTable":
        self._datatable = _coerce_datatable(value)
        return self._datatable

    def get_df(self) -> "DataTable":
        return self._datatable

    def set_datatable(self, value: Any) -> "DataTable":
        self._datatable = _coerce_datatable(value)
        return self._datatable

    def get_datatable(self) -> "DataTable":
        return self._datatable

    def set_data(self, key: str, value: Any) -> Any:
        self._data[key] = value
        return value

    def get_data(self, key: str, default: Any = None) -> Any:
        return self._data.get(key, default)

    def delete_data(self, key: str) -> None:
        self._data.pop(key, None)

    def clear_data(self) -> None:
        self._data.clear()

    def put(self, key: str, value: Any) -> Any:
        self.artifacts[key] = value
        return value

    def get(self, key: str, default: Any = None) -> Any:
        return self.artifacts.get(key, default)

    def delete(self, key: str) -> None:
        self.artifacts.pop(key, None)

    def require_df(self) -> Any:
        if self.datatable is None:
            raise ValueError("datatable is empty")
        return self.datatable

    def has_columns(self, *columns: str) -> bool:
        df = self.datatable
        if df is None or not hasattr(df, "columns"):
            return False
        current = set(str(column) for column in getattr(df, "columns", []))
        return all(column in current for column in columns)

    def require_columns(self, *columns: str) -> Any:
        if not self.has_columns(*columns):
            raise ValueError(f"missing required columns: {', '.join(columns)}")
        return self.datatable

    def snapshot_df(self, name: str = "default") -> Any:
        data = self.datatable
        if hasattr(data, "copy"):
            try:
                snapshot = data.copy(deep=True)
            except TypeError:
                snapshot = data.copy()
        else:
            snapshot = copy.deepcopy(data)
        self._df_snapshots[name] = snapshot
        return snapshot

    def restore_df(self, name: str = "default") -> Any:
        if name not in self._df_snapshots:
            raise KeyError(f"snapshot {name!r} not found")
        snapshot = self._df_snapshots[name]
        if hasattr(snapshot, "copy"):
            try:
                self.datatable = snapshot.copy(deep=True)
            except TypeError:
                self.datatable = snapshot.copy()
        else:
            self.datatable = copy.deepcopy(snapshot)
        return self.datatable

    def head(self, n: int = 5) -> Any:
        df = self.require_df()
        return df.head(n) if hasattr(df, "head") else df

    def tail(self, n: int = 5) -> Any:
        df = self.require_df()
        return df.tail(n) if hasattr(df, "tail") else df

    def _register_decorated_members(self) -> None:
        step_specs: dict[str, dict[str, Any]] = {}
        for cls in reversed(self.__class__.__mro__):
            for attr_name, attr_value in cls.__dict__.items():
                spec = getattr(attr_value, "_pipeline_step_spec", None)
                if spec is not None:
                    step_specs[attr_name] = spec

        ordered_step_names = list(getattr(self, "stages", []) or [])
        if not ordered_step_names:
            ordered_step_names = list(getattr(self, "pipelines", []) or [])
        if not ordered_step_names:
            ordered_step_names = list(step_specs.keys())

        seen_steps: set[str] = set()
        for step_name in ordered_step_names:
            if step_name in seen_steps:
                continue
            seen_steps.add(step_name)
            if not hasattr(self, step_name):
                raise AttributeError(f"pipeline step {step_name!r} is not defined on {self.__class__.__name__}")
            spec = step_specs.get(step_name, {})
            self.steps.append(
                FunctionStep(
                    name=spec.get("name", step_name),
                    fn=getattr(self, step_name),
                    retries=spec.get("retries", 0),
                    enabled=spec.get("enabled", True),
                    pass_data=False,
                    condition_hook=spec.get("condition"),
                    before_hook=spec.get("before"),
                    after_hook=spec.get("after"),
                    input_validator=spec.get("validate_input"),
                    output_validator=spec.get("validate_output"),
                    rollback_hook=spec.get("rollback"),
                    dump_csv=spec.get("dump_csv", True),
                )
            )

    def run(self, *, config: Optional[dict[str, Any]] = None) -> StepOutcome:
        input_data = self.load_data()
        if input_data is not None:
            self.datatable = input_data
        self.preprocess_data()
        current = self.datatable
        context = PipelineContext(
            pipeline=self,
            config={**self.config.as_dict(), **_config_to_dict(config)},
            input_data=input_data,
        )
        context.metadata["model"] = self.model
        self._runtime_context = context
        run_started_at = time.perf_counter()
        run_started_dt = datetime.now()
        step_records: list[dict[str, Any]] = []
        run_metadata: dict[str, Any] = {
            "run_id": context.run_id,
            "pipeline": self.__class__.__name__,
            "name": self.name,
            "input_rows": _df_rows(input_data),
            "input_columns": _df_columns(input_data),
        }
        try:
            _call_context_only(self.before_run, context)
            for step_index, step in enumerate(self.steps, start=1):
                step_before_rows = _df_rows(current)
                step_before_columns = _df_columns(current)
                snapshot_name = f"stage_{step_index:02d}_{step.name}"
                self.snapshot_df(snapshot_name)
                self.datatable = current
                outcome = step.execute(context, current)
                context.step_outcomes[step.name] = outcome
                if outcome.skipped:
                    self.logger.info("step skipped: {}", step.name)
                    step_records.append(
                        {
                            "index": step_index,
                            "step": step.name,
                            "status": "skipped",
                            "rows": f"{step_before_rows}->{step_before_rows}",
                            "added": "-",
                            "removed": "-",
                        }
                    )
                    continue
                if outcome.failed:
                    if outcome.error is not None:
                        self.logger.opt(exception=outcome.error).error("step failed: {}", step.name)
                    else:
                        self.logger.error("step failed: {}", step.name)
                    current = self._restore_stage_snapshot(step_index, step.name)
                    self.datatable = current
                    self._rollback(context, current, step)
                    self._run_rollback_actions(context)
                    step_records.append(
                        {
                            "index": step_index,
                            "step": step.name,
                            "status": "failed",
                            "rows": f"{step_before_rows}->{_df_rows(current)}",
                            "added": "-",
                            "removed": "-",
                        }
                    )
                    run_metadata.update(
                        {
                            "status": "failed",
                            "duration_seconds": time.perf_counter() - run_started_at,
                            "step_records": step_records,
                            "final_rows": _df_rows(self.datatable),
                            "final_columns": _df_columns(self.datatable),
                            "csv_output_path": self.config.get("csv_output_path"),
                            "db_output_path": self.config.get("db_output_path"),
                            "parquet_output_path": self.config.get("parquet_output_path"),
                            "summary_path": str(self._summary_log_path()),
                            "model": dict(vars(self.model)),
                        }
                    )
                    outcome.metadata.update(run_metadata)
                    markdown = self._build_summary_markdown(
                        context=context,
                        input_data=input_data,
                        started_at=run_started_at,
                        started_at_dt=run_started_dt,
                        status="failed",
                        step_records=step_records,
                        final_data=self.datatable,
                    )
                    self._append_summary_log(markdown)
                    _call_context_only(self.after_run, context)
                    return outcome
                if outcome.data is not None:
                    current = outcome.data
                elif self.datatable is not None:
                    current = self.datatable
                self.datatable = current
                current = self.datatable
                context.output_data = current
                self._dump_step_csv(step_index, step.name, current, context, step)
                step_after_rows = _df_rows(current)
                step_after_columns = _df_columns(current)
                added_columns = [column for column in step_after_columns if column not in step_before_columns]
                removed_columns = [column for column in step_before_columns if column not in step_after_columns]
                step_records.append(
                    {
                        "index": step_index,
                        "step": step.name,
                        "status": "success",
                        "rows": f"{step_before_rows}->{step_after_rows}",
                        "added": ", ".join(added_columns) or "-",
                        "removed": ", ".join(removed_columns) or "-",
                    }
                )
                self.logger.info("step success: {}", step.name)
            self.datatable = current
            self.postprocess_data()
            current = self.datatable
            self.save_outputs()
            run_metadata.update(
                {
                    "status": "success",
                    "duration_seconds": time.perf_counter() - run_started_at,
                    "step_records": step_records,
                    "final_rows": _df_rows(self.datatable),
                    "final_columns": _df_columns(self.datatable),
                    "csv_output_path": self.config.get("csv_output_path"),
                    "db_output_path": self.config.get("db_output_path"),
                    "parquet_output_path": self.config.get("parquet_output_path"),
                    "summary_path": str(self._summary_log_path()),
                    "model": dict(vars(self.model)),
                }
            )
            markdown = self._build_summary_markdown(
                context=context,
                input_data=input_data,
                started_at=run_started_at,
                started_at_dt=run_started_dt,
                status="success",
                step_records=step_records,
                final_data=self.datatable,
            )
            self._append_summary_log(markdown)
            _call_context_only(self.after_run, context)
            return StepOutcome(status=PipelineStatus.SUCCESS, data=self.datatable, metadata=run_metadata)
        finally:
            self._runtime_context = None

