"""Base pipeline skeleton for business workflows and dataframe pipelines.

This module intentionally stays small and extensible:
- `BaseStep` encapsulates one business step.
- `BasePipeline` orchestrates execution and lifecycle hooks.
- `@pipeline.step(...)` provides a decorator-friendly API.

The implementation is conservative on purpose so we can iterate on the
execution model, retry/rollback semantics, and dataframe-specific helpers
without forcing a large framework design too early.
"""

from __future__ import annotations

import inspect
import copy
import sqlite3
import time
from datetime import datetime
from pathlib import Path
import uuid
from dataclasses import dataclass, field
from enum import Enum
from types import SimpleNamespace
from typing import Any, Callable, Optional

from loguru import logger as loguru_logger

try:
    import yaml
except Exception:  # pragma: no cover - dependency should be installed
    yaml = None  # type: ignore[assignment]

__all__ = [
    "PipelineStatus",
    "StepOutcome",
    "PipelineContext",
    "BaseStep",
    "FunctionStep",
    "ConfigNamespace",
    "BasePipeline",
    "Pipeline",
]


_LOG_SINKS: dict[str, int] = {}


def _pipeline_log_path(class_name: str) -> Path:
    return (Path.cwd() / "logs" / f"{class_name}.log").resolve()


def _pipeline_data_dir(class_name: str) -> Path:
    return (Path.cwd() / "data" / class_name).resolve()


def _df_rows(data: Any) -> Optional[int]:
    try:
        return len(data)
    except Exception:
        return None


def _df_columns(data: Any) -> list[str]:
    columns = getattr(data, "columns", None)
    if columns is None:
        return []
    return [str(column) for column in columns]


def _model_items(model: Any) -> list[tuple[str, Any]]:
    if model is None:
        return []
    try:
        items = vars(model).items()
    except TypeError:
        return []
    return [(str(key), value) for key, value in items if not str(key).startswith("_")]


def _config_to_dict(config: Any) -> dict[str, Any]:
    if config is None:
        return {}
    if isinstance(config, ConfigNamespace):
        return config.as_dict()
    if isinstance(config, dict):
        return dict(config)
    if hasattr(config, "items"):
        try:
            return dict(config.items())
        except Exception:
            pass
    try:
        return dict(vars(config))
    except Exception:
        return {}


class ConfigNamespace(SimpleNamespace):
    """Namespace that also behaves like a lightweight mapping."""

    def get(self, key: str, default: Any = None) -> Any:
        return getattr(self, key, default)

    def setdefault(self, key: str, default: Any = None) -> Any:
        if not hasattr(self, key):
            setattr(self, key, default)
        return getattr(self, key)

    def update(self, other: Optional[dict[str, Any]] = None, **kwargs: Any) -> None:
        payload = dict(other or {})
        payload.update(kwargs)
        for key, value in payload.items():
            setattr(self, key, value)

    def as_dict(self) -> dict[str, Any]:
        return {key: value for key, value in vars(self).items() if not key.startswith("_")}

    def __contains__(self, key: object) -> bool:
        return isinstance(key, str) and hasattr(self, key)

    def __getitem__(self, key: str) -> Any:
        if not hasattr(self, key):
            raise KeyError(key)
        return getattr(self, key)

    def __setitem__(self, key: str, value: Any) -> None:
        setattr(self, key, value)


def _md_escape(value: Any) -> str:
    text = "-" if value is None else str(value)
    return text.replace("|", "\\|").replace("\n", " ")


def _pipeline_logger(class_name: str, pipeline_name: str) -> Any:
    log_path = _pipeline_log_path(class_name)
    sink_key = str(log_path)
    if sink_key not in _LOG_SINKS:
        log_path.parent.mkdir(parents=True, exist_ok=True)
        _LOG_SINKS[sink_key] = loguru_logger.add(
            log_path,
            level="INFO",
            rotation="10 MB",
            retention=14,
            encoding="utf-8",
            backtrace=False,
            diagnose=False,
            format=(
                "{time:YYYY-MM-DD HH:mm:ss} | {level:<8} | "
                "{extra[pipeline_class]}::{extra[pipeline_name]} | {message}"
            ),
        )
    return loguru_logger.bind(pipeline_class=class_name, pipeline_name=pipeline_name)


class PipelineStatus(str, Enum):
    SUCCESS = "success"
    FAILED = "failed"
    SKIPPED = "skipped"
    ROLLED_BACK = "rolled_back"


@dataclass(slots=True)
class StepOutcome:
    """Normalized result returned by step execution."""

    status: PipelineStatus = PipelineStatus.SUCCESS
    data: Any = None
    message: str = ""
    metadata: dict[str, Any] = field(default_factory=dict)
    error: Optional[BaseException] = None
    attempts: int = 1
    skipped_reason: Optional[str] = None

    @property
    def success(self) -> bool:
        return self.status == PipelineStatus.SUCCESS

    @property
    def skipped(self) -> bool:
        return self.status == PipelineStatus.SKIPPED

    @property
    def failed(self) -> bool:
        return self.status == PipelineStatus.FAILED

    @property
    def summary(self) -> dict[str, Any]:
        return self.metadata

    @property
    def rows(self) -> Optional[int]:
        value = self.metadata.get("final_rows")
        return int(value) if value is not None else None

    @property
    def columns(self) -> list[str]:
        value = self.metadata.get("final_columns")
        if isinstance(value, list):
            return [str(column) for column in value]
        if isinstance(value, tuple):
            return [str(column) for column in value]
        if value is None:
            return []
        return [str(value)]

    @property
    def db_path(self) -> Optional[str]:
        value = self.metadata.get("db_output_path")
        return str(value) if value is not None else None

    @property
    def parquet_path(self) -> Optional[str]:
        value = self.metadata.get("parquet_output_path")
        return str(value) if value is not None else None

    @property
    def summary_path(self) -> Optional[str]:
        value = self.metadata.get("summary_path")
        return str(value) if value is not None else None

    @property
    def csv_path(self) -> Optional[str]:
        value = self.metadata.get("csv_output_path")
        return str(value) if value is not None else None

    @property
    def duration_seconds(self) -> Optional[float]:
        value = self.metadata.get("duration_seconds")
        return float(value) if value is not None else None


@dataclass(slots=True)
class PipelineContext:
    """Runtime state shared across steps."""

    pipeline: "BasePipeline"
    run_id: str = field(default_factory=lambda: uuid.uuid4().hex)
    config: dict[str, Any] = field(default_factory=dict)
    metadata: dict[str, Any] = field(default_factory=dict)
    current_step: Optional[str] = None
    current_attempt: int = 0
    input_data: Any = None
    output_data: Any = None
    step_outcomes: dict[str, StepOutcome] = field(default_factory=dict)
    artifacts: dict[str, Any] = field(default_factory=dict)
    errors: list[BaseException] = field(default_factory=list)
    rollback_actions: list[tuple[str, Callable[[], Any]]] = field(default_factory=list)

    def put(self, key: str, value: Any) -> Any:
        self.artifacts[key] = value
        return value

    def get(self, key: str, default: Any = None) -> Any:
        return self.artifacts.get(key, default)

    def add_rollback_action(self, action: Callable[[], Any], label: Optional[str] = None) -> str:
        action_label = label or getattr(action, "__name__", "rollback_action")
        self.rollback_actions.append((action_label, action))
        return action_label


def _call_with_context(func: Callable[..., Any], context: PipelineContext, data: Any) -> Any:
    try:
        sig = inspect.signature(func)
    except (TypeError, ValueError):
        return func(data, context)

    params = list(sig.parameters.values())
    if not params:
        return func()
    if len(params) == 1:
        return func(data)
    return func(data, context)


def _call_context_only(func: Callable[..., Any], context: PipelineContext) -> Any:
    try:
        sig = inspect.signature(func)
    except (TypeError, ValueError):
        return func(context)

    params = list(sig.parameters.values())
    if not params:
        return func()
    return func(context)


@dataclass(slots=True)
class BaseStep:
    """Base class for one business step."""

    name: str
    enabled: bool = True
    retries: int = 0

    def condition(self, context: PipelineContext, data: Any) -> bool:
        return True

    def should_skip(self, context: PipelineContext, data: Any) -> bool:
        return not self.enabled or not self.condition(context, data)

    def before(self, context: PipelineContext, data: Any) -> Any:
        return data

    def validate_input(self, context: PipelineContext, data: Any) -> None:
        return None

    def run(self, context: PipelineContext, data: Any) -> Any:
        raise NotImplementedError

    def validate_output(self, context: PipelineContext, output: Any) -> None:
        return None

    def after(self, context: PipelineContext, output: Any) -> Any:
        return output

    def rollback(self, context: PipelineContext, data: Any, error: BaseException) -> Any:
        return data

    def execute(self, context: PipelineContext, data: Any) -> StepOutcome:
        if self.should_skip(context, data):
            return StepOutcome(status=PipelineStatus.SKIPPED, data=data, skipped_reason="condition-disabled")

        attempts = 0
        last_error: Optional[BaseException] = None
        current_data = data
        while attempts <= self.retries:
            attempts += 1
            context.current_step = self.name
            context.current_attempt = attempts
            try:
                prepared = self.before(context, current_data)
                self.validate_input(context, prepared)
                output = self.run(context, prepared)
                self.validate_output(context, output)
                output = self.after(context, output)
                return StepOutcome(
                    status=PipelineStatus.SUCCESS,
                    data=output,
                    attempts=attempts,
                )
            except BaseException as exc:  # pragma: no cover - execution path depends on user steps
                last_error = exc
                context.errors.append(exc)
                if attempts > self.retries:
                    break
        return StepOutcome(
            status=PipelineStatus.FAILED,
            data=current_data,
            error=last_error,
            attempts=attempts,
        )


@dataclass(slots=True)
class FunctionStep(BaseStep):
    """Function-backed step used by the decorator API."""

    fn: Callable[..., Any] = field(default=lambda data, context=None: data, repr=False)
    pass_data: bool = True
    input_validator: Optional[Callable[..., Any]] = field(default=None, repr=False)
    output_validator: Optional[Callable[..., Any]] = field(default=None, repr=False)
    before_hook: Optional[Callable[..., Any]] = field(default=None, repr=False)
    after_hook: Optional[Callable[..., Any]] = field(default=None, repr=False)
    rollback_hook: Optional[Callable[..., Any]] = field(default=None, repr=False)
    condition_hook: Optional[Callable[..., bool]] = field(default=None, repr=False)
    dump_csv: bool = True

    def condition(self, context: PipelineContext, data: Any) -> bool:
        if self.condition_hook is None:
            return super().condition(context, data)
        if self.pass_data:
            return bool(_call_with_context(self.condition_hook, context, data))
        return bool(_call_context_only(self.condition_hook, context))

    def before(self, context: PipelineContext, data: Any) -> Any:
        if self.before_hook is None:
            return super().before(context, data)
        if self.pass_data:
            return _call_with_context(self.before_hook, context, data)
        return _call_context_only(self.before_hook, context)

    def validate_input(self, context: PipelineContext, data: Any) -> None:
        if self.input_validator is None:
            return super().validate_input(context, data)
        if self.pass_data:
            _call_with_context(self.input_validator, context, data)
        else:
            _call_context_only(self.input_validator, context)

    def run(self, context: PipelineContext, data: Any) -> Any:
        if self.pass_data:
            return _call_with_context(self.fn, context, data)
        return _call_context_only(self.fn, context)

    def validate_output(self, context: PipelineContext, output: Any) -> None:
        if self.output_validator is None:
            return super().validate_output(context, output)
        if self.pass_data:
            _call_with_context(self.output_validator, context, output)
        else:
            _call_context_only(self.output_validator, context)

    def after(self, context: PipelineContext, output: Any) -> Any:
        if self.after_hook is None:
            return super().after(context, output)
        if self.pass_data:
            return _call_with_context(self.after_hook, context, output)
        return _call_context_only(self.after_hook, context)

    def rollback(self, context: PipelineContext, data: Any, error: BaseException) -> Any:
        if self.rollback_hook is None:
            return super().rollback(context, data, error)
        try:
            if self.pass_data:
                return _call_with_context(self.rollback_hook, context, data)
            return _call_context_only(self.rollback_hook, context)
        except TypeError:
            return _call_context_only(self.rollback_hook, context)


@dataclass(slots=True)
class BasePipeline:
    """Orchestrates a sequence of steps."""

    name: str = "pipeline"
    config: dict[str, Any] = field(default_factory=dict)
    logger: Any = field(default=None)
    steps: list[BaseStep] = field(default_factory=list)

    def __post_init__(self) -> None:
        self.config = ConfigNamespace(**_config_to_dict(self.config))
        if self.logger is None:
            self.logger = _pipeline_logger(self.__class__.__name__, self.name)

    def add_step(self, step: BaseStep) -> BaseStep:
        self.steps.append(step)
        return step

    def step(
        self,
        name: Optional[str] = None,
        *,
        retries: int = 0,
        enabled: bool = True,
        condition: Optional[Callable[..., bool]] = None,
        before: Optional[Callable[..., Any]] = None,
        after: Optional[Callable[..., Any]] = None,
        validate_input: Optional[Callable[..., Any]] = None,
        validate_output: Optional[Callable[..., Any]] = None,
        rollback: Optional[Callable[..., Any]] = None,
        dump_csv: bool = True,
    ) -> Callable[[Callable[..., Any]], FunctionStep]:
        """Decorator API for registering a function-backed step."""

        def decorator(fn: Callable[..., Any]) -> FunctionStep:
            step = FunctionStep(
                name=name or fn.__name__,
                fn=fn,
                retries=retries,
                enabled=enabled,
                pass_data=True,
                condition_hook=condition,
                before_hook=before,
                after_hook=after,
                input_validator=validate_input,
                output_validator=validate_output,
                rollback_hook=rollback,
                dump_csv=dump_csv,
            )
            self.add_step(step)
            return step

        return decorator

    def before_run(self, func: Callable[..., Any]) -> Callable[..., Any]:
        self.config.setdefault("_before_run", []).append(func)
        return func

    def after_run(self, func: Callable[..., Any]) -> Callable[..., Any]:
        self.config.setdefault("_after_run", []).append(func)
        return func

    def run(self, data: Any = None, *, config: Optional[dict[str, Any]] = None) -> StepOutcome:
        context = PipelineContext(pipeline=self, config={**self.config.as_dict(), **_config_to_dict(config)}, input_data=data)
        self._run_hooks("_before_run", context)
        current = data
        for step_index, step in enumerate(self.steps, start=1):
            outcome = step.execute(context, current)
            context.step_outcomes[step.name] = outcome
            if outcome.skipped:
                self.logger.info("step skipped: {}", step.name)
                continue
            if outcome.failed:
                if outcome.error is not None:
                    self.logger.opt(exception=outcome.error).error("step failed: {}", step.name)
                else:
                    self.logger.error("step failed: {}", step.name)
                self._rollback(context, current, step)
                self._run_hooks("_after_run", context)
                return outcome
            current = outcome.data
            context.output_data = current
            self._dump_step_csv(step_index, step.name, current, context, step)
            self.logger.info("step success: {}", step.name)
        self._run_hooks("_after_run", context)
        return StepOutcome(status=PipelineStatus.SUCCESS, data=current, metadata={"run_id": context.run_id})

    def _rollback(self, context: PipelineContext, current: Any, failed_step: BaseStep) -> None:
        for step in reversed(self.steps):
            if step is failed_step:
                break
            try:
                step.rollback(
                    context, current, context.errors[-1] if context.errors else RuntimeError("pipeline failed")
                )
            except Exception as exc:  # pragma: no cover - rollback depends on user code
                self.logger.exception("rollback failed for step {}", step.name)
                context.errors.append(exc)

    def _restore_stage_snapshot(self, step_index: int, step_name: str) -> Any:
        snapshot_name = f"stage_{step_index:02d}_{step_name}"
        try:
            self.restore_df(snapshot_name)
            self.logger.info("stage snapshot restored: {}", snapshot_name)
        except KeyError:
            self.logger.warning("stage snapshot missing: {}", snapshot_name)
        return self.data_df

    def _run_rollback_actions(self, context: PipelineContext) -> None:
        for label, action in reversed(context.rollback_actions):
            try:
                action()
                self.logger.info("rollback action executed: {}", label)
            except Exception as exc:  # pragma: no cover - rollback depends on user code
                self.logger.exception("rollback action failed: {}", label)
                context.errors.append(exc)

    def _run_hooks(self, key: str, context: PipelineContext) -> None:
        for hook in self.config.get(key, []):
            try:
                _call_context_only(hook, context)
            except Exception:  # pragma: no cover - hook failures depend on user code
                self.logger.exception("hook failed: {}", key)

    def _dump_step_csv(self, step_index: int, step_name: str, data: Any, context: PipelineContext, step: BaseStep) -> None:
        if not getattr(step, "dump_csv", True):
            return
        to_csv = getattr(data, "to_csv", None)
        if to_csv is None:
            return
        dump_dir = _pipeline_data_dir(self.__class__.__name__)
        dump_dir.mkdir(parents=True, exist_ok=True)
        dump_path = dump_dir / f"{step_index:02d}_{step_name}.csv"
        try:
            to_csv(dump_path, index=False)
            self.logger.info("step csv dumped: {}", dump_path)
        except Exception as exc:  # pragma: no cover - depends on user data
            self.logger.exception("step csv dump failed: {}", step_name)
            context.errors.append(exc)


class Pipeline(BasePipeline):
    """Public base class for business workflows.

    Subclasses declare step order through the `stages` class attribute and
    implement each step as a normal instance method.
    """

    stages: list[str] = []
    pipelines: list[str] = []

    def __init_subclass__(cls, **kwargs: Any) -> None:
        super().__init_subclass__(**kwargs)
        protected = {"preprocess_data", "postprocess_data", "save_outputs", "save_data_to_db"}
        overridden = protected.intersection(cls.__dict__)
        if overridden:
            raise TypeError(
                f"{cls.__name__} cannot override protected pipeline hooks: {', '.join(sorted(overridden))}"
            )

    @classmethod
    def step(
        cls,
        name: Optional[str] = None,
        *,
        retries: int = 0,
        enabled: bool = True,
        condition: Optional[Callable[..., bool]] = None,
        before: Optional[Callable[..., Any]] = None,
        after: Optional[Callable[..., Any]] = None,
        validate_input: Optional[Callable[..., Any]] = None,
        validate_output: Optional[Callable[..., Any]] = None,
        rollback: Optional[Callable[..., Any]] = None,
        dump_csv: bool = True,
    ) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
        """Optional metadata decorator for advanced step behavior."""

        def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
            fn._pipeline_step_spec = {  # type: ignore[attr-defined]
                "name": name or fn.__name__,
                "retries": retries,
                "enabled": enabled,
                "condition": condition,
                "before": before,
                "after": after,
                "validate_input": validate_input,
                "validate_output": validate_output,
                "rollback": rollback,
                "dump_csv": dump_csv,
            }
            return fn

        return decorator

    def __init__(
        self,
        name: str = "pipeline",
        *,
        config: Any = None,
        logger: Any = None,
    ) -> None:
        super().__init__(name=name, config=self._load_config_payload(config), logger=logger)
        self.data_df: Any = None
        self.model = SimpleNamespace()
        self.artifacts: dict[str, Any] = {}
        self._df_snapshots: dict[str, Any] = {}
        self._runtime_context: Optional[PipelineContext] = None
        self.steps = []
        self._register_decorated_members()

    def _load_config_payload(self, config: Any) -> dict[str, Any]:
        if config is None:
            pipeline_module = inspect.getmodule(self.__class__)
            module_file = getattr(pipeline_module, "__file__", None)
            if module_file:
                module_dir = Path(module_file).resolve().parent
                default_path = module_dir / "config.yaml"
                if default_path.exists():
                    config = default_path
                else:
                    return {}
            else:
                return {}
        if isinstance(config, (str, Path)):
            if yaml is None:  # pragma: no cover - dependency should be installed
                raise RuntimeError("pyyaml is required to load pipeline config files")
            config_path = Path(config).expanduser()
            if not config_path.is_absolute():
                pipeline_module = inspect.getmodule(self.__class__)
                module_file = getattr(pipeline_module, "__file__", None)
                if module_file:
                    module_dir = Path(module_file).resolve().parent
                    candidate = (module_dir / config_path).resolve()
                    if candidate.exists():
                        config_path = candidate
            config_path = config_path.resolve()
            with config_path.open("r", encoding="utf-8") as handle:
                payload = yaml.safe_load(handle) or {}
            if not isinstance(payload, dict):
                raise TypeError(f"pipeline config file must contain a mapping: {config_path}")
            payload.setdefault("_config_path", str(config_path))
            return payload
        if isinstance(config, ConfigNamespace):
            return config.as_dict()
        if isinstance(config, dict):
            return dict(config)
        if hasattr(config, "items"):
            return dict(config.items())
        return dict(vars(config))

    def before_run(self, ctx: PipelineContext) -> Any:
        return None

    def after_run(self, ctx: PipelineContext) -> Any:
        return None

    def load_data(self) -> Any:
        """Load the initial dataframe for `run()`.

        Subclasses override this hook to fetch data from CSV, YAML, DB, API,
        or any other source. The returned value is assigned to `self.data_df`.
        If a subclass does not override it, the pipeline starts from an empty
        dataframe.
        """

        try:
            import pandas as pd
        except Exception:  # pragma: no cover - pandas should be installed
            self.data_df = []
            return self.data_df

        self.data_df = pd.DataFrame()
        return self.data_df

    def preprocess_data(self) -> Any:
        """Fixed preprocessing hook that runs after `load_data()`."""

        return self.data_df

    def postprocess_data(self) -> Any:
        """Fixed postprocessing hook that runs after all stages."""

        return self.data_df

    def save_outputs(self) -> Any:
        """Persist the final `self.data_df` snapshot to CSV, SQLite, and Parquet.

        Subclasses do not override this hook. They can change the target via
        config keys:
        - `csv_path`
        - `db_path`
        - `db_table`
        - `db_if_exists`
        - `parquet_path`
        """

        df = self.data_df
        if df is None:
            return self.data_df

        data_dir = _pipeline_data_dir(self.__class__.__name__)
        data_dir.mkdir(parents=True, exist_ok=True)

        csv_path = self.config.get("csv_path")
        if csv_path is None:
            csv_path = data_dir / f"{self.__class__.__name__}.csv"
        else:
            csv_path = Path(csv_path).expanduser().resolve()
            csv_path.parent.mkdir(parents=True, exist_ok=True)

        try:
            df.to_csv(csv_path, index=False, sep="\t")
            self.config["csv_output_path"] = str(csv_path)
            self.logger.info("step csv saved: {}", csv_path)
        except Exception as exc:  # pragma: no cover - depends on user data
            self.logger.exception("step csv save failed: {}", csv_path)
            context = getattr(self, "_runtime_context", None)
            if context is not None:
                context.errors.append(exc)

        columns = _df_columns(df)
        if not columns:
            self.config["db_output_path"] = None
            self.config["parquet_output_path"] = None
            return self.data_df

        db_path = self.config.get("db_path")
        if db_path is None:
            db_path = data_dir / f"{self.__class__.__name__}.sqlite3"
        else:
            db_path = Path(db_path).expanduser().resolve()
            db_path.parent.mkdir(parents=True, exist_ok=True)

        table_name = str(self.config.get("db_table") or "final_data")
        if_exists = str(self.config.get("db_if_exists") or "replace")
        with sqlite3.connect(db_path) as conn:
            df.to_sql(table_name, conn, if_exists=if_exists, index=False)
        self.config["db_output_path"] = str(db_path)
        self.config["db_output_table"] = table_name
        self.logger.info("step db saved: {} -> {}", table_name, db_path)

        parquet_path = self.config.get("parquet_path")
        if parquet_path is None:
            parquet_path = db_path.with_suffix(".parquet")
        else:
            parquet_path = Path(parquet_path).expanduser().resolve()
            parquet_path.parent.mkdir(parents=True, exist_ok=True)

        try:
            df.to_parquet(parquet_path, index=False)
            self.config["parquet_output_path"] = str(parquet_path)
            self.logger.info("step parquet saved: {}", parquet_path)
        except Exception as exc:  # pragma: no cover - depends on optional parquet engine
            self.logger.exception("step parquet save failed: {}", parquet_path)
            context = getattr(self, "_runtime_context", None)
            if context is not None:
                context.errors.append(exc)
        return self.data_df

    def save_data_to_db(self) -> Any:
        """Backward-compatible alias for `save_outputs()`."""

        return self.save_outputs()

    def _summary_log_path(self) -> Path:
        summary_dir = _pipeline_data_dir(self.__class__.__name__)
        summary_dir.mkdir(parents=True, exist_ok=True)
        return summary_dir / "pipeline_summary.md"

    def _append_summary_log(self, markdown: str) -> None:
        summary_path = self._summary_log_path()
        if summary_path.exists() and summary_path.stat().st_size > 0:
            markdown = "\n\n---\n\n" + markdown.lstrip()
        with summary_path.open("a", encoding="utf-8") as handle:
            handle.write(markdown)

    def _build_summary_markdown(
        self,
        *,
        context: PipelineContext,
        input_data: Any,
        started_at: float,
        started_at_dt: datetime,
        status: str,
        step_records: list[dict[str, Any]],
        final_data: Any,
    ) -> str:
        duration_seconds = time.perf_counter() - started_at
        input_rows = _df_rows(input_data)
        input_columns = _df_columns(input_data)
        final_rows = _df_rows(final_data)
        final_columns = _df_columns(final_data)
        model_items = _model_items(self.model)

        lines = [
            "# Pipeline Run Summary",
            "",
            f"- **Pipeline:** `{_md_escape(self.__class__.__name__)}`",
            f"- **Name:** `{_md_escape(self.name)}`",
            f"- **Run ID:** `{_md_escape(context.run_id)}`",
            f"- **Status:** `{_md_escape(status)}`",
            f"- **Duration:** `{duration_seconds:.4f}s`",
            f"- **Input Rows:** `{_md_escape(input_rows)}`",
            f"- **Input Columns:** `{_md_escape(', '.join(input_columns) or '-')}`",
            f"- **Final Rows:** `{_md_escape(final_rows)}`",
            f"- **Final Columns:** `{_md_escape(', '.join(final_columns) or '-')}`",
            f"- **CSV Path:** `{_md_escape(self.config.get('csv_output_path', '-'))}`",
            f"- **DB Path:** `{_md_escape(self.config.get('db_output_path', '-'))}`",
            f"- **Parquet Path:** `{_md_escape(self.config.get('parquet_output_path', '-'))}`",
            "",
            "## Steps",
            "",
            "| # | Step | Status | Rows | Added Columns | Removed Columns |",
            "| --- | --- | --- | --- | --- | --- |",
        ]

        for record in step_records:
            lines.append(
                "| {index:02d} | {step} | {status} | {rows} | {added} | {removed} |".format(
                    index=record["index"],
                    step=_md_escape(record["step"]),
                    status=_md_escape(record["status"]),
                    rows=_md_escape(record["rows"]),
                    added=_md_escape(record["added"]),
                    removed=_md_escape(record["removed"]),
                )
            )

        lines.extend(
            [
                "",
                "## Notes",
                "",
                f"- **Started At:** `{_md_escape(started_at_dt.isoformat(timespec='seconds'))}`",
            ]
        )
        if model_items:
            lines.extend(["", "## Model", "", "| Key | Value |", "| --- | --- |"])
            for key, value in model_items:
                lines.append(f"| {_md_escape(key)} | {_md_escape(value)} |")
        return "\n".join(lines) + "\n"

    def load_csv(self, path: Any, **kwargs: Any) -> Any:
        """Load a CSV file into `self.data_df`."""

        import pandas as pd

        self.data_df = pd.read_csv(path, **kwargs)
        return self.data_df

    def load_yaml(self, path: Any, *, key: Optional[str] = None) -> Any:
        """Load a YAML file into `self.data_df`."""

        if yaml is None:  # pragma: no cover - dependency should be installed
            raise RuntimeError("pyyaml is required to load YAML files")
        with Path(path).expanduser().resolve().open("r", encoding="utf-8") as handle:
            payload = yaml.safe_load(handle)
        if key is not None and isinstance(payload, dict):
            payload = payload.get(key)
        self.data_df = payload
        return self.data_df

    def logging(self, message: Any, *args: Any, level: str = "info", **kwargs: Any) -> None:
        """Write a log message through the pipeline logger.

        The default sink writes to `logs/<ClassName>.log`, so subclasses can
        simply call `self.logging("message")` without setting up logging.
        """

        log_fn = getattr(self.logger, level.lower(), self.logger.info)
        log_fn(message, *args, **kwargs)

    def on_rollback(self, action: Callable[[], Any], *, label: Optional[str] = None) -> str:
        """Register a rollback action for external side effects."""

        context = self._runtime_context
        if context is None:
            raise RuntimeError("rollback actions can only be registered while the pipeline is running")
        return context.add_rollback_action(action, label=label)

    @property
    def df(self) -> Any:
        return self.data_df

    @df.setter
    def df(self, value: Any) -> None:
        self.data_df = value

    def set_df(self, value: Any) -> Any:
        self.data_df = value
        return value

    def get_df(self) -> Any:
        return self.data_df

    def put(self, key: str, value: Any) -> Any:
        self.artifacts[key] = value
        return value

    def get(self, key: str, default: Any = None) -> Any:
        return self.artifacts.get(key, default)

    def delete(self, key: str) -> None:
        self.artifacts.pop(key, None)

    def require_df(self) -> Any:
        if self.data_df is None:
            raise ValueError("data_df is empty")
        return self.data_df

    def has_columns(self, *columns: str) -> bool:
        df = self.data_df
        if df is None or not hasattr(df, "columns"):
            return False
        current = set(str(column) for column in getattr(df, "columns", []))
        return all(column in current for column in columns)

    def require_columns(self, *columns: str) -> Any:
        if not self.has_columns(*columns):
            raise ValueError(f"missing required columns: {', '.join(columns)}")
        return self.data_df

    def snapshot_df(self, name: str = "default") -> Any:
        data = self.data_df
        if hasattr(data, "copy"):
            try:
                snapshot = data.copy(deep=True)
            except TypeError:
                snapshot = data.copy()
        else:
            snapshot = copy.deepcopy(data)
        self._df_snapshots[name] = snapshot
        return snapshot

    def restore_df(self, name: str = "default") -> Any:
        if name not in self._df_snapshots:
            raise KeyError(f"snapshot {name!r} not found")
        snapshot = self._df_snapshots[name]
        if hasattr(snapshot, "copy"):
            try:
                self.data_df = snapshot.copy(deep=True)
            except TypeError:
                self.data_df = snapshot.copy()
        else:
            self.data_df = copy.deepcopy(snapshot)
        return self.data_df

    def head(self, n: int = 5) -> Any:
        df = self.require_df()
        return df.head(n) if hasattr(df, "head") else df

    def tail(self, n: int = 5) -> Any:
        df = self.require_df()
        return df.tail(n) if hasattr(df, "tail") else df

    def _register_decorated_members(self) -> None:
        step_specs: dict[str, dict[str, Any]] = {}
        for cls in reversed(self.__class__.__mro__):
            for attr_name, attr_value in cls.__dict__.items():
                spec = getattr(attr_value, "_pipeline_step_spec", None)
                if spec is not None:
                    step_specs[attr_name] = spec

        ordered_step_names = list(getattr(self, "stages", []) or [])
        if not ordered_step_names:
            ordered_step_names = list(getattr(self, "pipelines", []) or [])
        if not ordered_step_names:
            ordered_step_names = list(step_specs.keys())

        seen_steps: set[str] = set()
        for step_name in ordered_step_names:
            if step_name in seen_steps:
                continue
            seen_steps.add(step_name)
            if not hasattr(self, step_name):
                raise AttributeError(f"pipeline step {step_name!r} is not defined on {self.__class__.__name__}")
            spec = step_specs.get(step_name, {})
            self.steps.append(
                FunctionStep(
                    name=spec.get("name", step_name),
                    fn=getattr(self, step_name),
                    retries=spec.get("retries", 0),
                    enabled=spec.get("enabled", True),
                    pass_data=False,
                    condition_hook=spec.get("condition"),
                    before_hook=spec.get("before"),
                    after_hook=spec.get("after"),
                    input_validator=spec.get("validate_input"),
                    output_validator=spec.get("validate_output"),
                    rollback_hook=spec.get("rollback"),
                    dump_csv=spec.get("dump_csv", True),
                )
            )

    def run(self, *, config: Optional[dict[str, Any]] = None) -> StepOutcome:
        input_data = self.load_data()
        if input_data is not None:
            self.data_df = input_data
        self.preprocess_data()
        current = self.data_df
        context = PipelineContext(
            pipeline=self,
            config={**self.config.as_dict(), **_config_to_dict(config)},
            input_data=input_data,
        )
        context.metadata["model"] = self.model
        self._runtime_context = context
        run_started_at = time.perf_counter()
        run_started_dt = datetime.now()
        step_records: list[dict[str, Any]] = []
        run_metadata: dict[str, Any] = {
            "run_id": context.run_id,
            "pipeline": self.__class__.__name__,
            "name": self.name,
            "input_rows": _df_rows(input_data),
            "input_columns": _df_columns(input_data),
        }
        try:
            _call_context_only(self.before_run, context)
            for step_index, step in enumerate(self.steps, start=1):
                step_before_rows = _df_rows(current)
                step_before_columns = _df_columns(current)
                snapshot_name = f"stage_{step_index:02d}_{step.name}"
                self.snapshot_df(snapshot_name)
                self.data_df = current
                outcome = step.execute(context, current)
                context.step_outcomes[step.name] = outcome
                if outcome.skipped:
                    self.logger.info("step skipped: {}", step.name)
                    step_records.append(
                        {
                            "index": step_index,
                            "step": step.name,
                            "status": "skipped",
                            "rows": f"{step_before_rows}->{step_before_rows}",
                            "added": "-",
                            "removed": "-",
                        }
                    )
                    continue
                if outcome.failed:
                    if outcome.error is not None:
                        self.logger.opt(exception=outcome.error).error("step failed: {}", step.name)
                    else:
                        self.logger.error("step failed: {}", step.name)
                    current = self._restore_stage_snapshot(step_index, step.name)
                    self.data_df = current
                    self._rollback(context, current, step)
                    self._run_rollback_actions(context)
                    step_records.append(
                        {
                            "index": step_index,
                            "step": step.name,
                            "status": "failed",
                            "rows": f"{step_before_rows}->{_df_rows(current)}",
                            "added": "-",
                            "removed": "-",
                        }
                    )
                    run_metadata.update(
                        {
                            "status": "failed",
                            "duration_seconds": time.perf_counter() - run_started_at,
                            "step_records": step_records,
                            "final_rows": _df_rows(self.data_df),
                            "final_columns": _df_columns(self.data_df),
                            "csv_output_path": self.config.get("csv_output_path"),
                            "db_output_path": self.config.get("db_output_path"),
                            "parquet_output_path": self.config.get("parquet_output_path"),
                            "summary_path": str(self._summary_log_path()),
                            "model": dict(vars(self.model)),
                        }
                    )
                    outcome.metadata.update(run_metadata)
                    markdown = self._build_summary_markdown(
                        context=context,
                        input_data=input_data,
                        started_at=run_started_at,
                        started_at_dt=run_started_dt,
                        status="failed",
                        step_records=step_records,
                        final_data=self.data_df,
                    )
                    self._append_summary_log(markdown)
                    _call_context_only(self.after_run, context)
                    return outcome
                if outcome.data is not None:
                    current = outcome.data
                elif self.data_df is not None:
                    current = self.data_df
                self.data_df = current
                context.output_data = current
                self._dump_step_csv(step_index, step.name, current, context, step)
                step_after_rows = _df_rows(current)
                step_after_columns = _df_columns(current)
                added_columns = [column for column in step_after_columns if column not in step_before_columns]
                removed_columns = [column for column in step_before_columns if column not in step_after_columns]
                step_records.append(
                    {
                        "index": step_index,
                        "step": step.name,
                        "status": "success",
                        "rows": f"{step_before_rows}->{step_after_rows}",
                        "added": ", ".join(added_columns) or "-",
                        "removed": ", ".join(removed_columns) or "-",
                    }
                )
                self.logger.info("step success: {}", step.name)
            self.data_df = current
            self.postprocess_data()
            current = self.data_df
            self.save_outputs()
            run_metadata.update(
                {
                    "status": "success",
                    "duration_seconds": time.perf_counter() - run_started_at,
                    "step_records": step_records,
                    "final_rows": _df_rows(self.data_df),
                    "final_columns": _df_columns(self.data_df),
                    "csv_output_path": self.config.get("csv_output_path"),
                    "db_output_path": self.config.get("db_output_path"),
                    "parquet_output_path": self.config.get("parquet_output_path"),
                    "summary_path": str(self._summary_log_path()),
                    "model": dict(vars(self.model)),
                }
            )
            markdown = self._build_summary_markdown(
                context=context,
                input_data=input_data,
                started_at=run_started_at,
                started_at_dt=run_started_dt,
                status="success",
                step_records=step_records,
                final_data=self.data_df,
            )
            self._append_summary_log(markdown)
            _call_context_only(self.after_run, context)
            return StepOutcome(status=PipelineStatus.SUCCESS, data=self.data_df, metadata=run_metadata)
        finally:
            self._runtime_context = None
