"""
Pipeline class with | operator for chaining.

Supports both config-driven and programmatic pipeline construction.
Includes optional validation at extract (source) and load (target) stages.
"""

from __future__ import annotations

import logging
import uuid
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

import yaml

from pycharter.etl_generator.context import PipelineContext
from pycharter.etl_generator.protocols import Extractor, Loader, Transformer
from pycharter.etl_generator.result import BatchResult, LoadResult, PipelineResult
from pycharter.shared.errors import ErrorContext, get_error_context

logger = logging.getLogger(__name__)


def _unwrap_file_section(value: Any, section: str) -> Any:
    """If value is a dict with exactly one key equal to section, return the inner value.

    This lets callers pass a loaded YAML file as-is (e.g. config["transform"] = load_yaml(...))
    when the file has a single top-level key like "transform" or "jsonata", without double-nesting.
    """
    if value is None:
        return None
    if isinstance(value, dict) and list(value.keys()) == [section]:
        return value[section]
    return value


class Pipeline:
    """
    ETL Pipeline with | operator for chaining transformers.

    Programmatic usage:
        >>> pipeline = (
        ...     Pipeline(HTTPExtractor(url="..."))
        ...     | Rename({"old": "new"})
        ...     | PostgresLoader(...)
        ... )
        >>> result = await pipeline.run()

    Config-driven usage (variables and settings are first-class in all methods):
        >>> # From explicit files (extract, load, transform, variables, settings)
        >>> pipeline = Pipeline.from_config_files(
        ...     "configs/extract.yaml", "configs/load.yaml",
        ...     variables={"API_KEY": "secret"}
        ... )
        >>>
        >>> # From directory (extract.yaml, load.yaml, optional transform/settings/variables.yaml)
        >>> pipeline = Pipeline.from_config_dir("pipelines/users/")
        >>>
        >>> # From single file (all sections in one YAML)
        >>> pipeline = Pipeline.from_config_file("pipelines/users/pipeline.yaml")
        >>>
        >>> result = await pipeline.run()

    Async execution:
        run() is async. From a script use asyncio.run():
            asyncio.run(pipeline.run())
        From an async context (FastAPI, Jupyter) await directly:
            result = await pipeline.run()
        See pycharter/etl_generator/ASYNC_AND_EXECUTION.md for details.
    """

    def __init__(
        self,
        extractor: Extractor | None = None,
        transformers: list[Transformer] | None = None,
        loader: Loader | None = None,
        context: PipelineContext | None = None,
        name: str | None = None,
        transform_config: dict[str, Any] | list[dict[str, Any]] | None = None,
        # Validation configs
        extract_validation_config: dict[str, Any] | None = None,
        load_validation_config: dict[str, Any] | None = None,
        # Quality checks config (post-load)
        quality_checks_config: list[dict[str, Any]] | None = None,
        # Shared settings
        settings: dict[str, Any] | None = None,
        # Base directory for resolving relative paths
        base_dir: Path | None = None,
        # Raw configs for lineage dataset descriptors (set by _build_from_configs)
        extract_config_raw: dict[str, Any] | None = None,
        load_config_raw: dict[str, Any] | None = None,
    ):
        self.extractor = extractor
        self._transformers: list[Transformer] = (
            list(transformers) if transformers else []
        )
        self.loader = loader
        self.context = context or PipelineContext()
        self.name = name
        # For config-driven runs, store raw transform config and use apply_transforms
        self._transform_config = transform_config
        # Validation configs
        self._extract_validation_config = extract_validation_config
        self._load_validation_config = load_validation_config
        # Quality checks config (post-load)
        self._quality_checks_config = quality_checks_config
        # Shared settings from settings.yaml
        self._settings = settings or {}
        # Base directory for resolving file paths
        self._base_dir = base_dir
        # Run ID (set during run())
        self._run_id: str | None = None
        # Raw extract/load configs for OpenLineage dataset descriptors
        self._extract_config_raw = extract_config_raw
        self._load_config_raw = load_config_raw

    def __or__(self, other: Transformer | Loader) -> "Pipeline":
        """Chain transformer or set loader using | operator."""
        if isinstance(other, Loader):
            return Pipeline(
                extractor=self.extractor,
                transformers=self._transformers.copy(),
                loader=other,
                context=self.context,
                name=self.name,
                extract_validation_config=self._extract_validation_config,
                load_validation_config=self._load_validation_config,
                quality_checks_config=self._quality_checks_config,
                settings=self._settings,
                base_dir=self._base_dir,
            )
        else:
            new_transformers = self._transformers.copy()
            new_transformers.append(other)
            return Pipeline(
                extractor=self.extractor,
                transformers=new_transformers,
                loader=self.loader,
                context=self.context,
                name=self.name,
                extract_validation_config=self._extract_validation_config,
                load_validation_config=self._load_validation_config,
                quality_checks_config=self._quality_checks_config,
                settings=self._settings,
                base_dir=self._base_dir,
            )

    async def run(
        self,
        dry_run: bool = False,
        error_context: ErrorContext | None = None,
        **params: Any,
    ) -> PipelineResult:
        """
        Run the ETL pipeline.

        Args:
            dry_run: If True, extract and transform but do not load.
            error_context: Optional error context for handling failures.
                If not set, uses the default from get_error_context().
                In STRICT mode, extraction or load failures raise.
                In LENIENT/COLLECT mode, errors are logged and appended to result.errors.
            **params: Passed to extractor.extract() and loader.load().

        Returns:
            PipelineResult with counts and any errors.
        """
        run_id = str(uuid.uuid4())[:8]
        self._run_id = run_id
        start_time = datetime.now(timezone.utc)
        ctx = error_context or get_error_context()

        result = PipelineResult(
            pipeline_name=self.name,
            run_id=run_id,
            start_time=start_time,
        )

        if not self.extractor:
            result.success = False
            result.errors.append("No extractor configured")
            return result

        logger.info("[%s] Starting pipeline: %s", run_id, self.name or "unnamed")

        # Incremental extraction: inject watermark into context
        state_store = None
        existing_state = None
        inc_config = None
        max_watermark: str | None = None

        if self._extract_config_raw and isinstance(self._extract_config_raw, dict):
            inc_raw = self._extract_config_raw.get("incremental")
            if inc_raw and inc_raw.get("enabled", True):
                inc_config = inc_raw
                state_store = self._create_state_store()
                existing_state = state_store.get(self.name or "unnamed")
                current_wm = (
                    existing_state.watermark if existing_state else None
                ) or inc_config.get("initial_value", "")
                max_watermark = current_wm
                self.context.set("__watermark__", current_wm)
                logger.info("[%s] Incremental mode: watermark=%s", run_id, current_wm)

        # Lineage: build emitter and dataset descriptors (no-op when disabled)
        emitter, input_ds, output_ds = self._prepare_lineage()
        if emitter:
            run_id = str(uuid.uuid4())
            result.run_id = run_id

        # Create validators if configured
        extract_validator = self._create_extract_validator()
        load_validator = self._create_load_validator()

        # Accumulate loaded records for quality checks
        all_loaded_records: list[dict[str, Any]] = []
        quality_checker = self._create_quality_checker()

        # Lineage: emit START and capture payload for response
        if emitter:
            try:
                payload = emitter.start(self.name or "unnamed", run_id, input_ds)
                if payload:
                    result.lineage_events.append(payload)
            except Exception:
                logger.debug("Lineage START emission failed", exc_info=True)

        try:
            batch_index = 0
            async for batch in self.extractor.extract(**params):
                batch_result = BatchResult(batch_index=batch_index, rows_in=len(batch))
                result.rows_extracted += len(batch)

                # Source validation (after extract)
                if extract_validator:
                    try:
                        batch, quarantined, error_count = (
                            await extract_validator.validate(batch, run_id=run_id)
                        )
                        result.rows_quarantined_extract += len(quarantined)
                        if error_count > 0:
                            batch_result.errors.append(
                                f"Extract validation: {error_count} record(s) invalid"
                            )
                    except Exception as e:
                        # Validation error with on_error=fail
                        ctx.handle_error(str(e), e, category="extract_validation")
                        batch_result.errors.append(f"Extract validation failed: {e}")
                        result.success = False
                        result.errors.append(str(e))
                        break

                # Transform
                transformed = self._apply_transforms(batch)
                batch_result.rows_out = len(transformed)
                result.rows_transformed += len(transformed)

                # Track max watermark across batches
                if inc_config and transformed:
                    wm_field = inc_config["watermark_field"]
                    batch_max = max(
                        (str(r.get(wm_field, "")) for r in transformed),
                        default="",
                    )
                    if batch_max > (max_watermark or ""):
                        max_watermark = batch_max

                # Target validation (before load)
                if load_validator and transformed:
                    try:
                        transformed, quarantined, error_count = (
                            await load_validator.validate(transformed, run_id=run_id)
                        )
                        result.rows_quarantined_load += len(quarantined)
                        if error_count > 0:
                            batch_result.errors.append(
                                f"Load validation: {error_count} record(s) invalid"
                            )
                    except Exception as e:
                        # Validation error with on_error=fail
                        ctx.handle_error(str(e), e, category="load_validation")
                        batch_result.errors.append(f"Load validation failed: {e}")
                        result.success = False
                        result.errors.append(str(e))
                        break

                # Load
                load_success = True
                if not dry_run and self.loader and transformed:
                    try:
                        load_result = await self.loader.load(transformed, **params)
                        load_success = load_result.success
                        if load_result.success:
                            result.rows_loaded += load_result.rows_loaded
                            if quality_checker:
                                all_loaded_records.extend(transformed)
                        else:
                            msg = load_result.error or "Load failed"
                            ctx.handle_error(msg, category="load")
                            batch_result.errors.append(msg)
                            batch_result.rows_failed += len(transformed)
                    except Exception as e:
                        load_success = False
                        ctx.handle_error(str(e), e, category="load")
                        batch_result.errors.append(str(e))
                        batch_result.rows_failed += len(transformed)
                elif dry_run:
                    result.rows_loaded += len(transformed)

                # Acknowledge batch for message queue extractors
                if hasattr(self.extractor, "acknowledge"):
                    try:
                        await self.extractor.acknowledge(batch_index, load_success)
                    except Exception as ack_err:
                        logger.warning(
                            "[%s] Batch %d ack failed: %s",
                            run_id,
                            batch_index,
                            ack_err,
                        )

                result.batches_processed += 1
                result.batch_results.append(batch_result)
                batch_index += 1

        except Exception as e:
            result.success = False
            result.errors.append(str(e))
            ctx.handle_error(str(e), e, category="pipeline")
            logger.error("[%s] Pipeline error: %s", run_id, e)

        result.end_time = datetime.now(timezone.utc)
        result.duration_seconds = (result.end_time - start_time).total_seconds()
        result.rows_failed = sum(br.rows_failed for br in result.batch_results)

        # Post-load quality checks
        if quality_checker and result.success:
            last_load_result = LoadResult(
                success=True,
                rows_loaded=result.rows_loaded,
                rows_failed=result.rows_failed,
            )
            quality_report = quality_checker.run(all_loaded_records, last_load_result)
            result.quality_report = quality_report
            if not quality_report.passed:
                result.success = False
                for failure in quality_report.hard_failures:
                    result.errors.append(f"Quality check failed: {failure.message}")
            for warning in quality_report.warnings:
                logger.warning("[%s] Quality warning: %s", run_id, warning.message)

        if result.errors:
            result.success = False

        # Incremental: save updated watermark on success
        if state_store and result.success and max_watermark:
            from pycharter.etl_generator.state import PipelineState

            state_store.save(
                PipelineState(
                    pipeline_name=self.name or "unnamed",
                    watermark=max_watermark,
                    last_run_id=run_id,
                    last_run_at=datetime.now(timezone.utc).isoformat(),
                    run_count=((existing_state.run_count + 1) if existing_state else 1),
                )
            )

        # Lineage: emit COMPLETE or FAIL and capture payload for response
        if emitter:
            try:
                if result.success:
                    payload = emitter.complete(
                        self.name or "unnamed",
                        run_id,
                        input_ds,
                        output_ds,
                        result,
                    )
                else:
                    payload = emitter.fail(
                        self.name or "unnamed", run_id, input_ds, result
                    )
                if payload:
                    result.lineage_events.append(payload)
            except Exception:
                logger.debug("Lineage COMPLETE/FAIL emission failed", exc_info=True)

        logger.info(
            "[%s] Complete: extracted=%s, loaded=%s, "
            "quarantined_extract=%s, quarantined_load=%s",
            run_id,
            result.rows_extracted,
            result.rows_loaded,
            result.rows_quarantined_extract,
            result.rows_quarantined_load,
        )
        return result

    def _prepare_lineage(self):
        """Build lineage emitter and dataset descriptors from settings and raw configs.

        Returns (emitter, input_datasets, output_datasets).
        All three are None/empty when lineage is disabled or unavailable.
        """
        from pycharter.etl_generator.lineage import (
            create_lineage_emitter,
            dataset_from_extract_config,
            dataset_from_load_config,
        )

        emitter = create_lineage_emitter(self._settings)
        if emitter is None:
            return None, [], []

        input_ds = (
            [dataset_from_extract_config(self._extract_config_raw)]
            if self._extract_config_raw
            else []
        )
        output_ds = (
            [dataset_from_load_config(self._load_config_raw)]
            if self._load_config_raw
            else []
        )
        return emitter, input_ds, output_ds

    def _create_extract_validator(self):
        """Create extract validator if configured."""
        if not self._extract_validation_config:
            return None

        from pycharter.etl_generator.config_models import DLQConfig
        from pycharter.etl_generator.validation import create_etl_validator

        # Get default DLQ config from settings
        default_dlq = None
        if self._settings.get("dlq"):
            default_dlq = DLQConfig(**self._settings["dlq"])

        return create_etl_validator(
            validation_config=self._extract_validation_config,
            pipeline_name=self.name or "unnamed",
            stage="extract",
            metadata_store=None,  # Extract uses local schema, not store
            default_contract=None,
            default_dlq_config=default_dlq,
            base_dir=self._base_dir,
        )

    def _create_load_validator(self):
        """Create load validator if configured."""
        if not self._load_validation_config:
            return None

        from pycharter.etl_generator.config_models import DLQConfig
        from pycharter.etl_generator.validation import create_etl_validator
        from pycharter.metadata_store import MetadataStoreClient

        # Get default DLQ config from settings
        default_dlq = None
        if self._settings.get("dlq"):
            default_dlq = DLQConfig(**self._settings["dlq"])

        # Get default contract from settings
        default_contract = self._settings.get("contract")

        # Create metadata store if configured
        metadata_store = None
        if self._settings.get("metadata_store"):
            metadata_store = self._create_metadata_store()

        return create_etl_validator(
            validation_config=self._load_validation_config,
            pipeline_name=self.name or "unnamed",
            stage="load",
            metadata_store=metadata_store,
            default_contract=default_contract,
            default_dlq_config=default_dlq,
            base_dir=self._base_dir,
        )

    def _create_state_store(self):
        """Create state store from settings config."""
        from pycharter.etl_generator.state import create_state_store

        store_config = self._settings.get("state_store", {})
        return create_state_store(store_config)

    def _create_metadata_store(self) -> "MetadataStoreClient" | None:
        """Create metadata store from settings."""
        store_config = self._settings.get("metadata_store")
        if not store_config:
            return None

        store_type = store_config.get("type", "").lower()
        connection_string = store_config.get("connection_string")

        if not connection_string:
            logger.warning("metadata_store missing connection_string")
            return None

        if store_type == "postgres":
            from pycharter.metadata_store import PostgresMetadataStore

            return PostgresMetadataStore(connection_string)
        elif store_type == "sqlite":
            from pycharter.metadata_store import SQLiteMetadataStore

            return SQLiteMetadataStore(connection_string)
        elif store_type == "mongodb":
            from pycharter.metadata_store import MongoDBMetadataStore

            return MongoDBMetadataStore(connection_string)
        elif store_type == "redis":
            from pycharter.metadata_store import RedisMetadataStore

            return RedisMetadataStore(connection_string)
        elif store_type == "memory":
            from pycharter.metadata_store import InMemoryMetadataStore

            return InMemoryMetadataStore()
        else:
            logger.warning("Unknown metadata_store type: %s", store_type)
            return None

    def _create_quality_checker(self):
        """Create quality checker if quality checks are configured."""
        if not self._quality_checks_config:
            return None
        from pycharter.etl_generator.quality import create_quality_checker

        return create_quality_checker(
            self._quality_checks_config, pipeline_name=self.name or "unnamed"
        )

    def _apply_transforms(self, data: list[dict[str, Any]]) -> list[dict[str, Any]]:
        """Apply all transformers to data.

        For config-driven pipelines (when _transform_config is set), uses the unified
        apply_transforms() which supports simple_ops, jsonata, and custom_function.
        For programmatic pipelines (when _transformers is set), loops through the
        transformer chain.
        """
        # Config-driven path: use unified apply_transforms (supports jsonata, etc.)
        if self._transform_config:
            from pycharter.etl_generator.transformers import apply_transforms

            return apply_transforms(data, self._transform_config)

        # Programmatic path: loop through transformer instances
        result = data
        for transformer in self._transformers:
            result = transformer.transform(result)
        return result

    # =========================================================================
    # CONFIG-DRIVEN FACTORY METHODS
    # =========================================================================

    @classmethod
    def from_config_files(
        cls,
        extract: str | Path | dict[str, Any],
        load: str | Path | dict[str, Any],
        transform: str | Path | dict[str, Any] | list[dict[str, Any]] | None = None,
        variables: str | Path | dict[str, str] | None = None,
        settings: str | Path | dict[str, Any] | None = None,
        *,
        validate: bool = True,
        name: str | None = None,
        load_defaults: dict[str, Any] | None = None,
        base_dir: Path | None = None,
    ) -> "Pipeline":
        """
        Create pipeline from explicit file paths or dictionaries.

        Config inputs (same level): extract, load, transform, variables, settings.
        Each can be a path or dict; variables are used to resolve ${VAR} in others.

        Args:
            extract: Path or dict for extract config.
            load: Path or dict for load config.
            transform: Optional path or dict/list for transform config.
            variables: Optional path or dict for ${VAR} substitution.
            settings: Optional path or dict for shared settings (DLQ, metadata_store, etc.).
            validate: If True, validate configs against schemas.
            name: Optional pipeline name.
            load_defaults: Merged under load config (load wins).
            base_dir: Base for relative paths; default from extract path when possible.
        """
        variables = _resolve_variables(variables)
        if base_dir is None and isinstance(extract, (str, Path)):
            base_dir = Path(extract).parent.resolve()

        extract_config = _load_config_input(extract, variables)
        load_config = _load_config_input(load, variables)
        if load_defaults:
            load_config = {**load_defaults, **load_config}
        transform_config = (
            _load_config_input(transform, variables) if transform is not None else {}
        )
        settings_config = _resolve_settings(settings, variables)

        # Extract validation configs from extract/load
        extract_validation_config = (
            extract_config.pop("validation", None)
            if isinstance(extract_config, dict)
            else None
        )
        load_validation_config = (
            load_config.pop("validation", None)
            if isinstance(load_config, dict)
            else None
        )
        quality_checks_config = (
            load_config.pop("quality_checks", None)
            if isinstance(load_config, dict)
            else None
        )

        return cls._build_from_configs(
            extract_config=extract_config,
            transform_config=transform_config,
            load_config=load_config,
            variables=variables,
            validate=validate,
            name=name,
            extract_validation_config=extract_validation_config,
            load_validation_config=load_validation_config,
            quality_checks_config=quality_checks_config,
            settings=settings_config,
            base_dir=base_dir,
        )

    @classmethod
    def from_config_dir(
        cls,
        directory: str | Path,
        variables: str | Path | dict[str, str] | None = None,
        settings: str | Path | dict[str, Any] | None = None,
        *,
        validate: bool = True,
        name: str | None = None,
        load_defaults: dict[str, Any] | None = None,
        base_dir: Path | None = None,
    ) -> "Pipeline":
        """
        Create pipeline from a directory (extract.yaml, load.yaml, etc.).

        Config at same level: directory supplies extract, load, transform, variables, settings.
        Optional variables/settings args override or supplement directory files.

        Args:
            directory: Path to directory with extract.yaml, load.yaml, etc.
            variables: Optional path or dict; merged with directory variables.yaml (caller wins).
            settings: Optional path or dict; merged with directory settings.yaml (caller wins).
            validate: If True, validate configs.
            name: Optional pipeline name (default: directory name).
            load_defaults: Merged under load config (load wins).
            base_dir: Base for relative paths (default: directory).
        """
        directory = Path(directory)
        if not directory.is_dir():
            raise NotADirectoryError(f"Not a directory: {directory}")

        variables_file = directory / "variables.yaml"
        initial_vars = _load_variables_file(variables_file, {})
        variables = _resolve_variables(variables)
        variables = {**initial_vars, **variables}

        extract_file = directory / "extract.yaml"
        load_file = directory / "load.yaml"
        transform_file = directory / "transform.yaml"
        settings_file = directory / "settings.yaml"
        if not extract_file.exists():
            raise FileNotFoundError(f"Required file not found: {extract_file}")
        if not load_file.exists():
            raise FileNotFoundError(f"Required file not found: {load_file}")

        extract_config = _load_config_input(extract_file, variables)
        load_config = _load_config_input(load_file, variables)
        if load_defaults:
            load_config = {**load_defaults, **load_config}
        transform_config = (
            _load_config_input(transform_file, variables)
            if transform_file.exists()
            else {}
        )
        dir_settings = (
            _load_config_input(settings_file, variables)
            if settings_file.exists()
            else {}
        )
        if not isinstance(dir_settings, dict):
            dir_settings = {}
        settings_config = {**dir_settings, **_resolve_settings(settings, variables)}

        # Extract validation configs from extract/load
        extract_validation_config = (
            extract_config.pop("validation", None)
            if isinstance(extract_config, dict)
            else None
        )
        load_validation_config = (
            load_config.pop("validation", None)
            if isinstance(load_config, dict)
            else None
        )
        quality_checks_config = (
            load_config.pop("quality_checks", None)
            if isinstance(load_config, dict)
            else None
        )

        return cls._build_from_configs(
            extract_config=extract_config,
            transform_config=transform_config,
            load_config=load_config,
            variables=variables,
            validate=validate,
            name=name or directory.name,
            extract_validation_config=extract_validation_config,
            load_validation_config=load_validation_config,
            quality_checks_config=quality_checks_config,
            settings=settings_config,
            base_dir=(base_dir or directory).resolve(),
        )

    @classmethod
    def from_config_file(
        cls,
        path: str | Path,
        *,
        variables: dict[str, str] | None = None,
        validate: bool = True,
        name: str | None = None,
        load_defaults: dict[str, Any] | None = None,
        base_dir: Path | None = None,
    ) -> "Pipeline":
        """
        Create pipeline from a single config file (extract, load, transform, variables, settings as keys).

        Args:
            path: Path to pipeline YAML.
            variables: Optional overlay; merged with file variables (caller wins).
            validate: If True, validate config.
            name: Override name from file.
            load_defaults: Merged under load config (load wins).
            base_dir: Base for relative paths (default: path.parent).
        """
        path = Path(path)
        if not path.exists():
            raise FileNotFoundError(f"Config file not found: {path}")
        if not path.is_file():
            raise ValueError(
                f"Not a file: {path}. Use from_config_dir() for directories."
            )
        with open(path) as f:
            raw_content = f.read()
        raw_config = yaml.safe_load(raw_content) or {}
        file_vars = {}
        if isinstance(raw_config.get("variables"), dict):
            file_vars = {k: str(v) for k, v in raw_config["variables"].items()}
        variables = {**file_vars, **(variables or {})}
        context = PipelineContext(variables=variables)
        resolved_content = context.resolve(raw_content)
        config = yaml.safe_load(resolved_content) or {}
        if "extract" not in config:
            raise ValueError(f"Config file missing 'extract' section: {path}")
        if "load" not in config:
            raise ValueError(f"Config file missing 'load' section: {path}")

        extract_config = config["extract"]
        load_config = config["load"]
        if load_defaults:
            load_config = {**load_defaults, **load_config}
        extract_validation_config = (
            extract_config.pop("validation", None)
            if isinstance(extract_config, dict)
            else None
        )
        load_validation_config = (
            load_config.pop("validation", None)
            if isinstance(load_config, dict)
            else None
        )
        quality_checks_config = (
            load_config.pop("quality_checks", None)
            if isinstance(load_config, dict)
            else None
        )
        settings = {}
        for key in ("metadata_store", "dlq", "contract", "lineage"):
            if config.get(key):
                settings[key] = config[key]

        return cls._build_from_configs(
            extract_config=extract_config,
            transform_config=config.get("transform", {}),
            load_config=load_config,
            variables=variables,
            validate=validate,
            name=name if name is not None else config.get("name"),
            extract_validation_config=extract_validation_config,
            load_validation_config=load_validation_config,
            quality_checks_config=quality_checks_config,
            settings=settings,
            base_dir=(base_dir or path.parent).resolve(),
        )

    @classmethod
    def from_dict(
        cls,
        config: dict[str, Any],
        *,
        validate: bool = True,
        name: str | None = None,
        load_defaults: dict[str, Any] | None = None,
        base_dir: Path | None = None,
    ) -> "Pipeline":
        """
        Create pipeline from a config dict (extract, load, transform, variables, settings as keys).

        Args:
            config: Dict with extract, load, optional transform/variables/settings.
            validate: If True, validate config.
            name: Override config name.
            load_defaults: Merged under load config (load wins).
            base_dir: Base for relative paths.
        """
        if "extract" not in config:
            raise ValueError("Config dict missing 'extract' section")
        if "load" not in config:
            raise ValueError("Config dict missing 'load' section")

        variables = {}
        if isinstance(config.get("variables"), dict):
            variables = {k: str(v) for k, v in config["variables"].items()}

        context = PipelineContext(variables=variables)
        extract_config = context.resolve_dict(config["extract"])
        load_config = context.resolve_dict(config["load"])
        if load_defaults:
            load_config = {**load_defaults, **load_config}

        # Extract validation configs
        extract_validation_config = (
            extract_config.pop("validation", None)
            if isinstance(extract_config, dict)
            else None
        )
        load_validation_config = (
            load_config.pop("validation", None)
            if isinstance(load_config, dict)
            else None
        )
        quality_checks_config = (
            load_config.pop("quality_checks", None)
            if isinstance(load_config, dict)
            else None
        )

        # Build transform config: transform (simple ops), jsonata, and custom_function.
        # Normalize file-style input: a YAML file often has one top-level key (e.g. "transform"
        # or "jsonata"); unwrap so config["transform"] = load_yaml("transform.yaml") works.
        transform_config: dict[str, Any] = {}
        raw_transform_input = config.get("transform", {})

        if isinstance(raw_transform_input, dict) and "transform" in raw_transform_input:
            # Single file with "transform" (and optionally "jsonata"/"custom_function")
            raw_transform = raw_transform_input["transform"]
            jsonata_src = (
                config.get("jsonata")
                if config.get("jsonata") is not None
                else raw_transform_input.get("jsonata")
            )
            custom_src = (
                config.get("custom_function")
                if config.get("custom_function") is not None
                else raw_transform_input.get("custom_function")
            )
        else:
            raw_transform = _unwrap_file_section(raw_transform_input, "transform")
            jsonata_src = config.get("jsonata")
            custom_src = config.get("custom_function")

        if isinstance(raw_transform, list):
            transform_config["transform"] = [
                context.resolve_dict(item) if isinstance(item, dict) else item
                for item in raw_transform
            ]
        elif raw_transform:
            transform_config["transform"] = context.resolve_dict(raw_transform)

        jsonata_val = _unwrap_file_section(jsonata_src, "jsonata")
        if jsonata_val:
            transform_config["jsonata"] = context.resolve_dict(jsonata_val)

        custom_val = _unwrap_file_section(custom_src, "custom_function")
        if custom_val:
            transform_config["custom_function"] = context.resolve_dict(custom_val)

        # Build inline settings from top-level keys
        settings: dict[str, Any] = {}
        for key in ("metadata_store", "dlq", "lineage"):
            if config.get(key):
                val = config[key]
                settings[key] = (
                    context.resolve_dict(val) if isinstance(val, dict) else val
                )
        if config.get("contract"):
            settings["contract"] = config["contract"]

        return cls._build_from_configs(
            extract_config=extract_config,
            transform_config=transform_config,
            load_config=load_config,
            variables=variables,
            validate=validate,
            name=name if name is not None else config.get("name"),
            extract_validation_config=extract_validation_config,
            load_validation_config=load_validation_config,
            quality_checks_config=quality_checks_config,
            settings=settings,
            base_dir=base_dir,
        )

    @classmethod
    def _build_from_configs(
        cls,
        extract_config: dict[str, Any],
        transform_config: dict[str, Any] | list[dict[str, Any]],
        load_config: dict[str, Any],
        variables: dict[str, str] | None,
        validate: bool,
        name: str | None,
        extract_validation_config: dict[str, Any] | None = None,
        load_validation_config: dict[str, Any] | None = None,
        quality_checks_config: list[dict[str, Any]] | None = None,
        settings: dict[str, Any] | None = None,
        base_dir: Path | None = None,
    ) -> "Pipeline":
        """Internal method to build pipeline from resolved configs.

        Uses the unified apply_transforms path for transformations, which supports:
        - Simple ops: rename, convert, defaults, add, select, drop, filter
        - JSONata expressions
        - Custom functions

        Also supports validation at extract and load stages via extract_validation_config
        and load_validation_config parameters.
        """
        from pycharter.etl_generator.config_validator import ConfigValidator
        from pycharter.etl_generator.extractors.factory import ExtractorFactory
        from pycharter.etl_generator.loaders.factory import LoaderFactory

        # Validate if enabled
        if validate:
            validator = ConfigValidator(strict=True)
            validator.validate_extract(extract_config)
            if transform_config:
                # Wrap list in dict for validation
                if isinstance(transform_config, list):
                    validator.validate_transform({"transform": transform_config})
                else:
                    validator.validate_transform(transform_config)
            validator.validate_load(load_config)
            if settings:
                validator.validate_settings(settings)

        # Create context
        context = PipelineContext(variables=variables)

        # Create components via unified factories (picks up registered plugins)
        extractor = ExtractorFactory.create(extract_config) if extract_config else None
        loader_instance = LoaderFactory.create(load_config) if load_config else None

        # Load plugins from settings (config-driven plugin registration)
        if settings and settings.get("plugins"):
            from pycharter.etl_generator._plugins import load_plugins_from_settings

            load_plugins_from_settings(settings)

        # Pass raw transform config - will be used by apply_transforms in _apply_transforms
        return cls(
            extractor=extractor,
            transformers=None,  # No transformer objects for config-driven runs
            loader=loader_instance,
            context=context,
            name=name,
            transform_config=transform_config if transform_config else None,
            extract_validation_config=extract_validation_config,
            load_validation_config=load_validation_config,
            quality_checks_config=quality_checks_config,
            settings=settings or {},
            base_dir=base_dir,
            extract_config_raw=extract_config,
            load_config_raw=load_config,
        )


# =============================================================================
# HELPER FUNCTIONS
# =============================================================================


def _resolve_variables(
    variables: str | Path | dict[str, str] | None,
) -> dict[str, str]:
    """Return variables dict from path (to variables.yaml) or dict. Caller wins on merge."""
    if variables is None:
        return {}
    if isinstance(variables, (str, Path)):
        return _load_variables_file(Path(variables), {})
    return dict(variables)


def _resolve_settings(
    settings: str | Path | dict[str, Any] | None,
    variables: dict[str, str],
) -> dict[str, Any]:
    """Return settings dict from path or dict. Empty if None."""
    if settings is None:
        return {}
    loaded = _load_config_input(settings, variables)
    return loaded if isinstance(loaded, dict) else {}


def _load_config_input(
    config_input: str | Path | dict[str, Any] | list[dict[str, Any]],
    variables: dict[str, str],
) -> dict[str, Any] | list[dict[str, Any]]:
    """Load config from file path or return dict/list directly."""
    if isinstance(config_input, (dict, list)):
        return config_input

    path = Path(config_input)
    if not path.exists():
        raise FileNotFoundError(f"Config file not found: {path}")

    with open(path) as f:
        content = f.read()

    context = PipelineContext(variables=variables)
    content = context.resolve(content)
    return yaml.safe_load(content) or {}


def _load_variables_file(
    path: Path,
    caller_variables: dict[str, str],
) -> dict[str, str]:
    """Load a variables.yaml file and merge with caller-supplied variables.

    File variables are loaded first, then caller-supplied variables are overlaid
    so that the caller always takes precedence. All values are stringified.

    Args:
        path: Path to variables.yaml file.
        caller_variables: Variables supplied by the caller (take precedence).

    Returns:
        Merged variables dict with string values.

    Raises:
        ValueError: If the file does not contain a dict at the top level.
    """
    if not path.exists():
        return dict(caller_variables)

    with open(path) as f:
        raw = yaml.safe_load(f)

    if raw is None:
        return dict(caller_variables)

    if not isinstance(raw, dict):
        raise ValueError(
            f"variables.yaml must contain a mapping, got {type(raw).__name__}: {path}"
        )

    # File variables as strings, then overlay caller variables (caller wins)
    merged = {k: str(v) for k, v in raw.items()}
    merged.update(caller_variables)
    return merged
