"""
ETLOrchestrator - Run ETL from contract directory or separate extract/transform/load files.

This is the main entry point for config-driven ETL. It builds a Pipeline from
a contract directory, a single pipeline file, or explicit extract/transform/load
paths, and runs it, returning a result dict compatible with consumers like
statfyi-scraper-api.

Supports both single-run and bulk (multi-param) execution via ``run()`` and
``run_bulk()`` respectively.
"""

from __future__ import annotations

import asyncio
import logging
import re
from pathlib import Path
from typing import Any, AsyncIterator

import yaml

from pycharter.etl_generator.pipeline import Pipeline
from pycharter.etl_generator.rate_limiter import AsyncRateLimiter
from pycharter.etl_generator.result import LoadResult
from pycharter.shared.errors import PipelineConfigurationError

logger = logging.getLogger(__name__)


def _validate_data_list(data: Any, method: str) -> None:
    """Ensure data is a list for transform/load. Raises TypeError if not."""
    if not isinstance(data, list):
        raise TypeError(
            f"{method} expects a list of records, got {type(data).__name__}. "
            "For bulk flows, pass the concatenated list of extracted batches."
        )


def _load_result_to_dict(lr: LoadResult) -> dict[str, Any]:
    """Map LoadResult to consumer-friendly dict (total, inserted, updated, success, error)."""
    inserted = lr.inserted if lr.inserted is not None else lr.rows_loaded
    updated = lr.updated if lr.updated is not None else 0
    return {
        "total": lr.rows_loaded,
        "inserted": inserted,
        "updated": updated,
        "success": lr.success,
        "error": lr.error,
    }


def _context_to_variables(config_context: dict[str, Any]) -> dict[str, str | None]:
    """Convert config_context to string values for variable substitution."""
    if not config_context:
        return {}
    return {k: str(v) if v is not None else "" for k, v in config_context.items()}


class ETLOrchestrator:
    """
    Run ETL from a contract directory, a single pipeline file, or separate
    extract/transform/load files.

    Use PipelineFactory to discover pipeline names and create orchestrators,
    or pass explicit file paths:

        orchestrator = ETLOrchestrator(
            extract_file="configs/extract.yaml",
            transform_file="configs/transform.yaml",
            load_file="configs/load.yaml",
            config_context={"API_KEY": "..."},
        )
        result = await orchestrator.run()
    """

    def __init__(
        self,
        contract_dir: str | Path | None = None,
        contract_file: str | Path | None = None,
        extract_file: str | Path | None = None,
        transform_file: str | Path | None = None,
        load_file: str | Path | None = None,
        config_context: dict[str, Any] | None = None,
        verbose: bool = True,
        name: str | None = None,
        load_defaults: dict[str, Any] | None = None,
        settings_file: str | Path | None = None,
        variables_file: str | Path | None = None,
        **kwargs,
    ):
        """
        Initialize the orchestrator.

        Args:
            contract_dir: Path to directory containing extract.yaml, transform.yaml, load.yaml.
            contract_file: Path to single pipeline.yaml with all sections.
            extract_file: Path to extract config (use with load_file; overrides contract_dir/contract_file).
            transform_file: Optional path to transform config.
            load_file: Path to load config (use with extract_file).
            config_context: Variables for ${VAR} substitution (e.g. TARGET_DATABASE_URL, FMP_API_KEY).
            verbose: Enable verbose logging.
            name: Optional pipeline name (used when extract_file/load_file are set; default from extract_file stem).
            load_defaults: Default values merged under load.yaml config (e.g. database, ssh_tunnel).
                           Pipeline-specific load.yaml values take precedence.
            settings_file: Optional path to settings.yaml (used with from_config_files path).
            variables_file: Optional path to variables.yaml (used with from_config_files path).
            **kwargs: Ignored; for API compatibility with INTERFACES.md.
        """
        self.contract_dir = Path(contract_dir) if contract_dir else None
        self.contract_file = Path(contract_file) if contract_file else None
        self.extract_file = Path(extract_file) if extract_file else None
        self.transform_file = Path(transform_file) if transform_file else None
        self.load_file = Path(load_file) if load_file else None
        self.config_context = config_context or {}
        self.verbose = verbose
        self._name = name
        self._load_defaults = load_defaults
        self.settings_file = Path(settings_file) if settings_file else None
        self.variables_file = Path(variables_file) if variables_file else None
        self._pipeline: Pipeline | None = None

        # Require one of: contract_dir, contract_file, or (extract_file + load_file)
        if self.contract_dir is not None:
            if not self.contract_dir.is_dir():
                raise NotADirectoryError(
                    f"Contract directory not found: {self.contract_dir}"
                )
        elif self.contract_file is not None:
            if not self.contract_file.is_file():
                raise FileNotFoundError(
                    f"Contract file not found: {self.contract_file}"
                )
        elif self.extract_file is not None and self.load_file is not None:
            if not self.extract_file.is_file():
                raise FileNotFoundError(f"Extract file not found: {self.extract_file}")
            if not self.load_file.is_file():
                raise FileNotFoundError(f"Load file not found: {self.load_file}")
        else:
            raise ValueError(
                "Provide one of: contract_dir, contract_file, or both extract_file and load_file"
            )

    def _get_pipeline(self) -> Pipeline:
        """Build or return cached pipeline from contract directory, file, or separate configs."""
        if self._pipeline is not None:
            return self._pipeline
        variables = _context_to_variables(self.config_context)
        if self.contract_dir is not None:
            self._pipeline = Pipeline.from_config_dir(
                self.contract_dir,
                variables=variables or None,
                validate=True,
                name=self.contract_dir.name,
                load_defaults=self._load_defaults,
            )
        elif self.contract_file is not None:
            self._pipeline = Pipeline.from_config_file(
                self.contract_file,
                variables=variables or None,
                validate=True,
            )
        else:
            pipeline_name = self._name or (
                self.extract_file.stem if self.extract_file else "pipeline"
            )
            if self.variables_file:
                from pycharter.etl_generator.pipeline import _load_variables_file

                variables_arg = _load_variables_file(self.variables_file, variables)
            else:
                variables_arg = variables
            self._pipeline = Pipeline.from_config_files(
                str(self.extract_file),
                str(self.load_file),
                str(self.transform_file) if self.transform_file else None,
                variables_arg,
                str(self.settings_file) if self.settings_file else None,
                validate=True,
                name=pipeline_name,
                load_defaults=self._load_defaults,
                base_dir=self.extract_file.parent if self.extract_file else None,
            )
        return self._pipeline

    def ensure_ready(self, phase: str = "run") -> None:
        """
        Validate that the pipeline is configured for the requested phase.
        Call before extract/transform/load (or run) to fail fast with a clear error.

        Args:
            phase: One of "extract", "transform", "load", "run".
                   "run" checks extractor and loader (transform is optional).

        Raises:
            PipelineConfigurationError: If the pipeline cannot run the given phase.
        """
        pipeline = self._get_pipeline()
        if phase == "extract":
            if not pipeline.extractor:
                raise PipelineConfigurationError(
                    "Pipeline has no extractor; cannot run extract phase.",
                    phase="extract",
                )
        elif phase == "load":
            if not pipeline.loader:
                raise PipelineConfigurationError(
                    "Pipeline has no loader; cannot run load phase.",
                    phase="load",
                )
        elif phase == "run":
            if not pipeline.extractor:
                raise PipelineConfigurationError(
                    "Pipeline has no extractor; cannot run full pipeline.",
                    phase="run",
                )
            if not pipeline.loader:
                raise PipelineConfigurationError(
                    "Pipeline has no loader; cannot run full pipeline.",
                    phase="run",
                )
        # "transform" has no requirement (transform config can be empty)

    async def extract(self, **params: Any) -> AsyncIterator[list[dict[str, Any]]]:
        """
        Run only the extract phase, yielding batches of records.

        Each yielded value is a list of records (a batch). For symbol listing or
        bulk flows, concatenate batches to get the full list, then pass that list
        to transform() and load().

        The same pipeline instance is reused (cached); no need to rebuild between
        extract/transform/load calls.
        """
        self.ensure_ready("extract")
        pipeline = self._get_pipeline()
        async for batch in pipeline.extractor.extract(**params):
            yield batch

    def transform(
        self, data: list[dict[str, Any]], **params: Any
    ) -> list[dict[str, Any]]:
        """
        Run only the transform phase on in-memory data.

        Expects the flattened list of records (e.g. concatenation of all batches
        from extract()). Returns a single list of transformed records to pass to load().
        Empty list is valid and returns [].
        """
        _validate_data_list(data, "transform")
        pipeline = self._get_pipeline()
        return pipeline._apply_transforms(data)

    async def transform_async(
        self, data: list[dict[str, Any]], **params: Any
    ) -> list[dict[str, Any]]:
        """
        Async wrapper for transform. Currently delegates to sync transform();
        provided for forward compatibility when pipelines support async transformers.
        """
        return self.transform(data, **params)

    async def load(
        self,
        data: list[dict[str, Any]],
        session: Any = None,
        **params: Any,
    ) -> dict[str, Any]:
        """
        Run only the load phase on in-memory data.

        Expects the exact list returned by transform() (same order, no drop/add).
        session is optional and loader-dependent (e.g. PostgresLoader may use it;
        others may use connection from config). Returns a dict with total, inserted,
        updated, success, error. Empty data is valid; returns immediately without
        calling the loader.
        """
        _validate_data_list(data, "load")
        if not data:
            return {
                "total": 0,
                "inserted": 0,
                "updated": 0,
                "success": True,
                "error": None,
            }
        self.ensure_ready("load")
        pipeline = self._get_pipeline()
        load_params = {**params}
        if session is not None:
            load_params["session"] = session
        load_result = await pipeline.loader.load(data, **load_params)
        return _load_result_to_dict(load_result)

    async def run_phases(
        self,
        *,
        extract: bool = True,
        transform: bool = True,
        load: bool = True,
        data: list[dict[str, Any]] | None = None,
        dry_run: bool = False,
        session: Any = None,
        **params: Any,
    ) -> dict[str, Any]:
        """
        Run selected ETL phases in one call. Generalizes full run and bulk-style flows.

        When data is None: run extract (if extract=True), then transform (if transform=True),
        then load (if load=True). Same as run() when extract=transform=load=True.

        When data is provided: skip extract and use data as the input to transform (if
        transform=True), then load (if load=True). Useful for bulk flows that already
        have extracted data in memory.
        """
        if data is not None and not isinstance(data, list):
            raise TypeError("run_phases(data=...) expects a list of records.")
        if data is not None and not extract:
            # Use provided data as input to transform/load
            transformed = self.transform(data, **params) if transform else data
            if load and not dry_run:
                load_result = await self.load(transformed, session=session, **params)
                return {
                    "extraction": {"total_records": 0, "batches_processed": 0},
                    "transformation": {"total_records": len(transformed)},
                    "loading": load_result,
                    "success": load_result.get("success", True),
                    "error": load_result.get("error"),
                    "failed_batches": [],
                }
            return {
                "extraction": {"total_records": 0, "batches_processed": 0},
                "transformation": {"total_records": len(transformed)},
                "loading": {
                    "total": 0,
                    "inserted": 0,
                    "updated": 0,
                    "success": True,
                    "error": None,
                },
                "success": True,
                "error": None,
                "failed_batches": [],
            }
        # Full run (data is None or extract=True)
        return await self.run(dry_run=dry_run, session=session, **params)

    # ------------------------------------------------------------------
    # Config helpers
    # ------------------------------------------------------------------

    def get_extract_config(self) -> dict[str, Any]:
        """Return the raw extract.yaml config as a dict (cached)."""
        if not hasattr(self, "_extract_config_cache"):
            self._extract_config_cache: dict[str, Any] = {}
            if self.contract_dir is not None:
                path = self.contract_dir / "extract.yaml"
                if path.is_file():
                    with open(path) as f:
                        self._extract_config_cache = yaml.safe_load(f) or {}
        return self._extract_config_cache

    def get_path_param_names(self) -> list[str]:
        """Detect ``{param}`` placeholders in the extract endpoint."""
        endpoint = self.get_extract_config().get("api_endpoint", "")
        if not endpoint or "{" not in endpoint:
            return []
        return re.findall(r"\{(\w+)\}", endpoint)

    def get_default_param_values(self, param_name: str) -> list[str]:
        """Read ``default_param_values.<param_name>`` from extract config as a list."""
        cfg = self.get_extract_config().get("default_param_values", {})
        values = cfg.get(param_name, [])
        if isinstance(values, list):
            return [str(v).strip() for v in values if v]
        if values is not None:
            return [str(values).strip()]
        return []

    def get_input_params(self) -> dict[str, dict[str, Any]]:
        """Return ``input_params`` from extract config (param name → config dict)."""
        return self.get_extract_config().get("input_params", {})

    def get_default_for_param(self, param_name: str) -> str | None:
        """Return scalar default from ``default_param_values`` (first value if list)."""
        cfg = self.get_extract_config().get("default_param_values", {})
        value = cfg.get(param_name)
        if isinstance(value, list):
            return str(value[0]).strip() if value else None
        return str(value).strip() if value is not None else None

    def resolve_params(self, params: dict[str, Any]) -> dict[str, Any]:
        """
        Fill in missing required parameters from ``default_param_values``.

        Raises ``ValueError`` if a required ``input_params`` entry has no value
        and no default.
        """
        resolved = dict(params)
        input_params = self.get_input_params()
        path_params = self.get_path_param_names()

        # Check all declared input_params
        for name, cfg in input_params.items():
            if name in resolved:
                continue
            default = self.get_default_for_param(name)
            if default is not None:
                resolved[name] = default
            elif cfg.get("required", False):
                raise ValueError(
                    f"Missing required parameter '{name}'. "
                    f"Pass it via CLI (--{name} <value>), env var ({name.upper()}), "
                    f"or set default_param_values.{name} in extract.yaml."
                )

        # Also check path params not covered by input_params
        for name in path_params:
            if name in resolved or name in input_params:
                continue
            default = self.get_default_for_param(name)
            if default is not None:
                resolved[name] = default

        return resolved

    def validate_params(self, params: dict[str, Any]) -> list[str]:
        """
        Validate parameters without running the pipeline.

        Returns a list of error messages (empty = valid).
        """
        errors: list[str] = []
        input_params = self.get_input_params()
        path_params = self.get_path_param_names()

        for name, cfg in input_params.items():
            if name not in params and self.get_default_for_param(name) is None:
                if cfg.get("required", False):
                    errors.append(f"Missing required parameter: {name}")

        for name in path_params:
            if name not in params and name not in input_params:
                if self.get_default_for_param(name) is None:
                    errors.append(f"Missing path parameter: {name} (no default set)")

        return errors

    # ------------------------------------------------------------------
    # Single run
    # ------------------------------------------------------------------

    async def run(
        self,
        dry_run: bool = False,
        session: Any = None,
        error_threshold: float = 0.3,
        max_retries: int = 5,
        **params,
    ) -> dict[str, Any]:
        """
        Run the full ETL pipeline (extract → transform → load).

        Args:
            dry_run: If True, extract and transform but do not load.
            session: Optional DB session (currently unused; connection from config_context).
            error_threshold: Unused; for API compatibility.
            max_retries: Unused; for API compatibility.
            **params: Passed to extractor/loader (e.g. symbol, period, date).

        Returns:
            Dict with keys: extraction, transformation, loading, success, error, failed_batches.
        """
        self.ensure_ready("run")
        params = self.resolve_params(params)
        pipeline = self._get_pipeline()
        result = await pipeline.run(dry_run=dry_run, **params)

        # Map pycharter PipelineResult to consumer-friendly dict
        failed_batches = [
            {"batch_index": br.batch_index, "errors": br.errors}
            for br in result.batch_results
            if br.errors or br.rows_failed
        ]
        return {
            "extraction": {
                "total_records": result.rows_extracted,
                "batches_processed": result.batches_processed,
            },
            "transformation": {
                "total_records": result.rows_transformed,
                "batches_processed": result.batches_processed,
            },
            "loading": {
                "total_records": result.rows_loaded,
                "inserted": result.rows_loaded,
                "updated": 0,
                "batches_processed": result.batches_processed,
            },
            "success": result.success,
            "error": result.errors[0] if result.errors else None,
            "failed_batches": failed_batches,
        }

    # ------------------------------------------------------------------
    # Bulk (multi-param) run
    # ------------------------------------------------------------------

    async def run_bulk(
        self,
        param_name: str | None = None,
        param_values: list[str] | None = None,
        concurrency: int = 20,
        max_calls_per_minute: int = 300,
        dry_run: bool = False,
        session: Any = None,
        verbose: bool = True,
        **extra_params,
    ) -> dict[str, Any]:
        """
        Run the pipeline for many values of a single path parameter.

        Pattern: extract all in parallel → transform all → load all.

        Args:
            param_name: Path-parameter to iterate (auto-detected from ``api_endpoint`` if omitted).
            param_values: Values for *param_name*.  If ``None``, reads
                ``default_param_values.<param_name>`` from extract config.
            concurrency: Max parallel extraction tasks.
            max_calls_per_minute: Rate-limit ceiling across all tasks.
            dry_run: Skip the load phase when ``True``.
            session: Optional DB session forwarded to ``load()``.
            verbose: Emit progress log lines.
            **extra_params: Passed to every extraction call (e.g. ``start_date``, ``end_date``).

        Returns:
            Dict with ``success``, ``extraction``, ``transformation``, ``loading`` keys.
        """
        self.ensure_ready("run")

        # --- resolve fixed extra_params from defaults ---
        extra_params = self.resolve_params(extra_params)

        # --- resolve param_name ---
        if param_name is None:
            names = self.get_path_param_names()
            if not names:
                raise PipelineConfigurationError(
                    "Cannot auto-detect param_name: api_endpoint has no {param} placeholders.",
                    phase="run_bulk",
                )
            param_name = names[0]

        # --- resolve param_values ---
        if param_values is None:
            param_values = self.get_default_param_values(param_name)
        if not param_values:
            raise ValueError(
                f"No param_values for '{param_name}'. "
                "Pass them explicitly or set default_param_values in extract.yaml."
            )

        if verbose:
            logger.info(
                "Bulk pipeline starting: %d values for '%s' (concurrency=%d)",
                len(param_values),
                param_name,
                concurrency,
            )

        # --- Phase 1: parallel extraction ---
        rate_limiter = AsyncRateLimiter(max_calls_per_minute)
        semaphore = asyncio.Semaphore(concurrency)
        all_data: list[dict[str, Any]] = []
        failed: list[str] = []

        async def _extract_one(value: str) -> tuple:
            async with semaphore:
                await rate_limiter.acquire()
                try:
                    batches: list[dict[str, Any]] = []
                    async for batch in self.extract(
                        **{param_name: value, **extra_params}
                    ):
                        batches.extend(batch)
                    return value, batches
                except Exception as exc:
                    logger.debug(
                        "Extraction failed for %s=%s: %s", param_name, value, exc
                    )
                    return value, []

        results = await asyncio.gather(
            *[_extract_one(v) for v in param_values],
            return_exceptions=True,
        )

        for i, result in enumerate(results):
            if isinstance(result, Exception):
                failed.append(param_values[i])
                logger.warning(
                    "Extraction exception for %s: %s", param_values[i], result
                )
            else:
                value, data = result
                if data:
                    all_data.extend(data)
                else:
                    failed.append(value)

        if verbose:
            logger.info(
                "Extraction complete: %d records from %d params (%d failed)",
                len(all_data),
                len(param_values),
                len(failed),
            )

        if not all_data:
            return {
                "success": False,
                "error": "No data extracted",
                "extraction": {"total_records": 0, "failed": failed},
                "transformation": {"total_records": 0},
                "loading": {"total_records": 0},
            }

        # --- Phase 2: transform ---
        transformed = self.transform(all_data)
        if verbose:
            logger.info("Transformed %d records", len(transformed))

        # --- Phase 3: load ---
        if dry_run:
            load_result = {
                "total": len(transformed),
                "inserted": 0,
                "updated": 0,
                "success": True,
                "error": None,
            }
        else:
            load_result = await self.load(transformed, session=session)
            if verbose:
                logger.info(
                    "Loaded %d total (%d inserted, %d updated)",
                    load_result.get("total", 0),
                    load_result.get("inserted", 0),
                    load_result.get("updated", 0),
                )

        return {
            "success": True,
            "extraction": {
                "total_records": len(all_data),
                "params_processed": len(param_values),
                "params_failed": len(failed),
                "failed": failed[:20],
            },
            "transformation": {"total_records": len(transformed)},
            "loading": load_result,
        }
