"""
Pipeline Runner - Generic DLT Pipeline Execution

Orchestrates:
- Loading source configuration
- Resolving credentials from source config secrets via Key Vault
- Dynamically importing connector source (from installed package or file)
- Running dlt pipeline
- Recording sync history
"""

import importlib
import os
import sys
import importlib.util
import traceback
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Callable
from dataclasses import dataclass

import dlt

from .config_resolver import ConfigResolver
from .credential_resolver import CredentialResolver
from .observability import (
    setup_logging,
    SyncHistoryTracker,
    SyncRecord,
    ResourceResult,
)
from .schema_validator import validate_all, ValidationError
from .utils import get_base_path, get_connectors_path, file_exists, load_yaml


@dataclass
class PipelineConfig:
    """Configuration for a pipeline run."""
    source_name: str
    vault_url: str
    resources: Optional[List[str]] = None  # None = all enabled
    trigger_type: str = "manual"
    destination_path: str = "/lakehouse/default/Tables"
    base_path: Optional[str] = None


@dataclass
class TargetConfig:
    """Target destination configuration from source YAML."""
    workspace: str
    lakehouse: str
    schema_name: str

    @classmethod
    def from_dict(cls, data: Dict[str, Any]) -> Optional["TargetConfig"]:
        """Create TargetConfig from dictionary, returns None if required fields missing."""
        if not data:
            return None
        workspace = data.get("workspace")
        lakehouse = data.get("lakehouse")
        schema_name = data.get("schema_name")
        if all([workspace, lakehouse, schema_name]):
            return cls(workspace=workspace, lakehouse=lakehouse, schema_name=schema_name)
        return None

    def get_destination_path(self) -> str:
        """Build destination path for Fabric Lakehouse.

        Note: Do NOT include schema_name here - dlt appends dataset_name automatically.
        """
        return "/lakehouse/default/Tables"


class PipelineRunner:
    """
    Executes dlt pipelines based on YAML configuration.

    Supports two modes:
    1. Package mode: connectors installed via pip (vd-dlt[notion])
    2. File mode: connectors loaded from base_path directory (legacy)

    Usage:
        runner = PipelineRunner(
            vault_url="https://my-vault.vault.azure.net/",
        )
        result = runner.run("notion_wiki")
    """

    def __init__(
        self,
        vault_url: str,
        base_path: Optional[str] = None,
        destination_path: str = "/lakehouse/default/Tables",
    ):
        """
        Initialize pipeline runner.

        Args:
            vault_url: Azure Key Vault URL
            base_path: Base path for configs (default: from environment)
            destination_path: Path for dlt destination
        """
        self.vault_url = vault_url
        self.base_path = base_path or get_base_path()
        self.destination_path = destination_path

        self.config_resolver = ConfigResolver(self.base_path)
        self.cred_resolver = CredentialResolver(vault_url)
        self.sync_tracker = SyncHistoryTracker(f"{self.base_path}/state")

    def _setup_destination(self, destination_path: Optional[str] = None) -> None:
        """Configure filesystem destination for lakehouse."""
        path = destination_path or self.destination_path
        os.environ["DESTINATION__FILESYSTEM__BUCKET_URL"] = path
        os.environ["DESTINATION__FILESYSTEM__DELTALAKE_STORAGE_OPTIONS"] = (
            '{"allow_unsafe_rename": "true"}'
        )

    def _load_connector_defaults(self, connector_name: str) -> Dict[str, Any]:
        """
        Load connector defaults from installed schema package or file.

        Tries package import first (vd_dlt_{connector}_schema),
        falls back to file-based loading.

        Args:
            connector_name: Name of the connector

        Returns:
            Dictionary with 'default_sync' and 'resources' keys
        """
        # Try package-based loading first
        try:
            schema_module = importlib.import_module(f"vd_dlt_{connector_name}_schema")
            if hasattr(schema_module, "get_defaults"):
                return schema_module.get_defaults()
        except ImportError:
            pass

        # Fall back to file-based loading
        defaults_file = f"{self.base_path}/connectors/{connector_name}/defaults.yml"
        data = load_yaml(defaults_file)

        default_sync_config = data.get("default_sync", {})

        return {
            "default_sync": default_sync_config,
            "resources": data.get("resources", []),
        }

    def _load_connector_source(
        self,
        connector_name: str,
        credentials: Dict[str, str],
        source_config: Dict[str, Any],
        connector_defaults: Optional[Dict[str, Any]] = None,
    ):
        """
        Dynamically load and configure a connector source.

        Tries package import first (vd_dlt_{connector}),
        falls back to file-based loading.

        Args:
            connector_name: Name of the connector
            credentials: Resolved credentials
            source_config: Source configuration
            connector_defaults: Optional connector defaults

        Returns:
            Configured dlt source
        """
        connector_module = None

        # Try package-based loading first
        try:
            connector_module = importlib.import_module(f"vd_dlt_{connector_name}")
        except ImportError:
            pass

        # Fall back to file-based loading
        if connector_module is None:
            connectors_path = f"{self.base_path}/connectors"
            source_file = f"{connectors_path}/{connector_name}/source.py"

            if not file_exists(source_file):
                raise FileNotFoundError(
                    f"Connector '{connector_name}' not found. "
                    f"Install it with: pip install vd-dlt[{connector_name}]"
                )

            if connectors_path not in sys.path:
                sys.path.insert(0, connectors_path)

            spec = importlib.util.spec_from_file_location(
                f"connectors.{connector_name}.source",
                source_file,
            )
            connector_module = importlib.util.module_from_spec(spec)
            connector_module.__name__ = f"connectors.{connector_name}.source"
            sys.modules[connector_module.__name__] = connector_module
            spec.loader.exec_module(connector_module)

        if not hasattr(connector_module, "create_source"):
            raise AttributeError(
                f"Connector {connector_name} must have a create_source function"
            )

        import inspect
        sig = inspect.signature(connector_module.create_source)
        if "connector_defaults" in sig.parameters:
            return connector_module.create_source(credentials, source_config, connector_defaults)
        else:
            return connector_module.create_source(credentials, source_config)

    def run(
        self,
        source_name: str,
        resources: Optional[List[str]] = None,
        trigger_type: str = "manual",
    ) -> SyncRecord:
        """
        Run a pipeline for a source.

        Args:
            source_name: Name of the source configuration
            resources: Optional list of specific resources to run
            trigger_type: How this run was triggered

        Returns:
            SyncRecord with results
        """
        logger, log_path, correlation_id = setup_logging(source_name)
        logger.info(f"Starting pipeline for source: {source_name}")

        sync_record = self.sync_tracker.start_sync(
            source_name=source_name,
            correlation_id=correlation_id,
            trigger_type=trigger_type,
        )

        resource_results: Dict[str, ResourceResult] = {}
        overall_status = "succeeded"
        error_message = None
        start_time = datetime.now(timezone.utc)

        try:
            # Load source config
            logger.info("Loading source configuration")
            source_config = self.config_resolver.load_source_config(source_name)
            connector_name = source_config.get("connector", "rest_api_generic")

            # Resolve credentials from source config secrets via Key Vault
            logger.info("Resolving credentials from source secrets")
            credentials = self.cred_resolver.resolve_from_source_config(source_config)
            logger.info("Credentials resolved successfully")

            # Validate credentials and source config against schema
            logger.info("Validating configuration")
            is_valid, validation_errors = validate_all(
                connector_name, credentials, source_config, self.base_path
            )
            if not is_valid:
                raise ValidationError(validation_errors)
            logger.info("Configuration validated successfully")

            # Parse target config from source
            target_data = source_config.get("target", {})
            target_config = TargetConfig.from_dict(target_data)

            # Setup destination (use target config if available)
            if target_config:
                self._setup_destination(target_config.get_destination_path())
            else:
                self._setup_destination()

            # Load connector defaults
            logger.info(f"Loading connector defaults: {connector_name}")
            connector_defaults = self._load_connector_defaults(connector_name)

            # Load connector source
            logger.info(f"Loading connector: {connector_name}")
            source = self._load_connector_source(
                connector_name, credentials, source_config, connector_defaults
            )

            # Determine which resources to run
            if resources:
                resources_to_run = resources
            else:
                resources_to_run = self.config_resolver.get_enabled_resources(source_config)

            logger.info(f"Resources to sync: {resources_to_run}")

            # Create pipeline
            pipeline_name = source_config.get("pipeline_name", f"{source_name}_pipeline")
            if target_config:
                dataset_name = source_config.get("dataset_name", target_config.schema_name)
            else:
                dataset_name = source_config.get("dataset_name", source_name)
            table_prefix = source_config.get("table_prefix", "")

            logger.info(f"Pipeline: {pipeline_name}")
            logger.info(f"Dataset: {dataset_name}")
            if table_prefix:
                logger.info(f"Table prefix: {table_prefix}")

            pipeline = dlt.pipeline(
                pipeline_name=pipeline_name,
                destination="filesystem",
                dataset_name=dataset_name,
            )

            # Run the pipeline
            logger.info("Running pipeline...")

            write_disposition = "merge"
            if resources_to_run:
                first_resource_config = self.config_resolver.resolve_resource_config(
                    source_config, resources_to_run[0]
                )
                write_disposition = first_resource_config.get("write_disposition", "merge")

            load_info = pipeline.run(
                source,
                write_disposition=write_disposition,
                table_format="delta",
            )

            end_time = datetime.now(timezone.utc)
            duration_ms = int((end_time - start_time).total_seconds() * 1000)

            # Extract metrics from pipeline trace
            total_rows = 0
            try:
                row_counts = pipeline.last_trace.last_normalize_info.row_counts
                for table_name, rows in row_counts.items():
                    if not table_name.startswith("_dlt"):
                        total_rows += rows
                        resource_results[table_name] = ResourceResult(
                            resource_name=table_name,
                            status="succeeded",
                            rows_extracted=rows,
                            rows_loaded=rows,
                            duration_ms=duration_ms,
                        )
            except Exception as metrics_error:
                logger.warning(f"Could not extract row counts: {metrics_error}")

            logger.info(f"Pipeline completed: {total_rows} total rows loaded")
            overall_status = "succeeded"

        except Exception as e:
            overall_status = "failed"
            error_message = str(e)
            logger.error(f"Pipeline failed: {error_message}")
            logger.error(traceback.format_exc())

        # Complete sync tracking
        sync_record = self.sync_tracker.complete_sync(
            sync_record=sync_record,
            status=overall_status,
            resource_results=resource_results,
            error_message=error_message,
        )

        # Print summary
        print("\n" + "=" * 60)
        print(f"Pipeline: {source_name}")
        print(f"Status: {overall_status}")
        print(f"Correlation ID: {correlation_id}")
        print(f"Duration: {sync_record.duration_ms}ms")
        print(f"Rows loaded: {sync_record.total_rows_loaded}")
        print(f"Log file: {log_path}")
        print("=" * 60)

        return sync_record


def run_source(
    source_name: str,
    vault_url: str,
    resources: Optional[List[str]] = None,
    base_path: Optional[str] = None,
    destination_path: str = "/lakehouse/default/Tables",
) -> SyncRecord:
    """
    Convenience function to run a source pipeline.

    Args:
        source_name: Name of the source configuration
        vault_url: Azure Key Vault URL
        resources: Optional list of specific resources
        base_path: Optional base path override
        destination_path: Path for dlt destination

    Returns:
        SyncRecord with results
    """
    runner = PipelineRunner(
        vault_url=vault_url,
        base_path=base_path,
        destination_path=destination_path,
    )
    return runner.run(source_name, resources)
