"""
DLT Pipeline Runner (Simple)

Simple interface to run dlt pipelines from Fabric notebooks.
Source configuration is passed directly as a dictionary.
Secrets are resolved from Azure Key Vault.

This is the lightweight runner for direct notebook use.
For the full-featured runner, use PipelineRunner from vd_dlt.pipeline_runner.
"""

import importlib
import os
import sys
import yaml
from typing import Optional, Dict, Any


def run_pipeline(
    source_name: str,
    vault_url: str,
    destination_path: str = "/lakehouse/default/Tables",
    source_config: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
    """
    Run a dlt pipeline for the given source.

    Args:
        source_name: Name of the source config (e.g., "data_swim_notion")
        vault_url: Azure Key Vault URL
        destination_path: Path for dlt destination (from mounted lakehouse)
        source_config: Source config dict (required)

    Returns:
        Dict with pipeline results
    """
    import dlt

    if not source_config:
        raise ValueError("source_config is required")

    print(f"Source: {source_name}")

    connector_name = source_config.get('connector', 'notion')
    target = source_config.get('target', {})

    print(f"Connector: {connector_name}")
    print(f"Target: {target.get('workspace')}/{target.get('lakehouse')}/{target.get('schema_name')}")

    # Resolve credentials from source config 'secrets' dict
    secrets = source_config.get('secrets', {})
    credentials = {}

    print(f"\nResolving secrets from Key Vault")
    import notebookutils
    for key, secret_name in secrets.items():
        if isinstance(secret_name, str) and secret_name:
            # Strip legacy ${...} wrapper if present
            if secret_name.startswith("${") and secret_name.endswith("}"):
                secret_name = secret_name[2:-1]
            print(f"  Resolving: {key}")
            credentials[key] = notebookutils.credentials.getSecret(vault_url, secret_name)
        else:
            credentials[key] = secret_name

    # Load connector defaults from installed schema package
    connector_defaults = {}
    try:
        schema_module = importlib.import_module(f"vd_dlt_{connector_name}_schema")
        if hasattr(schema_module, "get_defaults"):
            connector_defaults = schema_module.get_defaults()
            print(f"\nLoaded connector defaults from vd_dlt_{connector_name}_schema")
    except ImportError:
        print(f"\nNo schema package found for {connector_name}, using empty defaults")

    # Import connector module from installed package
    try:
        connector_module = importlib.import_module(f"vd_dlt_{connector_name}")
        print(f"Loaded connector: vd_dlt_{connector_name}")
    except ImportError:
        raise ImportError(
            f"Connector package not found. Install with: pip install vd-dlt[{connector_name}]"
        )

    # Setup destination
    print(f"Using destination: {destination_path}")

    os.environ["DESTINATION__FILESYSTEM__BUCKET_URL"] = destination_path
    os.environ["DESTINATION__FILESYSTEM__DELTALAKE_STORAGE_OPTIONS"] = '{"allow_unsafe_rename": "true"}'

    # Create source
    print("\nCreating dlt source...")
    source = connector_module.create_source(
        credentials=credentials,
        source_config=source_config,
        connector_defaults=connector_defaults
    )

    # Run pipeline
    pipeline_name = source_config.get('pipeline_name', f"{source_name}_pipeline")
    dataset_name = target.get('schema_name', source_name)

    write_disposition = source_config.get('default_sync', {}).get('write_disposition', 'merge')

    print(f"\nRunning pipeline...")
    print(f"  Pipeline: {pipeline_name}")
    print(f"  Dataset: {dataset_name}")
    print(f"  Write disposition: {write_disposition}")

    pipeline = dlt.pipeline(
        pipeline_name=pipeline_name,
        destination="filesystem",
        dataset_name=dataset_name,
    )

    load_info = pipeline.run(
        source,
        write_disposition=write_disposition,
        table_format="delta"
    )

    # Get results
    results = {
        "status": "success",
        "pipeline_name": pipeline_name,
        "dataset_name": dataset_name,
        "destination": f"{destination_path}/{dataset_name}/",
        "load_info": str(load_info),
    }

    # Get row counts
    try:
        row_counts = pipeline.last_trace.last_normalize_info.row_counts
        total = 0
        table_counts = {}
        for table, rows in row_counts.items():
            if not table.startswith("_dlt"):
                table_counts[table] = rows
                total += rows
        results["row_counts"] = table_counts
        results["total_rows"] = total
    except Exception as e:
        results["row_counts_error"] = str(e)

    # Print summary
    print("\n" + "=" * 60)
    print("COMPLETED")
    print("=" * 60)
    print(load_info)

    if "row_counts" in results:
        print("\nRows loaded:")
        for table, rows in results["row_counts"].items():
            print(f"  {table}: {rows}")
        print(f"\nTotal: {results['total_rows']} rows")

    print(f"\nDestination: {results['destination']}")

    return results
