import sys
import os
import json
import yaml
import re
import typer
import subprocess
import importlib.util
import importlib
import shutil
import sqlalchemy as sa
from pathlib import Path
from typing import Optional, Callable, Set, Dict, Any, Iterable, Mapping
from sqlalchemy import inspect
from rich.console import Console
from rich.table import Table

# --- Core Library Imports ---
from sibi_flux.datacube.generator import (
    is_secure_path,
    check_drift,
    resolve_db_url,
    filter_global_imports,
    DatacubeRegistry,
    perform_discovery,
    generate_datacube_module_code,
    generate_field_map_files,
)
from sibi_flux.datacube.orchestrator import DiscoveryOrchestrator
from sibi_flux.datacube.field_factory import FieldMapFactory
from sibi_flux.init.rule_generator import RuleEngine
from sibi_flux.core.project import ProjectService

import sibi_flux.datacube.generator

app = typer.Typer(help="Sibi-Flux Data Cube Generator")
console = Console()

# --- Context Management ---


def _load_and_resolve_config(config_path: Path) -> dict:
    if not config_path.exists():
        return {}
        
    project_root = ProjectService.detect_project_root(config_path)
    
    # If the user passed a specific YAML file, we might want to respect its directory
    config_dir = config_path.parent if config_path.suffix in (".yaml", ".yml") else None
    
    return ProjectService.load_modular_config(project_root, config_dir=config_dir)


class CLIContext:
    def __init__(self):
        self.default_config: Optional[Path] = None
        self.field_translations_file: Optional[Path] = None
        self.valid_paths: list[str] = []
        self.valid_fieldmap_paths: list[str] = []
        self.params: dict = {}

    def configure(
        self,
        default_config: Path,
        field_translations_file: Path,
        valid_paths: list[str],
        valid_fieldmap_paths: list[str],
        params: Optional[dict] = None,
    ):
        self.default_config = default_config
        self.field_translations_file = field_translations_file
        self.valid_paths = valid_paths
        self.valid_fieldmap_paths = valid_fieldmap_paths
        self.params = params or {}

    def auto_configure(self):
        """Attempts to find defaults if not configured."""
        if self.default_config:
            return

        # Heuristic check for standard project layout
        # Case 1: Run from project root -> conf/discovery_params.yaml (New)
        # Case 2: Run from project root -> generators/datacubes/discovery_params.yaml (Legacy)
        
        candidate_conf = Path("conf/discovery_params.yaml")
        candidate_legacy = Path("generators/datacubes/discovery_params.yaml")
        
        candidate = None
        project_root = None
        
        if candidate_conf.exists():
            candidate = candidate_conf
            project_root = Path.cwd()
        elif candidate_legacy.exists():
            candidate = candidate_legacy
            project_root = Path.cwd()

        if candidate:
            # Use shared resolver to get normalization and project root paths
            raw_params = _load_and_resolve_config(candidate)

            self.configure(
                default_config=candidate.resolve(),
                field_translations_file=(
                    project_root
                    / "dataobjects/globals/global_field_translations.yaml"
                ).resolve(),
                valid_paths=[],
                valid_fieldmap_paths=[],
                params=raw_params,
            )
            console.print(f"[dim]Auto-configured context from {candidate}[/dim]")


context = CLIContext()
context.auto_configure()


def set_context_defaults(
    default_config: Path,
    field_translations_file: Path,
    valid_paths: list[str],
    valid_fieldmap_paths: list[str],
    params: Optional[dict] = None,
):
    """Configures the CLI context with project-specific defaults."""
    context.configure(
        default_config,
        field_translations_file,
        valid_paths,
        valid_fieldmap_paths,
        params,
    )

    # Ensure directories exist based on configured params
    if params:
        ProjectService.ensure_directories_exist(params, logger=console.log)


def _get_db_url_callback(
    registry: DatacubeRegistry, db_url_map: Optional[str], params: Optional[Dict[str, Any]] = None
) -> Callable[[str], str]:
    """Helper to create a callback that resolves DB URLs from CLI overrides or registry."""
    cli_urls = json.loads(db_url_map) if db_url_map else {}

    def get_url(conf_name: str) -> str:
        # 1. CLI Override
        if conf_name in cli_urls:
            return cli_urls[conf_name]
        
        # 2. Dynamic Resolution
        imports = registry.global_imports
        
        # Check specific import_spec from params
        if params and "databases" in params:
             for db in params.get("databases", []):
                  # Match by ID, Name, Ref or Obj
                  db_id = db.get("id")
                  db_name = db.get("name")
                  ref = db.get("connection_ref") or db.get("connection_obj")
                  
                  if db_id == conf_name or db_name == conf_name or ref == conf_name:
                       spec = db.get("import_spec")
                       if spec and isinstance(spec, dict):
                            # Prepend specific module
                            imports = [spec.get("module")] + imports
                       elif db.get("global_import"):
                            imports = [db.get("global_import")] + imports
                       elif ref and "." in ref:
                            # If connection_obj is a python path, use its parent module
                            imports = [".".join(ref.split(".")[:-1])] + imports
                       break
        
        lookup_name = conf_name.split(".")[-1]
        
        url = resolve_db_url(lookup_name, imports)
        if url:
            return url
        
        # Try full name just in case
        if lookup_name != conf_name:
             url = resolve_db_url(conf_name, imports)
             if url:
                  return url

        raise ValueError(
            f"Could not resolve DB URL for '{conf_name}' (as {lookup_name}). Provide via --db-urls or check imports."
        )

    return get_url


# --- Commands ---


@app.command()
def sync(
    config_file: Optional[Path] = typer.Option(None, "--config"),
    db_url_map: Optional[str] = typer.Option(
        None,
        "--db-urls",
        help="Optional JSON mapping. If omitted, tries to resolve from code.",
    ),
    force: bool = typer.Option(False, "--force", "-f"),
    env_file: Optional[Path] = typer.Option(
        None, "--env-file", "-e", help="Path to environment file"
    ),
    dry_run: bool = typer.Option(False, "--dry-run"),
) -> None:
    """Generates all Datacube classes based on the whitelists and field maps."""
    config_path = config_file or context.default_config
    if not config_path:
        console.print("[red]No config file specified and no default configured.[/red]")
        raise typer.Exit(code=1)

    # --- 1. Load Environment ---
    project_root = ProjectService.detect_project_root(config_path)
    ProjectService.setup_environment(project_root, env_file)

    # Start with empty/default registry
    config_data = _load_and_resolve_config(config_path)

    # Load existing Global Registry to preserve manual edits (e.g. custom_name)
    # We must flatten the grouped structure (by config object) into a single tables dict
    params = context.params or {}
    reg_rel_path = params.get("paths", {}).get("repositories", {}).get(
        "global_datacube_registry_file"
    ) or params.get("global_datacube_registry_file")

    if reg_rel_path:
        reg_file = Path(reg_rel_path)
        if not reg_file.is_absolute():
            reg_file = project_root / reg_rel_path
        
        console.print(f"[bold cyan]Registry:[/] {reg_file.relative_to(project_root)}")

        if reg_file.exists():
            try:
                with open(reg_file, "r") as rf:
                    existing_reg_data = yaml.safe_load(rf) or {}
                
                flat_tables = {}
                for grp, tbls in existing_reg_data.items():
                    if grp == "tables" and isinstance(tbls, dict):
                        # Modern format: tables are in 'tables' dict
                        for t, t_meta in tbls.items():
                            flat_tables[t] = t_meta
                        continue

                    if isinstance(tbls, dict) and grp != "global_imports":
                        # Legacy Scoped Format (grp is connection_obj)
                        for t, t_meta in tbls.items():
                            # Inject/Overwrite connection_obj only if implied by scope
                            t_meta["connection_obj"] = grp
                            flat_tables[t] = t_meta
                
                config_data["tables"] = flat_tables
                console.print(f"[green]✓ Loaded {len(flat_tables)} existing registry entries.[/green]")
            except Exception as e:
                console.print(f"[yellow]Warning: Could not load existing registry: {e}[/yellow]")

    registry = DatacubeRegistry(config_data, params=context.params)
    
    # Fix 2 Re-applied: Ensure global_imports are populated from params if registry loaded from file didn't have them
    # This is critical for Clean Project + Force flows where registry might be partial or JIT
    if not registry.global_imports and context.params.get("global_imports"):
        registry.global_imports = context.params.get("global_imports")

    # --- Aggregation Phase ---
    params = config_data
    databases = params.get("databases", [])

    # JIT DISCOVERY CHECK
    # If using simplified whitelist workflow, registry might be empty.
    # Auto-discover from whitelist in-memory.
    if not registry.tables:
        console.print(
            "[dim]Registry empty. Attempting JIT Discovery from Whitelists...[/dim]"
        )

        # Prepare URL resolver for orchestrator usage if needed
        import json

        cli_urls = json.loads(db_url_map) if db_url_map else {}

        for db in databases:
            db_name = db.get("name") or db.get("id") or "unknown"
            current_conn = db.get("connection_obj") or db.get("connection_ref") or db_name

            # Resolve whitelist/rules paths (reusing logic from discover command or simplifying?)
            # Orchestrator handles defaults if paths passed are relative/simple strings.
            # We need to resolve full paths to be safe, or trust Orchestrator logic.
            # Let's rely on params provided to orchestrator logic via context.params

            try:
                # Resolve DB URL
                import_spec = db.get("import_spec")
                if import_spec and isinstance(import_spec, dict):
                    imp = import_spec.get("module")
                else:
                    imp = db.get("global_import")
                db_imports = [imp] if imp else registry.global_imports

                # Helper to resolve
                if current_conn in cli_urls:
                    db_conn_str = cli_urls[current_conn]
                else:
                    db_conn_str = resolve_db_url(current_conn, db_imports)

                if not db_conn_str:
                    console.print(
                        f"[yellow]Skipping JIT discovery for {db_name}: No DB URL.[/yellow]"
                    )
                    continue

                # Initialize Orchestrator
                # We need to construct paths similar to 'discover' command logic
                # Or let Orchestrator defaults handle it.
                # Better to pass explicit defaults from params if available.

                disc_paths = params.get("paths", {}).get("discovery", {}) or params.get(
                    "discovery", {}
                )

                whitelist_file = (
                    db.get("whitelist_file")
                    or disc_paths.get("whitelist_file")
                    or params.get("whitelist_file")
                    or f"discovery_whitelist_{current_conn}.yaml"
                )
                rules_file = (
                    db.get("rules_file")
                    or disc_paths.get("rules_file")
                    or params.get("rules_file")
                    or f"discovery_rules_{current_conn}.yaml"
                )

                # Anchoring
                # project_root is already defined from ProjectService.detect_project_root(config_path)

                # Resolve Whitelist
                if Path(whitelist_file).is_absolute():
                    wl_path = whitelist_file
                else:
                    wl_path = str(project_root / whitelist_file)

                # Resolve Rules
                if Path(rules_file).is_absolute():
                    r_path = rules_file
                else:
                    r_path = str(project_root / rules_file)

                # console.print(f"DEBUG: {db_name} -> WL Path: {wl_path} (Exists: {Path(wl_path).exists()})")

                orchestrator = DiscoveryOrchestrator(
                    params=context.params,
                    rules_path=r_path,
                    whitelist_path=wl_path,
                    registry_path=str(config_path),  # Not saving, but needed for init?
                    db_connection_str=db_conn_str,
                    db_config=db,
                    project_root=project_root,
                )

                entries = orchestrator.discover()
                registry.merge_discovered(entries)

            except Exception as e:
                console.print(f"[red]JIT Discovery failed for {db_name}: {e}[/red]")

    # 0. Generate Field Maps (if enabled)
    # Check generation.enable_field_maps (defaults to True)
    if params.get("generation", {}).get("enable_field_maps", True):
        import json

        cli_urls = json.loads(db_url_map) if db_url_map else {}

        def get_url_safe(conf_name, db_config={}):
            if conf_name in cli_urls:
                return cli_urls[conf_name]
            
            # Try to resolve from db_config if provided
            imp_spec = db_config.get("import_spec")
            if imp_spec and isinstance(imp_spec, dict):
                module = imp_spec.get("module")
                symbol = imp_spec.get("symbol")
                if module and symbol:
                    imp = [f"from {module} import {symbol}"]
                    url = resolve_db_url(conf_name, imp)
                    if url:
                        return url
            
            # Match in databases list to find proper imports if db_config is incomplete
            lookup_name = conf_name.split(".")[-1]

            for d in databases:
                d_id = d.get("id")
                d_name = d.get("name")
                d_ref = d.get("connection_ref") or d.get("connection_obj")
                if d_id == conf_name or d_name == conf_name or d_ref == conf_name or d_id == lookup_name:
                    db_imp = d.get("global_import")
                    imp = [db_imp] if db_imp else registry.global_imports
                    if d_ref and "." in d_ref:
                        imp = [".".join(d_ref.split(".")[:-1])] + imp
                    
                    url = resolve_db_url(lookup_name, imp)
                    if url:
                         return url
                    break

            # Fallback to global imports
            url = resolve_db_url(lookup_name, registry.global_imports)
            return url

        generated_maps = _run_field_map_generation(
            context, config_path, databases, get_url_safe, force=force
        )

        if generated_maps:
            for table_name, mod_path in generated_maps.items():
                if table_name in registry.tables:
                    registry.tables[table_name]["field_map"] = mod_path
                else:
                    # Robust Lookup: Key might be a custom_name/alias in the registry
                    # but table_name is the raw table from whitelist/discovery
                    found = False
                    for reg_key, meta in registry.tables.items():
                        if meta.get("table") == table_name:
                            meta["field_map"] = mod_path
                            found = True
                            break
                    
                    if not found:
                        # Fallback: check if table_name WAS the custom key but table attribute is missing/different
                        # (Unlikely in current sync flow but safe)
                        pass

        # Ensure new modules are picked up
        importlib.invalidate_caches()

    # Inject valid paths for security from context

    # Inject valid paths for security from context
    # Also inject the resolved datacubes_dir since params determines it.

    # Resolving datacubes_dir locally just in case context.valid_paths misses it
    # (Context might rely on static registry or defaults, but params can contain overrides)
    dc_dir = params.get("paths", {}).get("target", {}).get("datacubes_dir")

    valid_paths = set(context.valid_paths)  # Use set for deduplication
    if dc_dir:
        # Resolve against project root if relative
        if Path(dc_dir).is_absolute():
            valid_paths.add(str(dc_dir))
        else:
            valid_paths.add(str(project_root / dc_dir))

    # Debug: Check if registry uses valid_paths correctly
    registry.valid_paths = list(valid_paths)
    registry.valid_fieldmap_paths = context.valid_fieldmap_paths

    get_url = _get_db_url_callback(registry, db_url_map, params=context.params)

    # Group tables by target file
    file_groups = registry.group_tables_by_file()

    # --- Force Clean (User Request) ---
    # If force is True, we wipe the target datacubes_dir to ensure no stale files remain.
    if force and not dry_run:
        if dc_dir and Path(dc_dir).is_dir():
             target_root = Path(dc_dir)
             if not target_root.is_absolute():
                  target_root = project_root / dc_dir
             
             from sibi_flux.core import SecureResource
             secure = SecureResource(project_root=project_root)
             
             # Safety: Ensure we aren't deleting root or critical paths
             if is_secure_path(str(target_root / "dummy"), registry.valid_paths) or target_root.name in ["dataobjects", "gencubes", "datacubes"]:
                 console.print(f"[bold red]Force Wipe:[/bold red] Cleaning {target_root}...")
                 for item in target_root.iterdir():
                     if item.name == "__init__.py":
                         continue
                     if item.is_dir():
                         shutil.rmtree(item)
                     else:
                         item.unlink()
             else:
                 console.print(f"[yellow]Skipping force wipe of {target_root} - Path security check precaution.[/yellow]")

    summary_table = Table(title="Sync Results")

    summary_table.add_column("File", style="magenta")
    summary_table.add_column("Classes", style="cyan")
    summary_table.add_column("Status")

    generated_registry = {}

    for file_path_str, items in file_groups.items():
        file_path = Path(file_path_str)
        if not file_path.is_absolute():
            file_path = project_root / file_path

        if not is_secure_path(file_path, registry.valid_paths):
            console.print(
                f"[bold red]Blocked:[/bold red] {file_path_str} is outside allowed paths."
            )
            continue

        is_append = False
        existing_content = ""

        # --- Registry Collection (Always run, even if skipped) ---
        # Structure: {conf_obj: {table_name: {class_name: ..., path: ...}}}
        for item in items:
            t_name = item[0]
            conf_obj = item[1]
            cls_n = item[4]
            # Calculate path relative to project root
            try:
                rel_path = file_path.relative_to(project_root)
            except Exception:
                rel_path = file_path

            if conf_obj not in generated_registry:
                generated_registry[conf_obj] = {}

            # Preserve custom_name from existing registry if present
            existing_meta = registry.get_table_details(t_name)
            custom_name = existing_meta.get("custom_name")

            entry_data = {
                "class_name": cls_n,
                "path": str(rel_path),
                "custom_name": custom_name,
            }

            generated_registry[conf_obj][t_name] = entry_data

        if file_path.exists() and not force:
            with open(file_path, "r") as f:
                existing_content = f.read()

            missing_items = []
            for item in items:
                # item is (table_name, conf_obj, base_cls, base_imp, cls_name)
                # Check if cls_name is already in file
                cls_name = item[4]
                if f"class {cls_name}" not in existing_content:
                    missing_items.append(item)

            if not missing_items:
                summary_table.add_row(
                    file_path_str,
                    str(len(items)),
                    "[yellow]Skipped (All Exist)[/yellow]",
                )
                continue

            items = missing_items
            is_append = True

        if dry_run:
            status = (
                "[blue]Dry Run (Append)[/blue]" if is_append else "[blue]Dry Run[/blue]"
            )
            summary_table.add_row(file_path_str, str(len(items)), status)
            continue

        # Prepare File Content
        imports_list, classes_code = generate_datacube_module_code(
            items=items,
            registry=registry,
            get_db_url_callback=get_url,
            logger=console.print,
        )
        imports = set(imports_list)

        # Collect used config objects for this file to filter imports
        used_configs = set(item[1] for item in items if item[1])  # item[1] is conf_obj
        filtered_global_imports = filter_global_imports(
            registry.global_imports, used_configs, ignored_prefixes=["solutions.conf"]
        )

        if not classes_code:
            status_msg = "[red]Failed (No Classes Generated)[/red]" if not is_append else "[red]Failed to Append[/red]"
            summary_table.add_row(file_path_str, "0", status_msg)
            continue

        if not is_append:
            # We are generating the field map with Mapping type hint, so we should allow it in the generator
            # but this file writes the datacube class.

            full_content = (
                sorted(list(imports))
                + filtered_global_imports
                + ["\n# --- Generated ---"]
                + classes_code
            )
            file_path.parent.mkdir(parents=True, exist_ok=True)
            with open(file_path, "w") as f:
                f.write("\n".join(full_content))
            status_msg = "[green]Generated[/green]"
        else:
            # Append only the new classes
            with open(file_path, "a") as f:
                f.write("\n\n" + "\n".join(classes_code))
            status_msg = f"[green]Appended {len(classes_code)} Classes[/green]"

        # Format using Ruff
        subprocess.run(
            ["uv", "run", "ruff", "format", str(file_path)], capture_output=True
        )
        summary_table.add_row(file_path_str, str(len(items)), status_msg)

    console.print(summary_table)

    # --- Write Datacube Registry ---
    reg_rel_path = params.get("paths", {}).get("repositories", {}).get(
        "global_datacube_registry_file"
    ) or params.get("global_datacube_registry_file")

    if reg_rel_path and generated_registry:
        try:
            if Path(reg_rel_path).is_absolute():
                reg_file = Path(reg_rel_path)
            else:
                reg_file = project_root / reg_rel_path

            reg_file.parent.mkdir(parents=True, exist_ok=True)

            # Group Logic Applied above.
            # Sort keys for stability
            reg_data = {
                k: dict(sorted(v.items()))
                for k, v in sorted(generated_registry.items())
            }

            with open(reg_file, "w") as f:
                yaml.dump(reg_data, f, sort_keys=False)

            total_tables = sum(len(tables) for tables in generated_registry.values())
            console.print(
                f"[green]Updated Datacube Registry at {reg_rel_path} ({total_tables} tables across {len(generated_registry)} groups)[/green]"
            )
        except Exception as e:
            console.print(f"[red]Failed to write Datacube Registry: {e}[/red]")


@app.command()
def discover(
    config_file: Optional[Path] = typer.Option(None, "--config"),
    db_conf: str = typer.Option(
        "replica_db_conf", help="Config object to use for discovery introspection"
    ),
    db_url_map: Optional[str] = typer.Option(None, "--db-urls"),
    env_file: Optional[Path] = typer.Option(
        None, "--env-file", "-e", help="Path to environment file"
    ),
    update: bool = typer.Option(
        False, "--update", help="Update the registry file in place"
    ),
    prune: bool = typer.Option(
        False,
        "--prune",
        help="Remove tables from registry if they are not in the discovery result",
    ),
    run_sync: bool = typer.Option(
        False, "--sync", help="Run sync immediately after update"
    ),
    dry_run: bool = typer.Option(
        False, "--dry-run", help="Preview changes without saving (overrides --update)"
    ),
    generate_fields: bool = typer.Option(
        False,
        "--generate-fields",
        help="Generate field_map files for discovered tables",
    ),
    force: bool = typer.Option(
        False, "--force", "-f", help="Force overwrite of existing field maps"
    ),
    fields_root: str = typer.Option(
        "solutions.conf.transforms.fields",
        "--fields-root",
        help="Python path root for field maps",
    ),
) -> None:
    config_path = config_file or context.default_config
    if not config_path:
        console.print("[red]No config file specified and no default configured.[/red]")
        raise typer.Exit(code=1)

    # --- 1. Load Environment ---
    project_root = ProjectService.detect_project_root(config_path)
    ProjectService.setup_environment(project_root, env_file)

    gen_config_path = config_path.parent / "generator_config.yaml"

    # Load Registry Config (Bootstrap if missing)
    if config_path.exists():
        config_data = _load_and_resolve_config(config_path)
    else:
        # If registry file doesn't exist (e.g. first run), initialize with minimal settings
        # We need "cubes_root_path" from params usually.
        # But wait, config_path IS the registry file path.
        console.print(
            f"[yellow]Registry file {config_path} not found. Initializing empty registry.[/yellow]"
        )
        config_path.parent.mkdir(parents=True, exist_ok=True)
        # Try to infer cubes_root_path from context params if available
        cubes_root = "dataobjects/gencubes"  # Default fallback
        if context.params and "paths" in context.params:
            cubes_root = (
                context.params.get("paths", {})
                .get("target", {})
                .get("datacubes_dir", cubes_root)
            )

        config_data = {"settings": {"cubes_root_path": cubes_root}}
        # We don't save it yet? Or should we?
        # DatacubeRegistry will use this data. If we save later, it's fine.

    registry = DatacubeRegistry(config_data)

    import json

    # Resolve DB URL
    cli_urls = json.loads(db_url_map) if db_url_map else {}

    def get_url(conf_name, db_config={}):
        if conf_name in cli_urls:
            return cli_urls[conf_name]
            
        # Try to resolve from db_config if provided
        imp_spec = db_config.get("import_spec")
        if imp_spec and isinstance(imp_spec, dict):
            module = imp_spec.get("module")
            symbol = imp_spec.get("symbol")
            if module and symbol:
                imp = [f"from {module} import {symbol}"]
                url = resolve_db_url(conf_name, imp)
                if url:
                    return url
        
        # Fallback to global imports
        url = resolve_db_url(conf_name, registry.global_imports)
        if url:
            return url
            
        raise ValueError(
            f"Could not resolve DB URL for '{conf_name}'. Provide via --db-urls or check imports."
        )

    # --- Initialize Global Field Registry (DEPRECATED) ---
    field_registry = None
    # if context.field_registry_path:
    #     field_registry = GlobalFieldRegistry(context.field_registry_path)

    # --- Core Workflow via Orchestrator ---

    # Remove global rules_path resolution from outside the loop
    # rules_path = config_path.parent / "discovery_rules.yaml" # MOVED INSIDE LOOP

    params = context.params
    databases = params.get("databases", [])

    # Fallback to single DB mode if no databases defined (Backwards Compatibility)
    if not databases:
        databases = [
            {
                "name": db_conf,
                "connection_obj": db_conf,
                # Use standard name if not defined
                "whitelist_file": "discovery_whitelist.yaml",
                "rules_file": "discovery_rules.yaml",
            }
        ]

    # Filter if user requested specific DB via CLI (using db_conf arg as filter name)
    # The `db_conf` argument defaults to "replica_db_conf".
    target_db_name = None

    aggregated_entries = {}
    last_orchestrator = None

    # Load existing all_tables data if accumulating
    # Load existing all_tables data if accumulating
    discovery_conf = params.get("paths", {}).get("discovery") or params.get("discovery", {})
    global_tables_file = (
        discovery_conf.get("all_tables_file")
        or params.get("all_tables_file")
        or "dataobjects/globals/all_tables.yaml"
    )
    
    # Handle absolute resolution (consistent with scan)
    if Path(global_tables_file).is_absolute():
        global_tables_path = Path(global_tables_file)
    else:
        global_tables_path = project_root / global_tables_file
    all_tables_data = {}
    if global_tables_path.exists():
        with open(global_tables_path, "r") as f:
            all_tables_data = yaml.safe_load(f) or {}

    for db_config in databases:
        db_name = db_config.get("id") or db_config.get("name", "unknown")
        conn_obj = db_config.get("connection_ref") or db_config.get("connection_obj")

        # Determine whitelist path
        wl_filename = db_config.get("whitelist_file")
        if not wl_filename:
            # Try global param fallback
            wl_filename = params.get("discovery", {}).get(
                "whitelist_file"
            ) or params.get("whitelist_file")

        if not wl_filename:
            # Default convention: discovery_whitelist_<db_name>.yaml
            wl_filename = f"discovery_whitelist_{conn_obj}.yaml"
        
        whitelist_path = project_root / wl_filename

        # Fallback 1: Auto-generated per-db (from scan/map fallback logic)
        if not whitelist_path.exists():
            fb = project_root / f"discovery_whitelist_{conn_obj}.yaml"
            if fb.exists():
                whitelist_path = fb
        
        # Fallback 2: Global whitelist.yaml (Generated by 'whitelist' command)
        if not whitelist_path.exists():
            global_wl = project_root / "whitelist.yaml"
            if global_wl.exists():
                console.print(f"[dim]Fallback: Using global {global_wl.name} for {db_name}[/dim]")
                whitelist_path = global_wl

        # Determine rules path
        rules_filename = db_config.get("rules_file")
        if not rules_filename:
            rules_filename = params.get("discovery", {}).get(
                "rules_file"
            ) or params.get("rules_file")

        if not rules_filename:
            rules_filename = f"discovery_rules_{conn_obj}.yaml"
        rules_path = project_root / rules_filename

        # Determine blacklist path
        bl_filename = db_config.get(
            "blacklist_file", f"discovery_blacklist_{conn_obj}.yaml"
        )
        blacklist_path = project_root / bl_filename

        console.print(f"[bold cyan]Discovering: {db_name} ({conn_obj})[/]")
        console.print(f"  [cyan]Rules:[/] {rules_path.name}")
        console.print(f"  [cyan]Whitelist:[/] {whitelist_path.name}")

        try:
            # Use specific import for this DB if available, else standard registry imports (which might be empty now)
            # Support both single string and list in config for robustness
            # Support proper import_spec from new config or legacy global_import
            import_spec = db_config.get("import_spec")
            if import_spec and isinstance(import_spec, dict):
                imp = import_spec.get("module")
            else:
                imp = db_config.get("global_import")
            db_imports = [imp] if imp else registry.global_imports
            if not db_imports and registry.global_imports:
                db_imports = registry.global_imports

            db_conn_str = get_url(conn_obj, db_config)
        except Exception as e:
            console.print(
                f"[red]Could not resolve connection {conn_obj} for {db_name}: {e}. Skipping.[/red]"
            )
            continue

        # Resolve Registry Path (Target)
        reg_rel_path = params.get("paths", {}).get("repositories", {}).get(
            "global_datacube_registry_file"
        ) or params.get("global_datacube_registry_file")
        
        real_registry_path = str(config_path) # Fallback to config if not defined (legacy behavior)
        if reg_rel_path:
             if Path(reg_rel_path).is_absolute():
                 real_registry_path = reg_rel_path
             else:
                 real_registry_path = str(project_root / reg_rel_path)

        orchestrator = DiscoveryOrchestrator(
            field_registry=field_registry,
            params=config_data,
            rules_path=str(rules_path),
            whitelist_path=str(whitelist_path),
            registry_path=real_registry_path,
            db_connection_str=db_conn_str,
            db_config=db_config,
            all_tables_path=str(global_tables_path),
            project_root=project_root,
        )

        try:
            entries = orchestrator.discover()
            aggregated_entries.update(entries)
            last_orchestrator = orchestrator

            # --- Capture Raw Tables for all_tables.yaml ---
            if hasattr(orchestrator, "raw_tables") and orchestrator.raw_tables:
                # Sort for consistency
                all_tables_data[conn_obj] = sorted(list(orchestrator.raw_tables))
                
                matched_count = len(entries)
                skipped_count = len(orchestrator.raw_tables) - matched_count
                
                console.print(f"  [green]✓ Matched {matched_count} tables[/green]")
                if skipped_count > 0:
                     console.print(f"  [dim]Skipped {skipped_count} unmatched tables[/dim]")
        except Exception as e:
            console.print(f"[red]Discovery failed for {db_name}: {e}[/red]")
            if not dry_run:
                raise  # Fail hard if not dry run? Or continue? Let's buffer errors?
                # For now, log and continue might result in partial registry which is bad (prune would wipe missing).
                # Fail safe:
                return

    # Aggregate global imports from ALL databases to ensure registry has them
    aggregated_global_imports = set(config_data.get("global_imports", []))
    for db in databases:
        if "global_import" in db:
            aggregated_global_imports.add(db["global_import"])

    # Save Aggregated Registry
    if last_orchestrator:
        console.print("")
        # Inject aggregated imports into the last orchestrator's update logic?
        # The orchestrator's save_registry loads existing, updates tables, and saves.
        # It DOES NOT currently update global_imports. We need to add that cap.

        # Helper manual update for now, or update Orchestrator to support it?
        last_orchestrator.save_registry(
            aggregated_entries,
            dry_run=dry_run,
            prune=prune or force,
            global_imports=list(aggregated_global_imports),
        )

        # Save all_tables.yaml
        if not dry_run and all_tables_data:
            with open(global_tables_path, "w") as f:
                yaml.dump(all_tables_data, f, sort_keys=False)
            console.print(
                f"[bold green]Updated {global_tables_file} with raw tables from providers.[/bold green]"
            )
        elif dry_run:
            console.print(
                f"[yellow]DRY RUN: Would update {global_tables_file} with {len(all_tables_data)} providers.[/yellow]"
            )

    # Save Registry changes (collected during discovery)
    if not dry_run and field_registry:
        field_registry.save()

    # Generate Field Maps (Bootstrap)
    if generate_fields and not dry_run:
        console.print("")
        console.rule("[bold blue]Generating Field Maps[/]")

        # Reload registry to ensure we have latest discovered tables
        with open(config_path, "r") as f:
            updated_config_data = yaml.safe_load(f)
        updated_registry = DatacubeRegistry(updated_config_data, params=context.params)

        # Convert python path to physical path
        phys_root = Path(fields_root.replace(".", "/"))

        # Group tables by connection to use correct inspector
        tables_by_conn = {}
        for t_name, t_data in updated_registry.tables.items():
            conn = t_data.get("connection_obj", updated_registry.default_connection_obj)
            if conn not in tables_by_conn:
                tables_by_conn[conn] = {}
            tables_by_conn[conn][t_name] = t_data

        for conn_obj, table_group in tables_by_conn.items():
            try:
                db_url = get_url(conn_obj)
                engine = sa.create_engine(db_url)
                inspector = inspect(engine)

                # Use the promoted Generator
                generate_field_map_files(
                    discovered_entries=table_group,
                    inspector=inspector,
                    root_path=phys_root,
                    force=force,
                    logger=console.print,
                    allowed_paths=(
                        updated_registry.valid_fieldmap_paths
                        if hasattr(updated_registry, "valid_fieldmap_paths")
                        else None
                    ),
                )
            except Exception as e:
                console.print(
                    f"[red]Error generating fields for connection {conn_obj}: {e}[/red]"
                )

    # Chained Sync
    if run_sync:
        if dry_run:
            console.print("[yellow]Skipping sync in dry-run mode.[/yellow]")
        elif not update:
            console.print(
                "[yellow]Sync skipped: Registry not updated (use --update to enable chaining).[/yellow]"
            )
        else:
            console.print("")
            console.rule("[bold blue]Auto-Syncing Datacubes[/]")
            # Call sync command directly with current context options
            sync(
                config_file=config_path,
                db_url_map=db_url_map,
                force=True,
                env_file=env_file,
                dry_run=False,
            )


@app.command()
def scan(
    config_file: Optional[Path] = typer.Option(None, "--config"),
    db_url_map: Optional[str] = typer.Option(None, "--db-urls"),
    env_file: Optional[Path] = typer.Option(None, "--env-file", "-e"),
    db_name: Optional[str] = typer.Option(
        None, "--db", help="Target specific database from params"
    ),
) -> None:
    """
    Introspects configured databases and dumps table lists to YAML.
    Step 2 of the workflow.
    """
    config_path = config_file or context.default_config
    if not config_path:
        console.print("[red]No config file specified and no default configured.[/red]")
        raise typer.Exit(code=1)

    # --- 1. Load Environment ---
    project_root = ProjectService.detect_project_root(config_path)
    ProjectService.setup_environment(project_root, env_file)

    config_data = _load_and_resolve_config(config_path)
    # Sync context for nested calls or legacy logic
    context.params = config_data
    registry = DatacubeRegistry(config_data, params=context.params)

    import json

    cli_urls = json.loads(db_url_map) if db_url_map else {}

    # Helper Resolution
    def get_url_safe(conf_name, db_config={}):
        # Strip dot-paths if passed
        lookup_name = conf_name.split(".")[-1]

        if lookup_name in cli_urls:
            return cli_urls[lookup_name]
        if conf_name in cli_urls:
            return cli_urls[conf_name]
            
        # Try to resolve from db_config if provided
        imp_spec = db_config.get("import_spec")
        if imp_spec and isinstance(imp_spec, dict):
            module = imp_spec.get("module")
            symbol = imp_spec.get("symbol")
            if module and symbol:
                imp = [f"from {module} import {symbol}"]
                url = resolve_db_url(lookup_name, imp)
                if url:
                    return url

        # Fallback to global imports
        url = resolve_db_url(lookup_name, registry.global_imports)
        return url

    params = config_data
    databases = params.get("databases", [])

    # Filter targets
    target_dbs = databases
    if db_name:
        target_dbs = [d for d in databases if d.get("name") == db_name]
        if not target_dbs:
            console.print(f"[red]Database '{db_name}' not found.[/red]")
            raise typer.Exit(code=1)

    console.print(f"[bold cyan]Scanning {len(target_dbs)} databases...[/bold cyan]")

    # Resolve global output file
    # First check nested discovery block (standard), then root (fallback)
    discovery_conf = params.get("paths", {}).get("discovery") or params.get("discovery", {})
    global_tables_file = (
        discovery_conf.get("all_tables_file") 
        or params.get("all_tables_file") 
        or "dataobjects/globals/all_tables.yaml"
    )
    
    # If using absolute path (resolved from context), use it directly
    # Otherwise treat as relative to PROJECT ROOT (not config file location)
    path_obj = Path(global_tables_file)
    if path_obj.is_absolute():
        global_tables_path = path_obj
    else:
        global_tables_path = project_root / global_tables_file
        
    # Ensure dir exists
    if not global_tables_path.parent.exists():
        global_tables_path.parent.mkdir(parents=True, exist_ok=True)

    # Load existing data to preserve config for DBs not being scanned
    all_tables_data = {}
    if global_tables_path.exists():
        with open(global_tables_path, "r") as f:
            all_tables_data = yaml.safe_load(f) or {}

    for i, db in enumerate(target_dbs, 1):
        db_id = db.get("id") or db.get("name") or "unknown"
        conn_obj = db.get("connection_obj") or db.get("connection_ref") or db_id

        console.print(f"\n[bold cyan][{i}/{len(target_dbs)}] Scanning: {db_id} ...[/bold cyan]")
        try:
            db_url = get_url_safe(conn_obj, db)
            if not db_url:
                console.print(f"  [red]Could not resolve URL for {conn_obj}[/red]")
                continue

            engine = sa.create_engine(db_url)
            inspector = inspect(engine)
            tables = sorted(inspector.get_table_names())

            # Update shared dictionary
            all_tables_data[conn_obj] = tables
            console.print(f"  [green]✓ Found {len(tables)} tables[/green]")

        except Exception as e:
            console.print(f"  [red]✗ Scan failed for {db_id}: {e}[/red]")
            if params.get("debug"):
                raise e

    # Persist aggregated result
    with open(global_tables_path, "w") as f:
        yaml.dump(all_tables_data, f, sort_keys=False)

    console.print(
        f"\n[bold green]Success: Updated global table list at {global_tables_path}[/bold green]"
    )


@app.command()
def drift(
    config_file: Optional[Path] = typer.Option(None, "--config"),
    db_url_map: Optional[str] = typer.Option(
        None,
        "--db-urls",
        help="Optional JSON mapping. If omitted, tries to resolve from code.",
    ),
    env_file: Optional[Path] = typer.Option(
        None, "--env-file", "-e", help="Path to environment file"
    ),
) -> None:
    """
    Checks for 'drift' between the generated Python classes and the DB schema.
    """
    config_path = config_file or context.default_config
    if not config_path:
        console.print("[red]No config file specified and no default configured.[/red]")
        raise typer.Exit(code=1)

    # --- 1. Load Environment ---
    project_root = ProjectService.detect_project_root(config_path)
    ProjectService.setup_environment(project_root, env_file)

    config_data = _load_and_resolve_config(config_path)

    registry = DatacubeRegistry(config_data)
    get_url = _get_db_url_callback(registry, db_url_map, params=config_data)
    cli_urls = json.loads(db_url_map) if db_url_map else {}

    drift_table = Table(title="Schema Drift Analysis")
    drift_table.add_column("Class", style="cyan")
    drift_table.add_column("Status", style="bold")
    drift_table.add_column("Issues", style="red")

    # The keys in processed_mappings correspond to the attribute names we need to check
    attribute_names = list(registry.processed_mappings.keys())

    for table_name, details in registry.tables.items():
        target = details.get("save_to_path", details.get("path"))
        if not target:
            drift_table.add_row(
                table_name, "[red]Config Error[/red]", "Missing save_to_path"
            )
            continue
        path = Path(target)
        if not path.exists():
            console.print(
                f"[yellow]Skipping {table_name}: File {path} not found.[/yellow]"
            )
            continue

        # 1. Determine Class Name
        provided_class_name = details.get("class_name")
        class_name = (
            provided_class_name
            if provided_class_name
            else "".join(w.capitalize() for w in table_name.split("_")) + "Dc"
        )

        # 2. Dynamically load the generated class from the file
        try:
            spec = importlib.util.spec_from_file_location(
                f"dynamic_mod_{table_name}", path
            )
            mod = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(mod)
            dc_class = getattr(mod, class_name)
        except Exception as e:
            drift_table.add_row(class_name, "[red]Load Error[/red]", str(e))
            continue

        # 3. Determine DB URL
        # Priority: CLI Override > Class Attribute > Registry Config
        db_url = None
        conf_obj = details.get(
            "connection_obj", details.get("config_obj", registry.default_connection_obj)
        )

        if conf_obj in cli_urls:
            db_url = cli_urls[conf_obj]
        elif hasattr(dc_class, "connection_url"):
            db_url = getattr(dc_class, "connection_url")
        elif hasattr(dc_class, "config") and isinstance(dc_class.config, dict):
            db_url = dc_class.config.get("connection_url")

        # Fallback to registry resolution
        if not db_url:
            try:
                db_url = get_url(conf_obj)
            except Exception:
                pass

        # 4. Introspect DB
        try:
            engine = sa.create_engine(db_url)
            inspector = inspect(engine)
            db_cols = {c["name"] for c in inspector.get_columns(table_name)}
        except Exception as e:
            drift_table.add_row(class_name, "[red]DB Error[/red]", repr(e))
            continue

        # 5. Extract Field Map (if any)
        field_map = getattr(dc_class, "field_map", None)
        if (
            not field_map
            and hasattr(dc_class, "config")
            and isinstance(dc_class.config, dict)
        ):
            field_map = dc_class.config.get("field_map")

        # 6. Check Drift
        issues = check_drift(dc_class, db_cols, attribute_names, field_map=field_map)

        if not issues:
            drift_table.add_row(class_name, "[green]Synced[/green]", "None")
        else:
            drift_table.add_row(class_name, "[yellow]DRIFT[/yellow]", "\n".join(issues))

    console.print(drift_table)


@app.command()
def propose_rules(
    config_file: Optional[Path] = typer.Option(None, "--config"),
    dry_run: bool = typer.Option(
        False, "--dry-run", help="Preview rules without saving"
    ),
    conf_override: Optional[str] = typer.Option(None, "--rules", help="Override rules file"),
    force: bool = typer.Option(False, "--force", "-f", help="Force overwrite existing rules"),
    add_missing: bool = typer.Option(False, "--add-missing", help="Review and add missing rules only"),
):
    """
    Analyzes all_tables.yaml and proposes new discovery rules.
    """
    config_path = config_file or context.default_config
    if not config_path:
        console.print("[red]No config file specified and no default configured.[/red]")
        raise typer.Exit(code=1)

    # Resolve paths via helper to ensure Project Root logic is applied
    project_root = ProjectService.detect_project_root(config_path)
    resolved_config = _load_and_resolve_config(config_path)

    # We rely on all_tables.yaml being generated by scan/discover
    # The resolved config will have absolute paths for these if the helper worked.
    params = context.params

    # Prefer resolved values if available, but check nested 'paths' structure first
    # Structure: paths -> discovery -> all_tables_file
    discovery_conf = params.get("paths", {}).get("discovery") or params.get("discovery", {})
    all_tables_file = (
        discovery_conf.get("all_tables_file")
        or params.get("all_tables_file")
        or "dataobjects/globals/all_tables.yaml"
    )

    path_obj = Path(all_tables_file)
    if path_obj.is_absolute():
        all_tables_path = path_obj
    else:
        all_tables_path = project_root / all_tables_file

    # Verify Existence
    if not all_tables_path.exists():
         console.print(f"[red]Error: {all_tables_path} not found. Run 'dc scan' first.[/red]")
         raise typer.Exit(code=1)

    # Rules File
    if conf_override:
        rules_path = Path(conf_override)
    else:
        raw_rules = (
            discovery_conf.get("rules_file")
            or params.get("rules_file")
            or "discovery_rules.yaml"
        )
        rules_path = Path(raw_rules)

    if not rules_path.is_absolute():
        rules_path = project_root / rules_path

    console.print(f"[bold cyan]Input:[/] {all_tables_path}")
    console.print(f"[bold cyan]Rules File:[/] {rules_path}")

    # Guard: Require Add-Missing if Rules Exist (Force is ignored for safety)
    if rules_path.exists() and not add_missing and not dry_run:
        console.print(f"\n[yellow]Rules file already exists.[/yellow]")
        console.print("[yellow]Skipping proposal generation to protect manual edits.[/yellow]")
        console.print("[dim]Use --add-missing to append new rules.[/dim]")
        return

    engine = RuleEngine(all_tables_path, rules_path)
    engine.load()
    updates = engine.propose_rules()

    total_rules = sum(len(rules) for rules in updates.values())

    if dry_run:
        console.print(f"\n[bold yellow]Proposed {total_rules} Updates:[/]")
        for conn, rules in updates.items():
            console.print(f"[cyan]{conn}:[/]")
            for r in rules:
                console.print(f"  - {r}")
    else:
        engine.save_proposal(updates)
        console.print(f"\n[bold green]✓ Successfully generated/updated {total_rules} rules.[/bold green]")


@app.command()
def match(
    config_file: Optional[Path] = typer.Option(None, "--config"),
    db_name: Optional[str] = typer.Option(
        None, "--db", help="Target specific database from params"
    ),
) -> None:
    """
    Applies discovery rules to scanned tables and generates whitelists (registry).
    Step 3 of the workflow.
    """
    config_path = config_file or context.default_config
    if not config_path:
        console.print("[red]No config file specified and no default configured.[/red]")
        raise typer.Exit(code=1)

    import yaml

    # Load Params
    project_root = ProjectService.detect_project_root(config_path)
    params = context.params
    databases = params.get("databases", [])
    folder_prefix = params.get("folder_prefix", "solutions/dataobjects/gencubes/")
    fields_suffix = params.get("fields_module_root", "fields")

    target_dbs = databases
    if db_name:
        target_dbs = [d for d in databases if d.get("name") == db_name]
        if not target_dbs:
            console.print(f"[red]Database '{db_name}' not found.[/red]")
            raise typer.Exit(code=1)

    for i, db in enumerate(target_dbs, 1):
        name = db.get("name")
        all_tables_file = db.get("all_tables_file")
        rules_file = db.get("rules_file")
        whitelist_file = db.get("whitelist_file")
        conn_obj = db.get("connection_obj")

        console.print(f"\n[bold cyan][{i}/{len(target_dbs)}] Matching: {name} ...[/bold cyan]")

        if not (all_tables_file and rules_file and whitelist_file):
            console.print(f"  [yellow]⚠ Skipping {name}: Missing file config.[/yellow]")
            continue
        
        console.print(f"  [cyan]Rules:[/] {Path(rules_file).name}")
        console.print(f"  [cyan]Whitelist:[/] {Path(whitelist_file).name}")

        # Re-resolve paths if relative
        tables_path = project_root / all_tables_file
        rules_path = project_root / rules_file
        out_path = project_root / whitelist_file

        # fields_suffix is a folder name (e.g. 'fields'), NOT a package.
        # However, if it mistakenly contains dots, we should prefer slashes for path arithmetic,
        # BUT only if it doesn't already exist as a directory (avoiding v1.0 splitting).
        sanitized_suffix = fields_suffix
        if "." in sanitized_suffix and "/" not in sanitized_suffix and not Path(sanitized_suffix).exists():
            sanitized_suffix = sanitized_suffix.replace(".", "/")
        
        # Path Composition
        db_domain = db.get("db_domain")
        import_base = Path(folder_prefix)
        if db_domain:
            import_base = import_base / db_domain
        
        import_base = import_base / sanitized_suffix

        try:
            import_base = import_base.relative_to(project_root)
        except ValueError:
            try:
                import_base = import_base.relative_to(Path.cwd())
            except ValueError:
                pass
        
        # Fields module base is a package string for the YAML 'field_map' entries
        fields_module_base = str(import_base).replace("/", ".")

        tables_path = project_root / all_tables_file
        rules_path = project_root / rules_file
        out_path = project_root / whitelist_file

        if not tables_path.exists():
            console.print(
                f"[red]Skipping {name}: {all_tables_file} not found. Run 'scan' first.[/red]"
            )
            continue

        if not rules_path.exists():
            continue

        # Load existing whitelist to preserve customizations
        existing_whitelist = {}
        if out_path.exists():
            with open(out_path, "r") as f:
                data = yaml.safe_load(f) or {}
                existing_whitelist = data.get("tables", {})

        with open(tables_path, "r") as f:
            all_tables = yaml.safe_load(f) or []

        with open(rules_path, "r") as f:
            rules_data = yaml.safe_load(f) or []

        # Match Logic
        console.print(
            f"[bold cyan]Matching: {name} ({len(all_tables)} tables)[/bold cyan]"
        )

        matches = {}
        matched_count = 0

        for table in sorted(all_tables):
            # Find first matching rule
            matched_rule = None
            for r in rules_data:
                pattern = r.get("pattern")
                mtype = r.get("match_type", "exact")

                is_match = False
                if mtype == "exact" and table == pattern:
                    is_match = True
                elif mtype == "prefix" and table.startswith(pattern):
                    is_match = True
                elif mtype == "regex":
                    import re

                    if re.search(pattern, table):
                        is_match = True

                if is_match:
                    matched_rule = r
                    break

            if matched_rule:
                # Construct Registry Entry
                # Resolve path using folder_prefix
                template = matched_rule.get("output_template", f"{table}_cubes.py")
                domain = matched_rule.get("domain", "common")

                # Path Construction: folder_prefix + db_domain + domain + output_template
                db_domain = db.get("db_domain", "")

                # Careful not to double slash if db_domain is empty, but Path handles it.
                # template should now be just filename per rule updates.
                full_path_obj = Path(folder_prefix)
                if db_domain:
                    full_path_obj = full_path_obj / db_domain

                full_path_obj = full_path_obj / domain / template
                full_path = str(full_path_obj)

                # Class Name Generation
                # Check if exists in old whitelist
                existing_entry = existing_whitelist.get(table, {})

                custom_name = existing_entry.get("custom_name")

                if custom_name:
                    class_name = custom_name
                elif existing_entry.get("class_name"):
                    class_name = existing_entry.get("class_name")
                else:
                    class_suffix = params.get("class_suffix", "Dc")
                    class_name = (
                        "".join(w.capitalize() for w in table.split("_")) + class_suffix
                    )

                field_map_template = matched_rule.get(
                    "field_map_template",
                    f"{fields_module_base}.{{domain}}.{{table}}.field_map",
                )

                # Construct defaults
                entry = {
                    "path": full_path,
                    "connection_obj": matched_rule.get("db_conn_override") or conn_obj,
                    "domain": domain,
                    "class_name": class_name,
                    # Ensure field_map is assigned
                    "field_map": field_map_template.format(domain=domain, table=table),
                }

                # Preserve custom_name if present, else default to None
                entry["custom_name"] = custom_name if custom_name else None

                # Preserve other fields if needed?
                # User asked specifically for keys on class_name preservation.
                # But generally we might want to respect other overrides?
                # For now, strict to class_name per request + generation logic.

                matches[table] = entry
                matched_count += 1

        # Write Whitelist (Registry)
        output_data = {"tables": matches}
        with open(out_path, "w") as f:
            yaml.dump(output_data, f, sort_keys=False)

        console.print(
            f"[green]Matched {matched_count} tables. Written to {out_path}[/green]"
        )


@app.command()
def map(
    config_file: Optional[Path] = typer.Option(None, "--config"),
    db_url_map: Optional[str] = typer.Option(None, "--db-urls"),
    env_file: Optional[Path] = typer.Option(None, "--env-file", "-e"),
    db_name: Optional[str] = typer.Option(
        None, "--db", help="Target specific database from params"
    ),
    force: bool = typer.Option(False, "--force", "-f"),
) -> None:
    """
    Generates YAML field maps for tables in the whitelist.
    Step 4 of the workflow.
    """
    config_path = config_file or context.default_config
    if not config_path:
        console.print("[red]No config file specified.[/red]")
        raise typer.Exit(code=1)
    
    
    # Load Config
    config_data = _load_and_resolve_config(config_path)
    if not context.params:
        context.params = {}
    context.params.update(config_data)

    # --- 1. Load Environment ---
    project_root = ProjectService.detect_project_root(config_path)
    ProjectService.setup_environment(project_root, env_file)

    import json

    cli_urls = json.loads(db_url_map) if db_url_map else {}
    registry = DatacubeRegistry({}, params=context.params)  # Dummy reg for imports

    def get_url_safe(conf_name, db_config={}):
        lookup_name = conf_name.split(".")[-1]
        if lookup_name in cli_urls:
            return cli_urls[lookup_name]
        if conf_name in cli_urls:
            return cli_urls[conf_name]
            
        # Try to resolve from db_config if provided
        imp_spec = db_config.get("import_spec")
        if imp_spec and isinstance(imp_spec, dict):
            module = imp_spec.get("module")
            symbol = imp_spec.get("symbol")
            if module and symbol:
                imp = [f"from {module} import {symbol}"]
                url = resolve_db_url(conf_name, imp)
                if url:
                    return url

        # Fallback to global imports
        lookup_name = conf_name.split(".")[-1]
        url = resolve_db_url(lookup_name, registry.global_imports)
        return url

    params = context.params
    databases = params.get("databases", [])

    target_dbs = databases
    if db_name:
        target_dbs = [d for d in databases if d.get("name") == db_name]

    console.print(f"[bold cyan]Mapping fields for {len(target_dbs)} databases...[/bold cyan]")

    _run_field_map_generation(
        context, config_path, target_dbs, get_url_safe, force=force
    )
    return


@app.command()
def prepare(
    config_file: Optional[Path] = typer.Option(None, "--config"),
    db_url_map: Optional[str] = typer.Option(None, "--db-urls"),
    env_file: Optional[Path] = typer.Option(None, "--env-file", "-e"),
    db_name: Optional[str] = typer.Option(
        None, "--db", help="Target specific database (filtering scan/whitelist)"
    ),
    force: bool = typer.Option(False, "--force", "-f", help="Force rebuild/overwrite"),
    dry_run: bool = typer.Option(
        False, "--dry-run", help="Preview changes without executing"
    ),
) -> None:
    """
    Phase 1: Discovery & Rules. Executes Scan -> Propose -> Whitelist.
    Used for introspecting DB and generating a draft whitelist.
    """
    console.rule("[bold cyan]Datacube Prepare (Discovery Phase)[/]")

    # Resolve config_path
    config_path = config_file or context.default_config
    if not config_path:
        console.print("[red]No config file specified and no default configured.[/red]")
        raise typer.Exit(code=1)

    # Resolve Context (Registry/Config Paths)
    project_root = ProjectService.detect_project_root(config_path)
    ProjectService.setup_environment(project_root, env_file)

    # Load Params into context
    if config_path and config_path.exists():
        try:
            config_data = _load_and_resolve_config(config_path)
            context.params = config_data
            context.default_config = config_path
            console.print(f"[dim]Loaded context from {config_path}[/dim]")
        except Exception as e:
            console.print(f"[yellow]Warning: Failed to load context from {config_path}: {e}[/yellow]")

    # Step 0a: Scan
    console.print(f"\n[bold cyan]Step 0a: Scan (Introspection)[/]")
    scan(
        config_file=config_file if config_file else None,
        db_url_map=db_url_map,
        env_file=env_file,
        db_name=db_name,
    )

    # Step 0b: Propose Rules
    console.print(f"\n[bold cyan]Step 0b: Propose Rules[/]")
    propose_rules(
        config_file=config_file if config_file else None,
        dry_run=dry_run,
        conf_override=db_name,
        force=force,
        add_missing=False,
    )

    # Step 0c: Generate Whitelist
    console.print(f"\n[bold cyan]Step 0c: Generate Whitelist[/]")
    whitelist(
        config_file=config_file,
        db_name=db_name,
        db_url_map=db_url_map,
        env_file=env_file,
        force=force,
    )

    console.rule("[bold cyan]Prepare Phase Completed[/]")


@app.command()
def apply(
    config_file: Optional[Path] = typer.Option(None, "--config"),
    db_url_map: Optional[str] = typer.Option(None, "--db-urls"),
    env_file: Optional[Path] = typer.Option(None, "--env-file", "-e"),
    db_name: Optional[str] = typer.Option(
        None, "--db", help="Target specific database (filtering map/sync)"
    ),
    app_name: Optional[str] = typer.Option(
        None, "--app-name", help="Run create_cubes for the specified application"
    ),
    force: bool = typer.Option(False, "--force", "-f", help="Force rebuild/overwrite"),
    dry_run: bool = typer.Option(
        False, "--dry-run", help="Preview changes without executing"
    ),
) -> None:
    """
    Phase 2: Implementation. Executes Map -> Discover -> Sync -> (Optional) Create Cubes.
    Used for generating code based on the finalized rules and whitelist.
    """
    console.rule("[bold green]Datacube Apply (Implementation Phase)[/]")

    # Resolve config_path
    config_path = config_file or context.default_config
    if not config_path:
        console.print("[red]No config file specified and no default configured.[/red]")
        raise typer.Exit(code=1)

    # Resolve Context
    project_root = ProjectService.detect_project_root(config_path)
    ProjectService.setup_environment(project_root, env_file)

    # Load Params into context
    if config_path and config_path.exists():
        try:
            config_data = _load_and_resolve_config(config_path)
            context.params = config_data
            context.default_config = config_path
            console.print(f"[dim]Loaded context from {config_path}[/dim]")
        except Exception as e:
            console.print(f"[yellow]Warning: Failed to load context from {config_path}: {e}[/yellow]")

    # Step 1: Map (Read Whitelist -> Update Repository)
    console.print(f"\n[bold cyan]Step 1: Map Generation (Field Translations)[/]")
    map(
        config_file=config_file,
        db_url_map=db_url_map,
        env_file=env_file,
        db_name=db_name,
        force=force,
    )

    # Step 2: Discover (Read Whitelist -> Update Registry)
    console.print(f"\n[bold cyan]Step 2: Discovery (Registry Update)[/]")
    discover(
        config_file=config_file,
        db_conf="replica_db_conf",
        db_url_map=db_url_map,
        env_file=env_file,
        update=True,
        prune=force,
        run_sync=False,
        dry_run=dry_run,
        generate_fields=False,
        force=force,
    )

    # Step 3: Sync (Code Generation)
    console.print(f"\n[bold cyan]Step 3: Sync (FastAPI / Pydantic Code Generation)[/]")
    sync(
        config_file=config_file,
        db_url_map=db_url_map,
        env_file=env_file,
        force=force,
        dry_run=dry_run,
    )

    # Step 4: Create Cubes (Optional)
    if app_name:
        console.print(f"\n[bold cyan]Step 4: Application Cubes ({app_name})[/]")
        if dry_run:
            console.print("[yellow]Skipping create_cubes in dry-run mode.[/yellow]")
        else:
            from sibi_flux.init.cube_extender import create_cubes
            create_cubes(app_name)

    console.rule("[bold green]Apply Phase Completed[/]")


@app.command()
def workflow(
    config_file: Optional[Path] = typer.Option(None, "--config"),
    db_url_map: Optional[str] = typer.Option(None, "--db-urls"),
    env_file: Optional[Path] = typer.Option(None, "--env-file", "-e"),
    db_name: Optional[str] = typer.Option(
        None, "--db", help="Target specific database (filtering map/sync)"
    ),
    app_name: Optional[str] = typer.Option(
        None, "--app-name", help="Run create_cubes for the specified application"
    ),
    force: bool = typer.Option(False, "--force", "-f", help="Force rebuild/overwrite"),
    dry_run: bool = typer.Option(
        False, "--dry-run", help="Preview changes without executing"
    ),
) -> None:
    """
    Executes the full Datacube workflow: Prepare (Discovery) -> Apply (Implementation).
    """
    console.rule("[bold magenta]Starting Full Datacube Workflow[/]")

    # 1. Prepare
    prepare(
        config_file=config_file,
        db_url_map=db_url_map,
        env_file=env_file,
        db_name=db_name,
        force=force,
        dry_run=dry_run,
    )

    # 2. Apply
    apply(
        config_file=config_file,
        db_url_map=db_url_map,
        env_file=env_file,
        db_name=db_name,
        app_name=app_name,
        force=force,
        dry_run=dry_run,
    )

    console.rule("[bold magenta]Workflow Completed[/]")


@app.command()
def init(
    config_file: Optional[Path] = typer.Option(None, "--config"),
    db_conf: Optional[str] = typer.Option(
        None, help="Config object to use for introspection"
    ),
    db_name: Optional[str] = typer.Option(
        None, "--db", help="Target specific database from params"
    ),
    db_url_map: Optional[str] = typer.Option(None, "--db-urls"),
    env_file: Optional[Path] = typer.Option(None, "--env-file", "-e"),
    dump_schema: Optional[Path] = typer.Option(
        None, "--dump-schema", help="Dump database schema"
    ),
    init_rules: Optional[Path] = typer.Option(
        None, "--init-rules", help="Initialize discovery rules from DB tables"
    ),
    reset: bool = typer.Option(
        False, "--reset", help="Reset registry and config to defaults"
    ),
) -> None:
    """
    Initializes configuration and schema dumps.
    """
    params = context.params

    # Determine Targets
    databases = params.get("databases", [])
    target_dbs = []

    if db_name:
        # Filter specific DB
        target_dbs = [d for d in databases if d.get("name") == db_name]
        if not target_dbs:
            console.print(f"[red]Database '{db_name}' not found in params.[/red]")
            raise typer.Exit(code=1)
    elif databases:
        # All DBs
        target_dbs = databases
    else:
        # Legacy Fallback
        target_db_conf = db_conf or params.get(
            "default_connection_obj", "replica_db_conf"
        )
        target_dbs = [
            {
                "name": target_db_conf,
                "connection_obj": target_db_conf,
                "whitelist_file": "discovery_whitelist.yaml",
            }
        ]

    # Validate db_conf override if provided (only if single target or legacy)
    if db_conf and not db_name and not databases:
        target_dbs[0]["connection_obj"] = db_conf

    # --- 1. Load Environment ---
    project_root = ProjectService.detect_project_root(config_path)
    ProjectService.setup_environment(project_root, env_file)

    # Resolve Context (Registry/Config Paths)
    # Be robust if config_file doesn't exist yet
    config_path = config_file or context.default_config

    if not config_path:
        console.print("[red]No config file target specified.[/red]")
        raise typer.Exit(code=1)

    import json

    cli_urls = json.loads(db_url_map) if db_url_map else {}

    # Helper to resolve URL without a full registry instance if file missing
    def resolve_url_safe(conf_name, db_config={}):
        if conf_name in cli_urls:
            return cli_urls[conf_name]
            
        # Try to resolve from db_config if provided
        imp_spec = db_config.get("import_spec")
        if imp_spec and isinstance(imp_spec, dict):
            module = imp_spec.get("module")
            symbol = imp_spec.get("symbol")
            if module and symbol:
                imp = [f"from {module} import {symbol}"]
                url = resolve_db_url(conf_name, imp)
                if url:
                    return url

        # Fallback to databases list or global imports
        # We need a registry instance to get global_imports
        # If config_path exists, load a minimal registry
        reg = None
        if config_path.exists():
            with open(config_path, "r") as f:
                data = yaml.safe_load(f)
            reg = DatacubeRegistry(data, params=context.params)

        for db in databases:
            if db.get("connection_obj") == conf_name or db.get("connection_ref") == conf_name:
                db_imp = db.get("global_import")
                imp = [db_imp] if db_imp else (reg.global_imports if reg else [])
                url = resolve_db_url(conf_name, imp)
                if url:
                    return url
        
        # Final fallback using global imports from the minimal registry if available, or empty list
        if reg:
            url = resolve_db_url(conf_name, reg.global_imports)
            if url:
                return url

        # Fallback: try raw resolve (might fail if imports missing)
        url = resolve_db_url(conf_name, [])
        if url:
            return url
        raise ValueError(
            "Cannot resolve DB URL. Please ensure registry exists or use --db-urls."
        )

    # 1. Reset / Initialize Files (Global)
    if reset:
        if typer.confirm("Are you sure you want to reset the registry?"):
            # We rely on params being provided by the wrapper/context now.
            if not params:
                console.print(
                    "[yellow]Warning: No params loaded from context. Defaults may be minimal.[/yellow]"
                )

            # Registry Default
            default_registry = {
                "global_imports": params.get("global_imports", []),
                "tables": {},
            }
            with open(config_path, "w") as f:
                yaml.dump(default_registry, f, sort_keys=False)
            console.print(f"[green]Reset {config_path}[/green]")

    # Loop over targets for DB-specific actions
    for db in target_dbs:
        db_name = db.get("name")
        db_name = db.get("name") or db.get("id")
        conn_obj = db.get("connection_obj") or db.get("connection_ref") or db_name

        try:
            db_url = resolve_url_safe(conn_obj, db)
            engine = sa.create_engine(db_url)
        except Exception as e:
            console.print(
                f"[red]Skipping {db_name}: Cannot check DB connection ({e})[/red]"
            )
            continue

        # 2. Dump Schema
        if dump_schema:
            from sibi_flux.datacube.generator import dump_db_schema

            console.print(f"[bold]Dumping schema for {db_name}...[/bold]")
            dump_db_schema(
                engine=engine,
                db_name=db_name,
                output_dir=dump_schema,
                logger=console.print,
            )

        # 3. Initialize Rules (Global/Merged?)
        if init_rules:
            insp = inspect(engine)
            tables = insp.get_table_names()

            target_path = init_rules
            if not target_path.is_absolute():
                target_path = project_root / target_path

            # Ensure parent dir exists
            if not target_path.parent.exists():
                target_path.parent.mkdir(parents=True, exist_ok=True)

            # Load existing to append?
            existing_rules = []
            if target_path.exists():
                with open(target_path, "r") as f:
                    existing_rules = yaml.safe_load(f) or []

            console.print(f"Appending rules for {db_name} tables...")

            new_rules = []
            for table in sorted(tables):
                # Check existence
                if any(r["pattern"] == table for r in existing_rules):
                    continue

                new_rules.append(
                    {
                        "pattern": table,
                        "match_type": "exact",
                        "domain": "common",
                        "output_template": f"common/{table}_cubes.py",
                        "db_conn_override": conn_obj,
                    }
                )

            all_rules = existing_rules + new_rules

            with open(target_path, "w") as f:
                yaml.dump(all_rules, f, sort_keys=False)
            console.print(f"[green]Rules updated for {db_name}.[/green]")


@app.command()
def whitelist(
    config_file: Optional[Path] = typer.Option(None, "--config"),
    db_name: Optional[str] = typer.Option(
        None, "--db", help="Target specific database from params"
    ),
    db_url_map: Optional[str] = typer.Option(None, "--db-urls"),
    env_file: Optional[Path] = typer.Option(None, "--env-file", "-e"),
    force: bool = typer.Option(False, "--force", "-f"),
) -> None:
    """
    Generates whitelist files based on discovery rules and database schema.
    """
    params = context.params
    databases = params.get("databases", [])

    # 1. Determine Targets
    target_dbs = []
    if db_name:
        target_dbs = [d for d in databases if d.get("name") == db_name]
        if not target_dbs:
            console.print(f"[red]Database '{db_name}' not found in params.[/red]")
            raise typer.Exit(code=1)
    elif databases:
        target_dbs = databases
    if not target_dbs:
        # Legacy
        target_db_conf = params.get("default_connection_obj", "replica_db_conf")
        target_dbs = [{"name": target_db_conf, "connection_obj": target_db_conf}]

    # 2. Env Load
    if env_file:
        env_path = env_file
    elif context.params and context.params.get("defaults", {}).get("env_file"):
        env_path = Path(context.params.get("defaults", {})["env_file"])
    else:
        env_path = Path(".env.linux")
    ProjectService.setup_environment(ProjectService.detect_project_root(), env_path)

    # 3. Config Path
    config_path = config_file or context.default_config
    if not config_path:
        console.print("[red]No config file target specified.[/red]")
        raise typer.Exit(code=1)

    import json

    cli_urls = json.loads(db_url_map) if db_url_map else {}

    config_data = _load_and_resolve_config(
        config_path
    )  # Since we need rules/paths resolved

    registry = DatacubeRegistry(config_data, params=context.params)

    def resolve_url_safe(conf_name, db_config):
        if db_url_map and conf_name in json.loads(db_url_map):
            return json.loads(db_url_map)[conf_name]
        
        imp_spec = db_config.get("import_spec")
        if imp_spec and isinstance(imp_spec, dict):
            module = imp_spec.get("module")
            symbol = imp_spec.get("symbol")
            if module and symbol:
                imp = [f"from {module} import {symbol}"]
            else:
                imp = registry.global_imports
        else:
            db_imp = db_config.get("global_import")
            imp = [db_imp] if db_imp else registry.global_imports
            
        return resolve_db_url(conf_name, imp)

    # 4. Iterate and Generate
    for db in target_dbs:
        db_name = db.get("name") or db.get("id")
        conn_obj = db.get("connection_obj") or db.get("connection_ref") or db_name

        try:
            db_url = resolve_url_safe(conn_obj, db)
            engine = sa.create_engine(db_url)
        except Exception as e:
            console.print(
                f"[red]Skipping {db_name}: Cannot check DB connection ({e})[/red]"
            )
            continue

        # 0. Try to load tables from all_tables.yaml (Scan Output)
        tables = []
        all_tables_file = (
            params.get("paths", {}).get("discovery", {}).get("all_tables_file")
            or params.get("all_tables_file")
            or "dataobjects/globals/all_tables.yaml"
        )
        # Resolve path
        project_root = ProjectService.detect_project_root(config_path)
        if not Path(all_tables_file).is_absolute():
            all_tables_path = project_root / all_tables_file
        else:
            all_tables_path = Path(all_tables_file)

        if all_tables_path.exists():
            try:
                with open(all_tables_path, "r") as f:
                    all_tables_data = yaml.safe_load(f) or {}
                    tables = all_tables_data.get(conn_obj) or []
            except Exception:
                pass

        if tables:
             console.print(f"  [green]✓ Loaded {len(tables)} tables from {all_tables_path.name}[/green]")
        else:
            # Fallback to Live Introspection
            console.print(f"  [yellow]⚠ Warning: {conn_obj} not found in all_tables.yaml. Falling back to live DB introspection.[/yellow]")
            insp = inspect(engine)
            tables = insp.get_table_names()

        # Determine path (Config Driven)
        discovery_cfg = params.get("paths", {}).get("discovery") or params.get(
            "discovery", {}
        )
        wl_file = (
            discovery_cfg.get("whitelist_file")
            or params.get("whitelist_file")
            or "whitelist.yaml"
        )
        target_path = Path(wl_file)
        if not target_path.is_absolute():
            target_path = project_root / target_path

        # Load Rules
        rules_file = db.get("rules_file")
        if not rules_file:
             rules_file = discovery_cfg.get("rules_file") or params.get("rules_file") or "discovery_rules.yaml"
        
        rules_path = Path(rules_file)
        if not rules_path.is_absolute():
            rules_path = project_root / rules_path

        console.print(f"  [cyan]Rules:[/] {rules_path.name}")
        console.print(f"  [cyan]Output:[/] {target_path.relative_to(project_root)}")
        rules_file = (
            discovery_cfg.get("rules_file")
            or params.get("rules_file")
            or "discovery_rules.yaml"
        )
        if Path(rules_file).is_absolute():
            rules_path = Path(rules_file)
        else:
            rules_path = project_root / rules_file

        filtered_tables = {}  # Default to ALL if no rules? No, tables is list.
        # If no rules, we might want to default to empty dicts for all tables?
        # Let's keep logic: if rules exist, filter.

        # 4. Use ConfigurationEngine for logic
        # Initialize engine with the resolved rules file
        from sibi_flux.datacube.config_engine import ConfigurationEngine

        # We need to construct a lightweight 'params' dict or use existing context.params
        # But we need to ensure 'engine' uses the correct 'rules_path' for THIS connection.

        # NOTE: ConfigEngine takes 'params' and 'rules_path'.
        # It handles scoped dictionary rules (via 'context_key') correctly.

        eng = ConfigurationEngine(
            context.params, rules_path=str(rules_path), context_key=conn_obj
        )

        filtered_tables = {}

        if rules_path.exists():
            for t in tables:
                # Resolve using engine (handles prefix, regex, template logic)
                # Pass mocked db_config for domain resolution logic inside engine
                mock_db_config = {
                    "db_domain": db.get("db_domain", "common"),
                    "connection_obj": conn_obj,
                }

                res = eng.resolve_table(t, db_config=mock_db_config)

                if res:
                    # Calculate relative paths
                    db_dom = db.get("db_domain")

                    # 1. Datacube Path Explicit
                    # Structure: datacubes_dir / [db_domain] / domain / template

                    # datacubes_dir resolved from params
                    dc_root_dir = (
                        context.params.get("paths", {})
                        .get("target", {})
                        .get("datacubes_dir", "dataobjects/gencubes")
                    )

                    dc_base = Path(dc_root_dir)
                    if db_dom:
                        dc_base = dc_base / db_dom

                    # Rule might provide 'output_template' (e.g. 'asm_cubes.py')
                    # We assume template is just the filename now per robust rules
                    template_name = res.get("output_template", Path(res["path"]).name)

                    full_dc_path = dc_base / res["domain"] / template_name

                    # 2. Field Map Path Explicit
                    # Structure: field_maps_dir / [db_domain] / domain / table.py

                    fields_root_dir = (
                        context.params.get("paths", {})
                        .get("target", {})
                        .get("field_maps_dir", "dataobjects/fields")
                    )

                    # Sanitize: if it contains dots but no slashes, it's likely a package-style string
                    if "." in fields_root_dir and "/" not in fields_root_dir and not Path(fields_root_dir).exists():
                        fields_root_dir = fields_root_dir.replace(".", "/")

                    fm_base = Path(fields_root_dir)
                    if db_dom:
                        fm_base = fm_base / db_dom

                    fm_path_full = fm_base / res["domain"] / f"{t}.py"

                    # Relativize logic
                    def _safe_rel(p):
                        try:
                            pp = Path(p)
                            # Use already resolved project_root
                            if pp.is_absolute():
                                try:
                                    return pp.relative_to(project_root)
                                except ValueError:
                                    return pp.resolve().relative_to(project_root.resolve())
                            return pp
                        except Exception:
                            return Path(p)

                    rel_dc_path = _safe_rel(full_dc_path)
                    rel_fm_path = _safe_rel(fm_path_full)

                    filtered_tables[t] = {
                        "domain": res["domain"],
                        "output_template": (
                            Path(res["path"]).name
                            if "output_template" not in res
                            else res.get("output_template", Path(res["path"]).name)
                        ),
                        "datacube_path": str(rel_dc_path),
                        "field_map_path": str(rel_fm_path),
                    }
                    # Recover template logic: if rule-based, template was used.
                    # But since we have the path, template is secondary.
                    # We'll stick to what we have.

        else:
            console.print(
                f"[yellow]No rules found for {conn_obj}. Whitelisting ALL tables (No Paths Calculated).[/yellow]"
            )
            filtered_tables = {t: {} for t in tables}

        # 5. Load Current Whitelist
        current_wl = {}
        if target_path.exists():
            try:
                with open(target_path, "r") as rf:
                    current_wl = yaml.safe_load(rf) or {}
            except:
                pass

        # Existing whitelist for this connection
        existing_tables_map = {}
        if conn_obj in current_wl:
            raw_wl = current_wl[conn_obj]
            if isinstance(raw_wl, list):
                # Upgrade legacy list to dict
                existing_tables_map = {t: {} for t in raw_wl}
            else:
                existing_tables_map = raw_wl.get("tables", {})

        # Calculate sets
        current_table_names = set(existing_tables_map.keys())
        new_table_names = set(filtered_tables.keys())

        # Sync Logic:
        # 1. Start with intersection (preserve config, but update rule-based defaults if missing?)
        # We want to keep manual overrides in existing map, but maybe refresh defaults?
        retained = current_table_names.intersection(new_table_names)

        # 2. Add new filtered tables (additions)
        added = new_table_names - current_table_names

        # 3. Removed (in current but not in filtered)
        removed = current_table_names - new_table_names

        # Construct new table map
        new_table_map = {}

        for t in retained:
            existing_meta = existing_tables_map.get(t, {})
            rule_meta = filtered_tables[t]
            
            # Strict Rewrite Logic (User Request)
            # We ONLY preserve 'custom_name' from existing entries.
            # All other metadata (domain, template, paths) must come strictly from rules/calculation.
            custom_name = existing_meta.get("custom_name")
            
            merged = rule_meta.copy()
            if custom_name:
                merged["custom_name"] = custom_name
                
            new_table_map[t] = merged

        # 2. Add new
        for t in added:
            new_table_map[t] = filtered_tables[t]

        # 3. Removed are omitted

        # Update structure
        # Inject Global Paths
        paths_cfg = params.get("paths", {}).get("target", {})

        # Helper to ensure relative paths
        def _to_rel(p):
            if not p:
                return p
            try:
                pp = Path(p)
                # Use already resolved project_root
                if pp.is_absolute():
                    try:
                        return str(pp.relative_to(project_root))
                    except ValueError:
                        return str(pp.resolve().relative_to(project_root.resolve()))
                return p
            except Exception:
                pass
            return p

        dc_dir = _to_rel(paths_cfg.get("datacubes_dir", "dataobjects/gencubes/"))
        fm_dir = _to_rel(
            paths_cfg.get("field_maps_dir", "dataobjects/gencubes/fields/")
        )

        # Get db_domain from config (fallback to db_name if missing)
        db_domain = db.get("db_domain") or db.get("name")

        rich_structure = {
            "db_domain": db_domain,
            "datacubes_dir": dc_dir,
            "field_maps_dir": fm_dir,
            "tables": dict(sorted(new_table_map.items())),  # Sort keys for stability
        }

        current_wl[conn_obj] = rich_structure

        with open(target_path, "w") as f:
            yaml.dump(current_wl, f, sort_keys=False)

        # Report
        console.print(f"[bold underline]Sync Report for {db_name}[/bold underline]")
        if added:
            console.print(
                f"[green]  + Added {len(added)} tables:[/green] {', '.join(sorted(list(added))[:5])}{'...' if len(added)>5 else ''}"
            )
        if removed:
            console.print(
                f"[red]  - Removed {len(removed)} tables:[/red] {', '.join(sorted(list(removed))[:5])}{'...' if len(removed)>5 else ''}"
            )
        if not added and not removed:
            console.print("[dim]  No changes.[/dim]")

        # Count keys directly
        final_count = len(rich_structure["tables"])
        console.print(f"[blue]  Total Whitelisted: {final_count}[/blue]")


def _run_field_map_generation(
    context, config_path, target_dbs, url_resolver, force=False
):
    """Shared logic for generating field map files."""
    params = context.params
    # Logic: modern nested > legacy flat > default
    folder_prefix = (
        params.get("paths", {}).get("target", {}).get("datacubes_dir")
        or params.get("folder_prefix")
        or "solutions/dataobjects/gencubes/"
    )

    # Priority: explicit 'field_maps_dir' > legacy suffix construction
    configured_fm_dir = params.get("paths", {}).get("target", {}).get("field_maps_dir")

    if configured_fm_dir:
        # If fm_dir is provided, it is the root for fields.
        # We don't append suffix to it unless it's just a root base?
        # Usually 'field_maps_dir' is the full relative path e.g. 'dataobjects/fields'
        fields_root_path_base = Path(configured_fm_dir)
        use_legacy_construction = False
    else:
        fields_suffix = (
            params.get("generation", {}).get("fields_subpackage")
            or params.get("fields_module_root")
            or "fields"
        )
        fields_root_path_base = Path(folder_prefix) / fields_suffix
        use_legacy_construction = True

    from sibi_flux.datacube.field_mapper import FieldTranslationManager

    # Initialize Manager & Load Global Repo
    # Initialize Manager & Load Global Repo
    repo_rel_path = (
        params.get("paths", {})
        .get("repositories", {})
        .get("global_field_repository_file")
        or params.get("global_repo_path")
        or "solutions/conf/global_field_repository.yaml"
    )

    # Resolve relative to project root if needed
    project_root = ProjectService.detect_project_root(config_path)
    if Path(repo_rel_path).is_absolute():
        global_repo_path = Path(repo_rel_path)
    else:
        global_repo_path = (project_root / repo_rel_path).resolve()

    manager = FieldTranslationManager()

    # 1. Load Repository (Definitions)
    if global_repo_path.exists():
        with open(global_repo_path, "r") as f:
            repo_data = yaml.safe_load(f) or []
            manager.load_from_list(repo_data)
        console.print(
            f"[green]Loaded {len(manager.fields)} fields from Global Repository.[/green]"
        )

    # 2. Load Translations (Overrides)
    trans_rel_path = params.get("paths", {}).get("repositories", {}).get(
        "global_field_translations_file"
    ) or params.get("global_field_translations_file")
    if trans_rel_path:
        if Path(trans_rel_path).is_absolute():
            trans_path = Path(trans_rel_path)
        else:
            trans_path = (project_root / trans_rel_path).resolve()

            if trans_path.exists():
                with open(trans_path, "r") as f:
                    trans_data = yaml.safe_load(f) or []
                    manager.load_from_list(trans_data)
                console.print(
                    f"[green]Loaded translations from {trans_rel_path}.[/green]"
                )

    # --- GLOBAL CLEAN BUILD ---
    if force:
        # Determine global field maps root to wipe
        # Logic matches default resolution used later
        tgt = params.get("paths", {}).get("target", {})
        fm_dir = tgt.get("field_maps_dir")

        # If not set in params, check if we can infer from default
        # But wait, whitelist overrides this per DB.
        # User said "fields folder is exclusive".
        # So we should wipe the configured 'field_maps_dir' from global params.

        if fm_dir:
            if Path(fm_dir).is_absolute():
                abs_fm_dir = Path(fm_dir)
            else:
                abs_fm_dir = (project_root / fm_dir).resolve()

            if abs_fm_dir.exists():
                from sibi_flux.core import SecureResource
                secure = SecureResource(project_root=project_root)
                
                console.print(
                    f"[bold red]Global Clean: Removing entire fields directory {abs_fm_dir}[/bold red]"
                )
                try:
                    # Enforce sandbox
                    secure.get_secure_path(abs_fm_dir)
                    import shutil
                    shutil.rmtree(abs_fm_dir)
                    
                    abs_fm_dir.mkdir(parents=True, exist_ok=True)
                    (abs_fm_dir / "__init__.py").touch()
                except Exception as e:
                    console.print(f"[red]Failed to clean global fields dir: {e}[/red]")
        else:
            console.print(
                "[yellow]Warning: Could not determine global field_maps_dir for clean build.[/yellow]"
            )

    generated_maps = {}
    
    for db in target_dbs:
        # console.print(f"DEBUG: Processing DB entry: {db} (Type: {type(db)})")
        if isinstance(db, str):
            console.print(
                f"[red]Error: Database entry is a string '{db}', expected dict. Check config.[/red]"
            )
            continue

        name = db.get("id") or db.get("name")
        conn_obj = db.get("connection_ref") or db.get("connection_obj")

        if not db.get("enable_field_map_generation", True) and not params.get(
            "generation", {}
        ).get("enable_field_maps", True):
            console.print(f"[dim]Skipping {name}: Field map generation disabled.[/dim]")
            continue

        # Language Settings
        source_lang = db.get("db_source_lang", "es")
        target_lang = db.get("db_target_lang", "en")

        # Path Composition
        db_domain = db.get("db_domain")

        if use_legacy_construction:
            fields_root_path = Path(folder_prefix)
            if db_domain:
                fields_root_path = fields_root_path / db_domain
            fields_root_path = fields_root_path / fields_suffix
        else:
            # Modern Logic: configured_fm_dir is root
            fields_root_path = fields_root_path_base
            if db_domain:
                fields_root_path = fields_root_path / db_domain

        # Resolve global whitelist path
        # Resolve global whitelist path
        # Try nested discovery param first, then legacy flat
        discovery_cfg = params.get("paths", {}).get("discovery") or params.get(
            "discovery", {}
        )
        global_whitelist_file = (
            discovery_cfg.get("whitelist_file")
            or params.get("whitelist_file")
            or "whitelist.yaml"
        )

        # If absolute, use directly (handled by gen_dc.py resolution), else resolve
        if Path(global_whitelist_file).is_absolute():
            wl_path = Path(global_whitelist_file)
        else:
            # Use Project Root anchoring
            wl_path = project_root / global_whitelist_file

        if not wl_path.exists():
            console.print(
                f"[yellow]Skipping {name}: Whitelist {global_whitelist_file} not found.[/yellow]"
            )
            continue

        with open(wl_path, "r") as f:
            registry_data = yaml.safe_load(f) or {}

        # Load Rules for Domain Inference
        rules_filename = db.get("rules_file")
        if not rules_filename:
            # Check deep keys
            disc = params.get("paths", {}).get("discovery", {})
            # console.print(f"DEBUG: Discovery Block: {disc}")
            rules_filename = disc.get("rules_file") or params.get("discovery", {}).get(
                "rules_file"
            )

        if not rules_filename:
            console.print(
                f"DEBUG: Fallback for {name}. Params keys: {disc.keys() if 'disc' in locals() else 'N/A'}"
            )
            rules_filename = f"discovery_rules_{conn_obj}.yaml"

        # Resolve path relative to project root properly
        if Path(rules_filename).is_absolute():
            rules_path = Path(rules_filename)
        else:
            rules_path = project_root / rules_filename

        rules = []
        # console.print(f"DEBUG: Checking rules path: {rules_path}")
        if rules_path.exists():
            with open(rules_path, "r") as f:
                rules = yaml.safe_load(f) or []

        # PROPERLY HANDLE DICT vs LIST RULES
        if isinstance(rules, dict):
            # Try to find specific rules for this connection
            candidates = [conn_obj, name]
            found = False
            for key in candidates:
                if key in rules:
                    rules = rules[key]
                    found = True
                    break
            if not found:
                rules = []

        # Support List or Dict Format
        scoped_data = registry_data.get(conn_obj, {})
        if isinstance(scoped_data, list):
            tables = {t: {} for t in scoped_data}
        else:
            tables = scoped_data.get("tables", {})

        if not tables:
            console.print(f"[dim]No tables in whitelist for {name}.[/dim]")
            continue

        # Override paths from whitelist if present
        # This allows whitelist to drive output location efficiently
        # Retrieve db_domain from whitelist entry (preferred) or registry config
        # Hoisted from loop for Clean Build capability
        wl_entry = registry_data.get(conn_obj, {})
        db_domain = wl_entry.get("db_domain") or db.get("db_domain") or db.get("name")

        wl_fields_dir = wl_entry.get("field_maps_dir")
        if wl_fields_dir:
            # Resolve relative to project root
            if Path(wl_fields_dir).is_absolute():
                fields_root_path = Path(wl_fields_dir)
            else:
                # Use cached or recalculated project_root
                if "project_root" not in locals():
                    try:
                        project_root = config_path.parent.parent.parent
                    except Exception:
                        project_root = Path.cwd()
                fields_root_path = project_root / wl_fields_dir
        else:
            # Fallback to global setting (calculated earlier, but we need to re-calc if not in scope or just reuse?)

            # Logic: modern nested > legacy flat > default
            folder_prefix = (
                params.get("paths", {}).get("target", {}).get("datacubes_dir")
                or params.get("folder_prefix")
                or "solutions/dataobjects/gencubes/"
            )
            fields_suffix = (
                params.get("generation", {}).get("fields_subpackage")
                or params.get("fields_module_root")
                or "fields"
            )
            # Resolve
            if not Path(folder_prefix).is_absolute():
                try:
                    prj_root = config_path.parent.parent.parent
                except Exception:
                    prj_root = Path.cwd()
                folder_prefix = str(prj_root / folder_prefix)

            # Construct
            fields_root_path = fields_root_path_base

        # Ensure Parent Chain has __init__.py (Backtrack to root)
        try:
            # Make sure fields_root_path itself exists
            fields_root_path.mkdir(parents=True, exist_ok=True)
            if not (fields_root_path / "__init__.py").exists():
                (fields_root_path / "__init__.py").touch()

            curr = fields_root_path.parent
            # Stop at project root (.) or root (/)
            while str(curr) != "." and str(curr) != "/" and len(curr.parts) > 0:
                if not (curr / "__init__.py").exists():
                    (curr / "__init__.py").touch()
                curr = curr.parent
        except Exception:
            pass

        # This is safe because db_domain is specific to this connection.
        db_target_root = fields_root_path / db_domain

        # Ensure Root Exists
        db_target_root.mkdir(parents=True, exist_ok=True)
        if not (db_target_root / "__init__.py").exists():
            (db_target_root / "__init__.py").touch()

        console.print(
            f"[bold cyan]Mapping fields for {name} ({len(tables)} tables) [Src:{source_lang} -> Tgt:{target_lang}]...[/bold cyan]"
        )

        try:
            db_url = url_resolver(conn_obj, db)
            engine = sa.create_engine(db_url)
            inspector = inspect(engine)

            for j, (table_name, details) in enumerate(tables.items(), 1):
                domain = details.get("domain")

                # If domain is missing (e.g. manual entry without rule), try fallback inference
                if not domain:
                    # Infer from rules
                    for r in rules:
                        pat = r.get("pattern")
                        mtype = r.get("match_type", "exact")
                        is_match = False
                        if mtype == "exact" and table_name == pat:
                            is_match = True
                        elif mtype == "prefix" and table_name.startswith(pat):
                            is_match = True
                        elif mtype == "regex":
                            import re

                            if re.search(pat, table_name):
                                is_match = True

                        if is_match:
                            domain = r.get("domain")
                            # console.print(f"DEBUG: {table_name} matched rule {pat} -> {domain}")
                            break

                if not domain:
                    # console.print(f"DEBUG: No match for {table_name}, defaulting to common")
                    domain = "common"

                # Target Resolution
                explicit_fm_path = details.get("field_map_path")
                if explicit_fm_path:
                    # Use Explicit Path from Whitelist
                    if Path(explicit_fm_path).is_absolute():
                        target_file = Path(explicit_fm_path)
                    else:
                        # Ensure project_root is available or fallback to cwd
                        if "project_root" not in locals():
                            try:
                                project_root = config_path.parent.parent.parent
                            except:
                                project_root = Path.cwd()
                        target_file = project_root / explicit_fm_path

                    target_dir = target_file.parent
                else:
                    # Fallback Logic
                    target_dir = db_target_root / domain
                    target_file = target_dir / f"{table_name}.py"

                # Ensure target directory exists for mapping
                target_dir.mkdir(parents=True, exist_ok=True)
                if not (target_dir / "__init__.py").exists():
                     (target_dir / "__init__.py").touch()

                # Skip if exists and not force
                if target_file.exists() and not force:
                    # console.print(f"DEBUG: Skipping existing {table_name}")
                    continue

                console.print(f"    [{j}/{len(tables)}] Mapping {table_name} -> {target_file.name}")

                try:
                    # Fix for SA 2.0 / Clickhouse: inspect connection, not engine
                    with engine.connect() as conn:
                        cols = inspect(conn).get_columns(table_name)
                    # console.print(f"DEBUG: Generating {table_name} with {len(cols)} columns")
                    field_map = {}

                    full_table_name = f"{db_domain}.{domain}.{table_name}"

                    for c in cols:
                        col_name = c["name"]
                        col_type = str(c["type"])

                        # 1. Register / Get Canonical Field
                        trans_msg = manager.register_field(
                            col_name, col_type, full_table_name
                        )
                        if trans_msg:
                            console.print(f"  [dim]{trans_msg}[/dim]")

                        # 2. Get Field Definition
                        fid = manager._generate_id(col_name, col_type)
                        field_def = manager.fields.get(fid)

                        if field_def:
                            # 3. Translate if needed
                            if target_lang not in field_def.aliases:
                                manager.translate_alias(fid, target_lang)

                            # 4. Determine Target Column
                            target_alias = (
                                field_def.aliases.get(target_lang)
                                or field_def.aliases.get("en")
                                or col_name
                            )
                            target_col = manager.generate_target_column(target_alias)

                            if not field_def.target_column:
                                field_def.target_column = target_col
                            else:
                                target_col = field_def.target_column

                            field_map[col_name] = target_col
                        else:
                            field_map[col_name] = col_name

                    # GENERATE PYTHON CODE USING FACTORY
                    lines = [
                        "from typing import Mapping, Any, List",
                        "from sibi_flux.datacube.field_factory import FieldMapFactory",
                        "",
                        "COLUMNS: List[str] = [",
                    ]
                    for col_name in field_map.keys():
                        lines.append(f'    "{col_name}",')
                    lines.append("]")

                    lines.append("")
                    lines.append(
                        f'field_map: Mapping[str, str] = FieldMapFactory.create("{full_table_name}", COLUMNS)'
                    )

                    # Generate metadata call
                    lines.append("")
                    lines.append(
                        f'metadata: Mapping[str, Any] = FieldMapFactory.create_metadata("{full_table_name}", COLUMNS)'
                    )

                    with open(target_file, "w") as f:
                        f.write("\n".join(lines))

                    # Calculate Import Path
                    try:
                        # Ensure we get relative path to project root (which should be sys.path root)
                        if "project_root" not in locals():
                            project_root = Path.cwd()
                        
                        rel_py_path = target_file.relative_to(project_root)
                        module_path = str(rel_py_path.with_suffix("")).replace("/", ".")
                        generated_maps[table_name] = f"{module_path}.field_map"
                    except ValueError:
                        # Fallback if outside project root?
                        pass
                    except Exception as e:
                        pass

                except Exception as e:
                    console.print(f"[red]Error processing {table_name}: {e}[/red]")
                    continue

        except Exception as e:
            console.print(f"[red]Error connecting/inspecting DB {name}: {e}[/red]")

    # --- Global Field Pruning ---
    # Collect all tables currently in the whitelists across all DBs
    all_active_usages = set()
    for db in target_dbs:
        conn = db.get("connection_ref") or db.get("connection_obj")
        # Reuse registry_data loaded in the loop
        scoped = registry_data.get(conn, {})
        if isinstance(scoped, list):
            tables_dict = {t: {} for t in scoped}
        else:
            tables_dict = scoped.get("tables", {})
        
        # Get db_domain (matching logic in loop)
        db_dom = scoped.get("db_domain") or db.get("db_domain") or db.get("name")
        
        for t_name, details in tables_dict.items():
            # Domain might be in details or inferred (matching logic in loop)
            # For pruning, we just need to ensure we cover what's in the repo
            # Field repo uses full_table_name: f"{db_domain}.{domain}.{table_name}"
            dom = details.get("domain") or "common" # Simple fallback for pruning match
            all_active_usages.add(f"{db_dom}.{dom}.{t_name}")

    pruned_count = manager.prune_fields(list(all_active_usages))
    if pruned_count > 0:
        console.print(f"[yellow]Pruned {pruned_count} unused fields from Global Repository.[/yellow]")

    # Save Updates to Global Repository
    try:
        manager.save_to_yaml(global_repo_path)
        console.print(
            f"[green]Updated Global Field Repository at {global_repo_path} ({len(manager.fields)} fields)[/green]"
        )
    except Exception as e:
        console.print(f"[red]Failed to save Global Field Repository: {e}[/red]")

    return generated_maps

# --- Aliases (User Friendly) ---
@app.command(name="create-app", help="Initialize a new Sibi Flux project")
def create_app_alias(
    project_name: str = typer.Argument(..., help="Name of the project to create")
):
    from sibi_flux.init.core import initialize_project
    initialize_project(project_name, lib=False, app=False)

app.command(name="propose-cubes", help="Alias for 'propose-rules'")(propose_rules)
app.command(name="create-cubes", help="Alias for 'sync'")(sync)


if __name__ == "__main__":
    app()
