"""
DataCube Generator module.

Handles introspection of database schemas and generation of Python DataCube classes.
"""

import sqlalchemy as sa
from pathlib import Path
from typing import Optional, Set, Any, Callable, Tuple, Dict
from collections.abc import Mapping, MutableMapping, Iterable, Sequence
from sqlalchemy import inspect
import re
import importlib
import sys
import os
from sibi_flux.utils.path_utils import PathUtils

# Cache for dynamically imported modules to avoid redundant disk I/O in loops
_MODULE_CACHE = {}

# --- Global Monkey Patch for Clickhouse-Connect / SQLA 2.0 Compatibility ---
try:
    import clickhouse_connect
    from clickhouse_connect.cc_sqlalchemy.dialect import ClickHouseDialect
    from sqlalchemy import text
    from clickhouse_connect.driver.binding import quote_identifier

    def patched_get_table_names(self, connection, schema=None, **kw):
        cmd = "SHOW TABLES"
        if schema:
            cmd += " FROM " + quote_identifier(schema)
        return [row.name for row in connection.execute(text(cmd))]

    ClickHouseDialect.get_table_names = patched_get_table_names

    # Register Dialect
    from sqlalchemy.dialects import registry

    registry.register(
        "clickhouse.connect",
        "clickhouse_connect.cc_sqlalchemy.dialect",
        "ClickHouseDialect",
    )
    registry.register(
        "clickhouse", "clickhouse_connect.cc_sqlalchemy.dialect", "ClickHouseDialect"
    )

except ImportError:
    pass
# ---------------------------------------------------------------------------


def get_sa_type_classes(
    type_names: Iterable[str], logger: Optional[Any] = None
) -> Tuple[Any, ...]:
    """Converts a list of strings to actual SQLAlchemy type classes."""
    classes = []
    for name in type_names:
        type_cls = getattr(sa, name, None)
        if type_cls:
            classes.append(type_cls)
        elif logger:
            logger.warning(f"SQLAlchemy type '{name}' not found.")
    return tuple(classes)


from sibi_flux.utils.security import is_secure_path


def generate_class_code(
    inspector: Optional[sa.engine.Inspector],
    table_name: str,
    config_obj: str,
    base_class: str,
    mappings: Mapping[str, tuple],
    class_name: Optional[str] = None,
    class_suffix: str = "Dc",
    backend: str = "sqlalchemy",
    field_map_var: Optional[str] = None,
    legacy_filters: bool = False,
    field_map_dict: Optional[Mapping[str, str]] = None,
    sticky_filters: Optional[Mapping[str, Any]] = None,
    column_metadata: Optional[Mapping[str, Any]] = None,
    metadata_var: Optional[str] = None,
) -> str:
    """Introspects DB table and generates Python class string. If inspector is None, generates shell."""

    # Initialize metadata dict based on the keys in our YAML mapping
    meta: dict[str, list[str]] = {label: [] for label in mappings.keys()}

    if inspector:
        try:
            columns = inspector.get_columns(table_name)
            for col in columns:
                col_name = col["name"]
                # Apply alias if map is provided
                if field_map_dict:
                    col_name = field_map_dict.get(col_name, col_name)

                for label, sa_classes in mappings.items():
                    if isinstance(col["type"], sa_classes):
                        meta[label].append(col_name)
        except (sa.exc.SQLAlchemyError, KeyError):
            # If introspection fails for specific table (e.g. missing), just skip columns
            pass

    if not class_name:
        class_name = (
            "".join(w.capitalize() for w in table_name.split("_")) + class_suffix
        )

    # Generate attribute lines for every label defined in the mapping
    metadata_lines = []
    for label, cols in meta.items():
        metadata_lines.append(f"    {label}: ClassVar[List[str]] = {repr(cols)}")

    # Configuration block extensions
    extra_config = []
    if field_map_var:
        extra_config.append(f"    field_map: ClassVar[Dict] = {field_map_var}")
    elif field_map_dict:
        extra_config.append(f"    field_map: ClassVar[Dict] = {repr(field_map_dict)}")

    if sticky_filters:
        extra_config.append(
            f"    sticky_filters: ClassVar[Dict] = {repr(sticky_filters)}"
        )

    # Metadata Lists
    if metadata_var:
        # Phase 5: Import-based Metadata (Factory)
        extra_config.append(
            f'    validation_schema: ClassVar[Mapping[str, str]] = {metadata_var}["validation_schema"]'
        )
        extra_config.append(
            f'    boolean_columns: List[str] = {metadata_var}["boolean_columns"]'
        )
        extra_config.append(
            f'    numeric_int_columns: List[str] = {metadata_var}["numeric_int_columns"]'
        )
        extra_config.append(
            f'    numeric_float_columns: List[str] = {metadata_var}["numeric_float_columns"]'
        )
        extra_config.append(
            f'    date_fields: List[str] = {metadata_var}["date_fields"]'
        )

    elif column_metadata:
        # Fallback: Inline Generation (Phase 4)
        bool_cols = []
        int_cols = []
        float_cols = []
        date_cols = []
        val_schema = {}

        for col, meta in column_metadata.items():
            # Meta is expected to be the pyarrow type string or a dict
            ptype = meta if isinstance(meta, str) else meta.get("type", "string")
            val_schema[col] = ptype

            if ptype == "boolean[pyarrow]":
                bool_cols.append(col)
            elif ptype in (
                "Int64[pyarrow]",
                "Int32[pyarrow]",
                "Int16[pyarrow]",
                "Int8[pyarrow]",
            ):
                int_cols.append(col)
            elif ptype in ("Float64[pyarrow]", "Float32[pyarrow]"):
                float_cols.append(col)
            elif ptype.startswith("timestamp") or ptype == "datetime64[ns, UTC]":
                date_cols.append(col)

        if bool_cols:
            extra_config.append(
                f"    boolean_columns: List[str] = {repr(sorted(bool_cols))}"
            )
        if int_cols:
            extra_config.append(
                f"    numeric_int_columns: List[str] = {repr(sorted(int_cols))}"
            )
        if float_cols:
            extra_config.append(
                f"    numeric_float_columns: List[str] = {repr(sorted(float_cols))}"
            )
        if date_cols:
            extra_config.append(
                f"    date_fields: List[str] = {repr(sorted(date_cols))}"
            )

        if val_schema:
            extra_config.append(
                f"    validation_schema: ClassVar[Mapping[str, str]] = {repr(val_schema)}"
            )

    attributes_str = "\n".join(metadata_lines)
    extra_config_str = "\n".join(extra_config)
    if extra_config_str:
        extra_config_str = "\n" + extra_config_str

    return f"""
class {class_name}({base_class}):
    df: Optional[dd.DataFrame] = None
    
    # Config
    backend: ClassVar[str] = "{backend}"
    connection_url: ClassVar[str] = {config_obj}.get("db_url")
    table: ClassVar[str] = "{table_name}"{extra_config_str}
    
    # Transformations & Metadata
{attributes_str}
"""


def check_drift(
    dc_class: Any,
    db_column_names: Iterable[str],
    attribute_names: Iterable[str],
    field_map: Optional[Mapping[str, str]] = None,
) -> list[str]:
    """
    Checks for drift between the datacube class metadata and database columns.
    Returns a list of issue strings.
    """
    code_cols: Set[str] = set()
    for attr in attribute_names:
        # Get the list from the class (default to empty list)
        cols = getattr(dc_class, attr, [])
        code_cols.update(cols)

    # Apply alias mapping to DB columns if present
    # Map: DB_NAME -> CODE_NAME
    if field_map:
        mapped_db_cols = set()
        for c in db_column_names:
            mapped_db_cols.add(field_map.get(c, c))  # Use alias if exists, else raw
        final_db_cols = mapped_db_cols
    else:
        final_db_cols = set(db_column_names)

    missing_in_code = final_db_cols - code_cols
    extra_in_code = code_cols - final_db_cols

    issues = []
    if missing_in_code:
        issues.append(f"DB has new cols: {missing_in_code}")
    if extra_in_code:
        issues.append(f"Code has stale cols: {extra_in_code}")

    return issues


def resolve_db_url(config_obj_name: str, imports: Iterable[str]) -> Optional[str]:
    """
    Attempts to resolve a database URL by dynamically importing the config object
    described in the import statements.
    """
    # Ensure current working directory is in path to allow local imports
    if os.getcwd() not in sys.path:
        sys.path.insert(0, os.getcwd())

    for imp in imports:
        # Simple parsing for "from module import var" or "from module import var1, var2"
        if imp.startswith("from ") and " import " in imp:
            parts = imp.split(" import ")
            module_name = parts[0][5:].strip()
            imported_vars = [v.strip() for v in parts[1].split(",")]

            if config_obj_name in imported_vars:
                try:
                    if module_name not in _MODULE_CACHE:
                        _MODULE_CACHE[module_name] = importlib.import_module(module_name)
                    mod = _MODULE_CACHE[module_name]
                    conf = getattr(mod, config_obj_name)
                    # Support dict-like (.get) or object attribute access
                    val = None
                    if hasattr(conf, "get"):
                        val = conf.get("db_url")
                    elif hasattr(conf, "db_url"):
                        val = conf.db_url
                    
                    if val and hasattr(val, "get_secret_value"):
                        return val.get_secret_value()
                    return val
                except (ImportError, AttributeError):
                    pass
        elif imp and not imp.startswith("from ") and " " not in imp:
            # Direct module import: "solutions.conf.credentials"
            try:
                if imp not in _MODULE_CACHE:
                    _MODULE_CACHE[imp] = importlib.import_module(imp)
                mod = _MODULE_CACHE[imp]
                conf = getattr(mod, config_obj_name, None)
                if conf:
                    val = None
                    if hasattr(conf, "get"):
                        val = conf.get("db_url")
                    elif hasattr(conf, "db_url"):
                        val = conf.db_url
                    
                    if val and hasattr(val, "get_secret_value"):
                        return val.get_secret_value()
                    return val
            except ImportError:
                pass
    return None
# resolve_db_url remains here as it's a core introspection utility.

# load_environment and ensure_directories_exist were moved to sibi_flux.core.project.ProjectService


def filter_global_imports(
    global_imports: Iterable[str],
    used_configs: Set[str],
    ignored_prefixes: Optional[Iterable[str]] = None,
) -> list[str]:
    """
    Filters global import lines, keeping only those that are used or generic.
    Renames imports if they contain multiple items but only some are used.
    """
    filtered = []
    ignored = ignored_prefixes or []
    for line in global_imports:
        if " import " in line and line.strip().startswith("from "):
            parts = line.split(" import ")
            prefix = parts[0]
            imported_vars = [v.strip() for v in parts[1].split(",")]

            # Check for intersection with used configs
            common = [v for v in imported_vars if v in used_configs]

            if common:
                # If we use some of the imported vars, keep only those
                filtered.append(f"{prefix} import {', '.join(common)}")
            else:
                # If we use NONE, check if it looks like a config import we should drop
                if any(p in prefix for p in ignored):
                    continue
                # Otherwise keep generic imports (e.g. typing, os, etc if they appeared here)
                filtered.append(line)
        else:
            filtered.append(line)
    return filtered


class DatacubeRegistry:
    """
    Encapsulates parsing logic for the datacube_registry.yaml.
    """

    def __init__(
        self, config_dict: MutableMapping[str, Any], params: Optional[dict] = None
    ):
        self.config = config_dict
        self.params = params or {}
        self.tables = self.config.setdefault("tables", {})
        self.global_imports = self.config.get("global_imports", [])
        if "global_imports" in self.params:
            self.global_imports.extend(self.params["global_imports"])
        # Deduplicate
        self.global_imports = sorted(list(set(self.global_imports)))

        # Resolve Mappings
        raw_mappings = self.config.get("type_mappings", {})
        self.processed_mappings = {
            label: get_sa_type_classes(types) for label, types in raw_mappings.items()
        }

        # Defaults (Registry > Params > Hardcoded)
        # Defaults (Registry > Params > Hardcoded)
        # PRIORITIZE: paths.target.datacubes_dir > target_datacubes_folder (legacy) > valid_libpaths (legacy)
        target_path = (
            self.params.get("paths", {}).get("target", {}).get("datacubes_dir")
        )
        if not target_path:
            # Legacy fallback
            target_path = self.config.get("target_datacubes_folder") or self.params.get(
                "target_datacubes_folder"
            )
        if not target_path:
            target_path = self.config.get("valid_libpaths") or self.params.get(
                "valid_libpaths"
            )

        self.valid_paths = []
        if target_path:
            if isinstance(target_path, str):
                self.valid_paths = [target_path]
            elif isinstance(target_path, list):
                self.valid_paths = target_path

        if not self.valid_paths:
            # Default fallback if nothing provided - risky but prevents crash on empty
            self.valid_paths = []

        # Field Map Paths
        tgt_fm = self.params.get("paths", {}).get("target", {}).get("field_maps_dir")
        if not tgt_fm:
            tgt_fm = self.config.get("target_fieldmaps_folder") or self.params.get(
                "target_fieldmaps_folder"
            )
        if not tgt_fm:
            tgt_fm = (
                self.config.get("valid_fieldmap_paths")
                or self.params.get("valid_fieldmap_paths")
                or []
            )

        if isinstance(tgt_fm, str):
            self.valid_fieldmap_paths = [tgt_fm]
        else:
            self.valid_fieldmap_paths = tgt_fm

        # Connection Defaults
        # Support fallback to default_connection_obj from params if not in registry
        self.default_connection_obj = self.config.get(
            "default_connection_obj",
            self.config.get(
                "default_config_obj", self.params.get("default_connection_obj")
            ),
        )

        self.default_base_class = (
            self.params.get("defaults", {}).get("base_datacube_class")
            or self.config.get("base_datacube_class")
            or self.params.get("base_datacube_class")
            or self.config.get("default_base_class")
            or self.params.get("default_base_class")
        )
        self.default_base_import = (
            self.params.get("defaults", {}).get("base_import")
            or self.config.get("default_base_import")
            or self.params.get("default_base_import")
        )
        self.default_backend = (
            self.params.get("defaults", {}).get("backend")
            or self.config.get("default_backend")
            or self.params.get("default_backend", "sqlalchemy")
        )
        self.class_suffix = (
            self.params.get("defaults", {}).get("class_suffix")
            or self.config.get("class_suffix")
            or self.params.get("class_suffix", "Dc")
        )
        self._enforce_custom_names()

    def _enforce_custom_names(self) -> None:
        """
        Ensures that if custom_name is set, it overrides class_name explicitly.
        """
        for table, meta in self.tables.items():
            if meta.get("custom_name"):
                meta["class_name"] = meta["custom_name"]

    def get_table_details(self, table_name: str) -> dict[str, Any]:
        return self.tables.get(table_name, {})

    def group_tables_by_file(self) -> dict[str, list[tuple]]:
        """
        Groups tables by their target file path.
        Returns: Dict[path_str, List[(table_name, conf_obj, base_cls, base_imp, cls_name)]]
        """
        file_groups: dict[str, list[tuple]] = {}
        for table_name, details in self.tables.items():
            target = details.get("save_to_path", details.get("path"))
            if not target:
                continue

            conf_obj = details.get(
                "connection_obj", details.get("config_obj", self.default_connection_obj)
            )
            base_cls = details.get("base_class", self.default_base_class)
            base_imp = details.get("import_from", self.default_base_import)
            cls_name = details.get("class_name")

            file_groups.setdefault(target, []).append(
                (table_name, conf_obj, base_cls, base_imp, cls_name)
            )
        return file_groups

    def merge_discovered(
        self, discovered_tables: Mapping[str, Mapping[str, Any]]
    ) -> None:
        """
        Merges discovered tables into the existing configuration.
        """
        for table, new_details in discovered_tables.items():
            if table in self.tables:
                existing = self.tables[table]
                for k, v in new_details.items():
                    if v is not None:
                        existing[k] = v
                    elif k not in existing:
                        existing[k] = v

                # Override class_name if custom_name is set
                if existing.get("custom_name"):
                    existing["class_name"] = existing["custom_name"]
                elif "class_name" not in existing:
                    existing["class_name"] = new_details.get("class_name")

                self.tables[table] = existing
            else:
                self.tables[table] = new_details

    def prune_tables(self, keep_tables: Iterable[str]) -> list[str]:
        """
        Removes tables from the registry that are not in the keep_tables list.
        """
        to_remove = [t for t in self.tables if t not in keep_tables]
        for t in to_remove:
            del self.tables[t]
        return to_remove

    def to_class_name(self, table_name: str) -> str:
        """
        Converts snake_case table name to CamelCase + Suffix.
        """
        return (
            "".join(w.capitalize() for w in table_name.split("_")) + self.class_suffix
        )

    def normalize_aliases(self, ignored_prefixes: Optional[list[str]] = None) -> None:
        pass

    def discover(self, allowed_paths: Optional[list[str]] = None) -> list[str]:
        return []


def perform_discovery(
    all_table_names: Iterable[str],
    match_rule_callback: Callable[[str], Optional[Mapping[str, Any]]],
    registry: DatacubeRegistry,
    whitelist: Optional[Mapping[str, Mapping[str, Any]]] = None,
    template: Optional[Mapping[str, Any]] = None,
    field_map_template: Optional[str] = None,
    default_connection: str = "default_db_conf",
) -> dict[str, dict[str, Any]]:
    """
    Executes the discovery logic:
    1. Filter by whitelist (if provided).
    2. Match tables against rules.
    3. Construct entries using (Template + Rule + Whitelist Override).
    """
    discovered = {}

    candidates = all_table_names
    if whitelist is not None:
        candidates = [t for t in all_table_names if t in whitelist]

    for table in candidates:
        rule = match_rule_callback(table)
        if not rule:
            continue

        path = rule["path"]
        class_name = registry.to_class_name(table)

        # entry = (template or {}).copy()
        # Refactoring Note: dict() creates a copy and supports Mapping
        entry = dict(template or {})

        if rule.get("domain") and field_map_template:
            # e.g. "solutions.conf.transforms.fields.{domain}.{table}.field_map"
            entry["field_map"] = field_map_template.format(
                domain=rule["domain"], table=table
            )

        entry.update(
            {
                "class_name": class_name,
                "connection_obj": rule.get("connection_obj") or default_connection,
                "save_to_path": path,
            }
        )

        if whitelist and table in whitelist:
            overrides = whitelist[table]
            if overrides:
                entry.update(overrides)

        discovered[table] = entry

    return discovered


def generate_field_map_files(
    discovered_entries: Mapping[str, Mapping[str, Any]],
    inspector: Any,
    root_path: Path,
    force: bool = False,
    logger: Optional[Any] = None,
    allowed_paths: Optional[Iterable[str]] = None,
) -> None:
    """
    Generates or updates Python field map files for discovered tables.
    Uses FieldMapFactory for fresh generation.
    """
    # Sanitize root_path: if it contains dots but no slashes, it's likely a package-style string
    root_path_str = str(root_path)
    if "." in root_path_str and "/" not in root_path_str and not Path(root_path_str).exists():
        root_path = Path(root_path_str.replace(".", "/"))

    for table, entry in discovered_entries.items():
        print(f"HARD DEBUG: Processing {table}")
        if logger:
            logger(f"DEBUG: Processing {table} for field map generation")
        # field_map = entry.get("field_map")
        # if not field_map:
        #    continue

        rule = entry
        domain = rule.get("domain", "common")
        target_dir = root_path / domain
        target_file = target_dir / f"{table}.py"

        # Security Check
        if allowed_paths:
            if not is_secure_path(str(target_file), allowed_paths):
                if logger:
                    logger(
                        f"[red]Blocked Field Map Generation: {target_file} is outside allowed paths.[/red]"
                    )
                continue

        target_dir.mkdir(parents=True, exist_ok=True)

        if not (target_dir / "__init__.py").exists():
            (target_dir / "__init__.py").touch()

        try:
            cols = inspector.get_columns(table)
            col_list = [c["name"] for c in cols]  # Preserve order

            if target_file.exists() and not force:
                if logger:
                    logger(
                        f"[yellow]Skipping {target_file}: Updating existing files to use Factory is not yet supported in this mode. Use --force to rewrite.[/yellow]"
                    )
            else:
                # --- Fresh Generation with Factory ---
                lines = [
                    "from typing import Mapping",
                    "from sibi_flux.datacube.field_factory import FieldMapFactory",
                    "",
                    f'field_map: Mapping[str, str] = FieldMapFactory.create("{table}", [',
                ]
                for col in col_list:
                    lines.append(f'    "{col}",')
                lines.append("])")

                with open(target_file, "w") as f:
                    f.write("\n".join(lines))

                if logger:
                    logger(f"[green]Generated {target_file} (using Factory)[/green]")

        except (OSError, sa.exc.SQLAlchemyError) as e:
            if logger:
                logger(f"[red]Failed to generate/sync fields for {table}: {e}[/red]")


def generate_datacube_module_code(
    items: Iterable[tuple],
    registry: DatacubeRegistry,
    get_db_url_callback: Callable[[str], str],
    logger: Optional[Any] = None,
) -> Tuple[list[str], list[str]]:
    """
    Generates the code components (imports, class definitions) for a single DataCube module file.
    Handles import aliasing to prevent variable shadowing.
    """
    # Refactoring Note: using concrete Dict in string literal is fine as it's code generation
    imports: Set[str] = {
        "from typing import Optional, List, ClassVar, Dict, Mapping",
        "import dask.dataframe as dd",
    }
    classes_code = []

    for table_name, conf_obj, base_cls, base_imp, cls_name in items:
        # Support "module:Class" format
        if ":" in base_imp and " " not in base_imp:
            mod, cls = base_imp.split(":", 1)
            imports.add(f"from {mod} import {cls}")
        else:
            imports.add(base_imp)

        # Config Object Import
        # We need to resolve import for conf_obj (e.g. clickhouse_conf)
        # Using registry.params which contains databases list with import_spec
        params = registry.params
        dbs = params.get("databases", [])
        for db in dbs:
            ref = db.get("connection_ref") or db.get("connection_obj")
            if ref == conf_obj:
               spec = db.get("import_spec")
               if spec and isinstance(spec, dict):
                   mod = spec.get("module")
                   sym = spec.get("symbol")
                   if mod and sym:
                       imports.add(f"from {mod} import {sym}")
               elif db.get("global_import"):
                   # Legacy or simple string import
                   imp_str = db.get("global_import")
                   if imp_str and sym and sym in imp_str:
                        imports.add(imp_str)
                   elif imp_str: # Fallback to add specific object if simple import
                       # Try to parse or just add "from ... import conf_obj" logic?
                       # If global_import is "from conf.credentials import *" -> hard to know
                       # If it is "from conf.credentials import replica_conf" -> easy
                       imports.add(imp_str)
               break

        details = registry.get_table_details(table_name)
        raw_name = details.get("table", table_name)
        field_map_str = details.get("field_map")
        sticky_filters = details.get("sticky_filters")
        column_metadata = details.get("column_metadata")

        field_map_var = None
        column_metadata_var = None
        legacy = False

        field_map_dict = details.get("field_map_dict")

        if field_map_str:
            if "." in field_map_str:
                mod, var = field_map_str.rsplit(".", 1)

                # REFACTOR: Import module as alias instead of individual variables
                # This is cleaner and more pythonic
                mod_alias = f"mod_{raw_name}"
                imports.add(f"import {mod} as {mod_alias}")

                field_map_var = f"{mod_alias}.{var}"
                legacy = True  # Still "legacy" in path sense, or "explicit" path

                # Try to load for introspection (Validation)
                try:
                    module = importlib.import_module(mod)
                    field_map_dict = getattr(module, var)
                    if not isinstance(field_map_dict, dict):
                        field_map_dict = None

                    # Check for metadata
                    if hasattr(module, "metadata"):
                        column_metadata_var = f"{mod_alias}.metadata"

                except (ImportError, AttributeError):
                    field_map_dict = None

        if field_map_str and not legacy:
            # This handles the case where we injected the path in CLI but didn't mark as legacy import
            if "." in field_map_str:
                mod, var = field_map_str.rsplit(".", 1)
                mod_alias = f"mod_{raw_name}"
                imports.add(f"import {mod} as {mod_alias}")

                field_map_var = f"{mod_alias}.{var}"
                column_metadata_var = f"{mod_alias}.metadata"
            else:
                if logger:
                    logger(
                        f"[yellow]Warning: Invalid field_map format '{field_map_str}' for {table_name}. Expected module.path.variable[/yellow]"
                    )

        # Fallback / Direct Injection
        if not field_map_dict:
            field_map_dict = details.get("field_map_dict")

        try:
            db_url = get_db_url_callback(conf_obj)
            engine = sa.create_engine(db_url)

            # Fix for SA 2.0 / Clickhouse: Inspect connection, not engine
            with engine.connect() as conn:
                inspector = inspect(conn)

                # Metadata Population Phase (Phase 4)
                # We want to populate `column_metadata` using the Global Repo if possible
                if not column_metadata:
                    try:
                        from sibi_flux.datacube.field_factory import FieldMapFactory

                        manager = FieldMapFactory._get_manager()

                        # Introspect columns (Optimization: We could reuse this for generate_class_code but API separation)
                        cols = inspector.get_columns(raw_name)
                        column_metadata = {}

                        for col in cols:
                            c_name = col["name"]
                            c_type = str(col["type"])
                            fid = manager._generate_id(c_name, c_type)
                            field_def = manager.fields.get(fid)

                            if field_def:
                                # Use target_column if available (this is the aliased name)
                                # Fallback to original name if not (though field definition usually implies an alias or identity)
                                key_name = field_def.target_column or c_name

                                # Use pyarrow mapping as canonical type for metadata lists
                                if "pyarrow" in field_def.mappings:
                                    raw_type = field_def.mappings["pyarrow"]

                                    # Map to DfValidator canonical types
                                    # This ensures validation_schema matches what DfValidator normalizes to
                                    type_map = {
                                        "string": "string[pyarrow]",
                                        "int64": "Int64[pyarrow]",
                                        "int32": "Int32[pyarrow]",
                                        "int16": "Int16[pyarrow]",
                                        "int8": "Int8[pyarrow]",
                                        "float64": "Float64[pyarrow]",
                                        "float32": "Float32[pyarrow]",
                                        "timestamp[ns]": "datetime64[ns, UTC]",
                                        "bool": "boolean[pyarrow]",
                                    }
                                    column_metadata[key_name] = type_map.get(
                                        raw_type, raw_type
                                    )
                                else:
                                    # Fallback or infer
                                    column_metadata[key_name] = field_def.type
                    except ImportError:
                        pass  # Factory not available
                    except Exception as e:
                        if logger:
                            logger(
                                f"[yellow]Warning: Metadata population failed for {table_name}: {e}[/yellow]"
                            )

                classes_code.append(
                    generate_class_code(
                        inspector,
                        raw_name, # Use raw table name instead of registry key
                        conf_obj,
                        base_cls,
                        registry.processed_mappings,
                        class_name=cls_name,
                        class_suffix=registry.class_suffix,
                        backend=registry.default_backend,
                        field_map_var=field_map_var,
                        legacy_filters=legacy,
                        field_map_dict=field_map_dict,
                        sticky_filters=sticky_filters,
                        column_metadata=column_metadata,
                        metadata_var=column_metadata_var,
                    )
                )
        except sa.exc.SQLAlchemyError as e:
            if logger:
                logger(
                    f"[yellow]Warning: Could not introspect {table_name} ({e}). Generating shell class.[/yellow]"
                )
            classes_code.append(
                generate_class_code(
                    None,
                    table_name,
                    conf_obj,
                    base_cls,
                    registry.processed_mappings,
                    class_name=cls_name,
                    class_suffix=registry.class_suffix,
                    backend=registry.default_backend,
                    field_map_var=field_map_var,
                    field_map_dict=field_map_dict,
                    sticky_filters=sticky_filters,
                )
            )

    return list(imports), classes_code


def dump_db_schema(
    engine: Any,
    db_name: str,
    output_dir: Path,
    whitelist: Optional[Mapping[str, Any]] = None,
    logger: Optional[Any] = None,
) -> None:
    """
    Introspects the database and dumps a comprehensive schema YAML file.
    Structure:
    <db_name>:
       <table_name>:
          columns: ...
          indexes: ...
          pk: ...
          foreign_keys: ...
    """
    inspector = inspect(engine)
    all_tables = inspector.get_table_names()

    if whitelist:
        tables_to_process = sorted([t for t in all_tables if t in whitelist])
    else:
        tables_to_process = sorted(all_tables)

    schema_data = {}

    for table in tables_to_process:
        try:
            # Columns
            columns = []
            for col in inspector.get_columns(table):
                columns.append({"name": col["name"], "type": str(col["type"])})

            schema_data[table] = {"columns": columns}
        except sa.exc.SQLAlchemyError as e:
            if logger:
                logger(f"[red]Error dumping schema for table {table}: {e}[/red]")
            schema_data[table] = {"error": str(e)}

    # Top level key is db_name
    final_output = {db_name: schema_data}

    output_dir.mkdir(parents=True, exist_ok=True)
    out_file = output_dir / f"schema_{db_name}.yaml"

    import yaml

    with open(out_file, "w") as f:
        yaml.dump(final_output, f, sort_keys=False, default_flow_style=False)

    if logger:
        logger(
            f"[green]Dumped schema to {out_file} ({len(tables_to_process)} tables)[/green]"
        )
