"""BLCE Kimball Model Proposer — proposes a complete DW model.

Merges dimension/fact candidates from E2E discovery, report analysis,
meeting notes, and consultant intake into a unified ProposedModel.
"""
from __future__ import annotations

import logging
import re
from typing import Any, Dict, List, Optional, Set

from .contracts import (
    E2EHandoff,
    ExtractedDimension,
    ExtractedKPI,
    IntakeQuestionnaire,
    MeasureValidation,
    MeetingNotesAnalysis,
    ProposedDimension,
    ProposedFact,
    ProposedModel,
    ReportAnalysis,
)
from .report_patterns import DIMENSION_KEYWORD_MAP, MEASURE_KEYWORD_MAP

logger = logging.getLogger(__name__)

# Regex patterns for column classification
_PK_SUFFIXES = ("_id", "_hid", "_tid", "_key", "_code", "_num", "_sk")
_DATE_PATTERNS = re.compile(r"(date|_dt$|_date$|_ts$|timestamp)", re.IGNORECASE)
_MEASURE_PATTERNS = re.compile(
    r"(amount|amt|qty|quantity|total|balance|rate|price|cost|count|sum|weight|volume)",
    re.IGNORECASE,
)

# SCD type detection patterns (P2.1)
_SCD2_COLUMN_PATTERNS = re.compile(
    r"(effective_from|effective_to|valid_from|valid_to|start_date|end_date|"
    r"is_current|current_flag|version|revision|row_effective|row_expiry|"
    r"begin_date|expire_date)", re.IGNORECASE)

_SCD0_TABLE_PATTERNS = re.compile(
    r"(dim_date|dim_calendar|dim_time|dim_product_component|"
    r"dim_status|dim_pay_code|dim_interest_type|dim_billing_category|"
    r"accounting_dates|payment_dates|service_dates)", re.IGNORECASE)


class KimballModelProposer:
    """Propose a complete Kimball star-schema model from multiple inputs."""

    def __init__(self) -> None:
        self._last_template_resolution: Dict[str, Any] = {
            "erp_type_input": "",
            "erp_type_resolved": "",
            "template_used": False,
            "template_resolution": "none",
            "detected_erp": "",
        }

    def get_last_template_resolution(self) -> Dict[str, Any]:
        """Return metadata from the last ERP template resolution decision."""
        return dict(self._last_template_resolution)

    def propose_model(
        self,
        handoff: E2EHandoff,
        questionnaire: Optional[IntakeQuestionnaire] = None,
        report_analyses: Optional[List[ReportAnalysis]] = None,
        notes_analyses: Optional[List[MeetingNotesAnalysis]] = None,
        erp_type: str = "",
        client_id: str = "",
    ) -> ProposedModel:
        """Propose dimensions, facts, and bus matrix.

        Args:
            handoff: E2E pipeline outputs.
            questionnaire: Completed intake questionnaire.
            report_analyses: Report analysis results.
            notes_analyses: Meeting notes analysis results.
            erp_type: ERP system type.
            client_id: Client identifier.

        Returns:
            ProposedModel with dimensions, facts, bus matrix.
        """
        # Resolve template ERP once up front so dims/facts use consistent selection.
        table_names = [
            p.get("table_name", "")
            for p in (handoff.table_profiles or [])
            if isinstance(p, dict) and p.get("table_name")
        ]
        resolved_erp = erp_type
        try:
            from src.data_modeling.erp_configs.templates import resolve_dm_template_erp

            self._last_template_resolution = resolve_dm_template_erp(erp_type, table_names)
            resolved_erp = self._last_template_resolution.get("erp_type_resolved", "") or erp_type
            logger.info(
                "Template resolution: input=%s resolved=%s mode=%s used=%s",
                erp_type or "",
                resolved_erp or "",
                self._last_template_resolution.get("template_resolution", "none"),
                self._last_template_resolution.get("template_used", False),
            )
        except Exception as exc:
            logger.debug("Template ERP resolution failed, using provided erp_type: %s", exc)
            self._last_template_resolution = {
                "erp_type_input": erp_type or "",
                "erp_type_resolved": erp_type or "",
                "template_used": False,
                "template_resolution": "none",
                "detected_erp": "",
            }

        # 1. Propose dimensions from all sources
        dimensions = self._propose_dimensions(
            handoff, questionnaire, report_analyses, notes_analyses, resolved_erp,
        )

        # 2. Propose facts from transaction tables + KPIs
        all_kpis = self._collect_kpis(questionnaire, report_analyses, notes_analyses)
        facts = self._propose_facts(
            handoff, dimensions, all_kpis, resolved_erp,
        )

        # 3. Generate bus matrix
        bus_matrix = self._generate_bus_matrix(dimensions, facts)

        # 4. Build measure dictionary
        measure_dict = self._build_measure_dictionary(all_kpis)

        # 5. Detect union patterns
        self._detect_union_patterns(facts, handoff)

        # 6. Map report items to proposed model (P1.1)
        mapping_report_dict = None
        if report_analyses:
            try:
                from .report_model_mapper import map_report_to_model

                model_tmp = ProposedModel(
                    client_id=client_id,
                    dimensions=dimensions,
                    facts=facts,
                )
                mapping = map_report_to_model(report_analyses, model_tmp)
                mapping_report_dict = mapping.model_dump()
            except Exception as exc:
                logger.debug("Report-to-model mapping failed: %s", exc)

        # 7. Detect hierarchies within dimensions (P1.4)
        self._detect_hierarchy_levels(dimensions, handoff)

        # 8. Auto-detect SCD types for dimensions (P2.1)
        self._detect_scd_types(dimensions, handoff)

        # 9. Validate measures against source table columns (P2.2)
        measure_validations = self._validate_measures(facts, handoff)

        return ProposedModel(
            client_id=client_id,
            dimensions=dimensions,
            facts=facts,
            bus_matrix=bus_matrix,
            measure_dictionary=measure_dict,
            mapping_report=mapping_report_dict,
            measure_validations=measure_validations,
        )

    # ------------------------------------------------------------------
    # Dimension proposal
    # ------------------------------------------------------------------

    def _propose_dimensions(
        self,
        handoff: E2EHandoff,
        questionnaire: Optional[IntakeQuestionnaire],
        report_analyses: Optional[List[ReportAnalysis]],
        notes_analyses: Optional[List[MeetingNotesAnalysis]],
        erp_type: str,
    ) -> List[ProposedDimension]:
        """Merge dimension candidates from all sources."""
        candidates: Dict[str, ProposedDimension] = {}

        # Source 1: E2E relationships via DimensionDetector
        self._dims_from_e2e(candidates, handoff)

        # Source 2: Report analyses
        for ra in (report_analyses or []):
            for ed in ra.dimensions:
                self._merge_dimension(candidates, ed)

        # Source 3: Meeting notes
        for mna in (notes_analyses or []):
            for ed in mna.dimensions:
                self._merge_dimension(candidates, ed)

        # Source 4: Questionnaire confirmed dims
        if questionnaire:
            self._dims_from_questionnaire(candidates, questionnaire)

        # Source 5: ERP template
        self._dims_from_erp_template(candidates, erp_type)

        return list(candidates.values())

    def _dims_from_e2e(
        self, candidates: Dict[str, ProposedDimension], handoff: E2EHandoff
    ) -> None:
        """Detect dimensions from E2E data using DimensionDetector + name patterns."""
        # Strategy 0: Enhanced table analyses from TableAnalyzer (P0.2)
        table_analyses = (handoff.run_summary or {}).get("table_analyses", {})
        natural_keys = (handoff.run_summary or {}).get("natural_keys", {})
        if table_analyses:
            for table, analysis in table_analyses.items():
                if analysis.get("role") != "reference":
                    continue
                key = table.upper()
                if key in candidates:
                    continue
                attrs = self._get_table_columns(handoff, table)
                domain = analysis.get("domain", "")
                biz_name = domain if domain and domain != "Unknown" else key.replace("DIM_", "").replace("_", " ").title()
                dim_name = key if key.startswith("DIM_") else f"DIM_{key}"
                candidates[key] = ProposedDimension(
                    name=dim_name,
                    business_name=biz_name,
                    source_tables=[table],
                    key_columns=natural_keys.get(table, []),
                    attributes=attrs,
                    rationale=f"Heuristic: {analysis.get('purpose_summary', 'reference table')}",
                    confidence=analysis.get("confidence", 0.65),
                )
            logger.info("Strategy 0 (TableAnalyzer): added %d dimension candidates", len([
                a for a in table_analyses.values() if a.get("role") == "reference"
            ]))

        # Strategy 1: DimensionDetector on relationships
        try:
            from src.data_modeling.dimension_detector import DimensionDetector

            detector = DimensionDetector()
            classification = detector.classify_tables(handoff.relationships)
            for table, cls in classification.items():
                if cls == "dimension":
                    key = table.upper()
                    if key not in candidates:
                        attrs = self._get_table_columns(handoff, table)
                        candidates[key] = ProposedDimension(
                            name=key if key.startswith("DIM_") else f"DIM_{key}",
                            business_name=key.replace("DIM_", "").replace("_", " ").title(),
                            source_tables=[table],
                            attributes=attrs,
                            rationale="Detected from E2E relationship analysis",
                            confidence=0.8,
                        )
        except Exception as exc:
            logger.debug("DimensionDetector unavailable: %s", exc)

        # Strategy 2: Fallback — detect tables with DIM_ prefix
        for profile in handoff.table_profiles:
            table = profile.get("table_name", "")
            key = table.upper()
            if key.startswith("DIM_") and key not in candidates:
                attrs = self._get_table_columns(handoff, table)
                candidates[key] = ProposedDimension(
                    name=key,
                    business_name=key.replace("DIM_", "").replace("_", " ").title(),
                    source_tables=[table],
                    attributes=attrs,
                    rationale="Detected from DIM_ table name prefix",
                    confidence=0.7,
                )

        # Strategy 3: Tables referenced as FK targets in relationships
        fk_targets: Set[str] = set()
        for rel in handoff.relationships:
            target = (rel.get("to_table", "") or rel.get("target_table", "")).upper()
            if target:
                fk_targets.add(target)

        for target in fk_targets:
            if target not in candidates:
                attrs = self._get_table_columns(handoff, target)
                row_count = 0
                for p in handoff.table_profiles:
                    if p.get("table_name", "").upper() == target:
                        row_count = p.get("row_count", 0) or 0
                        break
                # Low row count tables that are FK targets are likely dimensions
                if row_count < 100000:
                    dim_name = target if target.startswith("DIM_") else f"DIM_{target}"
                    candidates[dim_name] = ProposedDimension(
                        name=dim_name,
                        business_name=target.replace("DIM_", "").replace("_", " ").title(),
                        source_tables=[target],
                        attributes=attrs,
                        rationale="FK target table with low row count",
                        confidence=0.6,
                    )

        # Strategy 4 (P4.4): Column-name FK inference for dimension detection
        # When no explicit relationships exist, infer FK targets from column names
        if not fk_targets:
            try:
                from src.data_modeling.column_fk_inferer import infer_relationships_by_name

                table_columns: Dict[str, List[str]] = {}
                for p in handoff.table_profiles:
                    tbl = p.get("table_name", "")
                    cols = p.get("columns", [])
                    col_names = [
                        c.get("column_name", c.get("name", ""))
                        if isinstance(c, dict) else str(c)
                        for c in cols
                    ]
                    if tbl and col_names:
                        table_columns[tbl] = col_names

                inferred_rels = infer_relationships_by_name(table_columns, min_score=2)
                inferred_targets: Dict[str, int] = {}
                for rel in inferred_rels:
                    tgt = rel["target_table"].upper()
                    inferred_targets[tgt] = inferred_targets.get(tgt, 0) + 1

                for target, ref_count in inferred_targets.items():
                    # Skip tables already captured as candidates
                    dim_key = target if target.startswith("DIM_") else f"DIM_{target}"
                    if dim_key in candidates or target in candidates:
                        continue
                    # Tables referenced by multiple other tables as FK targets
                    if ref_count >= 1:
                        attrs = self._get_table_columns(handoff, target)
                        biz_name = target.replace("WP_", "").replace("DIM_", "").replace("_", " ").title()
                        candidates[dim_key] = ProposedDimension(
                            name=dim_key,
                            business_name=biz_name,
                            source_tables=[target],
                            attributes=attrs,
                            rationale=f"Inferred FK target (referenced by {ref_count} tables)",
                            confidence=min(0.7, 0.5 + ref_count * 0.05),
                        )
                logger.info("Strategy 4 (FK inference): added %d dimension candidates", len(inferred_targets))
            except Exception as exc:
                logger.debug("Column FK inference unavailable: %s", exc)

    def _merge_dimension(
        self,
        candidates: Dict[str, ProposedDimension],
        ed: ExtractedDimension,
    ) -> None:
        """Merge an ExtractedDimension into candidates."""
        key = ed.name.upper()
        if key in candidates:
            existing = candidates[key]
            existing.source_tables = list(
                set(existing.source_tables + ed.source_tables)
            )
            existing.attributes = list(
                set(existing.attributes + ed.attributes)
            )
            existing.confidence = min(1.0, existing.confidence + 0.1)
        else:
            candidates[key] = ProposedDimension(
                name=key if key.startswith("DIM_") else key,
                business_name=ed.business_name or key.replace("_", " ").title(),
                source_tables=ed.source_tables,
                key_columns=ed.key_columns,
                attributes=ed.attributes,
                hierarchy_levels=ed.hierarchy_levels,
                scd_type=ed.scd_type,
                rationale=f"Extracted from {ed.extracted_from}",
                confidence=ed.confidence,
            )

    def _dims_from_questionnaire(
        self,
        candidates: Dict[str, ProposedDimension],
        questionnaire: IntakeQuestionnaire,
    ) -> None:
        """Add confirmed dimensions from intake questionnaire."""
        for section_qs in questionnaire.sections.values():
            for q in section_qs:
                if q.question_id == "dim_confirmed" and (q.response or q.auto_value):
                    val = q.response or q.auto_value
                    for dim_name in val.split(","):
                        dim_name = dim_name.strip().upper()
                        if dim_name and dim_name not in candidates:
                            candidates[dim_name] = ProposedDimension(
                                name=dim_name,
                                business_name=dim_name.replace("_", " ").title(),
                                rationale="Confirmed in intake questionnaire",
                                confidence=0.9,
                            )
                        elif dim_name in candidates:
                            candidates[dim_name].confidence = min(
                                1.0, candidates[dim_name].confidence + 0.1
                            )

                if q.question_id == "dim_scd_tables" and q.response:
                    for tbl in q.response.split(","):
                        tbl = tbl.strip().upper()
                        if tbl in candidates:
                            candidates[tbl].scd_type = 2

    def _dims_from_erp_template(
        self,
        candidates: Dict[str, ProposedDimension],
        erp_type: str,
    ) -> None:
        """Add or merge dimensions from ERP DM template if available."""
        if not erp_type or erp_type == "unknown":
            return
        try:
            from src.data_modeling.erp_configs.templates import load_dm_template

            specs = load_dm_template(erp_type)
            for spec in (specs or []):
                name = getattr(spec, "target_table", "")
                if not name or not name.upper().startswith("DIM_"):
                    continue
                key = name.upper()
                if key in candidates:
                    # Merge template metadata into existing candidate
                    existing = candidates[key]
                    tmpl_sources = getattr(spec, "source_tables", [])
                    if tmpl_sources:
                        merged = list(dict.fromkeys(existing.source_tables + tmpl_sources))
                        existing.source_tables = merged
                    tmpl_cols = getattr(spec, "columns", [])
                    pk_cols = [
                        c.alias for c in tmpl_cols
                        if getattr(c, "kind", "") in ("str", "num")
                        and c.alias.upper().endswith(tuple(s.upper() for s in _PK_SUFFIXES))
                    ]
                    if pk_cols and not existing.natural_key:
                        existing.natural_key = pk_cols[0]
                    if tmpl_cols and not existing.hierarchy_levels:
                        level_cols = [
                            c.alias for c in tmpl_cols
                            if "LEVEL" in c.alias.upper() or "PARENT" in c.alias.upper()
                        ]
                        if level_cols:
                            existing.hierarchy_levels = level_cols
                    existing.confidence = min(1.0, existing.confidence + 0.15)
                    existing.rationale += f"; enriched from {erp_type} template"
                else:
                    candidates[key] = ProposedDimension(
                        name=key,
                        business_name=key.replace("DIM_", "").replace("_", " ").title(),
                        source_tables=getattr(spec, "source_tables", []),
                        rationale=f"From {erp_type} ERP template",
                        confidence=0.7,
                    )
        except Exception:
            pass

    # ------------------------------------------------------------------
    # Fact proposal
    # ------------------------------------------------------------------

    def _propose_facts(
        self,
        handoff: E2EHandoff,
        dimensions: List[ProposedDimension],
        kpis: List[ExtractedKPI],
        erp_type: str,
    ) -> List[ProposedFact]:
        """Propose fact tables from transaction tables and KPIs."""
        facts: Dict[str, ProposedFact] = {}
        dim_names = {d.name.upper() for d in dimensions}

        # Source 0: Enhanced table analyses from TableAnalyzer (P0.2)
        table_analyses = (handoff.run_summary or {}).get("table_analyses", {})
        natural_keys = (handoff.run_summary or {}).get("natural_keys", {})
        if table_analyses:
            for table, analysis in table_analyses.items():
                if analysis.get("role") != "transaction":
                    continue
                key = table.upper()
                fact_name = key if key.startswith("FACT_") else f"FACT_{key}"
                if fact_name in facts:
                    continue
                domain = analysis.get("domain", "")
                biz_name = domain if domain and domain != "Unknown" else key.replace("FACT_", "").replace("_", " ").title()
                keys = natural_keys.get(table, [])
                facts[fact_name] = ProposedFact(
                    name=fact_name,
                    business_name=biz_name,
                    grain=" x ".join(keys[:5]) if keys else "",
                    grain_columns=keys,
                    source_tables=[table],
                    rationale=f"Heuristic: {analysis.get('purpose_summary', 'transaction table')}",
                    confidence=analysis.get("confidence", 0.6),
                )

        # Source 1: High row-count tables with date columns (transaction pattern)
        for profile in handoff.table_profiles:
            table = profile.get("table_name", "")
            row_count = profile.get("row_count", 0) or 0
            columns = profile.get("columns", [])
            col_names = [
                c.get("column_name", c.get("name", "")) if isinstance(c, dict) else str(c)
                for c in columns
            ]

            has_date = any(_DATE_PATTERNS.search(c) for c in col_names)
            has_measure = any(_MEASURE_PATTERNS.search(c) for c in col_names)

            if has_date and has_measure and row_count > 1000:
                key = table.upper()
                fact_name = key if key.startswith("FACT_") else f"FACT_{key}"

                # Infer grain columns (FK-like columns)
                grain_cols = [c for c in col_names if c.lower().endswith(_PK_SUFFIXES)]
                # Infer measures
                measures = [c for c in col_names if _MEASURE_PATTERNS.search(c)]
                # Infer FK links to dimensions
                fks = [c for c in grain_cols if any(d in c.upper() for d in dim_names)]

                facts[fact_name] = ProposedFact(
                    name=fact_name,
                    business_name=key.replace("_", " ").title(),
                    grain=" x ".join(grain_cols[:5]) if grain_cols else "",
                    grain_columns=grain_cols,
                    measures=measures,
                    dimension_fks=fks,
                    source_tables=[table],
                    rationale=f"Transaction pattern: {row_count} rows, date + measures",
                    confidence=0.7,
                )

        # Source 1.5: FK-signature detection (P4.4)
        # Tables with high FK-column ratio + measure columns → likely facts
        for profile in (handoff.table_profiles or []):
            tname = profile.get("table_name", "").upper()
            fact_name = tname if tname.startswith("FACT_") else f"FACT_{tname}"
            if fact_name in facts or tname in dim_names:
                continue
            cols = profile.get("columns", [])
            if not cols:
                continue
            col_names = [
                c.get("column_name", c.get("name", "")) if isinstance(c, dict) else str(c)
                for c in cols
            ]
            fk_count = sum(
                1 for c in col_names
                if c.lower().endswith(_PK_SUFFIXES)
            )
            measure_count = sum(
                1 for c in col_names
                if _MEASURE_PATTERNS.search(c)
            )
            total = len(col_names)
            if total >= 4 and fk_count >= 2 and measure_count >= 1 and fk_count / total >= 0.25:
                measures = [c for c in col_names if _MEASURE_PATTERNS.search(c)]
                facts[fact_name] = ProposedFact(
                    name=fact_name,
                    business_name=tname.replace("WP_", "").replace("_", " ").title(),
                    source_tables=[tname],
                    measures=measures,
                    rationale="FK-signature detection: high FK ratio + measures",
                    confidence=0.55,
                )

        # Source 2: KPIs that may need dedicated facts
        for kpi in kpis:
            # Check if any existing fact covers this KPI's measures
            covered = any(
                kpi.name.lower() in " ".join(f.measures).lower()
                for f in facts.values()
            )
            if not covered and kpi.granularity:
                key = f"FACT_{kpi.name.upper()}"
                if key not in facts:
                    facts[key] = ProposedFact(
                        name=key,
                        business_name=kpi.business_name,
                        grain=kpi.granularity,
                        measures=[kpi.name],
                        rationale=f"Dedicated fact for KPI: {kpi.business_name}",
                        confidence=0.5,
                    )

        # Source 3: ERP template facts
        self._facts_from_erp_template(facts, erp_type)

        # Cross-validation confidence boost (P4.4)
        self._cross_validate(dimensions, list(facts.values()), handoff)

        return list(facts.values())

    def _facts_from_erp_template(
        self,
        facts: Dict[str, ProposedFact],
        erp_type: str,
    ) -> None:
        """Add or merge facts from ERP DM template if available."""
        if not erp_type or erp_type == "unknown":
            return
        try:
            from src.data_modeling.erp_configs.templates import load_dm_template

            specs = load_dm_template(erp_type)
            for spec in (specs or []):
                name = getattr(spec, "target_table", "")
                if not name or not (name.upper().startswith("FACT_") or name.upper().startswith("FCT_")):
                    continue
                key = name.upper()
                if key in facts:
                    # Merge template metadata into existing candidate
                    existing = facts[key]
                    tmpl_sources = getattr(spec, "source_tables", [])
                    if tmpl_sources:
                        merged = list(dict.fromkeys(existing.source_tables + tmpl_sources))
                        existing.source_tables = merged
                    tmpl_cols = getattr(spec, "columns", [])
                    if tmpl_cols:
                        measure_cols = [
                            c.alias for c in tmpl_cols
                            if getattr(c, "kind", "") == "num"
                            and not c.alias.upper().endswith(
                                tuple(s.upper() for s in _PK_SUFFIXES)
                            )
                        ]
                        if measure_cols:
                            existing.measures = list(
                                dict.fromkeys(existing.measures + measure_cols)
                            )
                        grain_cols = [
                            c.alias for c in tmpl_cols
                            if getattr(c, "kind", "") in ("str",)
                            and c.alias.upper().endswith(
                                tuple(s.upper() for s in _PK_SUFFIXES)
                            )
                        ]
                        if grain_cols and not existing.grain:
                            existing.grain = " x ".join(grain_cols[:5])
                    existing.confidence = min(1.0, existing.confidence + 0.15)
                    existing.rationale += f"; enriched from {erp_type} template"
                else:
                    biz = key.replace("FCT_", "").replace("FACT_", "").replace("_", " ").title()
                    facts[key] = ProposedFact(
                        name=key,
                        business_name=biz,
                        source_tables=getattr(spec, "source_tables", []),
                        rationale=f"From {erp_type} ERP template",
                        confidence=0.7,
                    )
        except Exception:
            pass

    # ------------------------------------------------------------------
    # Bus matrix
    # ------------------------------------------------------------------

    def _generate_bus_matrix(
        self,
        dimensions: List[ProposedDimension],
        facts: List[ProposedFact],
    ) -> Dict[str, Any]:
        """Build a bus matrix grid: facts × dimensions."""
        dim_names = [d.name for d in dimensions]
        matrix: Dict[str, Dict[str, bool]] = {}

        for fact in facts:
            row: Dict[str, bool] = {}
            for dim in dimensions:
                # Check if fact references this dimension
                linked = any(
                    dim.name.lower() in fk.lower()
                    for fk in fact.dimension_fks
                ) or any(
                    dim.name.replace("DIM_", "").lower() in gc.lower()
                    for gc in fact.grain_columns
                )
                row[dim.name] = linked
            matrix[fact.name] = row

        # Conformance: dimensions used by multiple facts
        dim_usage = {d: 0 for d in dim_names}
        for row in matrix.values():
            for d, linked in row.items():
                if linked:
                    dim_usage[d] = dim_usage.get(d, 0) + 1

        conformed = [d for d, count in dim_usage.items() if count >= 2]

        return {
            "dimensions": dim_names,
            "facts": [f.name for f in facts],
            "matrix": matrix,
            "conformed_dimensions": conformed,
            "fact_count": len(facts),
            "dimension_count": len(dimensions),
        }

    # ------------------------------------------------------------------
    # Measure dictionary
    # ------------------------------------------------------------------

    def _build_measure_dictionary(
        self, kpis: List[ExtractedKPI]
    ) -> Dict[str, str]:
        """Build canonical measure name → definition mapping."""
        dictionary: Dict[str, str] = {}
        for kpi in kpis:
            key = kpi.name.upper()
            if key not in dictionary:
                dictionary[key] = kpi.formula or kpi.business_name
        return dictionary

    # ------------------------------------------------------------------
    # Union detection
    # ------------------------------------------------------------------

    def _detect_union_patterns(
        self,
        facts: List[ProposedFact],
        handoff: E2EHandoff,
    ) -> None:
        """Detect facts that may need UNION ALL from multiple sources."""
        # Group source tables by prefix — if multiple tables feed the
        # same fact grain, flag for union
        for fact in facts:
            if len(fact.source_tables) > 1:
                fact.needs_union = True
                fact.union_sources = fact.source_tables

    # ------------------------------------------------------------------
    # Cross-validation (P4.4)
    # ------------------------------------------------------------------

    @staticmethod
    def _cross_validate(
        dimensions: List[ProposedDimension],
        facts: List[ProposedFact],
        handoff: E2EHandoff,
    ) -> None:
        """Boost confidence when dims/facts corroborate each other.

        - Dimension appearing as FK in >=2 facts → +0.1
        - Fact with FKs to >=3 dimensions → +0.1
        """
        # Build FK column→dim entity lookup from table profiles
        # Count how many facts reference each dimension via grain/FK columns
        dim_entity_map: Dict[str, ProposedDimension] = {}
        for d in dimensions:
            entity = d.name.upper().replace("DIM_", "")
            dim_entity_map[entity] = d

        dim_ref_counts: Dict[str, int] = {d.name: 0 for d in dimensions}
        for fact in facts:
            dim_hits = 0
            all_cols = (fact.grain_columns or []) + (fact.dimension_fks or [])
            matched_dims: Set[str] = set()
            for col in all_cols:
                col_upper = col.upper()
                for entity, dim in dim_entity_map.items():
                    if entity in col_upper and dim.name not in matched_dims:
                        matched_dims.add(dim.name)
                        dim_ref_counts[dim.name] = dim_ref_counts.get(dim.name, 0) + 1
                        dim_hits += 1
            if dim_hits >= 3:
                fact.confidence = min(1.0, fact.confidence + 0.1)

        for dim in dimensions:
            if dim_ref_counts.get(dim.name, 0) >= 2:
                dim.confidence = min(1.0, dim.confidence + 0.1)

    # ------------------------------------------------------------------
    # Helpers
    # ------------------------------------------------------------------

    def _collect_kpis(
        self,
        questionnaire: Optional[IntakeQuestionnaire],
        report_analyses: Optional[List[ReportAnalysis]],
        notes_analyses: Optional[List[MeetingNotesAnalysis]],
    ) -> List[ExtractedKPI]:
        """Collect all KPIs from all sources."""
        kpis: List[ExtractedKPI] = []
        for ra in (report_analyses or []):
            kpis.extend(ra.kpis)
        for mna in (notes_analyses or []):
            kpis.extend(mna.kpis)
        if questionnaire:
            for kpi_name in questionnaire.kpis:
                kpis.append(
                    ExtractedKPI(
                        name=kpi_name.upper().replace(" ", "_"),
                        business_name=kpi_name,
                        source_type="questionnaire",
                        confidence=0.8,
                    )
                )
        return kpis

    def _detect_hierarchy_levels(
        self,
        dimensions: List[ProposedDimension],
        handoff: E2EHandoff,
    ) -> None:
        """Detect and populate hierarchy_levels for each dimension (P1.4)."""
        try:
            from src.data_modeling.hierarchy_detector import detect_hierarchies

            # Build table→columns map from profiles
            table_columns: Dict[str, List[str]] = {}
            for p in handoff.table_profiles:
                tbl = p.get("table_name", "")
                cols = p.get("columns", [])
                col_names = [
                    c.get("column_name", c.get("name", ""))
                    if isinstance(c, dict) else str(c)
                    for c in cols
                ]
                if tbl and col_names:
                    table_columns[tbl] = col_names

            hierarchies = detect_hierarchies(
                table_columns=table_columns,
                relationships=handoff.relationships or None,
            )

            # Apply detected hierarchies to matching dimensions
            for dim in dimensions:
                for src in dim.source_tables:
                    levels = hierarchies.get(src.upper(), hierarchies.get(src, []))
                    if levels and not dim.hierarchy_levels:
                        dim.hierarchy_levels = levels
                        logger.info("Hierarchy detected for %s: %s", dim.name, levels)
        except Exception as exc:
            logger.debug("Hierarchy detection unavailable: %s", exc)

    # ------------------------------------------------------------------
    # SCD type detection (P2.1)
    # ------------------------------------------------------------------

    def _detect_scd_type(
        self, dim: ProposedDimension, handoff: E2EHandoff,
    ) -> int:
        """Detect SCD type for a single dimension.

        Returns 0 (fixed), 1 (overwrite), or 2 (history-tracked).
        Questionnaire overrides to SCD2 are never downgraded.
        """
        # If questionnaire already set SCD2, keep it
        if dim.scd_type >= 2:
            return dim.scd_type

        # SCD0: static reference tables by name pattern
        if _SCD0_TABLE_PATTERNS.search(dim.name):
            return 0

        # Check source table columns for SCD2 temporal indicators
        for src in dim.source_tables:
            cols = self._get_table_columns(handoff, src)
            for col in cols:
                if _SCD2_COLUMN_PATTERNS.search(col):
                    return 2

        # Default: SCD1
        return 1

    def _detect_scd_types(
        self,
        dimensions: List[ProposedDimension],
        handoff: E2EHandoff,
    ) -> None:
        """Auto-detect SCD types for all dimensions (P2.1)."""
        changes = 0
        for dim in dimensions:
            old_type = dim.scd_type
            dim.scd_type = self._detect_scd_type(dim, handoff)
            if dim.scd_type != old_type:
                changes += 1
                logger.info(
                    "SCD type for %s: %d → %d", dim.name, old_type, dim.scd_type,
                )
        if changes:
            logger.info("SCD auto-detection: updated %d dimensions", changes)

    # ------------------------------------------------------------------
    # Measure validation (P2.2)
    # ------------------------------------------------------------------

    def _validate_measures(
        self,
        facts: List[ProposedFact],
        handoff: E2EHandoff,
    ) -> List["MeasureValidation"]:
        """Validate that proposed measures exist in source tables with numeric types."""
        from .contracts import MeasureValidation

        _NUMERIC_TYPES = {
            "NUMBER", "INT", "INTEGER", "BIGINT", "SMALLINT", "TINYINT",
            "FLOAT", "DOUBLE", "REAL", "DECIMAL", "NUMERIC",
            "MONEY", "SMALLMONEY", "NUMBER(38,0)", "NUMBER(38,6)",
        }

        # Build lookup: TABLE_UPPER → {COL_UPPER: data_type}
        table_col_map: Dict[str, Dict[str, str]] = {}
        for p in handoff.table_profiles:
            tbl = p.get("table_name", "").upper()
            cols = p.get("columns", [])
            col_types: Dict[str, str] = {}
            for c in cols:
                if isinstance(c, dict):
                    cname = (c.get("column_name") or c.get("name", "")).upper()
                    ctype = (c.get("data_type") or c.get("type", "")).upper()
                else:
                    cname = str(c).upper()
                    ctype = ""
                if cname:
                    col_types[cname] = ctype
            if tbl:
                table_col_map[tbl] = col_types

        validations: List[MeasureValidation] = []
        for fact in facts:
            # Determine which source tables to check
            source_cols: Dict[str, str] = {}
            for src in fact.source_tables:
                source_cols.update(table_col_map.get(src.upper(), {}))

            for measure in fact.measures:
                mu = measure.upper()
                exists = mu in source_cols
                actual_type = source_cols.get(mu, "")
                base_type = actual_type.split("(")[0].strip()
                is_num = base_type in _NUMERIC_TYPES if exists else False

                warning = ""
                severity = "info"
                if not exists:
                    warning = f"Column '{measure}' not found in source tables {fact.source_tables}"
                    severity = "warning"
                elif not is_num:
                    warning = f"Column '{measure}' has non-numeric type '{actual_type}'"
                    severity = "warning"

                validations.append(MeasureValidation(
                    fact_name=fact.name,
                    measure_name=measure,
                    exists_in_source=exists,
                    is_numeric=is_num,
                    actual_data_type=actual_type,
                    warning=warning,
                    severity=severity,
                ))

        valid_count = sum(1 for v in validations if v.exists_in_source and v.is_numeric)
        logger.info(
            "Measure validation: %d/%d measures validated (exist + numeric)",
            valid_count, len(validations),
        )
        return validations

    # ------------------------------------------------------------------
    # Helpers
    # ------------------------------------------------------------------

    @staticmethod
    def _get_table_columns(
        handoff: E2EHandoff, table_name: str
    ) -> List[str]:
        """Extract column names from a table profile."""
        for p in handoff.table_profiles:
            if p.get("table_name", "").upper() == table_name.upper():
                columns = p.get("columns", [])
                return [
                    c.get("column_name", c.get("name", ""))
                    if isinstance(c, dict)
                    else str(c)
                    for c in columns
                ]
        return []
