"""BLCE Orchestrator — extended 22-phase pipeline for full engine execution.

Extends BLCEWorkflowController with additional integration phases:
intake, consultant_intake, catalog, report_processing, lineage, hierarchy,
persist, bus matrix, quality, model_generation, auto_build, artifact bundle,
and deployment.  Phases that depend on external platform services (catalog,
lineage, hierarchy, persist) gracefully degrade when those services are
unavailable.
"""
from __future__ import annotations

import logging
import time
from datetime import datetime
from typing import Any, Dict, List, Optional

from .config import BLCEConfig
from .contracts import BLCERunSummary, LogicArtifact
from .workflow_phase import BLCEPhaseResult, BLCEWorkflowController

logger = logging.getLogger(__name__)


# ---------------------------------------------------------------------------
# Extended phase order (22 phases)
# ---------------------------------------------------------------------------

BLCE_FULL_PHASE_ORDER = [
    "e2e_chain",             # 1. Load E2E pipeline outputs (if available)
    "intake",                # 2. Stage source files
    "consultant_intake",     # 3. Generate/process intake questionnaire
    "catalog_baseline",      # 4. Catalog scan for context
    "parse_sources",         # 5. SQL/Python/Excel parsing (inherited)
    "report_processing",     # 6. Extract KPIs/dims from reports/notes
    "reconciliation",        # 7. Reconcile Excel values vs warehouse
    "normalize",             # 8. Measure/filter/grain normalization (inherited)
    "hierarchy_attach",      # 9. Attach to hierarchy if available
    "cross_reference",       # 10. Cross-system entity mapping (inherited)
    "evidence_sample",       # 11. Build evidence sampling (inherited)
    "governance",            # 12. CORE/CUSTOM/CANDIDATE classification (inherited)
    "model_generation",      # 13. Propose Kimball model from all inputs
    "persist",               # 14. Persist artifacts + reconciliation
    "bus_matrix",            # 15. Generate bus matrix from artifacts
    "quality_recommend",     # 16. Generate quality expectations
    "skill_generation",      # 17. Auto-generate skills (inherited)
    "ai_enrich",             # 18. AI agent enrichment for low-confidence
    "agent_swarm",           # 19. Multi-provider agent routing
    "auto_build",            # 20. DDL + Wright + Quality + Semantic
    "artifact_bundle",       # 21. Generate reports and artifact bundle
    "deployment",            # 22. Execute generated DDL on Snowflake
]


class BLCEOrchestrator(BLCEWorkflowController):
    """Extended 22-phase BLCE pipeline.

    Inherits the 6 core phases from BLCEWorkflowController and adds
    15 integration phases for a complete end-to-end workflow.

    Phases:
     1. e2e_chain          — load E2E pipeline outputs (if available)
     2. intake             — stage and classify source files
     3. consultant_intake  — generate/process intake questionnaire
     4. catalog_baseline   — baseline catalog context
     5. parse_sources      — SQL/Python/Excel parsing (inherited)
     6. report_processing  — extract KPIs/dims from reports/notes
     7. normalize          — measure/filter/grain normalization (inherited)
     8. hierarchy_attach   — attach to DataBridge hierarchy
     9. cross_reference    — cross-system entity mapping (inherited)
    10. evidence_sample    — evidence sampling queries (inherited)
    11. governance         — CORE/CUSTOM/CANDIDATE classification (inherited)
    12. model_generation   — propose Kimball model from all inputs
    13. persist            — persist to graph/Snowflake
    14. bus_matrix          — generate Kimball bus matrix
    15. quality_recommend   — generate quality expectations
    16. skill_generation    — auto-generate skill prompts (inherited)
    17. ai_enrich           — AI agent enrichment pass
    18. agent_swarm         — multi-provider agent routing
    19. auto_build          — DDL + Wright + Quality + Semantic
    20. artifact_bundle     — generate reports
    21. deployment          — execute generated DDL on Snowflake (P5)
    """

    def __init__(self, config: Optional[BLCEConfig] = None):
        super().__init__(config)

    # ------------------------------------------------------------------
    # Override run_pipeline to use extended phase order
    # ------------------------------------------------------------------

    def run_full_engine(
        self,
        *,
        source_paths: Optional[List[str]] = None,
        artifacts: Optional[List[LogicArtifact]] = None,
        client_id: str = "",
        mode: str = "analyze_only",
        skip_phases: Optional[List[str]] = None,
    ) -> BLCERunSummary:
        """Run the full 22-phase BLCE engine.

        Args:
            source_paths: File paths to ingest (SQL, Python, Excel).
            artifacts: Pre-parsed artifacts (skips intake+parse).
            client_id: Client identifier.
            mode: "analyze_only" (no persist) or "full" (persist to SF/graph).
            skip_phases: Phase names to skip.

        Returns:
            BLCERunSummary with per-phase results.
        """
        # Configure source paths
        if source_paths:
            for p in source_paths:
                lp = p.lower()
                if lp.endswith(".sql"):
                    self.config.sql_paths.append(p)
                elif lp.endswith((".py", ".pyw")):
                    self.config.python_paths.append(p)
                elif lp.endswith((".xlsx", ".xls", ".xlsm")):
                    self.config.excel_paths.append(p)
                elif lp.endswith(".csv"):
                    self.config.csv_paths.append(p)
                elif lp.endswith((".dax", ".bim")):
                    self.config.dax_paths.append(p)
                elif lp.endswith((".mdx", ".mdxquery")):
                    self.config.mdx_paths.append(p)
                elif lp.endswith(".pdf"):
                    self.config.pdf_paths.append(p)

        # Build skip list
        skip = list(skip_phases or [])

        # If pre-parsed artifacts provided, skip intake + parse
        if artifacts:
            if "intake" not in skip:
                skip.append("intake")
            if "parse_sources" not in skip:
                skip.append("parse_sources")

        # In analyze_only mode, skip persist
        if mode == "analyze_only":
            if "persist" not in skip:
                skip.append("persist")

        return self.run_pipeline(
            phases=list(BLCE_FULL_PHASE_ORDER),
            skip=skip,
            artifacts=artifacts,
            client_id=client_id,
        )

    # ------------------------------------------------------------------
    # Parallel engine
    # ------------------------------------------------------------------

    def run_parallel_engine(
        self,
        *,
        source_paths: Optional[List[str]] = None,
        artifacts: Optional[List[LogicArtifact]] = None,
        client_id: str = "",
        mode: str = "analyze_only",
        skip_phases: Optional[List[str]] = None,
        max_workers: int = 4,
    ) -> BLCERunSummary:
        """Run the 22-phase pipeline with parallel execution of independent phases.

        Uses ``ParallelPipelineRunner`` and ``AgentRuntime`` to execute
        phases that share no data dependencies concurrently.
        """
        from .parallel_pipeline import ParallelPipelineRunner

        # Same setup as run_full_engine
        if source_paths:
            for p in source_paths:
                lp = p.lower()
                if lp.endswith(".sql"):
                    self.config.sql_paths.append(p)
                elif lp.endswith((".py", ".pyw")):
                    self.config.python_paths.append(p)
                elif lp.endswith((".xlsx", ".xls", ".xlsm")):
                    self.config.excel_paths.append(p)
                elif lp.endswith(".csv"):
                    self.config.csv_paths.append(p)
                elif lp.endswith((".dax", ".bim")):
                    self.config.dax_paths.append(p)
                elif lp.endswith((".mdx", ".mdxquery")):
                    self.config.mdx_paths.append(p)
                elif lp.endswith(".pdf"):
                    self.config.pdf_paths.append(p)

        skip = list(skip_phases or [])

        if artifacts:
            if "intake" not in skip:
                skip.append("intake")
            if "parse_sources" not in skip:
                skip.append("parse_sources")

        if mode == "analyze_only":
            if "persist" not in skip:
                skip.append("persist")

        # Build shared context (same as run_pipeline)
        context: Dict[str, Any] = {
            "client_id": client_id or self.config.client_id,
            "mode": mode,
        }
        if artifacts:
            context["artifacts"] = [a.model_dump() for a in artifacts]

        # Stash skip set for the runner
        self._parallel_skip = skip

        runner = ParallelPipelineRunner(
            self,
            max_workers=max_workers,
            timeout=self.config.runtime_timeout,
        )
        rt_result = runner.run(context)

        # Convert RuntimeResult → BLCERunSummary
        completed_phases = [
            d["name"] for d in rt_result.task_details
            if d["status"] == "completed"
        ]
        skipped_phases = list(skip) + [
            d["name"] for d in rt_result.task_details
            if d["status"] == "skipped"
        ]
        errors = [
            f"{d['name']}: {d['error']}" for d in rt_result.task_details
            if d["status"] == "failed"
        ]

        return BLCERunSummary(
            client_id=client_id,
            duration_seconds=rt_result.total_duration,
            phases_completed=completed_phases,
            phases_skipped=skipped_phases,
            artifacts_extracted=len(context.get("artifacts", [])),
            measures_found=sum(
                len(a.get("objects", {}).get("measures", []))
                for a in context.get("artifacts", [])
            ),
            errors=errors,
        )

    # ------------------------------------------------------------------
    # New phase implementations (Phase 8 additions)
    # ------------------------------------------------------------------

    def phase_intake(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 1: Stage and classify source files.

        Scans configured paths and categorizes files by type.
        """
        staged = []
        for p in self.config.sql_paths:
            staged.append({"path": p, "type": "sql", "status": "staged"})
        for p in self.config.python_paths:
            staged.append({"path": p, "type": "python", "status": "staged"})
        for p in self.config.excel_paths:
            staged.append({"path": p, "type": "excel", "status": "staged"})
        for p in self.config.csv_paths:
            staged.append({"path": p, "type": "csv", "status": "staged"})
        for p in self.config.dax_paths:
            staged.append({"path": p, "type": "dax", "status": "staged"})
        for p in self.config.mdx_paths:
            staged.append({"path": p, "type": "mdx", "status": "staged"})
        for p in self.config.pdf_paths:
            staged.append({"path": p, "type": "pdf", "status": "staged"})

        context["staged_files"] = staged
        return {
            "files_staged": len(staged),
            "by_type": {
                "sql": len(self.config.sql_paths),
                "python": len(self.config.python_paths),
                "excel": len(self.config.excel_paths),
                "csv": len(self.config.csv_paths),
                "dax": len(self.config.dax_paths),
                "mdx": len(self.config.mdx_paths),
                "pdf": len(self.config.pdf_paths),
            },
        }

    def phase_catalog_baseline(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 2: Baseline catalog context.

        Attempts to pull table metadata from the data catalog.
        Gracefully skips if catalog is not available.
        """
        try:
            from src.data_catalog.catalog_store import CatalogStore
            store = CatalogStore()
            assets = store.list_assets()
            context["catalog_assets"] = len(assets) if assets else 0
            return {
                "catalog_available": True,
                "assets_found": context["catalog_assets"],
            }
        except Exception:
            context["catalog_assets"] = 0
            return {
                "catalog_available": False,
                "reason": "Catalog not configured or unavailable",
            }

    def phase_hierarchy_attach(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 5: Attach artifacts to DataBridge hierarchy.

        Links measures and dependencies to hierarchy nodes if available.
        """
        arts = [LogicArtifact(**a) for a in context.get("artifacts", [])]
        if not arts:
            return {"status": "skipped", "reason": "No artifacts"}

        # Collect all dependency and measure names for hierarchy matching
        entity_names = set()
        for art in arts:
            for dep in art.objects.dependencies:
                if dep.name:
                    entity_names.add(dep.name.lower())
            for m in art.objects.measures:
                name = m.alias or m.name
                if name:
                    entity_names.add(name.lower())

        context["hierarchy_entities"] = list(entity_names)

        # Try to find matching hierarchy nodes
        try:
            from src.hierarchy.store import HierarchyStore
            h_store = HierarchyStore()
            projects = h_store.list_projects()
            matches = []
            for proj in (projects or []):
                for name in entity_names:
                    if name in proj.get("name", "").lower():
                        matches.append({
                            "entity": name,
                            "hierarchy_project": proj.get("name"),
                        })
            context["hierarchy_matches"] = matches
            return {
                "hierarchy_available": True,
                "entities_checked": len(entity_names),
                "matches_found": len(matches),
            }
        except Exception:
            context["hierarchy_matches"] = []
            return {
                "hierarchy_available": False,
                "entities_checked": len(entity_names),
                "reason": "Hierarchy store not available",
            }

    def phase_persist(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Persist artifacts and reconciliation outputs to configured targets.

        Stores BLCE run results using the persistence layer.
        """
        arts = context.get("artifacts", [])
        reconciliation_results = context.get("reconciliation_results", [])
        decisions = context.get("governance_decisions", [])
        skills = context.get("skills", [])

        if not arts and not reconciliation_results:
            return {"status": "skipped", "reason": "No artifacts or reconciliation results to persist"}

        persisted = {
            "artifacts": 0,
            "decisions": 0,
            "skills": 0,
            "targets": [],
        }

        # File persistence (always available)
        if self.config.persist_to_file:
            from pathlib import Path
            out_dir = Path(self.config.output_dir)
            out_dir.mkdir(parents=True, exist_ok=True)

            import json
            run_data = {
                "artifacts": arts,
                "governance_decisions": decisions,
                "skills": skills,
                "cross_references": context.get("cross_references", []),
                "normalized_measures": context.get("normalized_measures", []),
                "reconciliation_results": reconciliation_results,
                "reconciliation_plans": context.get("reconciliation_plans", []),
            }
            run_file = out_dir / f"blce_run_{context.get('client_id', 'unknown')}.json"
            run_file.write_text(json.dumps(run_data, indent=2, default=str),
                                encoding="utf-8")
            persisted["artifacts"] = len(arts)
            persisted["decisions"] = len(decisions)
            persisted["skills"] = len(skills)
            persisted["targets"].append("file")

        # Graph persistence (optional)
        if self.config.persist_to_graph:
            try:
                from src.knowledge_base.graph_manager import GraphManager
                gm = GraphManager()
                for art_dict in arts:
                    art = LogicArtifact(**art_dict)
                    gm.add_node(
                        node_id=art.artifact_id,
                        label=art.source_name or art.artifact_id,
                        node_type="blce_artifact",
                    )
                for rec in reconciliation_results:
                    rec_id = rec.get("reconciliation_id", "")
                    if not rec_id:
                        continue
                    gm.add_node(
                        node_id=rec_id,
                        label=rec.get("report_name", rec_id),
                        node_type="blce_reconciliation",
                    )
                persisted["targets"].append("graph")
            except Exception as exc:
                logger.debug("Graph persist skipped: %s", exc)

        # Snowflake persistence (optional)
        if self.config.persist_to_snowflake and self.config.dest_account:
            try:
                from src.persistence.snowflake_adapter import SnowflakePersistenceAdapter
                adapter = SnowflakePersistenceAdapter(
                    database=self.config.dest_database,
                    schema=self.config.dest_schema,
                )
                run_id = context.get("run_id") or f"blce_{context.get('client_id', 'unknown')}"
                adapter.save_run(
                    run_id,
                    {
                        "run_id": run_id,
                        "client_id": context.get("client_id", ""),
                        "artifacts": arts,
                        "reconciliation_results": reconciliation_results,
                        "reconciliation_plans": context.get("reconciliation_plans", []),
                        "governance_decisions": decisions,
                    },
                )
                persisted["targets"].append("snowflake")
            except Exception as exc:
                logger.debug("Snowflake persist skipped: %s", exc)

        return persisted

    def phase_bus_matrix(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 10: Generate Kimball bus matrix from artifacts.

        Maps measures to dimensions based on grain contracts and dependencies.
        """
        arts = [LogicArtifact(**a) for a in context.get("artifacts", [])]
        if not arts:
            return {"status": "skipped", "reason": "No artifacts"}

        # Build a lightweight bus matrix from artifact data
        facts = []
        dimensions = set()

        for art in arts:
            measures = [m.alias or m.name for m in art.objects.measures if m.name or m.alias]
            grains = [g.column for g in art.objects.grain_columns if g.column]

            if measures:
                facts.append({
                    "artifact_id": art.artifact_id,
                    "source": art.source_name or art.artifact_id,
                    "measures": measures,
                    "grain": grains,
                })
                dimensions.update(grains)

        bus_matrix = {
            "facts": facts,
            "dimensions": sorted(dimensions),
            "fact_count": len(facts),
            "dimension_count": len(dimensions),
        }

        context["bus_matrix"] = bus_matrix
        return bus_matrix

    def phase_quality_recommend(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 11: Generate quality expectations from artifact analysis.

        Infers data quality rules from measure types and filter patterns.
        """
        arts = [LogicArtifact(**a) for a in context.get("artifacts", [])]
        if not arts:
            return {"status": "skipped", "reason": "No artifacts"}

        from .agents import QualityPolicyAgent
        agent = QualityPolicyAgent()

        policies = []
        total_expectations = 0
        for art in arts:
            policy = agent.analyze(art)
            policies.append(policy.model_dump())
            total_expectations += len(policy.expectations)

        context["quality_policies"] = policies
        return {
            "policies_generated": len(policies),
            "total_expectations": total_expectations,
        }

    def phase_ai_enrich(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 13: AI agent enrichment for low-confidence artifacts.

        Runs ReportSemanticAgent on artifacts below the confidence threshold.
        """
        arts = [LogicArtifact(**a) for a in context.get("artifacts", [])]
        if not arts:
            return {"status": "skipped", "reason": "No artifacts"}

        from .agents import ReportSemanticAgent
        agent = ReportSemanticAgent(
            confidence_threshold=self.config.confidence_threshold,
        )

        enriched = []
        skipped = 0
        for art in arts:
            if art.confidence < self.config.confidence_threshold:
                analysis = agent.analyze(art)
                enriched.append(analysis.model_dump())
            else:
                skipped += 1

        context["semantic_analyses"] = enriched
        wizard_low_conf = 0
        for ra in context.get("report_analyses", []):
            for s in ra.get("wizard_suggestions", []) or []:
                if float(s.get("confidence", 0.0)) < self.config.excel_wizard_high_confidence_threshold:
                    wizard_low_conf += 1
        return {
            "enriched": len(enriched),
            "skipped_high_confidence": skipped,
            "threshold": self.config.confidence_threshold,
            "wizard_low_confidence_suggestions": wizard_low_conf,
        }

    def phase_e2e_chain(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 1: Load E2E pipeline outputs into BLCE context.

        If ``config.e2e_run_id`` or ``config.e2e_run_dir`` is set, loads
        the E2E handoff and converts table profiles into LogicArtifacts
        that seed ``context["artifacts"]``.
        """
        run_id = self.config.e2e_run_id
        run_dir = self.config.e2e_run_dir

        if not run_id and not run_dir:
            return {"status": "skipped", "reason": "No E2E run configured"}

        from .e2e_handoff import E2EHandoffLoader
        loader = E2EHandoffLoader(run_dir=run_dir, run_id=run_id)

        # Try directory-based loading first
        if run_dir:
            handoff = loader.load_from_directory(run_dir)
        else:
            # Try from context if available
            e2e_ctx = context.get("e2e_context", {})
            if e2e_ctx:
                handoff = loader.load_from_context(e2e_ctx)
            else:
                return {"status": "skipped", "reason": "No E2E context or directory"}

        artifacts = loader.convert_to_artifacts(handoff)

        # Seed the context with E2E-derived artifacts
        existing = context.get("artifacts", [])
        for art in artifacts:
            existing.append(art.model_dump())
        context["artifacts"] = existing
        context["e2e_handoff"] = handoff.model_dump()

        summary = loader.get_summary(handoff)
        return {
            "status": "loaded",
            "handoff_id": handoff.handoff_id,
            "artifacts_seeded": len(artifacts),
            "summary": summary,
        }

    # ------------------------------------------------------------------
    # New phase implementations (v0.2.0 — BLCE enhancements)
    # ------------------------------------------------------------------

    def phase_consultant_intake(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 3: Generate/process consultant intake questionnaire.

        Uses E2E handoff data to auto-populate a structured questionnaire.
        """
        handoff_dict = context.get("e2e_handoff")
        if not handoff_dict:
            return {"status": "skipped", "reason": "No E2E handoff in context"}

        from .contracts import E2EHandoff
        from .intake_generator import IntakeQuestionnaireGenerator

        handoff = E2EHandoff(**handoff_dict)
        generator = IntakeQuestionnaireGenerator()

        questionnaire = generator.generate(
            handoff,
            erp_type=self.config.erp_type_override,
            client_id=self.config.client_id,
        )

        context["questionnaire"] = questionnaire.model_dump()

        section_count = len(questionnaire.sections)
        auto_count = sum(
            1
            for qs in questionnaire.sections.values()
            for q in qs
            if q.auto_populated
        )

        return {
            "status": "generated",
            "questionnaire_id": questionnaire.questionnaire_id,
            "sections": section_count,
            "auto_populated_fields": auto_count,
            "erp_type": questionnaire.erp_type,
        }

    def phase_report_processing(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 6: Extract KPIs/dims from reports and meeting notes.

        Processes configured report and meeting notes files.
        """
        from .report_processor import ReportProcessor
        from .meeting_notes_processor import MeetingNotesProcessor

        rp = ReportProcessor(
            excel_wizard_enabled=self.config.excel_wizard_enabled,
            excel_wizard_use_copilot=self.config.excel_wizard_use_copilot,
            excel_wizard_gateway_url=self.config.excel_wizard_gateway_url,
            excel_wizard_copilot_mode=self.config.excel_wizard_copilot_mode,
            excel_wizard_high_confidence_threshold=self.config.excel_wizard_high_confidence_threshold,
            excel_wizard_review_threshold=self.config.excel_wizard_review_threshold,
            excel_wizard_max_chunks=self.config.excel_wizard_max_chunks,
        )
        mnp = MeetingNotesProcessor()

        report_analyses = []
        notes_analyses = []

        # Process report files
        for path in self.config.report_paths:
            try:
                analysis = rp.process_report_with_wizard(path)
                report_analyses.append(analysis.model_dump())
            except Exception as exc:
                logger.warning("Report processing failed for %s: %s", path, exc)

        # Process meeting notes
        for path in self.config.meeting_notes_paths:
            try:
                import pathlib
                text = pathlib.Path(path).read_text(encoding="utf-8")
                analysis = mnp.process_notes(text, file_path=path)
                notes_analyses.append(analysis.model_dump())
            except Exception as exc:
                logger.warning("Notes processing failed for %s: %s", path, exc)

        context["report_analyses"] = report_analyses
        context["notes_analyses"] = notes_analyses

        total_kpis = sum(len(ra.get("kpis", [])) for ra in report_analyses)
        total_kpis += sum(len(na.get("kpis", [])) for na in notes_analyses)
        total_dims = sum(len(ra.get("dimensions", [])) for ra in report_analyses)
        total_dims += sum(len(na.get("dimensions", [])) for na in notes_analyses)

        return {
            "reports_processed": len(report_analyses),
            "notes_processed": len(notes_analyses),
            "total_kpis_extracted": total_kpis,
            "total_dimensions_extracted": total_dims,
            "wizard_reports_enriched": sum(1 for ra in report_analyses if ra.get("excel_wizard_id")),
        }

    def phase_reconciliation(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Reconcile Excel report values vs warehouse outputs."""
        if not self.config.reconciliation_enabled:
            return {"status": "skipped", "reason": "Reconciliation disabled"}

        from .reconciliation import ExcelWarehouseReconciler, build_resolution_plan

        jobs = context.get("reconciliation_jobs", []) or list(self.config.reconciliation_jobs)
        if not jobs:
            return {"status": "skipped", "reason": "No reconciliation jobs configured"}

        reconciler = ExcelWarehouseReconciler()
        results = []
        plans = []
        errors = []
        live_queries_executed = 0

        for idx, job in enumerate(jobs):
            mode = str(job.get("mode", "static")).lower().strip()
            tolerance_pct = float(job.get("tolerance_pct", self.config.reconciliation_tolerance_pct))
            analysis_id = str(job.get("analysis_id", ""))
            report_name = str(job.get("report_name", f"reconciliation_job_{idx + 1}"))

            try:
                if mode == "static":
                    rec = reconciler.reconcile(
                        excel_values=job.get("excel_values", {}),
                        warehouse_values=job.get("warehouse_values", {}),
                        tolerance_pct=tolerance_pct,
                        analysis_id=analysis_id,
                        report_name=report_name,
                    )
                elif mode == "live":
                    from src.data_modeling.snowflake_pool import sf_pool

                    conn = sf_pool.get(
                        str(job.get("connection_name", "default")),
                        database=str(job.get("database", "")),
                        schema=str(job.get("schema", "")),
                    )
                    metric_queries = job.get("metric_queries", {}) or {}
                    warehouse_values = {}
                    for metric_key, sql in metric_queries.items():
                        cur = conn.cursor()
                        try:
                            cur.execute(str(sql))
                            row = cur.fetchone()
                            if row and row[0] is not None:
                                warehouse_values[str(metric_key)] = float(row[0])
                                live_queries_executed += 1
                        finally:
                            try:
                                cur.close()
                            except Exception:
                                pass

                    rec = reconciler.reconcile(
                        excel_values=job.get("excel_values", {}),
                        warehouse_values=warehouse_values,
                        tolerance_pct=tolerance_pct,
                        analysis_id=analysis_id,
                        report_name=report_name,
                    )
                else:
                    errors.append(f"job_{idx + 1}: unsupported mode '{mode}'")
                    continue

                plan = build_resolution_plan(rec, include_row_details=True)
                results.append(rec.model_dump())
                plans.append(plan)
            except Exception as exc:
                errors.append(f"job_{idx + 1}: {exc}")

        context["reconciliation_results"] = results
        context["reconciliation_plans"] = plans
        return {
            "jobs_received": len(jobs),
            "reconciliations_completed": len(results),
            "plans_generated": len(plans),
            "live_queries_executed": live_queries_executed,
            "errors": errors,
        }

    def phase_model_generation(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 12: Propose Kimball model from all collected inputs.

        Merges E2E, reports, notes, and questionnaire into a ProposedModel.
        """
        from .contracts import (
            E2EHandoff, IntakeQuestionnaire, MeetingNotesAnalysis, ReportAnalysis,
        )
        from .model_proposer import KimballModelProposer
        from .conflict_resolver import ConflictResolver

        handoff_dict = context.get("e2e_handoff")
        if not handoff_dict:
            return {"status": "skipped", "reason": "No E2E handoff in context"}

        handoff = E2EHandoff(**handoff_dict)

        # Reconstruct questionnaire if present
        q_dict = context.get("questionnaire")
        questionnaire = IntakeQuestionnaire(**q_dict) if q_dict else None

        # Reconstruct report/notes analyses
        report_analyses = [
            ReportAnalysis(**ra) for ra in context.get("report_analyses", [])
        ]
        report_analyses = self._apply_wizard_confidence_gate(report_analyses)
        notes_analyses = [
            MeetingNotesAnalysis(**na) for na in context.get("notes_analyses", [])
        ]

        proposer = KimballModelProposer()
        model = proposer.propose_model(
            handoff=handoff,
            questionnaire=questionnaire,
            report_analyses=report_analyses,
            notes_analyses=notes_analyses,
            erp_type=self.config.erp_type_override,
            client_id=self.config.client_id,
        )
        template_meta = proposer.get_last_template_resolution()

        # Detect conflicts
        resolver = ConflictResolver()
        all_kpis = []
        for ra in report_analyses:
            all_kpis.extend(ra.kpis)
        for na in notes_analyses:
            all_kpis.extend(na.kpis)
        conflicts = resolver.detect_conflicts(all_kpis)
        model.conflicts = conflicts

        context["proposed_model"] = model.model_dump()

        # P2 metrics
        scd_counts = {"scd0": 0, "scd1": 0, "scd2": 0}
        for d in model.dimensions:
            key = f"scd{d.scd_type}"
            scd_counts[key] = scd_counts.get(key, 0) + 1

        return {
            "status": "generated",
            "model_id": model.model_id,
            "erp_type_input": template_meta.get("erp_type_input", self.config.erp_type_override),
            "erp_type_resolved": template_meta.get("erp_type_resolved", ""),
            "template_used": template_meta.get("template_used", False),
            "template_resolution": template_meta.get("template_resolution", "none"),
            "dimensions": len(model.dimensions),
            "facts": len(model.facts),
            "bus_matrix_conformed": len(model.bus_matrix.get("conformed_dimensions", [])),
            "conflicts": len(model.conflicts),
            "measures_in_dictionary": len(model.measure_dictionary),
            "measure_validations": len(model.measure_validations),
            "scd_types": scd_counts,
        }

    def _apply_wizard_confidence_gate(
        self,
        report_analyses: List["ReportAnalysis"],
    ) -> List["ReportAnalysis"]:
        """Keep only approved/high-confidence wizard suggestions for modeling."""
        from .contracts import ExtractedDimension, ExtractedFilter, ExtractedKPI

        gated: List["ReportAnalysis"] = []
        high = self.config.excel_wizard_high_confidence_threshold
        review = self.config.excel_wizard_review_threshold

        for ra in report_analyses:
            allowed = []
            for suggestion in ra.wizard_suggestions:
                if suggestion.status == "approved":
                    allowed.append(suggestion)
                    continue
                if suggestion.confidence >= high:
                    allowed.append(suggestion)
                    continue
                if suggestion.confidence >= review:
                    # Medium-confidence suggestions are review-gated.
                    continue
                # Low-confidence suggestions are excluded.
            ra.wizard_suggestions = allowed

            # Materialize gated wizard suggestions into extracted entities.
            for suggestion in allowed:
                payload = suggestion.proposed_payload or {}
                if suggestion.suggestion_type == "kpi_definition":
                    ra.kpis.append(
                        ExtractedKPI(
                            name=payload.get("name", suggestion.title),
                            business_name=suggestion.title,
                            formula=payload.get("formula", ""),
                            source=ra.file_path,
                            source_type="report",
                            confidence=suggestion.confidence,
                        )
                    )
                elif suggestion.suggestion_type in ("dimension_candidate", "hierarchy_candidate"):
                    ra.dimensions.append(
                        ExtractedDimension(
                            name=payload.get("name", suggestion.title),
                            business_name=suggestion.title,
                            hierarchy_levels=payload.get("levels", []),
                            key_columns=payload.get("key_columns", []),
                            extracted_from="report",
                            confidence=suggestion.confidence,
                        )
                    )
                elif suggestion.suggestion_type == "filter_rule":
                    ra.filters.append(
                        ExtractedFilter(
                            description=suggestion.description or suggestion.title,
                            sql_expression=payload.get("sql_expression", ""),
                            applies_to=payload.get("applies_to", []),
                            source=ra.file_path,
                            confidence=suggestion.confidence,
                        )
                    )

            gated.append(ra)

        return gated

    def phase_auto_build(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 19: Generate DDL, Wright pipelines, quality expectations, semantic layer.

        Uses ProposedModel from phase_model_generation to auto-build artifacts.
        """
        from .contracts import ProposedModel
        from .ddl_generator import DDLGenerator
        from .wright_pipeline_builder import WrightPipelineBuilder
        from .quality_expectation_builder import QualityExpectationBuilder
        from .semantic_layer_builder import SemanticLayerBuilder

        model_dict = context.get("proposed_model")
        if not model_dict:
            return {"status": "skipped", "reason": "No proposed model in context"}

        model = ProposedModel(**model_dict)

        # P3.3: If any reviews exist, use only approved items for building
        has_reviews = any(
            d.review_status != "proposed" for d in model.dimensions
        ) or any(
            f.review_status != "proposed" for f in model.facts
        )
        if has_reviews:
            from .review_workflow import ReviewWorkflow
            model = ReviewWorkflow().get_approved_model(model)
            if not model.dimensions and not model.facts:
                return {"status": "skipped", "reason": "No approved items to build"}

        schema = self.config.target_schema

        ddl_gen = DDLGenerator()
        wright_gen = WrightPipelineBuilder()
        quality_gen = QualityExpectationBuilder()
        semantic_gen = SemanticLayerBuilder()

        ddls = ddl_gen.generate_all(model, schema)
        pipelines = wright_gen.build_all(model.facts, model.dimensions, schema)
        expectations = quality_gen.build_expectations(model)
        semantic = semantic_gen.build_semantic_layer(model)

        context["generated_ddls"] = [d.model_dump() for d in ddls]
        context["generated_pipelines"] = [p.model_dump() for p in pipelines]
        context["generated_expectations"] = [e.model_dump() for e in expectations]
        context["semantic_layer"] = [s.model_dump() for s in semantic]

        return {
            "ddl_objects": len(ddls),
            "wright_pipelines": len(pipelines),
            "quality_expectations": len(expectations),
            "semantic_entries": len(semantic),
        }

    def phase_agent_swarm(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 15: Multi-provider agent routing.

        Creates an AgentPool, routes semantic/xref/quality tasks to
        appropriate providers, and stores conversation traces.
        """
        from .agent_pool import AgentPool, AdaptiveOrchestrator
        from .conversation_store import ConversationStore

        pool = AgentPool()
        store = ConversationStore()

        arts = context.get("artifacts", [])
        if not arts:
            return {"status": "skipped", "reason": "No artifacts"}

        # Route representative tasks and record conversations
        task_types = ["semantic", "cross_reference", "quality"]
        results = []

        for task_type in task_types:
            conv_id = store.start_conversation(
                f"swarm_{task_type}", provider=pool.route_task(task_type),
            )
            store.add_turn(conv_id, "system", f"Routing {task_type} for {len(arts)} artifacts")

            scores = pool.score_providers(task_type)
            store.add_turn(conv_id, "assistant", f"Top provider: {scores[0].provider} ({scores[0].score:.2f})")
            store.end_conversation(conv_id, success=True)

            results.append({
                "task_type": task_type,
                "provider": scores[0].provider,
                "score": round(scores[0].score, 3),
                "conversation_id": conv_id,
            })

        context["agent_swarm_results"] = results
        return {
            "tasks_routed": len(results),
            "results": results,
        }

    def phase_artifact_bundle(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 14: Generate artifact bundle and reports.

        Produces a summary report of the entire BLCE run.
        """
        arts = context.get("artifacts", [])
        decisions = context.get("governance_decisions", [])
        skills = context.get("skills", [])
        bus_matrix = context.get("bus_matrix", {})
        policies = context.get("quality_policies", [])

        bundle = {
            "artifact_count": len(arts),
            "governance_decisions": len(decisions),
            "skills_generated": len(skills),
            "bus_matrix_facts": bus_matrix.get("fact_count", 0),
            "bus_matrix_dimensions": bus_matrix.get("dimension_count", 0),
            "quality_policies": len(policies),
            "total_expectations": sum(
                len(p.get("expectations", [])) for p in policies
            ),
        }

        # Classification breakdown
        core = sum(1 for d in decisions if d.get("classification") == "core")
        custom = sum(1 for d in decisions if d.get("classification") == "custom")
        candidate = sum(1 for d in decisions
                        if d.get("classification") == "candidate")
        bundle["classification_breakdown"] = {
            "core": core,
            "custom": custom,
            "candidate": candidate,
        }

        # P3.1: Generate proposal document if model exists
        model_dict = context.get("proposed_model")
        if model_dict:
            try:
                from .proposal_generator import ProposalGenerator
                from .contracts import ProposedModel
                model = ProposedModel(**model_dict)
                gen = ProposalGenerator()
                path = gen.generate_proposal(
                    model, client_name=self.config.client_id, run_id=self.run_id,
                )
                bundle["proposal_path"] = str(path)
            except Exception as exc:
                logger.warning("Proposal generation failed: %s", exc)

        context["artifact_bundle"] = bundle
        return bundle

    def phase_deployment(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Phase 21: Execute generated DDL on Snowflake.

        Reads generated_ddls from context (phase 19 output).
        Supports dry_run mode via context["dry_run"].
        """
        from .contracts import DeploymentSummary, GeneratedDDL
        from .ddl_executor import DDLExecutor

        ddl_dicts = context.get("generated_ddls")
        if not ddl_dicts:
            return {"status": "skipped", "reason": "No generated DDLs in context"}

        dry_run = context.get("dry_run", True)
        ddls = [GeneratedDDL(**d) for d in ddl_dicts]
        executor = DDLExecutor()

        # Get Snowflake connection (if not dry_run)
        connection = None
        if not dry_run:
            try:
                from src.data_modeling.snowflake_pool import sf_pool
                connection = sf_pool.get(
                    self.config.dest_account or "default",
                    database=self.config.dest_database or "",
                    schema=self.config.target_schema or "",
                )
            except Exception as exc:
                logger.warning("Snowflake connection failed: %s", exc)
                return {"status": "failed", "reason": f"Connection error: {exc}"}

        results = executor.execute_all(ddls, connection, dry_run=dry_run)

        summary = DeploymentSummary(
            run_id=context.get("run_id", ""),
            ddl_results=results,
            total_objects_deployed=sum(
                1 for r in results if r.status.value == "success"
            ),
            total_failures=sum(
                1 for r in results if r.status.value == "failed"
            ),
            total_execution_time_ms=sum(r.execution_time_ms for r in results),
        )

        context["deployment_summary"] = summary.model_dump()

        return {
            "deployed": summary.total_objects_deployed,
            "failed": summary.total_failures,
            "skipped": sum(1 for r in results if r.status.value == "skipped"),
            "execution_time_ms": summary.total_execution_time_ms,
            "dry_run": dry_run,
        }


