"""End-to-end integration tests for the full Orxaq pipeline.

Covers the complete lifecycle: load data -> discover DAG -> score solitons ->
simulate scenario -> export. All tests are deterministic (seed=42) and
use only stdlib + pytest.
"""

from __future__ import annotations

import csv
import json
import tempfile

from orxaq.experiences.dag_editor.export import DAGExporter
from orxaq.intelligence.simulator import (
    Scenario,
    ScenarioSimulator,
    Shock,
    SimulationResult,
)
from orxaq.kernel.types import CausalEdge, CausalRole, Entity, VariableType
from orxaq.kernel.world_dag import WorldDAG
from orxaq.orchestration.integration import OrxaqEngine

# ---------------------------------------------------------------------------
# TestFullPipeline — single end-to-end flow
# ---------------------------------------------------------------------------


class TestFullPipeline:
    """Run the complete pipeline in a single test method."""

    def test_full_pipeline_end_to_end(self):
        engine = OrxaqEngine()

        # Step 1: Load credit ontology
        dag = engine.load_credit_ontology()
        assert dag.node_count() == 19, "Ontology must have 19 variables"
        assert dag.edge_count() > 0, "Ontology must have edges"
        assert len(engine.audit_log) == 1

        # Step 2: Generate synthetic data
        data = engine.generate_synthetic(n=500, seed=42)
        assert len(data) == 500
        assert "credit_score" in data[0]
        assert "default_flag" in data[0]
        assert "unemployment_rate" in data[0]
        assert len(engine.audit_log) == 2

        # Step 3: Run quality gates
        qg_result = engine.run_quality_gates(data)
        assert qg_result.all_passed is True
        assert qg_result.overall_score > 0.9

        # Step 4: Profile data
        profile = engine.profile_data(data)
        assert profile.row_count == 500
        assert profile.column_count >= 19
        assert "credit_score" in profile.columns
        cs_prof = profile.columns["credit_score"]
        assert cs_prof.min_val is not None
        assert cs_prof.max_val is not None
        assert cs_prof.mean is not None

        # Step 5: Discover structure (small variable set for speed)
        discovery_vars = ["unemployment_rate", "income", "credit_score", "dpd_30"]
        consensus = engine.discover_structure(
            data,
            variables=discovery_vars,
            use_consensus=True,
        )
        assert hasattr(consensus, "edges")
        assert hasattr(consensus, "algorithms_used")
        assert "pc" in consensus.algorithms_used
        assert "ges" in consensus.algorithms_used

        # Step 6: Validate DAG against data
        report = engine.validate_dag(data)
        assert report.total_implications > 0
        assert isinstance(report.violation_rate, float)
        assert 0.0 <= report.violation_rate <= 1.0

        # Step 7: Fit PD model
        model = engine.fit_pd_model(data)
        assert model.is_fitted
        pd_val = model.predict_pd(data[0])
        assert 0.0 <= pd_val <= 1.0

        # Step 8: Compute ECL
        ecl_result = engine.compute_ecl(data)
        assert ecl_result.total_ecl >= 0
        assert ecl_result.horizon_quarters > 0
        assert ecl_result.pd_mean >= 0
        assert ecl_result.lgd > 0

        # Step 9: Stress test
        stress = engine.stress_test(data, "severely_adverse")
        assert stress["scenario"] == "severely_adverse"
        assert "base_pd" in stress
        assert "stress_pd" in stress
        assert "delta" in stress
        assert isinstance(stress["base_pd"], float)
        assert isinstance(stress["stress_pd"], float)

        # Step 10: Fairness audit
        decomp = engine.fairness_audit("age")
        assert decomp.protected_var == "age"
        assert decomp.outcome_var == "default_flag"
        assert isinstance(decomp.direct_effect, float)
        assert isinstance(decomp.total_effect, float)

        # Step 11: Export DAG
        exporter = DAGExporter(dag)

        json_str = exporter.to_json()
        parsed = json.loads(json_str)
        assert "nodes" in parsed
        assert "edges" in parsed
        assert len(parsed["nodes"]) == 19

        dot_str = exporter.to_dot()
        assert "digraph" in dot_str
        assert "rankdir=TB" in dot_str

        mermaid_str = exporter.to_mermaid()
        assert "graph TD" in mermaid_str

        # Step 12: Verify audit log captured all operations
        audit_entries = engine.audit_log.entries()
        assert (
            len(audit_entries) >= 6
        )  # ontology, synthetic, discovery, validate, pd, ecl, stress, fairness
        actions = [e.action for e in audit_entries]
        assert "ontology_loaded" in actions
        assert "synthetic_generated" in actions
        assert "discovery_run" in actions
        assert "dag_validated" in actions
        assert "pd_model_fit" in actions  # from compute_ecl which calls fit_pd_model
        assert "ecl_computed" in actions
        assert "stress_test" in actions
        assert "fairness_audit" in actions

        # Verify audit chain integrity
        assert engine.audit_log.verify() is True


# ---------------------------------------------------------------------------
# TestScenarioSimulation — direct simulator tests
# ---------------------------------------------------------------------------


class TestScenarioSimulation:
    """Test the ScenarioSimulator with controlled DAG inputs."""

    def _build_small_dag(self) -> WorldDAG:
        """Build a small 5-node DAG for testing propagation."""
        dag = WorldDAG()
        for name in ["gdp", "unemployment", "income", "dpd30", "default"]:
            dag.add_entity(
                Entity(
                    entity_id=name,
                    name=name,
                    causal_role=CausalRole.EXOGENOUS,
                    variable_type=VariableType.MACRO,
                )
            )
        dag.add_edge(CausalEdge(source="gdp", target="unemployment", confidence=0.8))
        dag.add_edge(CausalEdge(source="gdp", target="income", confidence=0.7))
        dag.add_edge(CausalEdge(source="unemployment", target="dpd30", confidence=0.85))
        dag.add_edge(CausalEdge(source="income", target="dpd30", confidence=0.75))
        dag.add_edge(CausalEdge(source="dpd30", target="default", confidence=0.9))
        return dag

    def test_single_shock_propagates(self):
        dag = self._build_small_dag()
        sim = ScenarioSimulator(dag, seed=42)
        scenario = Scenario(
            name="gdp_shock",
            description="GDP drops 4%",
            shocks=(Shock("gdp", -0.04, "absolute"),),
        )
        baseline = {
            "gdp": 0.02,
            "unemployment": 0.04,
            "income": 50000,
            "dpd30": 0.05,
            "default": 0.02,
        }
        result = sim.simulate(scenario, baseline)

        assert isinstance(result, SimulationResult)
        assert result.total_affected >= 1
        assert "gdp" in result.affected_variables
        # GDP shock should propagate to unemployment and income
        assert result.max_depth >= 1

    def test_multiple_shocks(self):
        dag = self._build_small_dag()
        sim = ScenarioSimulator(dag, seed=42)
        scenario = Scenario(
            name="multi_shock",
            description="GDP drop + unemployment spike",
            shocks=(
                Shock("gdp", -0.04, "absolute"),
                Shock("unemployment", 0.10, "absolute"),
            ),
        )
        baseline = {
            "gdp": 0.02,
            "unemployment": 0.04,
            "income": 50000,
            "dpd30": 0.05,
            "default": 0.02,
        }
        result = sim.simulate(scenario, baseline)

        assert result.total_affected >= 2
        assert "gdp" in result.affected_variables
        assert "unemployment" in result.affected_variables

    def test_affected_variables_and_depths(self):
        dag = self._build_small_dag()
        sim = ScenarioSimulator(dag, seed=42)
        scenario = Scenario(
            name="upstream_shock",
            description="Shock at root propagates through all layers",
            shocks=(Shock("gdp", -0.05, "absolute"),),
        )
        baseline = {
            "gdp": 0.02,
            "unemployment": 0.04,
            "income": 50000,
            "dpd30": 0.05,
            "default": 0.02,
        }
        result = sim.simulate(scenario, baseline)

        # Propagate 3+ levels: gdp -> {unemployment,income} -> dpd30 -> default
        assert result.max_depth >= 2
        assert result.impact_score > 0.0
        assert result.impact_score <= 1.0

    def test_impact_score_bounded(self):
        dag = self._build_small_dag()
        sim = ScenarioSimulator(dag, seed=42)
        scenario = Scenario(
            name="extreme",
            description="Extreme shock",
            shocks=(Shock("gdp", -1.0, "absolute"),),
        )
        baseline = {
            "gdp": 0.02,
            "unemployment": 0.04,
            "income": 50000,
            "dpd30": 0.05,
            "default": 0.02,
        }
        result = sim.simulate(scenario, baseline)
        assert 0.0 <= result.impact_score <= 1.0

    def test_scenario_serialization(self):
        scenario = Scenario(
            name="test",
            description="test scenario",
            shocks=(Shock("gdp", -0.04),),
            severity="adverse",
            horizon_quarters=8,
        )
        d = scenario.to_dict()
        restored = Scenario.from_dict(d)
        assert restored.name == scenario.name
        assert restored.severity == scenario.severity
        assert len(restored.shocks) == len(scenario.shocks)
        assert restored.shocks[0].variable == "gdp"
        assert restored.horizon_quarters == 8

    def test_deterministic_with_seed(self):
        dag = self._build_small_dag()
        scenario = Scenario(
            name="det_test",
            description="Determinism check",
            shocks=(Shock("gdp", -0.04, "absolute"),),
        )
        baseline = {
            "gdp": 0.02,
            "unemployment": 0.04,
            "income": 50000,
            "dpd30": 0.05,
            "default": 0.02,
        }

        r1 = ScenarioSimulator(dag, seed=42).simulate(scenario, baseline)
        r2 = ScenarioSimulator(dag, seed=42).simulate(scenario, baseline)

        assert r1.total_affected == r2.total_affected
        assert r1.impact_score == r2.impact_score
        for var in r1.affected_variables:
            assert abs(r1.affected_variables[var] - r2.affected_variables[var]) < 1e-12


# ---------------------------------------------------------------------------
# TestDataIngestionPipeline — CSV -> profile -> quality gates
# ---------------------------------------------------------------------------


class TestDataIngestionPipeline:
    """Test the data ingestion flow from CSV through profiling and quality gates."""

    def test_csv_ingest_profile_quality(self):
        engine = OrxaqEngine()

        # Step 1: Create temp CSV with credit data columns
        with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False, newline="") as f:
            writer = csv.writer(f)
            writer.writerow(
                [
                    "credit_score",
                    "dti",
                    "income",
                    "loan_amount",
                    "interest_rate",
                    "default_flag",
                ]
            )
            for i in range(50):
                writer.writerow(
                    [
                        650 + i * 2,  # credit_score
                        0.3 + i * 0.005,  # dti
                        40000 + i * 500,  # income
                        150000 + i * 1000,  # loan_amount
                        0.04 + i * 0.001,  # interest_rate
                        1 if i % 10 == 0 else 0,  # default_flag
                    ]
                )
            path = f.name

        # Step 2: Ingest
        dataset = engine.ingest_csv(path)
        assert dataset.row_count == 50
        assert dataset.connector_type == "csv"

        # Step 3: Profile the data
        data_list = dataset.to_list()
        profile = engine.profile_data(data_list)
        assert profile.row_count == 50
        assert profile.column_count == 6

        # Step 4: Run quality gates
        qg_result = engine.run_quality_gates(data_list)
        assert qg_result.all_passed is True

        # Step 5: Verify catalog was updated
        catalog_summary = engine.catalog.summary()
        assert catalog_summary["total_datasets"] >= 1
        assert catalog_summary["total_rows"] >= 50

    def test_ingest_and_audit_trail(self):
        engine = OrxaqEngine()

        with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False, newline="") as f:
            writer = csv.writer(f)
            writer.writerow(["a", "b"])
            writer.writerow(["1", "2"])
            writer.writerow(["3", "4"])
            path = f.name

        engine.ingest_csv(path)
        entries = engine.audit_log.entries()
        assert any(e.action == "data_ingested" for e in entries)
        ingested = [e for e in entries if e.action == "data_ingested"][0]
        assert "rows=2" in ingested.details


# ---------------------------------------------------------------------------
# TestDiscoveryValidationLoop — discover -> validate round-trip
# ---------------------------------------------------------------------------


class TestDiscoveryValidationLoop:
    """Test that discovered structure can be validated against data."""

    def test_discovery_then_validation(self):
        engine = OrxaqEngine()

        # Step 1: Load ontology and generate data
        engine.load_credit_ontology()
        data = engine.generate_synthetic(n=500, seed=42)

        # Step 2: Discover structure from data (small subset for speed)
        discovery_vars = ["unemployment_rate", "income", "credit_score", "dpd_30"]
        consensus = engine.discover_structure(data, variables=discovery_vars, use_consensus=True)
        assert len(consensus.edges) >= 0  # may find 0 or more edges
        assert consensus.algorithms_used == ("pc", "ges")

        # Step 3: Validate the ontology DAG against data
        report = engine.validate_dag(data)
        assert report.total_implications > 0
        assert isinstance(report.violation_rate, float)
        # For synthetic data generated from the ontology, violation rate
        # should be reasonable (not 100% violations)
        assert report.violation_rate < 1.0

    def test_pc_only_discovery(self):
        engine = OrxaqEngine()
        engine.load_credit_ontology()
        data = engine.generate_synthetic(n=300, seed=42)

        result = engine.discover_structure(
            data,
            variables=["unemployment_rate", "income", "credit_score"],
            use_consensus=False,
        )
        assert hasattr(result, "skeleton")
        assert hasattr(result, "ci_tests_run")

    def test_discovery_with_larger_variable_set(self):
        engine = OrxaqEngine()
        engine.load_credit_ontology()
        data = engine.generate_synthetic(n=500, seed=42)

        variables = [
            "unemployment_rate",
            "income",
            "credit_score",
            "dti",
            "dpd_30",
            "default_flag",
        ]
        consensus = engine.discover_structure(data, variables=variables, use_consensus=True)
        assert consensus.algorithms_used == ("pc", "ges")
        # With more variables, consensus should find at least some edges
        # in data generated from a known causal structure
        assert isinstance(consensus.edges, tuple)


# ---------------------------------------------------------------------------
# TestConcurrentOperations — engine isolation
# ---------------------------------------------------------------------------


class TestConcurrentOperations:
    """Verify that multiple engine instances do not interfere."""

    def test_two_engines_isolated(self):
        engine_a = OrxaqEngine()
        engine_b = OrxaqEngine()

        # Load ontology only in engine A
        dag_a = engine_a.load_credit_ontology()
        assert dag_a.node_count() == 19
        assert engine_b.world_dag.node_count() == 0

        # Generate data only in engine A
        data_a = engine_a.generate_synthetic(n=100, seed=42)
        assert len(data_a) == 100

        # Engine B has independent audit log
        assert len(engine_a.audit_log) == 2
        assert len(engine_b.audit_log) == 0

    def test_engines_with_different_data(self):
        engine_a = OrxaqEngine()
        engine_b = OrxaqEngine()

        engine_a.load_credit_ontology()
        engine_b.load_credit_ontology()

        data_a = engine_a.generate_synthetic(n=100, seed=42)
        data_b = engine_b.generate_synthetic(n=200, seed=99)

        assert len(data_a) == 100
        assert len(data_b) == 200

        # Catalogs are independent
        assert engine_a.catalog.summary()["total_datasets"] == 0
        assert engine_b.catalog.summary()["total_datasets"] == 0

    def test_independent_audit_chains(self):
        engine_a = OrxaqEngine()
        engine_b = OrxaqEngine()

        engine_a.load_credit_ontology()
        engine_b.load_credit_ontology()
        engine_a.generate_synthetic(n=10, seed=42)

        assert len(engine_a.audit_log) == 2
        assert len(engine_b.audit_log) == 1

        # Both chains are independently valid
        assert engine_a.audit_log.verify() is True
        assert engine_b.audit_log.verify() is True


# ---------------------------------------------------------------------------
# TestEdgeCases — boundary conditions
# ---------------------------------------------------------------------------


class TestEdgeCases:
    """Test edge cases and boundary conditions."""

    def test_empty_data_quality_gates(self):
        engine = OrxaqEngine()
        result = engine.run_quality_gates([])
        assert result.all_passed is True
        assert result.overall_score == 1.0

    def test_empty_data_profile(self):
        engine = OrxaqEngine()
        profile = engine.profile_data([])
        assert profile.row_count == 0
        assert profile.column_count == 0

    def test_single_row_data(self):
        engine = OrxaqEngine()
        engine.load_credit_ontology()
        data = engine.generate_synthetic(n=1, seed=42)
        assert len(data) == 1

        profile = engine.profile_data(data)
        assert profile.row_count == 1
        assert profile.column_count >= 19

        qg = engine.run_quality_gates(data)
        assert qg.all_passed is True

    def test_all_null_columns(self):
        engine = OrxaqEngine()
        data = [
            {"a": None, "b": None, "c": 1.0},
            {"a": None, "b": None, "c": 2.0},
            {"a": None, "b": None, "c": 3.0},
        ]
        profile = engine.profile_data(data)
        assert profile.columns["a"].null_count == 3
        assert profile.columns["a"].null_pct == 1.0
        assert profile.columns["c"].null_count == 0

        # Quality gates should fail for null columns
        qg = engine.run_quality_gates(data)
        assert qg.all_passed is False

    def test_large_n_synthetic(self):
        engine = OrxaqEngine()
        engine.load_credit_ontology()
        data = engine.generate_synthetic(n=1500, seed=42)
        assert len(data) == 1500

        profile = engine.profile_data(data)
        assert profile.row_count == 1500

        qg = engine.run_quality_gates(data)
        assert qg.all_passed is True

    def test_status_after_operations(self):
        engine = OrxaqEngine()
        st = engine.status()
        assert st["dag"]["nodes"] == 0
        assert st["audit_entries"] == 0

        engine.load_credit_ontology()
        engine.generate_synthetic(n=10, seed=42)
        st = engine.status()
        assert st["dag"]["nodes"] == 19
        assert st["audit_entries"] == 2

    def test_stress_test_all_scenarios(self):
        engine = OrxaqEngine()
        engine.load_credit_ontology()
        data = engine.generate_synthetic(n=200, seed=42)

        for scenario_name in ["baseline", "adverse", "severely_adverse"]:
            result = engine.stress_test(data, scenario_name)
            assert result["scenario"] == scenario_name
            assert isinstance(result["base_pd"], float)
            assert isinstance(result["stress_pd"], float)
            assert isinstance(result["delta"], float)

    def test_dag_export_dict_format(self):
        engine = OrxaqEngine()
        dag = engine.load_credit_ontology()
        exporter = DAGExporter(dag)

        d = exporter.to_dict()
        assert isinstance(d, dict)
        assert "nodes" in d
        assert "edges" in d
        assert len(d["nodes"]) == 19

        # Each node has required fields
        for node in d["nodes"]:
            assert "id" in node
            assert "name" in node
            assert "causal_role" in node
            assert "variable_type" in node

        # Each edge has required fields
        for edge in d["edges"]:
            assert "source" in edge
            assert "target" in edge
            assert "confidence" in edge
            assert "mechanism" in edge

    def test_simulator_no_baseline(self):
        """ScenarioSimulator handles None baseline gracefully."""
        dag = WorldDAG()
        dag.add_entity(Entity(entity_id="x", name="X"))
        dag.add_entity(Entity(entity_id="y", name="Y"))
        dag.add_edge(CausalEdge(source="x", target="y", confidence=0.8))

        sim = ScenarioSimulator(dag, seed=42)
        scenario = Scenario(
            name="no_baseline",
            description="Test with no baseline",
            shocks=(Shock("x", 1.0, "absolute"),),
        )
        result = sim.simulate(scenario, baseline=None)
        assert isinstance(result, SimulationResult)
        assert result.total_affected >= 1

    def test_monte_carlo_simulation(self):
        """Monte Carlo simulation produces stable aggregate statistics."""
        dag = WorldDAG()
        for name in ["a", "b", "c"]:
            dag.add_entity(Entity(entity_id=name, name=name))
        dag.add_edge(CausalEdge(source="a", target="b", confidence=0.9))
        dag.add_edge(CausalEdge(source="b", target="c", confidence=0.8))

        sim = ScenarioSimulator(dag, seed=42)
        scenario = Scenario(
            name="mc_test",
            description="Monte Carlo test",
            shocks=(Shock("a", 1.0, "absolute"),),
        )
        baseline = {"a": 0.0, "b": 0.5, "c": 0.3}
        mc_result = sim.monte_carlo(scenario, baseline, n_simulations=20)

        assert mc_result["scenario"] == "mc_test"
        assert mc_result["n_simulations"] == 20
        assert "variable_stats" in mc_result
        assert "impact_score_mean" in mc_result
        assert 0.0 <= mc_result["impact_score_mean"] <= 1.0
