"""Integration tests: end-to-end cross-module workflows.

Tests realistic multi-module workflows that exercise the full GreyMatter
knowledge pipeline:
- Session -> Observation -> Pattern escalation -> Knowledge promotion
- Standards seeding -> Context engine -> Session context injection
- Work item lifecycle -> Session binding -> Pattern evidence tracking
- Recall decay -> Review -> Promotion -> Re-scan verification
- Soul document evolution across sessions
- Quality checks on knowledge from multiple sources
"""

import json
import sqlite3
from datetime import datetime, timedelta, timezone

import pytest

from greymatter_core.db import SQLiteDatabase
from greymatter_core.context import ContextEngine
from greymatter_core.documents import DocumentStore
from greymatter_core.export import KnowledgeExporter, KnowledgeImporter
from greymatter_core.knowledge import KnowledgeStore
from greymatter_core.patterns import PatternManager
from greymatter_core.quality import QualityEngine
from greymatter_core.recall import (
    C_FLOOR,
    CONFIRM_FACTOR,
    DEFAULT_STABILITY,
    MAX_STABILITY_DAYS,
    REVIEW_THRESHOLD,
    ProactiveRecall,
)
from greymatter_core.sessions import SessionManager
from greymatter_core.soul import SoulManager
from greymatter_core.standards import StandardsManager
from greymatter_core.work_items import WorkItemManager


# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------

@pytest.fixture
def db():
    """Shared in-memory database for end-to-end tests."""
    database = SQLiteDatabase(":memory:")
    yield database
    database.close()


@pytest.fixture
def knowledge(db):
    return KnowledgeStore(db)


@pytest.fixture
def patterns(db):
    return PatternManager(db)


@pytest.fixture
def sessions(db):
    return SessionManager(db)


@pytest.fixture
def work_items(db):
    return WorkItemManager(db)


@pytest.fixture
def quality(db):
    return QualityEngine(db)


@pytest.fixture
def standards(db):
    return StandardsManager(db)


@pytest.fixture
def doc_store(db):
    return DocumentStore(db)


@pytest.fixture
def context_engine(db, doc_store, standards, patterns):
    return ContextEngine(db, doc_store, standards, patterns)


@pytest.fixture
def soul(tmp_path):
    return SoulManager(tmp_path / "soul" / "default.md")


@pytest.fixture
def recall_db(tmp_path):
    """File-based SQLite DB with full schema for recall integration tests."""
    path = str(tmp_path / "recall_e2e.db")
    conn = sqlite3.connect(path)
    conn.executescript("""
        CREATE TABLE IF NOT EXISTS knowledge_entries (
            id TEXT PRIMARY KEY,
            entry_type TEXT DEFAULT 'reference',
            title TEXT NOT NULL,
            content TEXT DEFAULT '',
            domain TEXT DEFAULT 'general',
            scope TEXT DEFAULT 'cross-cutting',
            tags TEXT,
            valence TEXT DEFAULT 'neutral',
            applied_count INTEGER DEFAULT 0,
            confidence TEXT DEFAULT 'medium',
            pruned_at TEXT,
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS observations (
            id TEXT PRIMARY KEY,
            session_id TEXT DEFAULT 'test',
            trigger TEXT DEFAULT 'test',
            category TEXT DEFAULT 'insight',
            title TEXT NOT NULL,
            content TEXT DEFAULT '',
            domains TEXT DEFAULT 'general',
            confidence REAL DEFAULT 0.5,
            stability_days REAL DEFAULT 45.0,
            last_confirmed_at TEXT,
            review_count INTEGER DEFAULT 0,
            last_reviewed_at TEXT,
            review_source TEXT,
            knowledge_id TEXT,
            pattern_id TEXT,
            promoted_at TEXT,
            pruned INTEGER DEFAULT 0,
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS patterns (
            id TEXT PRIMARY KEY,
            type TEXT DEFAULT 'process',
            title TEXT NOT NULL,
            domain TEXT DEFAULT 'general',
            observation TEXT DEFAULT '',
            confidence TEXT DEFAULT 'low',
            confirmations INTEGER DEFAULT 0,
            last_applied_at TEXT,
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );
    """)
    conn.close()
    yield path


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def _seed_old_entries(path: str, days_old: int = 120):
    """Seed a recall DB with entries old enough to trigger decay."""
    conn = sqlite3.connect(path)
    old_ts = (datetime.now(timezone.utc) - timedelta(days=days_old)).isoformat()

    conn.execute(
        """INSERT INTO knowledge_entries
           (id, title, content, domain, entry_type, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, ?, ?)""",
        ("kn-e2e-001", "Circuit breaker pattern", "Open/half-open/closed states",
         "orchestration", "expertise", old_ts, old_ts),
    )
    conn.execute(
        """INSERT INTO observations
           (id, session_id, title, content, category, domains,
            confidence, stability_days, last_confirmed_at, created_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
        ("obs-e2e-001", "sess-1", "Always validate token expiry",
         "Check JWT exp claim before trusting", "gotcha", "security",
         0.6, 45.0, old_ts, old_ts),
    )
    conn.execute(
        """INSERT INTO patterns
           (id, title, domain, observation, type, confirmations,
            last_applied_at, created_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
        ("pat-e2e-001", "TDD First", "testing", "Write tests before code",
         "process", 0, old_ts, old_ts),
    )
    conn.commit()
    conn.close()


def _seed_promotable_observation(path: str, obs_id: str, title: str,
                                  content: str, category: str = "gotcha",
                                  confidence: float = 0.85,
                                  review_count: int = 3,
                                  session_id: str = "sess-promo"):
    """Seed an observation that meets direct promotion criteria."""
    conn = sqlite3.connect(path)
    old_ts = (datetime.now(timezone.utc) - timedelta(days=90)).isoformat()
    conn.execute(
        """INSERT INTO observations
           (id, session_id, title, content, category, domains,
            confidence, stability_days, review_count,
            last_confirmed_at, created_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
        (obs_id, session_id, title, content, category, "orchestration",
         confidence, 45.0, review_count, old_ts, old_ts),
    )
    conn.commit()
    conn.close()


# ---------------------------------------------------------------------------
# 1. Standards -> Context Engine -> Session Pipeline
# ---------------------------------------------------------------------------

class TestStandardsContextIntegration:
    """Tests the standards -> context engine -> session context flow."""

    def test_seeded_standards_appear_in_context(self, standards, context_engine):
        """Standards seeded with defaults appear in context engine output."""
        count = standards.seed_defaults()
        assert count > 0

        ctx = context_engine.build_session_context(max_tokens=50000)
        payload = json.loads(ctx["quality_standards"])
        assert len(payload) > 0

        categories = {s["category"] for s in payload}
        assert "security" in categories

    def test_deactivated_standards_excluded_from_context(self, standards, context_engine):
        """Deactivating a standard removes it from context payload."""
        standards.seed_defaults()

        active_before = standards.get_active(severity_filter=["critical"])
        std_id = active_before[0]["id"]
        deactivated_rule = active_before[0]["rule"]

        standards.deactivate(std_id)

        ctx = context_engine.build_session_context()
        payload = json.loads(ctx["quality_standards"])
        rules_in_payload = {s["rule"] for s in payload}
        assert deactivated_rule not in rules_in_payload

    def test_context_includes_patterns_after_escalation(
        self, standards, patterns, sessions, context_engine
    ):
        """Patterns that escalate to medium+ confidence appear in context."""
        standards.seed_defaults()
        pat = patterns.create(
            type="process", domain="testing",
            title="Property-based testing", observation="Use hypothesis for invariants",
        )

        # Low-confidence pattern should NOT appear in session load
        ctx_before = context_engine.build_session_context()
        pat_ids_before = [p["id"] for p in ctx_before["active_patterns"]]
        assert pat["id"] not in pat_ids_before

        # Escalate to medium
        for _ in range(3):
            session = sessions.start(project="greymatter")
            patterns.add_evidence(pat["id"], session_id=session["id"])
            sessions.end(session["id"])

        ctx_after = context_engine.build_session_context()
        pat_ids_after = [p["id"] for p in ctx_after["active_patterns"]]
        assert pat["id"] in pat_ids_after

    def test_context_with_documents_and_patterns(
        self, standards, doc_store, patterns, sessions, context_engine
    ):
        """Context engine combines documents, standards, and patterns."""
        standards.seed_defaults()

        doc_store.create(
            doc_type="design",
            title="Architecture Decision Record",
            content="We chose SQLite for local persistence because..." * 10,
            project="greymatter",
        )

        pat = patterns.create(
            type="process", domain="architecture",
            title="ADR for decisions", observation="Record architecture decisions as ADRs",
        )
        for _ in range(4):
            session = sessions.start(project="greymatter")
            patterns.add_evidence(pat["id"], session_id=session["id"])
            sessions.end(session["id"])

        ctx = context_engine.build_session_context(project="greymatter")
        assert ctx["quality_standards"]
        assert ctx["active_patterns"]
        assert len(ctx["active_patterns"]) >= 1

    def test_context_request_tracking_and_relevance(
        self, db, standards, context_engine, sessions
    ):
        """Context requests are recorded and can receive relevance scores."""
        standards.seed_defaults()
        session = sessions.start(project="greymatter")

        context_engine.record_request(
            session_id=session["id"],
            query="How to handle database migrations?",
            doc_ids=["doc-001", "doc-002"],
            tokens=3500,
        )

        rows = db.execute(
            "SELECT * FROM context_requests WHERE session_id = ?",
            (session["id"],),
        )
        assert len(rows) == 1
        assert rows[0]["tokens_served"] == 3500
        assert json.loads(rows[0]["documents_served"]) == ["doc-001", "doc-002"]

        context_engine.evaluate_relevance(rows[0]["id"], score=0.85)
        updated = db.execute_one(
            "SELECT relevance_score FROM context_requests WHERE id = ?",
            (rows[0]["id"],),
        )
        assert updated["relevance_score"] == 0.85


# ---------------------------------------------------------------------------
# 2. Recall -> Review -> Promotion Pipeline
# ---------------------------------------------------------------------------

class TestRecallPromotionPipeline:
    """Tests the full recall -> review -> direct promotion -> verification flow."""

    def test_direct_promotion_via_review(self, recall_db):
        """High-confidence observation is promoted to knowledge after 3+ reviews."""
        _seed_promotable_observation(
            recall_db,
            obs_id="obs-promo-001",
            title="Always use parameterized queries",
            content="SQL injection prevention requires parameterized queries",
            category="gotcha",
            confidence=0.76,
            review_count=2,
        )

        recall = ProactiveRecall(db_path=recall_db)
        try:
            # One more review should push confidence > 0.8 and review_count >= 3
            recall.record_review("obs-promo-001", review_source="test")

            conn = sqlite3.connect(recall_db)
            conn.row_factory = sqlite3.Row
            obs = dict(conn.execute(
                "SELECT confidence, review_count, knowledge_id FROM observations WHERE id = ?",
                ("obs-promo-001",),
            ).fetchone())
            conn.close()

            assert obs["review_count"] == 3
            assert obs["confidence"] > 0.8

            # Direct promotion should have triggered
            conn = sqlite3.connect(recall_db)
            conn.row_factory = sqlite3.Row
            kn = conn.execute(
                "SELECT * FROM knowledge_entries WHERE id = ?",
                ("kn-promo-001",),
            ).fetchone()
            conn.close()

            assert kn is not None
            kn_dict = dict(kn)
            assert kn_dict["title"] == "Always use parameterized queries"
            assert "SQL injection" in kn_dict["content"]
        finally:
            recall.close()

    def test_promoted_observation_marked_with_knowledge_id(self, recall_db):
        """After promotion, observation has knowledge_id and promoted_at set."""
        _seed_promotable_observation(
            recall_db,
            obs_id="obs-link-001",
            title="Validate input at system boundaries",
            content="External inputs must be validated before processing",
            category="correction",
            confidence=0.82,
            review_count=3,
        )

        recall = ProactiveRecall(db_path=recall_db)
        try:
            # Already meets thresholds; next review triggers promotion
            recall.record_review("obs-link-001", review_source="test")

            conn = sqlite3.connect(recall_db)
            conn.row_factory = sqlite3.Row
            obs = dict(conn.execute(
                "SELECT knowledge_id, promoted_at FROM observations WHERE id = ?",
                ("obs-link-001",),
            ).fetchone())
            conn.close()

            assert obs["knowledge_id"] == "kn-link-001"
            assert obs["promoted_at"] is not None
        finally:
            recall.close()

    def test_batch_promote_processes_multiple(self, recall_db):
        """batch_promote_high_confidence processes multiple eligible observations."""
        for i in range(5):
            _seed_promotable_observation(
                recall_db,
                obs_id=f"obs-batch-{i:03d}",
                title=f"Batch promotable insight number {i}",
                content=f"Detailed content for batch promotion test observation {i}",
                category="insight",
                confidence=0.85 + (i * 0.01),
                review_count=3 + i,
            )

        recall = ProactiveRecall(db_path=recall_db)
        try:
            result = recall.batch_promote_high_confidence(limit=10)
            assert result["scanned"] == 5
            assert result["promoted"] >= 3
            assert result["promoted"] + result["skipped"] == result["scanned"]
        finally:
            recall.close()

    def test_review_then_rescan_shows_fewer_due(self, recall_db):
        """Reviewing an item resets its decay clock, removing it from due list."""
        _seed_old_entries(recall_db, days_old=120)

        recall = ProactiveRecall(db_path=recall_db)
        try:
            due_before = recall.get_due_for_review(limit=50)
            assert len(due_before) > 0

            # Review all due items
            for item in due_before:
                recall.record_review(item["id"], source_table=item["source_table"])

            due_after = recall.get_due_for_review(limit=50)
            assert len(due_after) < len(due_before)
        finally:
            recall.close()

    def test_recall_prompt_urgency_levels(self, recall_db):
        """format_recall_prompt classifies entries by urgency correctly."""
        recall = ProactiveRecall(db_path=recall_db)
        try:
            entries = [
                {"id": "kn-1", "source_table": "knowledge_entries",
                 "entry_type": "expertise", "title": "Critical item",
                 "domain": "testing", "retrievability": 0.2,
                 "days_since_review": 200, "stability": 30,
                 "content_preview": "Very old knowledge"},
                {"id": "obs-1", "source_table": "observations",
                 "entry_type": "gotcha", "title": "High urgency item",
                 "domain": "security", "retrievability": 0.4,
                 "days_since_review": 100, "stability": 45,
                 "content_preview": "Somewhat old"},
                {"id": "pat-1", "source_table": "patterns",
                 "entry_type": "process", "title": "Due item",
                 "domain": "testing", "retrievability": 0.6,
                 "days_since_review": 60, "stability": 60,
                 "content_preview": "Approaching threshold"},
            ]

            prompt = recall.format_recall_prompt(entries)
            assert "[CRITICAL]" in prompt
            assert "[HIGH]" in prompt
            assert "[DUE]" in prompt
            assert "Critical item" in prompt
            assert "High urgency item" in prompt
            assert "Due item" in prompt
        finally:
            recall.close()


# ---------------------------------------------------------------------------
# 3. Session -> Work Item -> Knowledge Capture Pipeline
# ---------------------------------------------------------------------------

class TestSessionKnowledgePipeline:
    """Tests the full session -> work -> discover -> capture -> quality pipeline."""

    def test_full_session_knowledge_capture_workflow(
        self, sessions, work_items, knowledge, patterns, quality
    ):
        """Simulate a complete agent session that captures knowledge.

        Flow: start session -> claim work -> discover patterns -> create
        knowledge -> quality check -> end session.
        """
        session = sessions.start(project="greymatter")
        work = work_items.create(
            title="Investigate rate limiting",
            priority=10,
            domain_tags="api,performance",
            project="greymatter",
        )
        work_items.update_status(work["id"], "in_progress")

        pat = patterns.create(
            type="process", domain="api",
            title="Token bucket for rate limiting",
            observation="Token bucket algorithm is simpler than leaky bucket for our use case",
        )
        patterns.add_evidence(
            pat["id"],
            session_id=session["id"],
            work_item_id=work["id"],
        )

        knowledge.create(
            entry_type="gotcha",
            title="Rate limiter must be per-agent",
            content="Shared rate limiter across agents causes cascading throttles",
            domain="api",
            tags="rate-limiting,agents",
        )

        knowledge.create(
            entry_type="expertise",
            title="Token bucket implementation",
            content="Use asyncio.Semaphore with periodic token refill",
            domain="api",
            tags="rate-limiting,async",
        )

        report = quality.full_check()
        assert report.total_entries >= 2

        work_items.update_status(work["id"], "completed")
        sessions.end(session["id"], summary="Investigated rate limiting patterns")

        ended = sessions.get(session["id"])
        assert ended["ended_at"] is not None
        assert ended["summary"] == "Investigated rate limiting patterns"

        completed_work = work_items.get(work["id"])
        assert completed_work["status"] == "completed"

        api_knowledge = knowledge.search(domain="api")
        assert len(api_knowledge) >= 2

    def test_session_checkpoint_and_state(self, sessions):
        """Session checkpointing preserves context for recovery."""
        session = sessions.start(project="greymatter")
        sessions.update_state(session["id"], "active", active_work_item_id="wi-001")

        cp = sessions.checkpoint(
            session["id"],
            context_summary="Working on rate limiter tests",
            open_items=json.dumps(["wi-001"]),
        )
        assert "checkpoint_id" in cp

        latest = sessions.get_latest_checkpoint(session["id"])
        assert latest is not None
        assert latest["context_summary"] == "Working on rate limiter tests"

    def test_session_compaction_counter(self, sessions):
        """Compaction counter tracks how many times context was compacted."""
        session = sessions.start(project="greymatter")
        c1 = sessions.increment_compaction(session["id"])
        c2 = sessions.increment_compaction(session["id"])
        assert c2 == c1 + 1


# ---------------------------------------------------------------------------
# 4. Export -> Import -> Quality Verification Pipeline
# ---------------------------------------------------------------------------

class TestExportImportQualityPipeline:
    """Tests export/import with quality verification on imported data."""

    def test_import_then_quality_check(self, db, knowledge, quality):
        """Imported knowledge undergoes quality checking like native entries."""
        knowledge.create(
            entry_type="expertise",
            title="SQLite connection pooling",
            content="Use a single WAL-mode connection for reads and writes",
            domain="database",
        )
        knowledge.create(
            entry_type="expertise",
            title="SQLite connection management",
            content="Manage connections carefully with context managers",
            domain="database",
        )

        exporter = KnowledgeExporter(db)
        data = exporter.to_json(domain="database")

        target_db = SQLiteDatabase(":memory:")
        try:
            importer = KnowledgeImporter(target_db)
            result = importer.from_json(data)
            assert result["imported"] == 2

            target_quality = QualityEngine(target_db)
            report = target_quality.full_check()
            assert report.total_entries == 2
        finally:
            target_db.close()

    def test_export_import_preserves_domain_consistency(self, db, knowledge):
        """Round-trip preserves domain tags for domain-scoped searches."""
        for domain in ("security", "database", "api"):
            knowledge.create(
                entry_type="expertise",
                title=f"{domain.title()} best practice",
                content=f"Important practice for {domain}",
                domain=domain,
            )

        exporter = KnowledgeExporter(db)
        data = exporter.to_json()

        target_db = SQLiteDatabase(":memory:")
        try:
            importer = KnowledgeImporter(target_db)
            importer.from_json(data)

            target_knowledge = KnowledgeStore(target_db)
            for domain in ("security", "database", "api"):
                results = target_knowledge.search(domain=domain)
                assert len(results) == 1, f"Expected 1 entry for domain {domain}"
                assert results[0]["domain"] == domain
        finally:
            target_db.close()


# ---------------------------------------------------------------------------
# 5. Soul Evolution Across Multi-Session Workflow
# ---------------------------------------------------------------------------

class TestSoulEvolutionIntegration:
    """Tests soul document evolution across realistic session patterns."""

    def test_soul_grows_with_session_activity(self, soul, sessions, patterns):
        """Soul document accumulates preferences and moments across sessions."""
        soul.create_initial_soul("Keith")

        # Session 1: Discover a coding preference
        s1 = sessions.start(project="greymatter")
        soul.add_learned_preference("Prefers explicit error handling over exceptions")
        soul.increment_session_count()
        sessions.end(s1["id"], summary="Established error handling preferences")

        # Session 2: Capture a relationship moment
        s2 = sessions.start(project="greymatter")
        soul.add_relationship_moment("Resolved a complex state machine bug together")
        soul.increment_session_count()
        sessions.end(s2["id"], summary="Debugged coordinator state machine")

        # Session 3: Add another preference
        s3 = sessions.start(project="salesos")
        soul.add_learned_preference("Uses pytest parametrize for data-driven tests")
        soul.increment_session_count()
        sessions.end(s3["id"], summary="Added SalesOS test coverage")

        content = soul.load()
        assert "Prefers explicit error handling" in content
        assert "Resolved a complex state machine bug" in content
        assert "Uses pytest parametrize" in content

        recent = sessions.get_recent(limit=5)
        assert len(recent) == 3

    def test_soul_section_integrity_after_mutations(self, soul):
        """Soul sections remain extractable after multiple mutations."""
        soul.create_initial_soul("Keith")
        soul.add_learned_preference("Pref 1")
        soul.add_learned_preference("Pref 2")
        soul.add_relationship_moment("Moment 1")
        soul.increment_session_count()

        identity = soul.get_section("Identity")
        assert identity is not None
        assert "Keith" in identity

        # Preferences and moments should be in their respective sections
        content = soul.load()
        assert "Pref 1" in content
        assert "Pref 2" in content
        assert "Moment 1" in content


# ---------------------------------------------------------------------------
# 6. Standards Enforcement Integration
# ---------------------------------------------------------------------------

class TestStandardsEnforcementIntegration:
    """Tests that quality standards are actionable and correctly structured."""

    def test_all_default_standards_seed_correctly(self, standards):
        """All default standards seed without errors and are queryable."""
        count = standards.seed_defaults()
        assert count >= 20  # We have 20+ default standards

        critical = standards.get_active(severity_filter=["critical"])
        important = standards.get_active(severity_filter=["important"])
        assert len(critical) > 0
        assert len(important) > 0

    def test_standards_idempotent_seeding(self, standards):
        """Seeding twice does not create duplicates."""
        count1 = standards.seed_defaults()
        count2 = standards.seed_defaults()
        assert count2 == 0  # All already exist

    def test_context_payload_is_valid_json(self, standards):
        """Standards context payload is valid JSON array."""
        standards.seed_defaults()
        payload_str = standards.get_context_payload()
        payload = json.loads(payload_str)
        assert isinstance(payload, list)
        assert len(payload) > 0
        for item in payload:
            assert "category" in item
            assert "rule" in item
            assert "severity" in item

    def test_standards_severity_filtering(self, standards):
        """Severity filter correctly separates critical from minor."""
        standards.seed_defaults()

        critical = standards.get_active(severity_filter=["critical"])
        minor = standards.get_active(severity_filter=["minor"])

        critical_ids = {s["id"] for s in critical}
        minor_ids = {s["id"] for s in minor}
        assert not critical_ids & minor_ids  # No overlap


# ---------------------------------------------------------------------------
# 7. Incremental Consolidation Integration
# ---------------------------------------------------------------------------

class TestIncrementalConsolidationIntegration:
    """Tests cluster-based observation consolidation through the review path."""

    def test_cluster_consolidation_with_multiple_sessions(self, recall_db):
        """3+ similar observations from 2+ sessions consolidate on review."""
        conn = sqlite3.connect(recall_db)
        old_ts = (datetime.now(timezone.utc) - timedelta(days=90)).isoformat()

        # Seed 3 similar observations from 2 different sessions
        for i, session_id in enumerate(["sess-a", "sess-a", "sess-b"]):
            conn.execute(
                """INSERT INTO observations
                   (id, session_id, title, content, category, domains,
                    confidence, stability_days, review_count,
                    last_confirmed_at, created_at)
                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (f"obs-cluster-{i:03d}", session_id,
                 f"Always validate JWT claims part {i}",
                 f"JWT validation detail {i} -- check exp, iss, aud",
                 "gotcha", "security",
                 0.72, 45.0, 1, old_ts, old_ts),
            )
        conn.commit()
        conn.close()

        recall = ProactiveRecall(db_path=recall_db)
        try:
            # Review the trigger observation -- should fire consolidation
            recall.record_review("obs-cluster-000", review_source="test")

            conn = sqlite3.connect(recall_db)
            conn.row_factory = sqlite3.Row

            # Check if consolidation created a knowledge entry
            kn = conn.execute(
                "SELECT * FROM knowledge_entries WHERE id LIKE 'kn-cluster%'"
            ).fetchone()

            # Check if observations were linked
            linked = conn.execute(
                "SELECT COUNT(*) as cnt FROM observations "
                "WHERE knowledge_id IS NOT NULL AND id LIKE 'obs-cluster%'"
            ).fetchone()
            conn.close()

            if kn is not None:
                # Consolidation succeeded
                kn_dict = dict(kn)
                assert "JWT" in kn_dict["title"]
                assert linked["cnt"] >= 2
        finally:
            recall.close()


# ---------------------------------------------------------------------------
# 8. Work Item Review Lifecycle
# ---------------------------------------------------------------------------

class TestWorkItemReviewLifecycle:
    """Tests the full work item lifecycle including review states."""

    def test_work_item_review_cycle(self, work_items):
        """Work item goes through: queued -> in_progress -> pending_review -> approved."""
        work = work_items.create(
            title="Implement rate limiter",
            description="Add token bucket rate limiting to dispatch daemon",
            priority=10,
            domain_tags="api,performance",
        )
        assert work["status"] == "queued"

        work_items.update_status(work["id"], "in_progress")
        work_items.update_status(work["id"], "pending_review")

        reviewed = work_items.update_review(
            work["id"],
            status="approved",
            review_notes="LGTM -- tests pass, rate limiting works correctly",
        )
        assert reviewed is not None
        assert reviewed["status"] == "approved"

    def test_work_item_needs_revision_cycle(self, work_items):
        """Work item can be sent back for revision."""
        work = work_items.create(title="Fix auth bug", priority=1)
        work_items.update_status(work["id"], "in_progress")
        work_items.update_status(work["id"], "pending_review")

        revised = work_items.update_review(
            work["id"],
            status="needs_revision",
            review_notes="Missing edge case handling for expired tokens",
        )
        assert revised is not None
        assert revised["status"] == "needs_revision"

    def test_work_item_count_by_status(self, work_items):
        """Count correctly filters by status."""
        work_items.create(title="Task 1", priority=10)
        work_items.create(title="Task 2", priority=20)
        w3 = work_items.create(title="Task 3", priority=30)
        work_items.update_status(w3["id"], "in_progress")

        assert work_items.count(status="queued") == 2
        assert work_items.count(status="in_progress") == 1
        assert work_items.count() == 3


# ---------------------------------------------------------------------------
# 9. Pattern Lineage and From-Observations
# ---------------------------------------------------------------------------

class TestPatternLineageIntegration:
    """Tests pattern creation from observations with lineage tracking."""

    def test_pattern_from_observations_with_lineage(self, db, knowledge, patterns):
        """create_from_observations records the source observation IDs."""
        obs1 = knowledge.create(
            entry_type="observation",
            title="Agents retry on 429 errors",
            content="Observed in 3 different sessions",
            domain="api",
        )
        obs2 = knowledge.create(
            entry_type="observation",
            title="Rate limit responses need backoff",
            content="Exponential backoff prevents thundering herd",
            domain="api",
        )

        pat = patterns.create_from_observations(
            type="process",
            domain="api",
            title="Exponential backoff on rate limits",
            observation="Agents should use exponential backoff for 429 responses",
            source_observation_ids=[obs1["id"], obs2["id"]],
            session_id="sess-lineage-001",
        )

        assert pat["id"].startswith("pat-")
        assert pat["confidence"] == "low"

        retrieved = patterns.get(pat["id"])
        assert retrieved is not None
        assert retrieved["title"] == "Exponential backoff on rate limits"

    def test_pattern_summary_and_count(self, patterns, sessions):
        """list_summary and count provide accurate views."""
        for i in range(5):
            p = patterns.create(
                type="process", domain="testing",
                title=f"Pattern {i}", observation=f"Observation {i}",
            )
            if i < 3:
                for _ in range(3):
                    s = sessions.start(project="greymatter")
                    patterns.add_evidence(p["id"], session_id=s["id"])
                    sessions.end(s["id"])

        total = patterns.count()
        assert total == 5

        medium_count = patterns.count(confidence="medium")
        assert medium_count == 3

        summary = patterns.list_summary(limit=10)
        assert summary["total"] == 5


# ---------------------------------------------------------------------------
# 10. Full Pipeline: Create -> Export -> Import -> Recall -> Review
# ---------------------------------------------------------------------------

class TestFullKnowledgePipeline:
    """End-to-end: create knowledge, export, import to new DB, verify decay."""

    def test_exported_knowledge_subject_to_recall_in_target(
        self, db, knowledge, tmp_path
    ):
        """Knowledge exported and imported into a new DB decays and can be reviewed."""
        # Create knowledge in source
        knowledge.create(
            entry_type="expertise",
            title="State machine testing strategy",
            content="Test every valid and invalid state transition explicitly",
            domain="testing",
        )

        # Export from source
        exporter = KnowledgeExporter(db)
        data = exporter.to_json()

        # Import into file-based target (needed for ProactiveRecall)
        target_path = str(tmp_path / "target.db")
        target_db = SQLiteDatabase(target_path)
        try:
            importer = KnowledgeImporter(target_db)
            result = importer.from_json(data)
            assert result["imported"] == 1

            # Backdate the entry to simulate aging
            conn = sqlite3.connect(target_path)
            old_ts = (datetime.now(timezone.utc) - timedelta(days=120)).isoformat()
            conn.execute(
                "UPDATE knowledge_entries SET created_at = ?, updated_at = ?",
                (old_ts, old_ts),
            )
            conn.commit()
            conn.close()
        finally:
            target_db.close()

        # Now check recall against the target DB
        recall = ProactiveRecall(db_path=target_path)
        try:
            due = recall.get_due_for_review(limit=10)
            assert len(due) >= 1
            assert any("State machine" in item["title"] for item in due)

            # Review it
            entry = next(item for item in due if "State machine" in item["title"])
            reviewed = recall.record_review(entry["id"])
            assert reviewed is True

            # After review, should no longer be due
            due_after = recall.get_due_for_review(limit=10)
            due_titles = [item["title"] for item in due_after]
            assert "State machine testing strategy" not in due_titles
        finally:
            recall.close()
