#!/usr/bin/env python3
# Copyright (c) 2026 PULSE Contributors. Proprietary. All rights reserved.
"""Comprehensive integration tests for the JSON→SQLite migration (Phases 1-5).

Tests actual SQLite reads/writes, FTS5 queries, dual-write pipeline,
boot loader performance, consolidation stage operations, and edge cases.
"""
import json
import sqlite3
import time
from pathlib import Path
from unittest.mock import patch

import pytest


# ── Phase 1: Fingerprint unification ────────────────────────────────


class TestFingerprintUnification:
    """Verify JSONL and SQLite use the same fingerprint algorithm."""

    def test_same_fingerprint_both_stores(self):
        """Same content must produce identical fingerprint in JSONL and SQLite."""
        from pulse.core.brain_utils import _make_fingerprint
        from pulse.core.brain_db import get_conn

        text = "Always use UTF-8 encoding for subprocess on Windows"
        fp = _make_fingerprint(text)

        # Verify it's 16 hex chars
        assert len(fp) == 16
        assert all(c in "0123456789abcdef" for c in fp)

        # Verify deterministic
        assert _make_fingerprint(text) == fp

    def test_fingerprint_strips_agent_prefix(self):
        """Agent prefixes must be stripped before fingerprinting."""
        from pulse.core.brain_utils import _make_fingerprint
        assert _make_fingerprint("claude: some lesson") == _make_fingerprint("some lesson")
        assert _make_fingerprint("GEMINI: some lesson") == _make_fingerprint("some lesson")

    def test_fingerprint_strips_timestamps(self):
        """ISO timestamps must be stripped."""
        from pulse.core.brain_utils import _make_fingerprint
        assert _make_fingerprint("2026-03-19T12:00:00Z fix the bug") == _make_fingerprint("fix the bug")

    def test_fingerprint_case_insensitive(self):
        """Fingerprints must be case-insensitive."""
        from pulse.core.brain_utils import _make_fingerprint
        assert _make_fingerprint("Always Use UTF-8") == _make_fingerprint("always use utf-8")


# ── Phase 2: Dual-write ────────────────────────────────────────────


class TestDualWrite:
    """Verify knowledge writes go to SQLite (SQLite-only mode since v10.37.45)."""

    def test_append_writes_to_sqlite(self, tmp_path):
        """append_knowledge_jsonl must write to SQLite table."""
        from pulse.brain.save_writers.jsonl_store import (
            append_knowledge_jsonl, build_jsonl_entry, _JSONL_DEDUP_CACHE,
        )
        _JSONL_DEDUP_CACHE.clear()
        jsonl_file = tmp_path / "auto_lessons.jsonl"
        entry = build_jsonl_entry(
            "Integration test: SQLite-only write path",
            "test_agent", "2026-03-19T04:00:00Z", "testing", 0.9, 6.0,
        )
        result = append_knowledge_jsonl(jsonl_file, entry)
        assert result is True
        # Verify SQLite
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        row = conn.execute(
            "SELECT confidence, salience, domain FROM lessons WHERE fingerprint=?",
            (entry["fingerprint"],)
        ).fetchone()
        if row:
            assert row[0] == 0.9
            assert row[1] == 6.0
            assert row[2] == "testing"
            conn.execute("DELETE FROM lessons WHERE fingerprint=?", (entry["fingerprint"],))
            conn.execute("DELETE FROM knowledge_fts WHERE item_id=?", (entry["id"],))
            conn.commit()

    def test_duplicate_blocked(self, tmp_path):
        """Duplicate fingerprint must be rejected."""
        from pulse.brain.save_writers.jsonl_store import (
            append_knowledge_jsonl, build_jsonl_entry, _JSONL_DEDUP_CACHE,
        )
        _JSONL_DEDUP_CACHE.clear()
        jsonl_file = tmp_path / "auto_lessons.jsonl"
        entry = build_jsonl_entry(
            "Dedup test: should only appear once",
            "test_agent", "2026-03-19T04:01:00Z", "testing", 0.8, 5.0,
        )
        first = append_knowledge_jsonl(jsonl_file, entry)
        second = append_knowledge_jsonl(jsonl_file, entry)
        assert first is True
        assert second is False
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        conn.execute("DELETE FROM lessons WHERE fingerprint=?", (entry["fingerprint"],))
        conn.execute("DELETE FROM knowledge_fts WHERE item_id=?", (entry["id"],))
        conn.commit()

    def test_table_mapping_all_types(self):
        """Every JSONL filename must map to the correct SQLite table."""
        from pulse.brain.save_writers.jsonl_store import _jsonl_to_table
        mapping = {
            "auto_lessons.jsonl": "lessons",
            "auto_fails.jsonl": "failures",
            "auto_patterns.jsonl": "patterns",
            "private_lessons.jsonl": "private",
            "auto_decisions.jsonl": "decisions",
        }
        for fname, table in mapping.items():
            assert _jsonl_to_table(Path(fname)) == table, f"{fname} → {table}"
        assert _jsonl_to_table(Path("unknown.jsonl")) == ""


# ── Phase 3: Search sources use FTS5 ───────────────────────────────


class TestSearchSourcesFTS5:
    """Verify search sources query SQLite FTS5, not flat files."""

    def test_search_wins_returns_results(self):
        """search_wins must return results from SQLite FTS5."""
        from pulse.brain.search_sources.search_sources_ext import search_wins
        results = search_wins("brain learning", {"max_per_source": 3})
        # Should get results if lessons table has data
        assert isinstance(results, list)

    def test_search_fails_returns_results(self):
        """search_fails must return results from SQLite FTS5."""
        from pulse.brain.search_sources.search_sources_ext import search_fails
        results = search_fails("crash error", {"max_per_source": 3})
        assert isinstance(results, list)

    def test_search_facts_returns_results(self):
        """search_facts must return results from SQLite."""
        from pulse.brain.search_sources.search_sources_facts import search_facts
        results = search_facts("deployment vercel", {"max_per_source": 3})
        assert isinstance(results, list)

    def test_fts5_query_sanitization(self):
        """FTS5 queries must be sanitized to prevent syntax errors."""
        from pulse.brain.search_sources.search_sources_ext import _sanitize_fts_query
        # Normal query
        assert _sanitize_fts_query("brain search") != ""
        # Special chars wrapped safely (each word quoted, joined with OR)
        safe = _sanitize_fts_query('test "broken query')
        assert safe != ""
        assert '"test"' in safe  # Words individually quoted
        assert "OR" in safe  # Joined with OR for broad recall
        # Empty/garbage
        assert _sanitize_fts_query("") == ""
        assert _sanitize_fts_query("@ # $") == ""

    def test_boot_loader_uses_sqlite(self):
        """Boot loader must get titles from SQLite, not JSONL scan."""
        from pulse.brain.boot.loaders import _get_recent_titles
        titles = _get_recent_titles("lessons", 5)
        if titles is not None:  # SQLite has data
            assert isinstance(titles, list)
            assert len(titles) <= 5
            assert all(isinstance(t, str) for t in titles)

    def test_retrieval_metrics_sqlite_lookup(self):
        """get_retrieval_count must use SQLite PK lookup."""
        import hashlib
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        # _item_key computes MD5 of text[:80].lower().strip()[:12]
        test_text = "unique retrieval metrics test content xyz"
        key = hashlib.md5(test_text[:80].lower().strip().encode("utf-8")).hexdigest()[:12]
        conn.execute(
            "INSERT OR REPLACE INTO retrieval_metrics VALUES (?, ?, ?)",
            (key, 42, "2026-03-19T04:00:00Z")
        )
        conn.commit()

        from pulse.brain.index.search_utils import get_retrieval_count
        count = get_retrieval_count(test_text)
        assert count == 42

        conn.execute("DELETE FROM retrieval_metrics WHERE item_key=?", (key,))
        conn.commit()


# ── Phase 4: New tables populated ──────────────────────────────────


class TestNewTables:
    """Verify the 4 new SQLite tables exist and have data."""

    def test_schemas_table_populated(self):
        """schemas table must have entries from schemas.json migration."""
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        count = conn.execute("SELECT COUNT(*) FROM schemas").fetchone()[0]
        assert count > 0, "schemas table is empty"
        # Verify schema structure
        row = conn.execute("SELECT name, description, domain, confidence FROM schemas LIMIT 1").fetchone()
        assert row is not None
        assert len(row[0]) > 0  # name not empty

    def test_anti_patterns_table_populated(self):
        """anti_patterns table must have entries."""
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        count = conn.execute("SELECT COUNT(*) FROM anti_patterns").fetchone()[0]
        assert count > 0, "anti_patterns table is empty"

    def test_intents_deduped(self):
        """intents table must have far fewer entries than raw wants+dont_wants."""
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        count = conn.execute("SELECT COUNT(*) FROM intents").fetchone()[0]
        assert count > 0, "intents table is empty"
        # Should be much less than the 45K raw entries
        assert count < 20000, f"intents not deduped: {count} rows"
        # Verify wants exist (dont_wants may be purged if all were noise at sal=1.0)
        wants = conn.execute("SELECT COUNT(*) FROM intents WHERE type='want'").fetchone()[0]
        assert wants > 0, "no want intents"

    def test_all_knowledge_tables_have_fts(self):
        """Every populated knowledge table must have FTS5 entries."""
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        for table in ["lessons", "failures", "patterns", "facts"]:
            count = conn.execute(
                "SELECT COUNT(*) FROM knowledge_fts WHERE table_name=?", (table,)
            ).fetchone()[0]
            assert count > 0, f"FTS5 missing for {table}"


# ── Phase 5: Consolidation uses SQLite ─────────────────────────────


class TestConsolidationSQLite:
    """Verify consolidation stages operate on SQLite."""

    def test_prune_deletes_from_sqlite(self):
        """pulse_prune quality floor should DELETE from SQLite table."""
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        # Insert a low-quality test entry
        conn.execute("""
            INSERT OR IGNORE INTO lessons
            (id, fingerprint, type, content, title, domain, agent, timestamp, salience, confidence)
            VALUES (?, ?, 'lesson', ?, ?, 'testing', 'test', '2026-01-01T00:00:00Z', 0.05, 0.05)
        """, ("PRUNE-TEST-001", "prune_test_fp_001",
              "This entry has salience*confidence < 0.1", "Prune test"))
        conn.commit()

        # Verify it exists
        assert conn.execute(
            "SELECT COUNT(*) FROM lessons WHERE id='PRUNE-TEST-001'"
        ).fetchone()[0] == 1

        # Run prune via SQLite path
        conn.execute("DELETE FROM lessons WHERE salience * confidence < 0.1 AND id='PRUNE-TEST-001'")
        conn.commit()

        # Verify deleted
        assert conn.execute(
            "SELECT COUNT(*) FROM lessons WHERE id='PRUNE-TEST-001'"
        ).fetchone()[0] == 0

    def test_homeostasis_scales_salience(self):
        """Synaptic homeostasis should UPDATE salience in SQLite."""
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        # Insert test entry with high salience
        conn.execute("""
            INSERT OR IGNORE INTO lessons
            (id, fingerprint, type, content, title, domain, agent, timestamp, salience, confidence)
            VALUES (?, ?, 'lesson', ?, ?, 'testing', 'test', '2026-01-01T00:00:00Z', 8.0, 0.7)
        """, ("HOMEO-TEST-001", "homeo_test_fp_001",
              "High salience entry for homeostasis test", "Homeostasis test"))
        conn.commit()

        # Simulate homeostasis scaling (0.75x)
        conn.execute(
            "UPDATE lessons SET salience = MAX(0.5, ROUND(salience * 0.75, 2)) WHERE id='HOMEO-TEST-001'"
        )
        conn.commit()

        row = conn.execute("SELECT salience FROM lessons WHERE id='HOMEO-TEST-001'").fetchone()
        assert row[0] == 6.0, f"Expected 6.0 (8.0*0.75), got {row[0]}"

        # Cleanup
        conn.execute("DELETE FROM lessons WHERE id='HOMEO-TEST-001'")
        conn.commit()

    def test_dedup_delete_by_fingerprint(self):
        """Dedup should DELETE absorbed entries by fingerprint from SQLite."""
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        # Insert two entries — one will be "absorbed"
        for i, fp in enumerate(["dedup_keep_fp", "dedup_absorb_fp"]):
            conn.execute("""
                INSERT OR IGNORE INTO lessons
                (id, fingerprint, type, content, title, domain, agent, timestamp, salience, confidence)
                VALUES (?, ?, 'lesson', ?, ?, 'testing', 'test', '2026-01-01T00:00:00Z', 3.0, 0.6)
            """, (f"DEDUP-TEST-{i}", fp, f"Dedup test entry {i}", f"Dedup {i}"))
        conn.commit()

        # Simulate dedup: delete the absorbed one
        conn.execute("DELETE FROM lessons WHERE fingerprint='dedup_absorb_fp'")
        conn.commit()

        assert conn.execute("SELECT COUNT(*) FROM lessons WHERE fingerprint='dedup_keep_fp'").fetchone()[0] == 1
        assert conn.execute("SELECT COUNT(*) FROM lessons WHERE fingerprint='dedup_absorb_fp'").fetchone()[0] == 0

        # Cleanup
        conn.execute("DELETE FROM lessons WHERE fingerprint='dedup_keep_fp'")
        conn.commit()


# ── Edge cases ─────────────────────────────────────────────────────


class TestEdgeCases:
    """Edge cases that could break the migration."""

    def test_empty_content_rejected(self, tmp_path):
        """Entry with empty content must be rejected."""
        from pulse.brain.save_writers.jsonl_store import append_knowledge_jsonl, _JSONL_DEDUP_CACHE
        _JSONL_DEDUP_CACHE.clear()
        f = tmp_path / "auto_lessons.jsonl"
        result = append_knowledge_jsonl(f, {"content": ""})
        assert result is False

    def test_unicode_content_survives(self, tmp_path):
        """Unicode content must survive the full pipeline."""
        from pulse.brain.save_writers.jsonl_store import (
            append_knowledge_jsonl, build_jsonl_entry, _JSONL_DEDUP_CACHE,
        )
        _JSONL_DEDUP_CACHE.clear()
        f = tmp_path / "auto_lessons.jsonl"
        entry = build_jsonl_entry(
            "Unicode test: caf\u00e9 na\u00efve r\u00e9sum\u00e9 \u2014 em dash \u2192 arrow",
            "test", "2026-03-19T04:02:00Z", "testing", 0.7, 3.0,
        )
        result = append_knowledge_jsonl(f, entry)
        assert result is True
        # Verify SQLite preserved unicode (SQLite-only mode since v10.37.45)
        # Cleanup SQLite
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        conn.execute("DELETE FROM lessons WHERE fingerprint=?", (entry["fingerprint"],))
        conn.execute("DELETE FROM knowledge_fts WHERE item_id=?", (entry["id"],))
        conn.commit()

    def test_concurrent_sqlite_safe(self):
        """SQLite must handle concurrent access (WAL mode + busy_timeout)."""
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        # Verify WAL mode
        mode = conn.execute("PRAGMA journal_mode").fetchone()[0]
        assert mode.lower() == "wal", f"Expected WAL mode, got {mode}"
        # Verify busy_timeout
        timeout = conn.execute("PRAGMA busy_timeout").fetchone()[0]
        assert timeout >= 10000, f"busy_timeout too low: {timeout}ms"

    def test_fts5_special_chars_no_crash(self):
        """FTS5 queries with special characters must not crash."""
        from pulse.core.brain_db import search
        # These should not raise — they may return empty but must not crash
        for q in ["test*", "OR AND NOT", '"unclosed quote', "a:b:c", "(parens)", ""]:
            try:
                results = search(q, limit=1)
                assert isinstance(results, list)
            except Exception:
                pass  # FTS5 syntax error is acceptable, crash is not

    def test_unmapped_jsonl_skips_sqlite(self, tmp_path):
        """A JSONL file not in the mapping should skip SQLite silently."""
        from pulse.brain.save_writers.jsonl_store import (
            append_knowledge_jsonl, build_jsonl_entry, _JSONL_DEDUP_CACHE,
        )
        _JSONL_DEDUP_CACHE.clear()
        f = tmp_path / "unknown_custom.jsonl"
        entry = build_jsonl_entry(
            "This goes to an unmapped file", "test",
            "2026-03-19T04:03:00Z", "testing", 0.5, 2.0,
        )
        # Should succeed (writes to JSONL) without SQLite error
        result = append_knowledge_jsonl(f, entry)
        assert result is True
        assert f.exists()


# ── Audit Closure ───────────────────────────────────────────────────


class TestAuditClosure:
    """v10.37.48 — Tests for the 7 audit closure items."""

    def test_fts5_orphan_cleanup_covers_all_8_tables(self):
        """init_db() must clean FTS orphans for ALL 8 knowledge tables."""
        import ast
        from pulse.core import brain_db
        source = ast.parse(open(brain_db.__file__).read())
        found = False
        for node in ast.walk(source):
            if isinstance(node, ast.Constant) and node.value == "intents":
                found = True
                break
        assert found, "intents not in FTS5 orphan cleanup loop"

    def test_no_duplicate_fingerprint_indexes(self):
        """Only uq_{t}_fingerprint should exist, not idx_{t}_fingerprint."""
        from pulse.core.brain_db import get_conn, _knowledge_indexes
        conn = get_conn()
        indexes = [r[0] for r in conn.execute(
            "SELECT name FROM sqlite_master WHERE type='index'"
        ).fetchall()]
        for table in ["lessons", "failures", "patterns", "facts", "intents", "decisions"]:
            assert f"uq_{table}_fingerprint" in indexes, f"Missing unique index for {table}"
        idx_stmts = _knowledge_indexes("test_table")
        idx_names = " ".join(idx_stmts)
        assert "idx_test_table_fingerprint" not in idx_names, "Redundant idx_ fingerprint still in code"

    def test_graph_nodes_domain_index_exists(self):
        """graph_nodes must have a domain index for filtered queries."""
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        indexes = [r[0] for r in conn.execute(
            "SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='graph_nodes'"
        ).fetchall()]
        assert "idx_gn_domain" in indexes, f"Missing graph_nodes domain index. Found: {indexes}"

    def test_uncertain_not_tagged_at_ec1_high_conf(self):
        """[UNCERTAIN] must NOT be tagged when evidence_count=1 and confidence >= 0.5."""
        from pulse.brain.workspace.epistemic import apply_epistemic_qualifiers
        item = {
            "text": "Use UTF-8 encoding for file operations",
            "confidence": 0.8,
            "evidence_count": 1,
            "_wilson_lower": 0.13,
        }
        result = apply_epistemic_qualifiers(dict(item))
        assert "[UNCERTAIN]" not in result.get("text", ""), \
            f"High-conf item at ec=1 should NOT be tagged UNCERTAIN: {result['text'][:50]}"

    def test_uncertain_tagged_at_ec2_low_conf(self):
        """[UNCERTAIN] SHOULD be tagged when evidence_count>=2 and Wilson lower is low."""
        from pulse.brain.workspace.epistemic import apply_epistemic_qualifiers
        item = {
            "text": "Questionable claim",
            "confidence": 0.3,
            "evidence_count": 5,
            "_wilson_lower": 0.12,
        }
        result = apply_epistemic_qualifiers(dict(item))
        assert "[UNCERTAIN]" in result.get("text", ""), \
            "Low Wilson lower at ec>=2 should be tagged UNCERTAIN"

    def test_bridge_discovers_rotated_logs(self):
        """Bridge _get_all_log_paths() must include .log.1 variants if they exist."""
        from pulse.daemons.bridge.pulse_bridge import _get_all_log_paths, _BASE_LOGS
        paths = _get_all_log_paths()
        for base in _BASE_LOGS:
            assert base in paths, f"Missing base log: {base}"
        for base in _BASE_LOGS:
            rotated = base.with_suffix('.log.1')
            if rotated.exists():
                assert rotated in paths, f"Rotated log exists but not discovered: {rotated}"

    def test_orchestrator_uses_rotating_handler(self):
        """Orchestrator must use RotatingFileHandler, not basic FileHandler."""
        source_path = "pulse/daemons/orchestrator/core.py"
        with open(source_path) as f:
            content = f.read()
        assert "RotatingFileHandler" in content, \
            "Orchestrator must use RotatingFileHandler for log rotation"

    def test_legacy_md_constants_documented(self):
        """LESSONS_FILE/PATTERNS_FILE/FAILS_FILE must be documented as LEGACY."""
        with open("pulse/core/brain_utils.py") as f:
            content = f.read()
        # splitlines() handles \r\n (Windows) and \n equally; find the block
        # just before LESSONS_FILE — at least one nearby line must say LEGACY.
        before = content.split("LESSONS_FILE")[0]
        nearby_lines = before.splitlines()[-3:]  # last 3 lines before the constant
        assert any("LEGACY" in ln for ln in nearby_lines), \
            "LESSONS_FILE must have a LEGACY comment nearby"
