"""SQLite-backed storage for PersistentHandler.

Provides persistent storage for all TrustMem data structures using SQLite.
Schema uses SQLite-compatible type mappings.

Usage:
    store = SQLiteStore("path/to/db.sqlite")  # file-based
    store = SQLiteStore(":memory:")            # in-memory (tests)
"""
from __future__ import annotations

import hashlib
import json
import logging
import math
import os
import sqlite3
import struct
import threading
from datetime import datetime, timezone
from typing import Any

logger = logging.getLogger(__name__)


_SCHEMA_SQL = """
CREATE TABLE IF NOT EXISTS memories (
    memory_id       TEXT PRIMARY KEY,
    content_hash    TEXT,
    user_id         TEXT DEFAULT NULL,
    content         TEXT NOT NULL,
    tags            TEXT DEFAULT '[]',
    metadata        TEXT DEFAULT '{}',
    confidence      TEXT DEFAULT 'medium' CHECK(confidence IN ('high', 'medium', 'low')),
    source_trust    TEXT DEFAULT 'derived' CHECK(source_trust IN ('verified', 'input', 'derived')),
    memory_type     TEXT NOT NULL DEFAULT 'semantic' CHECK(memory_type IN ('semantic', 'document')),
    created_at      TEXT,
    updated_at      TEXT,
    last_evaluated_at TEXT
);

CREATE TABLE IF NOT EXISTS memory_provenance (
    provenance_id     TEXT PRIMARY KEY,
    child_memory_id   TEXT NOT NULL REFERENCES memories(memory_id) ON DELETE CASCADE,
    parent_memory_id  TEXT REFERENCES memories(memory_id) ON DELETE CASCADE,
    user_id           TEXT,
    relationship      TEXT NOT NULL DEFAULT 'contradicts',
    created_at        TEXT
);

CREATE TABLE IF NOT EXISTS memory_audit_trail (
    audit_id             TEXT PRIMARY KEY,
    memory_id            TEXT NOT NULL,
    event_type           TEXT NOT NULL,
    old_confidence       TEXT,
    new_confidence       TEXT,
    triggered_by         TEXT,
    details              TEXT DEFAULT '{}',
    created_at           TEXT,
    sequence_number      INTEGER UNIQUE,
    record_hash          TEXT,
    prev_hash            TEXT
);

CREATE TABLE IF NOT EXISTS entity_nodes (
    entity_id        TEXT PRIMARY KEY,
    name             TEXT NOT NULL,
    entity_type      TEXT DEFAULT 'other',
    user_id          TEXT DEFAULT NULL,
    confidence       TEXT DEFAULT 'medium',
    source_memory_id TEXT REFERENCES memories(memory_id) ON DELETE SET NULL,
    mention_count    INTEGER DEFAULT 1,
    metadata         TEXT DEFAULT '{}',
    is_active        INTEGER DEFAULT 1,
    created_at       TEXT,
    updated_at       TEXT
);

CREATE TABLE IF NOT EXISTS entity_edges (
    edge_id          TEXT PRIMARY KEY,
    subject_id       TEXT NOT NULL REFERENCES entity_nodes(entity_id) ON DELETE CASCADE,
    subject_name     TEXT NOT NULL,
    predicate        TEXT NOT NULL,
    object_id        TEXT NOT NULL REFERENCES entity_nodes(entity_id) ON DELETE CASCADE,
    object_name      TEXT NOT NULL,
    user_id          TEXT DEFAULT NULL,
    confidence       TEXT DEFAULT 'medium',
    source_memory_id TEXT REFERENCES memories(memory_id) ON DELETE SET NULL,
    mention_count    INTEGER DEFAULT 1,
    metadata         TEXT DEFAULT '{}',
    is_active        INTEGER DEFAULT 1,
    created_at       TEXT,
    updated_at       TEXT
);

CREATE TABLE IF NOT EXISTS memory_vectors (
    id       TEXT PRIMARY KEY,
    vector   BLOB NOT NULL,
    payload  TEXT DEFAULT '{}'
);

CREATE TABLE IF NOT EXISTS memory_access_log (
    access_id       TEXT PRIMARY KEY,
    event_type      TEXT NOT NULL,
    user_id         TEXT,
    accessor        TEXT DEFAULT 'system',
    query           TEXT,
    filters         TEXT DEFAULT '{}',
    memory_ids      TEXT DEFAULT '[]',
    result_count    INTEGER DEFAULT 0,
    created_at      TEXT NOT NULL,
    sequence_number INTEGER UNIQUE,
    record_hash     TEXT,
    prev_hash       TEXT
);

CREATE TABLE IF NOT EXISTS memory_tokens (
    token TEXT NOT NULL,
    memory_id TEXT NOT NULL,
    PRIMARY KEY (token, memory_id)
);
CREATE INDEX IF NOT EXISTS idx_memory_tokens_token ON memory_tokens(token);

"""

_INDEX_SQL = """
-- Performance indexes (run after migrations to ensure columns exist)
CREATE INDEX IF NOT EXISTS idx_memories_user
    ON memories(user_id);
CREATE INDEX IF NOT EXISTS idx_memories_confidence
    ON memories(confidence);
CREATE INDEX IF NOT EXISTS idx_provenance_child
    ON memory_provenance(child_memory_id);
CREATE INDEX IF NOT EXISTS idx_provenance_parent
    ON memory_provenance(parent_memory_id);
CREATE INDEX IF NOT EXISTS idx_entity_edges_user_active
    ON entity_edges(user_id, is_active);
CREATE INDEX IF NOT EXISTS idx_entity_edges_subject
    ON entity_edges(user_id, is_active, subject_name COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_entity_edges_object
    ON entity_edges(user_id, is_active, object_name COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_entity_nodes_user_name
    ON entity_nodes(user_id, is_active, name COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_audit_trail_memory
    ON memory_audit_trail(memory_id);
CREATE INDEX IF NOT EXISTS idx_memories_user_created
    ON memories(user_id, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_provenance_relationship
    ON memory_provenance(relationship);
CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_content_hash
    ON memories(content_hash) WHERE content_hash IS NOT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_audit_sequence
    ON memory_audit_trail(sequence_number) WHERE sequence_number IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_memories_source_trust
    ON memories(source_trust);
CREATE INDEX IF NOT EXISTS idx_memories_type
    ON memories(memory_type);
CREATE INDEX IF NOT EXISTS idx_memories_expires
    ON memories(expires_at);
CREATE INDEX IF NOT EXISTS idx_access_log_user
    ON memory_access_log(user_id);
CREATE INDEX IF NOT EXISTS idx_access_log_accessor
    ON memory_access_log(accessor);
CREATE INDEX IF NOT EXISTS idx_access_log_created
    ON memory_access_log(created_at);
CREATE UNIQUE INDEX IF NOT EXISTS idx_access_log_sequence
    ON memory_access_log(sequence_number) WHERE sequence_number IS NOT NULL;
"""

# Append-only triggers for tamper-proof audit trail
_AUDIT_TRIGGER_SQL = """
CREATE TRIGGER IF NOT EXISTS audit_trail_no_update
BEFORE UPDATE ON memory_audit_trail
BEGIN
    SELECT RAISE(ABORT, 'UPDATE on memory_audit_trail is prohibited (tamper-proof audit trail)');
END;

CREATE TRIGGER IF NOT EXISTS audit_trail_no_delete
BEFORE DELETE ON memory_audit_trail
BEGIN
    SELECT RAISE(ABORT, 'DELETE on memory_audit_trail is prohibited (tamper-proof audit trail)');
END;

CREATE TRIGGER IF NOT EXISTS access_log_no_update
BEFORE UPDATE ON memory_access_log
BEGIN
    SELECT RAISE(ABORT, 'UPDATE on memory_access_log is prohibited (tamper-proof access log)');
END;

CREATE TRIGGER IF NOT EXISTS access_log_no_delete
BEFORE DELETE ON memory_access_log
BEGIN
    SELECT RAISE(ABORT, 'DELETE on memory_access_log is prohibited (tamper-proof access log)');
END;
"""



_AUDIT_HASH_GENESIS = "GENESIS"


def _compute_audit_hash(
    memory_id: str,
    event_type: str,
    old_confidence: str | None,
    new_confidence: str | None,
    triggered_by: str | None,
    details_json: str,
    created_at: str | None,
    prev_hash: str,
) -> str:
    """Compute SHA-256 hash for an audit trail record (hash chain link)."""
    payload = (
        f"{memory_id}|{event_type}|{old_confidence}|{new_confidence}|"
        f"{triggered_by}|{details_json}|{created_at}|{prev_hash}"
    )
    return hashlib.sha256(payload.encode("utf-8")).hexdigest()


_ACCESS_HASH_GENESIS = "GENESIS"


def _compute_access_hash(
    access_id: str,
    event_type: str,
    user_id: str | None,
    accessor: str,
    query: str | None,
    filters_json: str,
    memory_ids_json: str,
    result_count: int,
    created_at: str,
    prev_hash: str,
) -> str:
    """Compute SHA-256 hash for an access log record (hash chain link)."""
    payload = (
        f"{access_id}|{event_type}|{user_id}|{accessor}|"
        f"{query}|{filters_json}|{memory_ids_json}|{result_count}|"
        f"{created_at}|{prev_hash}"
    )
    return hashlib.sha256(payload.encode("utf-8")).hexdigest()


def _json_dumps(obj: Any) -> str:
    """Serialize to JSON string for storage."""
    if isinstance(obj, str):
        return obj
    return json.dumps(obj, ensure_ascii=False, default=str)


def _json_loads(s: str | None, default: Any = None) -> Any:
    """Deserialize JSON string from storage.

    Returns *default* (instead of the raw string) on parse failure to
    guarantee consistent return types for callers expecting list/dict.
    """
    if s is None:
        return default
    try:
        return json.loads(s)
    except (json.JSONDecodeError, TypeError):
        logger.warning("JSON deserialization failed, returning default: %.80r", s)
        return default


def _is_cjk(char: str) -> bool:
    """Check if character is CJK (Chinese/Japanese/Korean)."""
    cp = ord(char)
    return (
        0x4E00 <= cp <= 0x9FFF      # CJK Unified Ideographs
        or 0x3040 <= cp <= 0x309F   # Hiragana
        or 0x30A0 <= cp <= 0x30FF   # Katakana
        or 0xAC00 <= cp <= 0xD7AF   # Hangul
        or 0x3400 <= cp <= 0x4DBF   # CJK Extension A
        or 0xF900 <= cp <= 0xFAFF   # CJK Compatibility Ideographs
    )


def _better_tokenize(text: str) -> list[str]:
    """Better-trigram tokenization: word-boundary aware with CJK single-char support.

    - Words < 3 chars: kept as-is
    - CJK characters: each character is a separate token
    - Other words >= 3 chars: trigram decomposition
    - All tokens lowercased
    """
    if not text:
        return []
    tokens: list[str] = []
    for word in text.lower().split():
        chars = list(word)
        i = 0
        while i < len(chars):
            if _is_cjk(chars[i]):
                tokens.append(chars[i])
                i += 1
            else:
                # Collect non-CJK run
                run_start = i
                while i < len(chars) and not _is_cjk(chars[i]):
                    i += 1
                run = word[run_start:i]
                if len(run) < 3:
                    tokens.append(run)
                else:
                    for j in range(len(run) - 2):
                        tokens.append(run[j : j + 3])
    return tokens


# ── Column whitelists for update operations ──────────────────────
# Only columns in these sets may be passed to the corresponding
# update_*() method.  Any unexpected column name raises ValueError
# to prevent SQL injection via dynamic column names.
_MEMORY_UPDATABLE_COLUMNS: frozenset[str] = frozenset({
    "content_hash", "confidence", "tags", "metadata",
    "updated_at", "source_trust", "last_evaluated_at",
    "expires_at",
})


# Columns whose changes should be auto-recorded in audit trail
_AUDITABLE_MEMORY_COLUMNS: frozenset[str] = frozenset({
    "confidence",
})


class SQLiteStore:
    """SQLite-backed persistent storage for TrustMem.

    Supports optional encryption via ``encryption_key`` (requires
    ``pysqlcipher3`` to be installed).  When ``db_path`` is a file,
    permissions are restricted to ``0o600`` (owner read/write only).
    """

    def __init__(
        self,
        db_path: str = ":memory:",
        encryption_key: str | None = None,
    ) -> None:
        if db_path != ":memory:":
            db_dir = os.path.dirname(db_path)
            if db_dir:
                os.makedirs(db_dir, exist_ok=True)

        self._db_path = db_path
        self._lock = threading.RLock()
        self._in_transaction = False

        # ── Optional SQLCipher encryption ──
        if encryption_key:
            try:
                from pysqlcipher3 import dbapi2 as sqlcipher  # type: ignore[import-untyped]
                self._conn = sqlcipher.connect(db_path, check_same_thread=False)
                safe_key = encryption_key.replace("'", "''")
                self._conn.execute(f"PRAGMA key='{safe_key}'")
            except ImportError:
                raise RuntimeError(
                    "pysqlcipher3 is required for encrypted databases. "
                    "Install it with: pip install pysqlcipher3"
                )
        else:
            self._conn = sqlite3.connect(db_path, check_same_thread=False)

        self._conn.row_factory = sqlite3.Row
        self._conn.execute("PRAGMA journal_mode=WAL")
        self._conn.execute("PRAGMA foreign_keys=ON")
        self._create_tables()

        # ── Harden file permissions (owner read/write only) ──
        if db_path != ":memory:" and os.path.exists(db_path):
            try:
                os.chmod(db_path, 0o600)
            except OSError:
                pass  # Best-effort (may fail on some platforms)

    def _create_tables(self) -> None:
        with self._lock:
            self._conn.executescript(_SCHEMA_SQL)
            self._conn.commit()
        self._run_migrations()
        with self._lock:
            self._conn.executescript(_INDEX_SQL)
            self._conn.executescript(_AUDIT_TRIGGER_SQL)
            self._conn.commit()
            # Backfill token index for any memories not yet indexed
            self._backfill_tokens()

    def _run_migrations(self) -> None:
        """Idempotent schema migrations.

        Each migration checks if it has already been applied before executing.
        New migrations should be appended at the end with an incremental
        version number.
        """
        with self._lock:
            # Ensure schema_version table exists
            self._conn.execute(
                "CREATE TABLE IF NOT EXISTS schema_version ("
                "  version INTEGER PRIMARY KEY,"
                "  applied_at TEXT NOT NULL"
                ")"
            )
            row = self._conn.execute(
                "SELECT COALESCE(MAX(version), 0) FROM schema_version"
            ).fetchone()
            current_version = row[0] if row else 0

            # --- Migration 1: Align default trust_score/zone with ZTM constants ---
            if current_version < 1:
                # No ALTER needed — defaults only affect new rows via schema.
                # Existing rows already have explicit values from add_memory().
                self._conn.execute(
                    "INSERT INTO schema_version (version, applied_at) VALUES (1, ?)",
                    (datetime.now(timezone.utc).isoformat(),),
                )

            # --- Migration 2: Drop memory_type column ---
            if current_version < 2:
                # SQLite doesn't support DROP COLUMN before 3.35.0.
                # Check if column exists first (new DBs won't have it).
                cols = [
                    row[1] for row in
                    self._conn.execute("PRAGMA table_info(memories)").fetchall()
                ]
                if "memory_type" in cols:
                    try:
                        self._conn.execute("ALTER TABLE memories DROP COLUMN memory_type")
                    except sqlite3.OperationalError:
                        # SQLite < 3.35.0: column remains but is unused
                        logger.info("SQLite does not support DROP COLUMN; memory_type column remains but is unused")
                self._conn.execute(
                    "INSERT INTO schema_version (version, applied_at) VALUES (2, ?)",
                    (datetime.now(timezone.utc).isoformat(),),
                )

            # --- Migration 3: ZTM v2 — confidence-based model ---
            if current_version < 3:
                cols = [
                    row[1] for row in
                    self._conn.execute("PRAGMA table_info(memories)").fetchall()
                ]
                # Add confidence column if missing
                if "confidence" not in cols:
                    self._conn.execute(
                        "ALTER TABLE memories ADD COLUMN confidence TEXT DEFAULT 'medium'"
                    )
                    # Map old trust_score to confidence
                    if "trust_score" in cols:
                        self._conn.execute(
                            "UPDATE memories SET confidence = 'high' WHERE trust_score >= 0.70"
                        )
                        self._conn.execute(
                            "UPDATE memories SET confidence = 'low' WHERE trust_score <= 0.20"
                        )
                # Drop legacy columns (best-effort)
                for col in ("trust_score", "zone", "is_contaminated", "corroboration_count"):
                    if col in cols:
                        try:
                            self._conn.execute(f"ALTER TABLE memories DROP COLUMN {col}")
                        except sqlite3.OperationalError:
                            pass  # SQLite < 3.35.0
                # Drop legacy tables (best-effort)
                self._conn.execute("DROP TABLE IF EXISTS corroboration_map")
                self._conn.execute("DROP TABLE IF EXISTS contradiction_cache")
                self._conn.execute(
                    "INSERT INTO schema_version (version, applied_at) VALUES (3, ?)",
                    (datetime.now(timezone.utc).isoformat(),),
                )

            # --- Migration 4: Entity system — trust_score/zone → confidence ---
            if current_version < 4:
                for table in ("entity_nodes", "entity_edges"):
                    cols = [
                        row[1] for row in
                        self._conn.execute(f"PRAGMA table_info({table})").fetchall()
                    ]
                    # Add confidence column if missing
                    if "confidence" not in cols:
                        self._conn.execute(
                            f"ALTER TABLE {table} ADD COLUMN confidence TEXT DEFAULT 'medium'"
                        )
                        # Map existing trust_score to confidence
                        if "trust_score" in cols:
                            self._conn.execute(
                                f"UPDATE {table} SET confidence = 'high' WHERE trust_score >= 0.60"
                            )
                            self._conn.execute(
                                f"UPDATE {table} SET confidence = 'low' WHERE trust_score < 0.40"
                            )
                    # Drop legacy columns (best-effort)
                    for col in ("trust_score", "zone"):
                        if col in cols:
                            try:
                                self._conn.execute(f"ALTER TABLE {table} DROP COLUMN {col}")
                            except sqlite3.OperationalError:
                                pass  # SQLite < 3.35.0
                self._conn.execute(
                    "INSERT INTO schema_version (version, applied_at) VALUES (4, ?)",
                    (datetime.now(timezone.utc).isoformat(),),
                )

            # --- Migration 5: Tamper-proof audit trail (hash chain + append-only) ---
            if current_version < 5:
                audit_cols = [
                    row[1] for row in
                    self._conn.execute("PRAGMA table_info(memory_audit_trail)").fetchall()
                ]

                needs_rebuild = "sequence_number" not in audit_cols

                if needs_rebuild:
                    # Check if FK CASCADE exists
                    fk_rows = self._conn.execute(
                        "PRAGMA foreign_key_list(memory_audit_trail)"
                    ).fetchall()
                    has_fk_cascade = any(
                        r[2] == "memories" for r in fk_rows
                    )

                    if has_fk_cascade:
                        # Rebuild table to remove FK CASCADE
                        # (audit records should survive memory deletion)
                        self._conn.execute(
                            "ALTER TABLE memory_audit_trail RENAME TO _old_audit_trail"
                        )
                        self._conn.execute("""
                            CREATE TABLE memory_audit_trail (
                                audit_id         TEXT PRIMARY KEY,
                                memory_id        TEXT NOT NULL,
                                event_type       TEXT NOT NULL,
                                old_confidence   TEXT,
                                new_confidence   TEXT,
                                triggered_by     TEXT,
                                details          TEXT DEFAULT '{}',
                                created_at       TEXT,
                                sequence_number  INTEGER UNIQUE,
                                record_hash      TEXT,
                                prev_hash        TEXT
                            )
                        """)
                        self._conn.execute("""
                            INSERT INTO memory_audit_trail
                                (audit_id, memory_id, event_type, old_confidence,
                                 new_confidence, triggered_by, details, created_at)
                            SELECT audit_id, memory_id, event_type, old_confidence,
                                   new_confidence, triggered_by, details, created_at
                            FROM _old_audit_trail
                            ORDER BY created_at ASC, audit_id ASC
                        """)
                        self._conn.execute("DROP TABLE _old_audit_trail")
                    else:
                        # Just add columns
                        for col, typedef in [
                            ("sequence_number", "INTEGER UNIQUE"),
                            ("record_hash", "TEXT"),
                            ("prev_hash", "TEXT"),
                        ]:
                            if col not in audit_cols:
                                self._conn.execute(
                                    f"ALTER TABLE memory_audit_trail ADD COLUMN {col} {typedef}"
                                )

                    # Backfill hash chain for existing records
                    rows = self._conn.execute(
                        "SELECT audit_id, memory_id, event_type, old_confidence, "
                        "new_confidence, triggered_by, details, created_at "
                        "FROM memory_audit_trail ORDER BY created_at ASC, audit_id ASC"
                    ).fetchall()

                    prev_hash = _AUDIT_HASH_GENESIS
                    for seq, row in enumerate(rows, start=1):
                        details_json = row[6] if row[6] else "{}"
                        record_hash = _compute_audit_hash(
                            memory_id=row[1],
                            event_type=row[2],
                            old_confidence=row[3],
                            new_confidence=row[4],
                            triggered_by=row[5],
                            details_json=details_json,
                            created_at=row[7],
                            prev_hash=prev_hash,
                        )
                        self._conn.execute(
                            "UPDATE memory_audit_trail SET sequence_number=?, "
                            "record_hash=?, prev_hash=? WHERE audit_id=?",
                            (seq, record_hash, prev_hash, row[0]),
                        )
                        prev_hash = record_hash

                self._conn.execute(
                    "INSERT INTO schema_version (version, applied_at) VALUES (5, ?)",
                    (datetime.now(timezone.utc).isoformat(),),
                )

            # --- Migration 6: Source Trust Level ---
            if current_version < 6:
                cols = [
                    row[1] for row in
                    self._conn.execute("PRAGMA table_info(memories)").fetchall()
                ]
                if "source_trust" not in cols:
                    self._conn.execute(
                        "ALTER TABLE memories ADD COLUMN source_trust TEXT DEFAULT 'derived'"
                    )
                    # All existing memories were LLM-extracted → "derived" is correct default
                self._conn.execute(
                    "INSERT INTO schema_version (version, applied_at) VALUES (6, ?)",
                    (datetime.now(timezone.utc).isoformat(),),
                )

            # --- Migration 7: TTL (expires_at) ---
            if current_version < 7:
                cols = [
                    row[1] for row in
                    self._conn.execute("PRAGMA table_info(memories)").fetchall()
                ]
                if "expires_at" not in cols:
                    self._conn.execute(
                        "ALTER TABLE memories ADD COLUMN expires_at TEXT"
                    )
                self._conn.execute(
                    "INSERT INTO schema_version (version, applied_at) VALUES (7, ?)",
                    (datetime.now(timezone.utc).isoformat(),),
                )

            # --- Migration 8: Add user_id column ---
            if current_version < 8:
                cols = [
                    row[1] for row in
                    self._conn.execute("PRAGMA table_info(memories)").fetchall()
                ]
                if "user_id" not in cols:
                    self._conn.execute(
                        "ALTER TABLE memories ADD COLUMN user_id TEXT DEFAULT NULL"
                    )
                self._conn.execute(
                    "INSERT INTO schema_version (version, applied_at) VALUES (8, ?)",
                    (datetime.now(timezone.utc).isoformat(),),
                )

            # --- Migration 10: Add memory_type column ---
            if current_version < 10:
                cols = [
                    row[1] for row in
                    self._conn.execute("PRAGMA table_info(memories)").fetchall()
                ]
                if "memory_type" not in cols:
                    self._conn.execute(
                        "ALTER TABLE memories ADD COLUMN memory_type TEXT NOT NULL DEFAULT 'semantic'"
                    )
                self._conn.execute(
                    "INSERT INTO schema_version (version, applied_at) VALUES (10, ?)",
                    (datetime.now(timezone.utc).isoformat(),),
                )

            self._conn.commit()

    def _backfill_tokens(self) -> None:
        """Populate token index for memories not yet indexed."""
        try:
            rows = self._conn.execute(
                "SELECT m.memory_id, m.content FROM memories m "
                "WHERE m.memory_id NOT IN (SELECT DISTINCT memory_id FROM memory_tokens)"
            ).fetchall()
            for row in rows:
                memory_id, content = row[0], row[1]
                tokens = _better_tokenize(content)
                for token in set(tokens):
                    self._conn.execute(
                        "INSERT OR IGNORE INTO memory_tokens (token, memory_id) VALUES (?, ?)",
                        (token, memory_id),
                    )
            if rows:
                self._conn.commit()
        except Exception as e:
            logger.warning("Token backfill failed: %s", e)

    def close(self) -> None:
        self._conn.close()

    def begin_transaction(self) -> None:
        """Begin explicit transaction (caller must call commit/rollback).

        While in transaction, individual put_* methods skip their own commit(),
        deferring to the caller's commit_transaction/rollback_transaction.
        Raises RuntimeError if already in a transaction (nesting not supported).
        """
        self._lock.acquire()
        if self._in_transaction:
            self._lock.release()
            raise RuntimeError("Nested transactions are not supported")
        self._in_transaction = True
        self._conn.execute("BEGIN IMMEDIATE")

    def commit_transaction(self) -> None:
        """Commit and release lock."""
        try:
            self._in_transaction = False
            self._conn.commit()
        finally:
            self._lock.release()

    def rollback_transaction(self) -> None:
        """Rollback and release lock."""
        try:
            self._in_transaction = False
            self._conn.rollback()
        finally:
            self._lock.release()

    # ================================================================
    # memories
    # ================================================================

    def put_memory(self, memory: dict) -> None:
        with self._lock:
            self._conn.execute(
                """INSERT OR REPLACE INTO memories
                   (memory_id, content_hash, user_id, content,
                    tags, metadata, confidence,
                    source_trust, memory_type,
                    created_at, updated_at,
                    last_evaluated_at, expires_at)
                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""",
                (
                    memory["memory_id"],
                    memory.get("content_hash"),
                    memory.get("user_id"),
                    memory["content"],
                    _json_dumps(memory.get("tags", [])),
                    _json_dumps(memory.get("metadata", {})),
                    memory.get("confidence", "medium"),
                    memory.get("source_trust", "derived"),
                    memory.get("memory_type", "semantic"),
                    memory.get("created_at"),
                    memory.get("updated_at"),
                    memory.get("last_evaluated_at"),
                    memory.get("expires_at"),
                ),
            )
            # Sync token index
            self._conn.execute(
                "DELETE FROM memory_tokens WHERE memory_id = ?",
                (memory["memory_id"],),
            )
            tokens = _better_tokenize(memory["content"])
            for token in set(tokens):
                self._conn.execute(
                    "INSERT OR IGNORE INTO memory_tokens (token, memory_id) VALUES (?, ?)",
                    (token, memory["memory_id"]),
                )
            if not self._in_transaction:
                self._conn.commit()

    def get_memory(self, memory_id: str) -> dict | None:
        row = self._conn.execute(
            "SELECT * FROM memories WHERE memory_id = ?", (memory_id,)
        ).fetchone()
        return self._row_to_memory(row) if row else None

    def delete_memory(self, memory_id: str) -> bool:
        """Delete a memory. FK ON DELETE CASCADE handles related records."""
        with self._lock:
            cur = self._conn.execute(
                "DELETE FROM memories WHERE memory_id = ?", (memory_id,)
            )
            # Remove from token index
            self._conn.execute(
                "DELETE FROM memory_tokens WHERE memory_id = ?", (memory_id,)
            )
            self._conn.commit()
            return cur.rowcount > 0

    # ── Selective Forgetting ────────────────────────────────────

    def find_memories_by_entity(
        self,
        entity: str,
        user_id: str | None = None,
        strict: bool = False,
    ) -> list[dict]:
        """Find all memories related to an entity.

        Search strategy:
          1. entity_nodes.name matching entity → source_memory_id
          2. memories.content LIKE '%entity%' (text match)

        Args:
            strict: If True, match user_id exactly (no merge with NULL).
                    Used by forget() to avoid deleting org-shared data.
        """
        params: list = [entity, f"%{entity}%"]
        user_clause = ""
        if user_id:
            user_clause = "AND m.user_id = ?"
            params.append(user_id)
        elif strict:
            user_clause = "AND m.user_id IS NULL"

        sql = f"""
            SELECT DISTINCT m.memory_id, m.content, m.user_id,
                   m.confidence, m.source_trust, m.created_at
            FROM memories m
            WHERE (
                m.memory_id IN (
                    SELECT en.source_memory_id
                    FROM entity_nodes en
                    WHERE en.name = ? COLLATE NOCASE
                      AND en.source_memory_id IS NOT NULL
                )
                OR m.content LIKE ?
            )
            {user_clause}
            ORDER BY m.created_at
        """
        rows = self._conn.execute(sql, params).fetchall()
        return [dict(r) for r in rows]

    def find_entity_nodes_by_name(
        self,
        entity: str,
        user_id: str | None = None,
    ) -> list[dict]:
        """Find entity nodes matching the given name (strict user_id match)."""
        if user_id:
            rows = self._conn.execute(
                "SELECT * FROM entity_nodes WHERE name = ? COLLATE NOCASE AND user_id = ?",
                (entity, user_id),
            ).fetchall()
        else:
            rows = self._conn.execute(
                "SELECT * FROM entity_nodes WHERE name = ? COLLATE NOCASE AND user_id IS NULL",
                (entity,),
            ).fetchall()
        return [dict(r) for r in rows]

    def find_entity_edges_by_name(
        self,
        entity: str,
        user_id: str | None = None,
    ) -> list[dict]:
        """Find entity edges where subject or object matches (strict user_id match)."""
        if user_id:
            rows = self._conn.execute(
                "SELECT * FROM entity_edges "
                "WHERE (subject_name = ? COLLATE NOCASE OR object_name = ? COLLATE NOCASE) "
                "AND user_id = ?",
                (entity, entity, user_id),
            ).fetchall()
        else:
            rows = self._conn.execute(
                "SELECT * FROM entity_edges "
                "WHERE (subject_name = ? COLLATE NOCASE OR object_name = ? COLLATE NOCASE) "
                "AND user_id IS NULL",
                (entity, entity),
            ).fetchall()
        return [dict(r) for r in rows]

    def hard_delete_entity_nodes(
        self,
        entity: str,
        user_id: str | None = None,
    ) -> int:
        """Physically delete entity nodes (strict user_id match). For GDPR erasure."""
        with self._lock:
            if user_id:
                cur = self._conn.execute(
                    "DELETE FROM entity_nodes WHERE name = ? COLLATE NOCASE AND user_id = ?",
                    (entity, user_id),
                )
            else:
                cur = self._conn.execute(
                    "DELETE FROM entity_nodes WHERE name = ? COLLATE NOCASE AND user_id IS NULL",
                    (entity,),
                )
            self._conn.commit()
            return cur.rowcount

    def hard_delete_entity_edges(
        self,
        entity: str,
        user_id: str | None = None,
    ) -> int:
        """Physically delete entity edges (strict user_id match). For GDPR erasure."""
        with self._lock:
            if user_id:
                cur = self._conn.execute(
                    "DELETE FROM entity_edges "
                    "WHERE (subject_name = ? COLLATE NOCASE OR object_name = ? COLLATE NOCASE) "
                    "AND user_id = ?",
                    (entity, entity, user_id),
                )
            else:
                cur = self._conn.execute(
                    "DELETE FROM entity_edges "
                    "WHERE (subject_name = ? COLLATE NOCASE OR object_name = ? COLLATE NOCASE) "
                    "AND user_id IS NULL",
                    (entity, entity),
                )
            self._conn.commit()
            return cur.rowcount

    def cleanup_expired(self, cutoff: str) -> list[str]:
        """Return memory_ids where expires_at IS NOT NULL AND expires_at < cutoff."""
        rows = self._conn.execute(
            "SELECT memory_id FROM memories WHERE expires_at IS NOT NULL AND expires_at < ?",
            (cutoff,),
        ).fetchall()
        return [r[0] for r in rows]

    def update_memory(self, memory_id: str, updates: dict) -> None:
        if not updates:
            return
        # ── Column whitelist ──
        disallowed = set(updates.keys()) - _MEMORY_UPDATABLE_COLUMNS - {"memory_id"}
        if disallowed:
            raise ValueError(
                f"Columns not allowed in update_memory: {sorted(disallowed)}"
            )
        # ── Snapshot old values for audit trail ──
        auditable_changes = _AUDITABLE_MEMORY_COLUMNS & set(updates.keys())
        old_values: dict[str, Any] = {}
        if auditable_changes:
            old_mem = self.get_memory(memory_id)
            if old_mem:
                old_values = {k: old_mem.get(k) for k in auditable_changes}
        sets = []
        vals = []
        for k, v in updates.items():
            if k == "memory_id":
                continue
            if k in ("tags", "metadata"):
                v = _json_dumps(v)
            sets.append(f"{k} = ?")
            vals.append(v)
        if not sets:
            return
        vals.append(memory_id)
        with self._lock:
            own_transaction = not self._in_transaction
            if own_transaction:
                self._conn.execute("BEGIN IMMEDIATE")
            try:
                self._conn.execute(
                    f"UPDATE memories SET {', '.join(sets)} WHERE memory_id = ?",
                    vals,
                )
                # ── Auto audit trail (same transaction, with hash chain) ──
                if old_values:
                    import uuid as _uuid
                    now_iso = datetime.now(timezone.utc).isoformat()
                    for col in auditable_changes:
                        old_val = old_values.get(col)
                        new_val = updates.get(col)
                        if old_val != new_val:
                            self.put_audit_trail({
                                "audit_id": str(_uuid.uuid4()),
                                "memory_id": memory_id,
                                "event_type": f"update_{col}",
                                "old_confidence": old_values.get("confidence"),
                                "new_confidence": updates.get("confidence"),
                                "triggered_by": "sqlite_store.update_memory",
                                "details": {"column": col, "old": old_val, "new": new_val},
                                "created_at": now_iso,
                            })
                if own_transaction:
                    self._conn.commit()
            except Exception:
                if own_transaction:
                    self._conn.rollback()
                raise

    def list_memories_by_user(
        self, user_id: str, limit: int = 50, offset: int = 0,
    ) -> list[dict]:
        rows = self._conn.execute(
            "SELECT * FROM memories WHERE user_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?",
            (user_id, limit, offset),
        ).fetchall()
        return [self._row_to_memory(r) for r in rows]

    def list_memories_multi(
        self,
        user_id: str | None = None,
        scope_filter: bool = True,
        limit: int = 50,
        offset: int = 0,
        memory_type: str | None = None,
    ) -> list[dict]:
        """List memories with scope filter.

        When scope_filter=True (default, for get_all):
          - user_id specified: returns user's memories + org-shared (user_id IS NULL)
          - user_id is None: returns only user_id IS NULL memories
        When scope_filter=False (for list_all admin):
          - Returns all memories matching user without scope check
        """
        conditions: list[str] = []
        params: list = []

        if scope_filter:
            if user_id is not None:
                # Return user's own memories + org-shared (user_id IS NULL)
                conditions.append("(user_id = ? OR user_id IS NULL)")
                params.append(user_id)
            else:
                # Only org-shared memories
                conditions.append("user_id IS NULL")
        else:
            # Admin mode (list_all) - no scope filter
            if user_id is not None:
                conditions.append("user_id = ?")
                params.append(user_id)

        if memory_type is not None:
            conditions.append("memory_type = ?")
            params.append(memory_type)

        if conditions:
            where = "WHERE " + " AND ".join(conditions)
        else:
            where = ""
        rows = self._conn.execute(
            f"SELECT * FROM memories {where} ORDER BY created_at DESC LIMIT ? OFFSET ?",
            (*params, limit, offset),
        ).fetchall()
        return [self._row_to_memory(r) for r in rows]

    def search_tokens(
        self,
        query: str,
        user_id: str | None = None,
        memory_type: str | None = None,
        limit: int = 50,
    ) -> list[dict]:
        """Token-based keyword search using inverted index.

        When user_id is provided, returns user's memories + org-shared (user_id IS NULL).
        When user_id is None, returns only org-shared memories (user_id IS NULL).

        Returns list of dicts with memory_id and match_count.
        """
        query_tokens = _better_tokenize(query)
        if not query_tokens:
            return []

        placeholders = ",".join(["?"] * len(query_tokens))

        conditions = ["t.token IN (" + placeholders + ")"]
        params: list[Any] = list(query_tokens)

        if user_id is not None:
            conditions.append("(m.user_id = ? OR m.user_id IS NULL)")
            params.append(user_id)
        else:
            conditions.append("m.user_id IS NULL")

        if memory_type is not None:
            conditions.append("m.memory_type = ?")
            params.append(memory_type)

        where = " AND ".join(conditions)
        rows = self._conn.execute(
            f"SELECT t.memory_id, COUNT(*) as match_count "
            f"FROM memory_tokens t "
            f"JOIN memories m ON t.memory_id = m.memory_id "
            f"WHERE {where} "
            f"GROUP BY t.memory_id "
            f"ORDER BY match_count DESC "
            f"LIMIT ?",
            (*params, limit),
        ).fetchall()

        return [{"memory_id": r[0], "match_count": r[1]} for r in rows]

    def all_memories(self) -> list[dict]:
        rows = self._conn.execute("SELECT * FROM memories").fetchall()
        return [self._row_to_memory(r) for r in rows]

    def get_memories_by_ids(self, ids: list[str]) -> dict[str, dict]:
        if not ids:
            return {}
        placeholders = ",".join("?" for _ in ids)
        rows = self._conn.execute(
            f"SELECT * FROM memories WHERE memory_id IN ({placeholders})", ids
        ).fetchall()
        return {r["memory_id"]: self._row_to_memory(r) for r in rows}

    def count_memories(self) -> int:
        row = self._conn.execute("SELECT COUNT(*) FROM memories").fetchone()
        return row[0] if row else 0

    def all_memory_ids(self, batch_size: int = 1000) -> list[str]:
        """Get all memory IDs without loading full records."""
        rows = self._conn.execute(
            "SELECT memory_id FROM memories ORDER BY created_at",
        ).fetchall()
        return [r[0] for r in rows]

    def get_memory_stats_sql(self, user_id: str | None = None) -> dict:
        """Get memory statistics using SQL aggregation (no full table load)."""
        conditions: list[str] = []
        params: list = []
        if user_id:
            conditions.append("user_id = ?")
            params.append(user_id)
        where = ("WHERE " + " AND ".join(conditions)) if conditions else ""
        params_t = tuple(params)

        total = self._conn.execute(
            f"SELECT COUNT(*) FROM memories {where}", params_t,
        ).fetchone()[0]

        conf_rows = self._conn.execute(
            f"SELECT confidence, COUNT(*) FROM memories {where} GROUP BY confidence", params_t,
        ).fetchall()
        confidence_dist = {row[0]: row[1] for row in conf_rows}

        trust_rows = self._conn.execute(
            f"SELECT source_trust, COUNT(*) FROM memories {where} GROUP BY source_trust", params_t,
        ).fetchall()
        source_trust_dist = {row[0]: row[1] for row in trust_rows}

        type_rows = self._conn.execute(
            f"SELECT memory_type, COUNT(*) FROM memories {where} GROUP BY memory_type", params_t,
        ).fetchall()
        memory_type_dist = {row[0]: row[1] for row in type_rows}

        return {
            "total_memories": total,
            "confidence_distribution": confidence_dist,
            "source_trust_distribution": source_trust_dist,
            "memory_type_distribution": memory_type_dist,
        }

    def count_memories_by_user(self, user_id: str) -> int:
        row = self._conn.execute(
            "SELECT COUNT(*) FROM memories WHERE user_id = ?", (user_id,)
        ).fetchone()
        return row[0] if row else 0

    def _row_to_memory(self, row: sqlite3.Row) -> dict:
        d = dict(row)
        d["tags"] = _json_loads(d.get("tags", "[]"), default=[])
        d["metadata"] = _json_loads(d.get("metadata", "{}"), default={})
        return d

    # ================================================================
    # provenance
    # ================================================================

    def put_provenance(self, record: dict) -> None:
        with self._lock:
            self._conn.execute(
                """INSERT OR REPLACE INTO memory_provenance
                   (provenance_id, child_memory_id, parent_memory_id,
                    user_id,
                    relationship, created_at)
                   VALUES (?,?,?,?,?,?)""",
                (
                    record["provenance_id"],
                    record["child_memory_id"],
                    record.get("parent_memory_id"),
                    record.get("user_id"),
                    record.get("relationship", "derived_from"),
                    record.get("created_at"),
                ),
            )
            if not self._in_transaction:
                self._conn.commit()

    def get_provenance_by_child(self, child_id: str) -> list[dict]:
        rows = self._conn.execute(
            "SELECT * FROM memory_provenance WHERE child_memory_id = ?",
            (child_id,),
        ).fetchall()
        return [dict(r) for r in rows]

    def get_provenance_by_parent(self, parent_id: str) -> list[dict]:
        rows = self._conn.execute(
            "SELECT * FROM memory_provenance WHERE parent_memory_id = ?",
            (parent_id,),
        ).fetchall()
        return [dict(r) for r in rows]

    def update_provenance(self, provenance_id: str, updates: dict) -> None:
        """Update fields on a provenance record."""
        if not updates:
            return
        cols = ", ".join(f"{k} = ?" for k in updates)
        vals = list(updates.values()) + [provenance_id]
        with self._lock:
            self._conn.execute(
                f"UPDATE memory_provenance SET {cols} WHERE provenance_id = ?",
                vals,
            )

    def all_provenance(self) -> list[dict]:
        rows = self._conn.execute("SELECT * FROM memory_provenance").fetchall()
        return [dict(r) for r in rows]

    def get_contradictions(
        self, memory_id: str, limit: int = 20, offset: int = 0,
    ) -> list[dict]:
        """Get contradicts provenance edges for a memory (as child or parent).

        Returns dicts with opponent_memory_id, role, relationship, created_at.
        Joins with memories table to get opponent's current confidence.
        """
        rows = self._conn.execute(
            """
            SELECT
                p.parent_memory_id AS opponent_memory_id,
                'loser' AS role,
                m.confidence AS opponent_confidence,
                p.created_at AS created_at
            FROM memory_provenance p
            LEFT JOIN memories m ON m.memory_id = p.parent_memory_id
            WHERE p.child_memory_id = ? AND p.relationship = 'contradicts'
            UNION ALL
            SELECT
                p.child_memory_id AS opponent_memory_id,
                'winner' AS role,
                m.confidence AS opponent_confidence,
                p.created_at AS created_at
            FROM memory_provenance p
            LEFT JOIN memories m ON m.memory_id = p.child_memory_id
            WHERE p.parent_memory_id = ? AND p.relationship = 'contradicts'
            ORDER BY 4 DESC
            LIMIT ? OFFSET ?
            """,
            (memory_id, memory_id, limit, offset),
        ).fetchall()
        return [dict(r) for r in rows]

    def get_provenance_links(
        self, memory_id: str, limit: int = 20, offset: int = 0,
    ) -> list[dict]:
        """Get non-contradicts provenance edges for a memory (derived_from etc).

        Returns edges where memory is child (has parents) or parent (has children).
        """
        rows = self._conn.execute(
            """
            SELECT
                relationship,
                parent_memory_id,
                child_memory_id,
                'parent' AS direction,
                created_at AS created_at
            FROM memory_provenance
            WHERE child_memory_id = ? AND relationship != 'contradicts'
            UNION ALL
            SELECT
                relationship,
                parent_memory_id,
                child_memory_id,
                'child' AS direction,
                created_at AS created_at
            FROM memory_provenance
            WHERE parent_memory_id = ? AND relationship != 'contradicts'
            ORDER BY 5 DESC
            LIMIT ? OFFSET ?
            """,
            (memory_id, memory_id, limit, offset),
        ).fetchall()
        return [dict(r) for r in rows]

    # ================================================================
    # audit_trail
    # ================================================================

    def put_audit_trail(self, entry: dict) -> dict:
        """Append an audit trail record with hash chain linking.

        Returns the entry dict enriched with sequence_number, record_hash, prev_hash.
        """
        details_json = _json_dumps(entry.get("details", {}))
        memory_id = entry["memory_id"]
        event_type = entry["event_type"]
        old_confidence = entry.get("old_confidence")
        new_confidence = entry.get("new_confidence")
        triggered_by = entry.get("triggered_by")
        created_at = entry.get("created_at")

        with self._lock:
            # Get latest chain tail
            row = self._conn.execute(
                "SELECT sequence_number, record_hash FROM memory_audit_trail "
                "ORDER BY sequence_number DESC LIMIT 1"
            ).fetchone()

            if row and row[0] is not None:
                seq = row[0] + 1
                prev_hash = row[1]
            else:
                seq = 1
                prev_hash = _AUDIT_HASH_GENESIS

            record_hash = _compute_audit_hash(
                memory_id=memory_id,
                event_type=event_type,
                old_confidence=old_confidence,
                new_confidence=new_confidence,
                triggered_by=triggered_by,
                details_json=details_json,
                created_at=created_at,
                prev_hash=prev_hash,
            )

            self._conn.execute(
                """INSERT INTO memory_audit_trail
                   (audit_id, memory_id, event_type, old_confidence, new_confidence,
                    triggered_by, details, created_at,
                    sequence_number, record_hash, prev_hash)
                   VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
                (
                    entry["audit_id"],
                    memory_id,
                    event_type,
                    old_confidence,
                    new_confidence,
                    triggered_by,
                    details_json,
                    created_at,
                    seq,
                    record_hash,
                    prev_hash,
                ),
            )
            if not self._in_transaction:
                self._conn.commit()

        entry["sequence_number"] = seq
        entry["record_hash"] = record_hash
        entry["prev_hash"] = prev_hash
        return entry

    def get_audit_trail(self, memory_id: str, limit: int = 50, offset: int = 0) -> list[dict]:
        rows = self._conn.execute(
            "SELECT * FROM memory_audit_trail WHERE memory_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?",
            (memory_id, limit, offset),
        ).fetchall()
        result = []
        for r in rows:
            d = dict(r)
            d["details"] = _json_loads(d.get("details", "{}"), default={})
            result.append(d)
        return result

    def all_audit_trail(self) -> list[dict]:
        rows = self._conn.execute("SELECT * FROM memory_audit_trail").fetchall()
        result = []
        for r in rows:
            d = dict(r)
            d["details"] = _json_loads(d.get("details", "{}"), default={})
            result.append(d)
        return result

    def verify_audit_chain(self) -> dict:
        """Verify the entire audit trail hash chain for tamper detection.

        Checks that:
        1. The first record has prev_hash == GENESIS
        2. Each subsequent record's prev_hash matches the previous record_hash
        3. Each record_hash is correctly computed from its fields

        Returns:
            {"valid": bool, "total_records": int, "verified": int,
             "first_broken_at": int | None, "error": str | None}
        """
        rows = self._conn.execute(
            "SELECT audit_id, memory_id, event_type, old_confidence, "
            "new_confidence, triggered_by, details, created_at, "
            "sequence_number, record_hash, prev_hash "
            "FROM memory_audit_trail ORDER BY sequence_number ASC"
        ).fetchall()

        total = len(rows)
        if total == 0:
            return {
                "valid": True, "total_records": 0, "verified": 0,
                "first_broken_at": None, "error": None,
            }

        prev_expected_hash = _AUDIT_HASH_GENESIS

        for i, row in enumerate(rows):
            seq = row[8]
            stored_hash = row[9]
            stored_prev = row[10]
            details_json = row[6] if row[6] else "{}"

            # Check prev_hash linkage
            if stored_prev != prev_expected_hash:
                return {
                    "valid": False, "total_records": total, "verified": i,
                    "first_broken_at": seq,
                    "error": (
                        f"Chain broken at sequence {seq}: "
                        f"prev_hash mismatch (expected {prev_expected_hash[:16]}..., "
                        f"got {stored_prev[:16] if stored_prev else 'None'}...)"
                    ),
                }

            # Recompute and verify record_hash
            computed = _compute_audit_hash(
                memory_id=row[1],
                event_type=row[2],
                old_confidence=row[3],
                new_confidence=row[4],
                triggered_by=row[5],
                details_json=details_json,
                created_at=row[7],
                prev_hash=stored_prev,
            )
            if computed != stored_hash:
                return {
                    "valid": False, "total_records": total, "verified": i,
                    "first_broken_at": seq,
                    "error": (
                        f"Hash mismatch at sequence {seq}: "
                        f"record may have been tampered with"
                    ),
                }

            # Check sequence continuity
            expected_seq = i + 1
            if seq != expected_seq:
                return {
                    "valid": False, "total_records": total, "verified": i,
                    "first_broken_at": seq,
                    "error": (
                        f"Sequence gap: expected {expected_seq}, got {seq} "
                        f"(records may have been deleted)"
                    ),
                }

            prev_expected_hash = stored_hash

        return {
            "valid": True, "total_records": total, "verified": total,
            "first_broken_at": None, "error": None,
        }

    def count_audit_trail_before(self, before: str) -> int:
        """Count audit trail records with created_at < *before* (ISO 8601)."""
        row = self._conn.execute(
            "SELECT COUNT(*) FROM memory_audit_trail WHERE created_at < ?",
            (before,),
        ).fetchone()
        return row[0]

    def delete_audit_trail_before(self, before: str) -> int:
        """Disabled: deletion would break hash chain integrity.

        Raises ValueError unconditionally.
        """
        raise ValueError(
            "Cannot delete audit trail records: hash chain integrity would be broken. "
            "Use count_audit_trail_before() to count records."
        )

    # ================================================================
    # memory_access_log (Read Audit)
    # ================================================================

    def put_access_log(self, entry: dict) -> dict:
        """Append an access log record with hash chain linking.

        Returns the entry dict enriched with sequence_number, record_hash, prev_hash.
        """
        access_id = entry["access_id"]
        event_type = entry["event_type"]
        user_id = entry.get("user_id")
        accessor = entry.get("accessor", "system")
        query = entry.get("query")
        filters_json = _json_dumps(entry.get("filters", {}))
        memory_ids_json = _json_dumps(entry.get("memory_ids", []))
        result_count = entry.get("result_count", 0)
        created_at = entry["created_at"]

        with self._lock:
            # Get latest chain tail
            row = self._conn.execute(
                "SELECT sequence_number, record_hash FROM memory_access_log "
                "ORDER BY sequence_number DESC LIMIT 1"
            ).fetchone()

            if row and row[0] is not None:
                seq = row[0] + 1
                prev_hash = row[1]
            else:
                seq = 1
                prev_hash = _ACCESS_HASH_GENESIS

            record_hash = _compute_access_hash(
                access_id=access_id,
                event_type=event_type,
                user_id=user_id,
                accessor=accessor,
                query=query,
                filters_json=filters_json,
                memory_ids_json=memory_ids_json,
                result_count=result_count,
                created_at=created_at,
                prev_hash=prev_hash,
            )

            self._conn.execute(
                """INSERT INTO memory_access_log
                   (access_id, event_type, user_id, accessor, query,
                    filters, memory_ids, result_count, created_at,
                    sequence_number, record_hash, prev_hash)
                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?)""",
                (
                    access_id, event_type, user_id, accessor, query,
                    filters_json, memory_ids_json, result_count, created_at,
                    seq, record_hash, prev_hash,
                ),
            )
            if not self._in_transaction:
                self._conn.commit()

        entry["sequence_number"] = seq
        entry["record_hash"] = record_hash
        entry["prev_hash"] = prev_hash
        return entry

    def get_access_log(
        self,
        *,
        memory_id: str | None = None,
        accessor: str | None = None,
        since: str | None = None,
        limit: int = 50,
        offset: int = 0,
    ) -> list[dict]:
        """Query access log records with optional filters."""
        conditions: list[str] = []
        params: list[Any] = []

        if memory_id:
            # Search within memory_ids JSON array
            conditions.append("memory_ids LIKE ?")
            params.append(f'%"{memory_id}"%')
        if accessor:
            conditions.append("accessor = ?")
            params.append(accessor)
        if since:
            conditions.append("created_at >= ?")
            params.append(since)

        where = " AND ".join(conditions) if conditions else "1=1"
        sql = (
            f"SELECT * FROM memory_access_log WHERE {where} "
            f"ORDER BY created_at DESC LIMIT ? OFFSET ?"
        )
        params.extend([limit, offset])

        rows = self._conn.execute(sql, params).fetchall()
        result = []
        for r in rows:
            d = dict(r)
            d["filters"] = _json_loads(d.get("filters", "{}"), default={})
            d["memory_ids"] = _json_loads(d.get("memory_ids", "[]"), default=[])
            result.append(d)
        return result

    def verify_access_chain(self) -> dict:
        """Verify the access log hash chain for tamper detection.

        Supports PURGE markers: if a record's prev_hash starts with
        ``"PURGE:"``, it is treated as a valid chain origin.

        Returns:
            {"valid": bool, "total_records": int, "verified": int,
             "first_broken_at": int | None, "error": str | None,
             "purge_points": list[dict]}
        """
        rows = self._conn.execute(
            "SELECT access_id, event_type, user_id, accessor, query, "
            "filters, memory_ids, result_count, created_at, "
            "sequence_number, record_hash, prev_hash "
            "FROM memory_access_log ORDER BY sequence_number ASC"
        ).fetchall()

        total = len(rows)
        purge_points: list[dict] = []
        if total == 0:
            return {
                "valid": True, "total_records": 0, "verified": 0,
                "first_broken_at": None, "error": None,
                "purge_points": purge_points,
            }

        prev_expected_hash = _ACCESS_HASH_GENESIS

        for i, row in enumerate(rows):
            seq = row[9]
            stored_hash = row[10]
            stored_prev = row[11]
            filters_json = row[5] if row[5] else "{}"
            memory_ids_json = row[6] if row[6] else "[]"

            # Check prev_hash linkage (allow PURGE markers)
            if stored_prev != prev_expected_hash:
                if stored_prev and stored_prev.startswith("PURGE:"):
                    purge_points.append({
                        "sequence": seq, "marker": stored_prev,
                    })
                else:
                    return {
                        "valid": False, "total_records": total, "verified": i,
                        "first_broken_at": seq,
                        "error": (
                            f"Chain broken at sequence {seq}: "
                            f"prev_hash mismatch"
                        ),
                        "purge_points": purge_points,
                    }

            # Recompute and verify record_hash
            computed = _compute_access_hash(
                access_id=row[0],
                event_type=row[1],
                user_id=row[2],
                accessor=row[3],
                query=row[4],
                filters_json=filters_json,
                memory_ids_json=memory_ids_json,
                result_count=row[7],
                created_at=row[8],
                prev_hash=stored_prev,
            )
            if computed != stored_hash:
                return {
                    "valid": False, "total_records": total, "verified": i,
                    "first_broken_at": seq,
                    "error": (
                        f"Hash mismatch at sequence {seq}: "
                        f"record may have been tampered with"
                    ),
                    "purge_points": purge_points,
                }

            prev_expected_hash = stored_hash

        return {
            "valid": True, "total_records": total, "verified": total,
            "first_broken_at": None, "error": None,
            "purge_points": purge_points,
        }

    def purge_access_log(self, before: str) -> dict:
        """Purge access log records older than *before* (ISO 8601).

        Physically deletes records and rewrites the chain boundary
        with a PURGE marker for auditability.

        Returns:
            {"purged": int, "before": str, "marker": str | None}
        """
        with self._lock:
            # Count records to purge
            count_row = self._conn.execute(
                "SELECT COUNT(*) FROM memory_access_log WHERE created_at < ?",
                (before,),
            ).fetchone()
            purge_count = count_row[0]

            if purge_count == 0:
                return {"purged": 0, "before": before, "marker": None}

            # Find the last record being purged (for PURGE marker)
            last_purged = self._conn.execute(
                "SELECT record_hash FROM memory_access_log "
                "WHERE created_at < ? ORDER BY sequence_number DESC LIMIT 1",
                (before,),
            ).fetchone()
            last_purged_hash = last_purged[0] if last_purged else "unknown"

            # Find the first surviving record
            first_surviving = self._conn.execute(
                "SELECT sequence_number FROM memory_access_log "
                "WHERE created_at >= ? ORDER BY sequence_number ASC LIMIT 1",
                (before,),
            ).fetchone()

            now_iso = datetime.now(timezone.utc).isoformat()
            marker = f"PURGE:{now_iso}:{purge_count}:{last_purged_hash}"

            # Drop triggers temporarily
            self._conn.execute("DROP TRIGGER IF EXISTS access_log_no_update")
            self._conn.execute("DROP TRIGGER IF EXISTS access_log_no_delete")

            # Delete old records
            self._conn.execute(
                "DELETE FROM memory_access_log WHERE created_at < ?",
                (before,),
            )

            # Rewrite prev_hash of first surviving record
            if first_surviving:
                surviving_seq = first_surviving[0]
                self._conn.execute(
                    "UPDATE memory_access_log SET prev_hash = ? "
                    "WHERE sequence_number = ?",
                    (marker, surviving_seq),
                )

                # Recompute record_hash for the updated record
                row = self._conn.execute(
                    "SELECT access_id, event_type, user_id, accessor, query, "
                    "filters, memory_ids, result_count, created_at "
                    "FROM memory_access_log WHERE sequence_number = ?",
                    (surviving_seq,),
                ).fetchone()
                if row:
                    new_hash = _compute_access_hash(
                        access_id=row[0],
                        event_type=row[1],
                        user_id=row[2],
                        accessor=row[3],
                        query=row[4],
                        filters_json=row[5] if row[5] else "{}",
                        memory_ids_json=row[6] if row[6] else "[]",
                        result_count=row[7],
                        created_at=row[8],
                        prev_hash=marker,
                    )
                    self._conn.execute(
                        "UPDATE memory_access_log SET record_hash = ? "
                        "WHERE sequence_number = ?",
                        (new_hash, surviving_seq),
                    )

                    # Rewrite next record's prev_hash to match new hash
                    next_row = self._conn.execute(
                        "SELECT sequence_number FROM memory_access_log "
                        "WHERE sequence_number > ? ORDER BY sequence_number ASC LIMIT 1",
                        (surviving_seq,),
                    ).fetchone()
                    if next_row:
                        next_seq = next_row[0]
                        self._conn.execute(
                            "UPDATE memory_access_log SET prev_hash = ? "
                            "WHERE sequence_number = ?",
                            (new_hash, next_seq),
                        )
                        # Recompute that record's hash too
                        nr = self._conn.execute(
                            "SELECT access_id, event_type, user_id, accessor, query, "
                            "filters, memory_ids, result_count, created_at "
                            "FROM memory_access_log WHERE sequence_number = ?",
                            (next_seq,),
                        ).fetchone()
                        if nr:
                            next_hash = _compute_access_hash(
                                access_id=nr[0], event_type=nr[1],
                                user_id=nr[2], accessor=nr[3], query=nr[4],
                                filters_json=nr[5] if nr[5] else "{}",
                                memory_ids_json=nr[6] if nr[6] else "[]",
                                result_count=nr[7], created_at=nr[8],
                                prev_hash=new_hash,
                            )
                            self._conn.execute(
                                "UPDATE memory_access_log SET record_hash = ? "
                                "WHERE sequence_number = ?",
                                (next_hash, next_seq),
                            )

            # Recreate triggers
            self._conn.executescript("""
CREATE TRIGGER IF NOT EXISTS access_log_no_update
BEFORE UPDATE ON memory_access_log
BEGIN
    SELECT RAISE(ABORT, 'UPDATE on memory_access_log is prohibited (tamper-proof access log)');
END;

CREATE TRIGGER IF NOT EXISTS access_log_no_delete
BEFORE DELETE ON memory_access_log
BEGIN
    SELECT RAISE(ABORT, 'DELETE on memory_access_log is prohibited (tamper-proof access log)');
END;
""")
            self._conn.commit()

        return {"purged": purge_count, "before": before, "marker": marker}

    def reset_access_log(self) -> None:
        """Clear all access log records (development/testing only)."""
        with self._lock:
            self._conn.execute("DROP TRIGGER IF EXISTS access_log_no_update")
            self._conn.execute("DROP TRIGGER IF EXISTS access_log_no_delete")
            self._conn.execute("DELETE FROM memory_access_log")
            self._conn.executescript("""
CREATE TRIGGER IF NOT EXISTS access_log_no_update
BEFORE UPDATE ON memory_access_log
BEGIN
    SELECT RAISE(ABORT, 'UPDATE on memory_access_log is prohibited (tamper-proof access log)');
END;

CREATE TRIGGER IF NOT EXISTS access_log_no_delete
BEFORE DELETE ON memory_access_log
BEGIN
    SELECT RAISE(ABORT, 'DELETE on memory_access_log is prohibited (tamper-proof access log)');
END;
""")
            self._conn.commit()

    # ================================================================
    # entity_nodes
    # ================================================================

    def put_entity(self, entity: dict) -> None:
        with self._lock:
            self._conn.execute(
                """INSERT INTO entity_nodes
                   (entity_id, name, entity_type, user_id,
                    confidence, source_memory_id,
                    mention_count, metadata, is_active, created_at, updated_at)
                   VALUES (?,?,?,?,?,?,?,?,?,?,?)
                   ON CONFLICT(entity_id) DO UPDATE SET
                    name = excluded.name,
                    entity_type = excluded.entity_type,
                    confidence = excluded.confidence,
                    source_memory_id = excluded.source_memory_id,
                    mention_count = excluded.mention_count,
                    metadata = excluded.metadata,
                    is_active = excluded.is_active,
                    updated_at = excluded.updated_at""",
                (
                    entity["entity_id"],
                    entity["name"],
                    entity.get("entity_type", "other"),
                    entity.get("user_id"),
                    entity.get("confidence", "medium"),
                    entity.get("source_memory_id"),
                    entity.get("mention_count", 1),
                    _json_dumps(entity.get("metadata", {})),
                    1 if entity.get("is_active", True) else 0,
                    entity.get("created_at"),
                    entity.get("updated_at"),
                ),
            )
            if not self._in_transaction:
                self._conn.commit()

    def get_entity(self, entity_id: str) -> dict | None:
        row = self._conn.execute(
            "SELECT * FROM entity_nodes WHERE entity_id = ?", (entity_id,)
        ).fetchone()
        return self._row_to_entity(row) if row else None

    def get_entity_by_name(self, name: str, user_id: str | None = None) -> dict | None:
        if user_id is not None:
            # Return user's entity first; fall back to org-shared
            row = self._conn.execute(
                "SELECT * FROM entity_nodes WHERE LOWER(name) = LOWER(?) AND (user_id = ? OR user_id IS NULL) AND is_active = 1 ORDER BY CASE WHEN user_id IS NULL THEN 1 ELSE 0 END LIMIT 1",
                (name, user_id),
            ).fetchone()
        else:
            row = self._conn.execute(
                "SELECT * FROM entity_nodes WHERE LOWER(name) = LOWER(?) AND user_id IS NULL AND is_active = 1",
                (name,),
            ).fetchone()
        return self._row_to_entity(row) if row else None

    def delete_entity(self, entity_id: str) -> bool:
        with self._lock:
            cur = self._conn.execute(
                "UPDATE entity_nodes SET is_active = 0, updated_at = ? WHERE entity_id = ?",
                (datetime.now(timezone.utc).isoformat(), entity_id),
            )
            self._conn.commit()
            return cur.rowcount > 0

    def _row_to_entity(self, row: sqlite3.Row) -> dict:
        d = dict(row)
        d["metadata"] = _json_loads(d.get("metadata", "{}"), default={})
        d["is_active"] = bool(d.get("is_active", 1))
        return d

    # ================================================================
    # entity_edges
    # ================================================================

    def put_entity_edge(self, edge: dict) -> None:
        with self._lock:
            self._conn.execute(
                """INSERT INTO entity_edges
                   (edge_id, subject_id, subject_name, predicate,
                    object_id, object_name, user_id,
                    confidence, source_memory_id,
                    mention_count, metadata,
                    is_active, created_at, updated_at)
                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
                   ON CONFLICT(edge_id) DO UPDATE SET
                    subject_id = excluded.subject_id,
                    subject_name = excluded.subject_name,
                    predicate = excluded.predicate,
                    object_id = excluded.object_id,
                    object_name = excluded.object_name,
                    confidence = excluded.confidence,
                    source_memory_id = excluded.source_memory_id,
                    mention_count = excluded.mention_count,
                    metadata = excluded.metadata,
                    is_active = excluded.is_active,
                    updated_at = excluded.updated_at""",
                (
                    edge["edge_id"],
                    edge["subject_id"],
                    edge["subject_name"],
                    edge["predicate"],
                    edge["object_id"],
                    edge["object_name"],
                    edge.get("user_id"),
                    edge.get("confidence", "medium"),
                    edge.get("source_memory_id"),
                    edge.get("mention_count", 1),
                    _json_dumps(edge.get("metadata", {})),
                    1 if edge.get("is_active", True) else 0,
                    edge.get("created_at"),
                    edge.get("updated_at"),
                ),
            )
            if not self._in_transaction:
                self._conn.commit()

    def get_entity_edge(self, edge_id: str) -> dict | None:
        row = self._conn.execute(
            "SELECT * FROM entity_edges WHERE edge_id = ?", (edge_id,)
        ).fetchone()
        return self._row_to_edge(row) if row else None

    def delete_entity_edge(self, edge_id: str) -> bool:
        with self._lock:
            cur = self._conn.execute(
                "UPDATE entity_edges SET is_active = 0, updated_at = ? WHERE edge_id = ?",
                (datetime.now(timezone.utc).isoformat(), edge_id),
            )
            self._conn.commit()
            return cur.rowcount > 0

    def search_entity_edges(
        self,
        user_id: str | None = None,
        entity_name: str = "",
        predicate: str | None = None,
        direction: str = "both",
        limit: int = 20,
        partial_match: bool = False,
    ) -> list[dict]:
        """Search edges connected to an entity by name."""
        name_lower = entity_name.strip().lower()
        conditions = ["e.is_active = 1"]
        params: list = []
        if user_id is not None:
            conditions.append("(e.user_id = ? OR e.user_id IS NULL)")
            params.append(user_id)
        else:
            conditions.append("e.user_id IS NULL")

        if partial_match:
            escaped = name_lower.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
            like_val = f"%{escaped}%"
            op, val = "LIKE", like_val
        else:
            op, val = "=", name_lower

        escape_clause = " ESCAPE '\\'" if partial_match else ""
        if direction == "outgoing":
            conditions.append(f"LOWER(e.subject_name) {op} ?{escape_clause}")
            params.append(val)
        elif direction == "incoming":
            conditions.append(f"LOWER(e.object_name) {op} ?{escape_clause}")
            params.append(val)
        else:  # both
            conditions.append(f"(LOWER(e.subject_name) {op} ?{escape_clause} OR LOWER(e.object_name) {op} ?{escape_clause})")
            params.extend([val, val])

        if predicate:
            conditions.append("e.predicate = ?")
            params.append(predicate)

        params.append(limit)
        where = " AND ".join(conditions)
        query = f"SELECT * FROM entity_edges e WHERE {where} ORDER BY e.mention_count DESC LIMIT ?"

        rows = self._conn.execute(query, params).fetchall()
        return [self._row_to_edge(r) for r in rows]

    def list_entity_edges(self, user_id: str | None = None, limit: int = 100) -> list[dict]:
        """List all active edges for a user scope."""
        if user_id is not None:
            rows = self._conn.execute(
                "SELECT * FROM entity_edges WHERE (user_id = ? OR user_id IS NULL) AND is_active = 1 "
                "ORDER BY created_at DESC LIMIT ?",
                (user_id, limit),
            ).fetchall()
        else:
            rows = self._conn.execute(
                "SELECT * FROM entity_edges WHERE user_id IS NULL AND is_active = 1 "
                "ORDER BY created_at DESC LIMIT ?",
                (limit,),
            ).fetchall()
        return [self._row_to_edge(r) for r in rows]

    def get_entity_stats(self, user_id: str | None = None) -> dict:
        """Return entity/edge counts for a user scope."""
        if user_id is not None:
            node_params: tuple = (user_id,)
            node_where = "(user_id = ? OR user_id IS NULL) AND is_active = 1"
        else:
            node_params = ()
            node_where = "user_id IS NULL AND is_active = 1"

        node_count = self._conn.execute(
            f"SELECT COUNT(*) FROM entity_nodes WHERE {node_where}",
            node_params,
        ).fetchone()[0]
        edge_count = self._conn.execute(
            f"SELECT COUNT(*) FROM entity_edges WHERE {node_where}",
            node_params,
        ).fetchone()[0]

        conf_rows = self._conn.execute(
            f"SELECT confidence, COUNT(*) FROM entity_nodes WHERE {node_where} GROUP BY confidence",
            node_params,
        ).fetchall()
        confidence_dist = {r[0]: r[1] for r in conf_rows}

        type_rows = self._conn.execute(
            f"SELECT entity_type, COUNT(*) FROM entity_nodes WHERE {node_where} GROUP BY entity_type ORDER BY COUNT(*) DESC",
            node_params,
        ).fetchall()
        type_dist = {r[0]: r[1] for r in type_rows}

        return {
            "user_id": user_id,
            "total_entities": node_count,
            "total_edges": edge_count,
            "confidence_distribution": confidence_dist,
            "entity_type_distribution": type_dist,
        }

    def _row_to_edge(self, row: sqlite3.Row) -> dict:
        d = dict(row)
        d["metadata"] = _json_loads(d.get("metadata", "{}"), default={})
        d["is_active"] = bool(d.get("is_active", 1))
        return d

    # ================================================================
    # clear / reset
    # ================================================================

    def clear_all(self) -> None:
        tables = [
            "memories", "memory_provenance", "memory_audit_trail",
            "entity_nodes", "entity_edges", "memory_vectors",
            "memory_access_log",
        ]
        with self._lock:
            # Temporarily disable append-only triggers for dev reset
            self._conn.execute("DROP TRIGGER IF EXISTS audit_trail_no_update")
            self._conn.execute("DROP TRIGGER IF EXISTS audit_trail_no_delete")
            self._conn.execute("DROP TRIGGER IF EXISTS access_log_no_update")
            self._conn.execute("DROP TRIGGER IF EXISTS access_log_no_delete")
            for t in tables:
                self._conn.execute(f"DELETE FROM {t}")
            # Clear token index
            self._conn.execute("DELETE FROM memory_tokens")
            # Re-create triggers
            self._conn.executescript(_AUDIT_TRIGGER_SQL)
            self._conn.commit()

    # ================================================================
    # Vector CRUD (for SQLiteVectorStore adapter)
    # ================================================================

    def upsert_vector(self, id: str, vector: list[float], payload: dict) -> None:
        blob = struct.pack(f"<{len(vector)}f", *vector)
        payload_json = json.dumps(payload, ensure_ascii=False)
        with self._lock:
            self._conn.execute(
                "INSERT OR REPLACE INTO memory_vectors (id, vector, payload) "
                "VALUES (?, ?, ?)",
                (id, blob, payload_json),
            )
            self._conn.commit()

    def search_vectors(
        self,
        query_vector: list[float],
        limit: int = 10,
        filters: dict[str, Any] | None = None,
    ) -> list[dict]:
        rows = self._conn.execute(
            "SELECT id, vector, payload FROM memory_vectors"
        ).fetchall()
        results = []
        # Extract special filter keys (don't mutate caller's dict)
        if filters:
            filters = dict(filters)
            filters.pop("min_zone", None)  # legacy: ignored
        for row in rows:
            payload = json.loads(row["payload"] or "{}")
            if filters and not all(
                payload.get(k) == v for k, v in filters.items()
            ):
                continue
            n = len(row["vector"]) // 4
            vec = list(struct.unpack(f"<{n}f", row["vector"]))
            score = _cosine_similarity(query_vector, vec)
            results.append({"id": row["id"], "score": score, "payload": payload})
        results.sort(key=lambda x: x["score"], reverse=True)
        return results[:limit]

    def delete_vector(self, id: str) -> None:
        with self._lock:
            self._conn.execute("DELETE FROM memory_vectors WHERE id = ?", (id,))
            self._conn.commit()

    def update_vector_payload(self, id: str, payload: dict) -> None:
        with self._lock:
            row = self._conn.execute(
                "SELECT payload FROM memory_vectors WHERE id = ?", (id,)
            ).fetchone()
            if row is None:
                return
            existing = json.loads(row["payload"] or "{}")
            existing.update(payload)
            self._conn.execute(
                "UPDATE memory_vectors SET payload = ? WHERE id = ?",
                (json.dumps(existing, ensure_ascii=False), id),
            )
            self._conn.commit()

    def count_vectors(self) -> int:
        row = self._conn.execute(
            "SELECT COUNT(*) AS cnt FROM memory_vectors"
        ).fetchone()
        return row["cnt"] if row else 0


def _cosine_similarity(a: list[float], b: list[float]) -> float:
    dot = sum(x * y for x, y in zip(a, b))
    norm_a = math.sqrt(sum(x * x for x in a))
    norm_b = math.sqrt(sum(x * x for x in b))
    if norm_a == 0 or norm_b == 0:
        return 0.0
    return dot / (norm_a * norm_b)
