"""NEUROVAULT storage engine - extends CHRONOS with knowledge graph."""

from __future__ import annotations

import logging
import json
import os
import sqlite3
import time
import hashlib
from pathlib import Path
from typing import Any, Optional

try:
    import numpy as np
except Exception:  # pragma: no cover
    np = None  # type: ignore

from neurovault.models import Entity, MemoryCluster, Relationship

logger = logging.getLogger(__name__)

SCHEMA_PATH = Path(__file__).parent / "schema.sql"


def _ser_emb(emb: np.ndarray | None) -> bytes | None:
    if np is None:
        return None
    return emb.astype(np.float32).tobytes() if emb is not None else None


def _des_emb(data: bytes | None) -> np.ndarray | None:
    if np is None:
        return None
    return np.frombuffer(data, dtype=np.float32).copy() if data else None


def _content_hash(content: str) -> str:
    return hashlib.sha256(content.encode("utf-8")).hexdigest()[:16] if content else ""


class NeurovaultStorage:
    """SQLite storage for NEUROVAULT knowledge graph + cognitive state."""

    def __init__(self, db_path: str | Path):
        self.db_path = Path(db_path)
        self._conn: Optional[sqlite3.Connection] = None

    @property
    def conn(self) -> sqlite3.Connection:
        if self._conn is None:
            self.db_path.parent.mkdir(parents=True, exist_ok=True)
            try:
                os.chmod(self.db_path.parent, 0o700)
            except Exception as exc:
                logger.debug("Unable to chmod storage directory %s: %s", self.db_path.parent, exc)
            self._conn = sqlite3.connect(str(self.db_path), check_same_thread=False)
            self._conn.row_factory = sqlite3.Row
            self._conn.execute("PRAGMA journal_mode = WAL")
            self._conn.execute("PRAGMA foreign_keys = ON")
            try:
                os.chmod(self.db_path, 0o600)
            except Exception as exc:
                logger.debug("Unable to chmod storage file %s: %s", self.db_path, exc)
        return self._conn

    def initialize_schema(self) -> None:
        script = SCHEMA_PATH.read_text()
        try:
            self.conn.executescript(script)
            return
        except sqlite3.OperationalError as exc:
            # Compatibility path: CHRONOS may pre-create memories table without v0.3 columns.
            if "no such column: content_hash" not in str(exc):
                raise

        for sql in (
            "ALTER TABLE memories ADD COLUMN content_hash TEXT DEFAULT ''",
            "ALTER TABLE memories ADD COLUMN device_origin TEXT DEFAULT 'local'",
        ):
            try:
                self.conn.execute(sql)
            except sqlite3.OperationalError:
                pass

        self.conn.executescript(script)

    def close(self) -> None:
        if self._conn:
            self._conn.close()
            self._conn = None

    # ================================================================
    # CHRONOS Mirror Helpers
    # ================================================================

    def upsert_memory_snapshot(
        self,
        memory_id: bytes,
        persona_id: bytes,
        content: str,
        source: str,
        importance: float,
        metadata: dict[str, Any] | None,
        created_at: int,
        is_active: bool = True,
        device_origin: str = "local",
    ) -> None:
        payload = (
            memory_id,
            persona_id,
            content,
            source,
            max(0.0, min(float(importance), 1.0)),
            json.dumps(metadata or {}),
            _content_hash(content),
            str(device_origin or "local"),
            int(created_at),
            1 if is_active else 0,
        )
        try:
            self.conn.execute(
                """
                INSERT INTO memories
                (id, persona_id, content, source, importance, metadata, embedding, content_hash, device_origin, created_at, is_active)
                VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?)
                ON CONFLICT(id) DO UPDATE SET
                    persona_id = excluded.persona_id,
                    content = excluded.content,
                    source = excluded.source,
                    importance = excluded.importance,
                    metadata = excluded.metadata,
                    content_hash = excluded.content_hash,
                    device_origin = excluded.device_origin,
                    created_at = excluded.created_at,
                    is_active = excluded.is_active
                """,
                payload,
            )
            return
        except sqlite3.OperationalError:
            pass

        # Backward compatibility with pre-v0.3 schema.
        self.conn.execute(
            """
            INSERT INTO memories (id, persona_id, content, source, importance, metadata, embedding, created_at, is_active)
            VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?)
            ON CONFLICT(id) DO UPDATE SET
                persona_id = excluded.persona_id,
                content = excluded.content,
                source = excluded.source,
                importance = excluded.importance,
                metadata = excluded.metadata,
                created_at = excluded.created_at,
                is_active = excluded.is_active
            """,
            (
                memory_id,
                persona_id,
                content,
                source,
                max(0.0, min(float(importance), 1.0)),
                json.dumps(metadata or {}),
                int(created_at),
                1 if is_active else 0,
            ),
        )

    def sync_active_memories(self, persona_id: bytes, records: list[dict[str, Any]]) -> None:
        active_ids: list[bytes] = []
        for record in records:
            memory_id = record.get("id")
            if isinstance(memory_id, bytearray):
                memory_id = bytes(memory_id)
            if not isinstance(memory_id, bytes):
                continue

            self.upsert_memory_snapshot(
                memory_id=memory_id,
                persona_id=persona_id,
                content=str(record.get("content", "")),
                source=str(record.get("source", "unknown")),
                importance=float(record.get("importance", 0.5)),
                metadata=record.get("metadata", {}) if isinstance(record.get("metadata"), dict) else {},
                created_at=int(record.get("created_at", int(time.time() * 1000))),
                is_active=True,
            )
            active_ids.append(memory_id)

        if active_ids:
            placeholders = ",".join("?" for _ in active_ids)
            self.conn.execute(
                f"""
                UPDATE memories
                SET is_active = 0
                WHERE persona_id = ?
                  AND id NOT IN ({placeholders})
                """,
                (persona_id, *active_ids),
            )
        else:
            self.conn.execute(
                "UPDATE memories SET is_active = 0 WHERE persona_id = ?",
                (persona_id,),
            )
        self.conn.commit()

    # ================================================================
    # Entity Operations
    # ================================================================

    def upsert_entity(
        self,
        name: str,
        entity_type: str,
        confidence: float = 0.8,
        seen_at: int | None = None,
    ) -> Entity:
        """Create or update an entity."""
        now = seen_at or int(time.time() * 1000)
        entity = Entity(name=name, entity_type=entity_type, first_seen=now, last_seen=now)

        existing = self.conn.execute(
            "SELECT * FROM entities WHERE canonical_name = ? AND entity_type = ?",
            (entity.canonical_name, entity_type),
        ).fetchone()

        if existing:
            self.conn.execute(
                """UPDATE entities SET mention_count = mention_count + 1,
                   last_seen = ?, importance = MIN(importance + ?, 1.0)
                   WHERE id = ?""",
                (now, max(0.005, min(confidence, 0.05)), existing["id"]),
            )
            self.conn.commit()
            entity.id = existing["id"]
            entity.mention_count = existing["mention_count"] + 1
            entity.importance = min(existing["importance"] + 0.01, 1.0)
            return entity

        self.conn.execute(
            """INSERT INTO entities (id, name, canonical_name, entity_type,
               description, embedding, mention_count, first_seen, last_seen,
               importance, metadata) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            (
                entity.id,
                entity.name,
                entity.canonical_name,
                entity_type,
                "",
                None,
                1,
                now,
                now,
                max(0.1, min(confidence, 1.0)),
                "{}",
            ),
        )
        self.conn.commit()
        return entity

    def get_entity(self, entity_id: bytes) -> Entity:
        row = self.conn.execute("SELECT * FROM entities WHERE id = ?", (entity_id,)).fetchone()
        if not row:
            raise ValueError("Entity not found")
        return self._row_to_entity(row)

    def find_entities_by_name(self, name: str) -> list[Entity]:
        rows = self.conn.execute(
            "SELECT * FROM entities WHERE canonical_name LIKE ? ORDER BY mention_count DESC",
            (f"%{name.lower()}%",),
        ).fetchall()
        return [self._row_to_entity(r) for r in rows]

    def list_entities(
        self,
        entity_type: str | None = None,
        limit: int = 50,
        sort: str = "mention_count",
    ) -> list[Entity]:
        sort_map = {
            "mention_count": "mention_count",
            "importance": "importance",
            "last_seen": "last_seen",
            "first_seen": "first_seen",
            "name": "name",
        }
        order_by = sort_map.get(sort, "mention_count")

        query = "SELECT * FROM entities"
        params: list[object] = []
        if entity_type:
            query += " WHERE entity_type = ?"
            params.append(entity_type)
        query += f" ORDER BY {order_by} DESC LIMIT ?"
        params.append(max(1, min(limit, 1000)))

        rows = self.conn.execute(query, params).fetchall()
        return [self._row_to_entity(r) for r in rows]

    def _row_to_entity(self, row: sqlite3.Row) -> Entity:
        return Entity(
            id=row["id"],
            name=row["name"],
            canonical_name=row["canonical_name"],
            entity_type=row["entity_type"],
            description=row["description"],
            embedding=_des_emb(row["embedding"]),
            mention_count=row["mention_count"],
            first_seen=row["first_seen"],
            last_seen=row["last_seen"],
            importance=row["importance"],
            metadata=json.loads(row["metadata"]) if row["metadata"] else {},
        )

    # ================================================================
    # Relationship Operations
    # ================================================================

    def upsert_relationship(
        self,
        source_name: str,
        target_name: str,
        relation_type: str,
        strength: float = 0.5,
    ) -> None:
        source_entities = self.find_entities_by_name(source_name)
        target_entities = self.find_entities_by_name(target_name)
        if not source_entities or not target_entities:
            return

        source_id = source_entities[0].id
        target_id = target_entities[0].id
        now = int(time.time() * 1000)

        existing = self.conn.execute(
            """SELECT * FROM relationships
               WHERE source_entity_id = ? AND target_entity_id = ? AND relation_type = ?""",
            (source_id, target_id, relation_type),
        ).fetchone()

        if existing:
            new_strength = min(existing["strength"] + 0.05, 1.0)
            self.conn.execute(
                """UPDATE relationships SET evidence_count = evidence_count + 1,
                   strength = ?, last_seen = ? WHERE id = ?""",
                (new_strength, now, existing["id"]),
            )
        else:
            rel = Relationship(
                source_entity_id=source_id,
                target_entity_id=target_id,
                relation_type=relation_type,
                strength=strength,
            )
            self.conn.execute(
                """INSERT INTO relationships (id, source_entity_id, target_entity_id,
                   relation_type, strength, evidence_count, first_seen, last_seen, metadata)
                   VALUES (?, ?, ?, ?, ?, 1, ?, ?, '{}')""",
                (rel.id, source_id, target_id, relation_type, strength, now, now),
            )
        self.conn.commit()

    def get_relationships_from(
        self,
        entity_id: bytes,
        min_strength: float = 0.0,
    ) -> list[Relationship]:
        rows = self.conn.execute(
            """SELECT * FROM relationships
               WHERE source_entity_id = ? AND strength >= ?
               ORDER BY strength DESC""",
            (entity_id, min_strength),
        ).fetchall()
        return [self._row_to_rel(r) for r in rows]

    def get_relationships_to(
        self,
        entity_id: bytes,
        min_strength: float = 0.0,
    ) -> list[Relationship]:
        rows = self.conn.execute(
            """SELECT * FROM relationships
               WHERE target_entity_id = ? AND strength >= ?
               ORDER BY strength DESC""",
            (entity_id, min_strength),
        ).fetchall()
        return [self._row_to_rel(r) for r in rows]

    def _row_to_rel(self, row: sqlite3.Row) -> Relationship:
        return Relationship(
            id=row["id"],
            source_entity_id=row["source_entity_id"],
            target_entity_id=row["target_entity_id"],
            relation_type=row["relation_type"],
            strength=row["strength"],
            evidence_count=row["evidence_count"],
            first_seen=row["first_seen"],
            last_seen=row["last_seen"],
            metadata=json.loads(row["metadata"]) if row["metadata"] else {},
        )

    # ================================================================
    # Memory-Entity Links
    # ================================================================

    def link_memory_entity(self, memory_id: bytes, entity_id: bytes, confidence: float = 0.8) -> None:
        self.conn.execute(
            "INSERT OR IGNORE INTO memory_entities (memory_id, entity_id, confidence) VALUES (?, ?, ?)",
            (memory_id, entity_id, confidence),
        )
        self.conn.commit()

    def get_entities_for_memory(self, memory_id: bytes) -> list[Entity]:
        rows = self.conn.execute(
            """SELECT e.* FROM entities e
               JOIN memory_entities me ON e.id = me.entity_id
               WHERE me.memory_id = ?""",
            (memory_id,),
        ).fetchall()
        return [self._row_to_entity(r) for r in rows]

    def get_memories_for_entity(self, entity_id: bytes) -> list[tuple[bytes, float]]:
        rows = self.conn.execute(
            "SELECT memory_id, confidence FROM memory_entities WHERE entity_id = ?",
            (entity_id,),
        ).fetchall()
        return [(r["memory_id"], r["confidence"]) for r in rows]

    # ================================================================
    # Processing Log
    # ================================================================

    def is_processed(self, memory_id: bytes, phase: str) -> bool:
        row = self.conn.execute(
            "SELECT 1 FROM processing_log WHERE memory_id = ? AND phase = ?",
            (memory_id, phase),
        ).fetchone()
        return row is not None

    def mark_processed(self, memory_id: bytes, phase: str) -> None:
        self.conn.execute(
            "INSERT OR IGNORE INTO processing_log (memory_id, phase, processed_at) VALUES (?, ?, ?)",
            (memory_id, phase, int(time.time() * 1000)),
        )
        self.conn.commit()

    def get_unprocessed_memory_ids(self, phase: str, limit: int = 500) -> list[bytes]:
        rows = self.conn.execute(
            """SELECT m.id FROM memories m
               LEFT JOIN processing_log p ON m.id = p.memory_id AND p.phase = ?
               WHERE p.memory_id IS NULL AND m.is_active = 1
               ORDER BY m.created_at DESC LIMIT ?""",
            (phase, limit),
        ).fetchall()
        return [r["id"] for r in rows]

    # ================================================================
    # Ontology / Concept Hierarchy
    # ================================================================

    def insert_concept_hierarchy(
        self,
        hid: bytes,
        child_id: bytes,
        parent_id: bytes,
        relation_type: str,
        confidence: float,
        source: str,
        created_at: int,
    ) -> None:
        """Insert or ignore a concept hierarchy edge."""
        self.conn.execute(
            """
            INSERT OR IGNORE INTO concept_hierarchy
            (id, child_entity_id, parent_entity_id, relation_type, confidence, source, created_at)
            VALUES (?, ?, ?, ?, ?, ?, ?)
            """,
            (
                hid,
                child_id,
                parent_id,
                relation_type,
                float(confidence),
                source,
                int(created_at),
            ),
        )
        self.conn.commit()

    def get_hierarchy_parents(self, entity_id: bytes, relation_type: str = "IS-A") -> list[tuple[bytes, str, float]]:
        """Return parent entity links for one child entity."""
        rows = self.conn.execute(
            """
            SELECT parent_entity_id, relation_type, confidence
            FROM concept_hierarchy
            WHERE child_entity_id = ? AND relation_type = ?
            """,
            (entity_id, relation_type),
        ).fetchall()
        return [(r["parent_entity_id"], str(r["relation_type"]), float(r["confidence"])) for r in rows]

    def get_hierarchy_children(self, entity_id: bytes, relation_type: str = "IS-A") -> list[tuple[bytes, str, float]]:
        """Return child entity links for one parent entity."""
        rows = self.conn.execute(
            """
            SELECT child_entity_id, relation_type, confidence
            FROM concept_hierarchy
            WHERE parent_entity_id = ? AND relation_type = ?
            """,
            (entity_id, relation_type),
        ).fetchall()
        return [(r["child_entity_id"], str(r["relation_type"]), float(r["confidence"])) for r in rows]

    def get_concept_count(self) -> int:
        """Return concept hierarchy edge count."""
        row = self.conn.execute("SELECT COUNT(*) AS cnt FROM concept_hierarchy").fetchone()
        return int(row["cnt"] if row else 0)

    # ================================================================
    # Plugin State
    # ================================================================

    def upsert_plugin_state(
        self,
        plugin_id: str,
        enabled: bool = True,
        config: dict[str, Any] | None = None,
        last_poll: int = 0,
        error_count: int = 0,
    ) -> None:
        """Insert or update plugin runtime state."""
        now = int(time.time() * 1000)
        self.conn.execute(
            """
            INSERT INTO plugin_state (plugin_id, enabled, config, last_poll, error_count, installed_at)
            VALUES (?, ?, ?, ?, ?, ?)
            ON CONFLICT(plugin_id) DO UPDATE SET
                enabled = excluded.enabled,
                config = excluded.config,
                last_poll = excluded.last_poll,
                error_count = excluded.error_count
            """,
            (
                plugin_id,
                1 if enabled else 0,
                json.dumps(config or {}),
                int(last_poll),
                int(error_count),
                now,
            ),
        )
        self.conn.commit()

    def list_plugin_state(self) -> list[dict[str, Any]]:
        """List persisted plugin states."""
        rows = self.conn.execute(
            "SELECT plugin_id, enabled, config, last_poll, error_count, installed_at FROM plugin_state"
        ).fetchall()
        out: list[dict[str, Any]] = []
        for row in rows:
            payload: dict[str, Any]
            try:
                payload = json.loads(row["config"]) if row["config"] else {}
            except Exception:
                payload = {}
            out.append(
                {
                    "plugin_id": str(row["plugin_id"]),
                    "enabled": bool(row["enabled"]),
                    "config": payload,
                    "last_poll": int(row["last_poll"]),
                    "error_count": int(row["error_count"]),
                    "installed_at": int(row["installed_at"]),
                }
            )
        return out

    # ================================================================
    # Embedding Helpers
    # ================================================================

    def get_recent_embeddings(self, limit: int = 20) -> list[dict[str, Any]]:
        """Get recent memory embeddings for semantic deduplication."""
        rows = self.conn.execute(
            """
            SELECT id, embedding
            FROM memories
            WHERE embedding IS NOT NULL
            ORDER BY created_at DESC
            LIMIT ?
            """,
            (max(1, min(limit, 1000)),),
        ).fetchall()

        results: list[dict[str, Any]] = []
        for row in rows:
            emb = _des_emb(row["embedding"])
            if emb is None:
                continue
            results.append({"id": row["id"], "embedding": emb.tolist() if hasattr(emb, "tolist") else list(emb)})
        return results

    # ================================================================
    # Cluster Operations
    # ================================================================

    def upsert_cluster(self, memory_ids: list[bytes], centroid_embedding: Any) -> None:
        now = int(time.time() * 1000)
        cluster = MemoryCluster(memory_ids=memory_ids, centroid_embedding=centroid_embedding)
        self.conn.execute(
            """INSERT OR REPLACE INTO clusters
               (id, centroid_embedding, memory_count, created_at, updated_at)
               VALUES (?, ?, ?, ?, ?)""",
            (cluster.id, _ser_emb(centroid_embedding), len(memory_ids), now, now),
        )
        for mid in memory_ids:
            self.conn.execute(
                "INSERT OR IGNORE INTO cluster_memories (cluster_id, memory_id) VALUES (?, ?)",
                (cluster.id, mid),
            )
        self.conn.commit()

    def get_clusters_without_insights(self) -> list[MemoryCluster]:
        rows = self.conn.execute("SELECT * FROM clusters WHERE insight_id IS NULL").fetchall()
        clusters: list[MemoryCluster] = []
        for r in rows:
            mem_rows = self.conn.execute(
                "SELECT memory_id FROM cluster_memories WHERE cluster_id = ?",
                (r["id"],),
            ).fetchall()
            clusters.append(
                MemoryCluster(
                    id=r["id"],
                    memory_ids=[mr["memory_id"] for mr in mem_rows],
                    centroid_embedding=_des_emb(r["centroid_embedding"]),
                    created_at=r["created_at"],
                    updated_at=r["updated_at"],
                )
            )
        return clusters

    def get_top_clusters(self, limit: int = 10) -> list[MemoryCluster]:
        rows = self.conn.execute(
            "SELECT * FROM clusters ORDER BY memory_count DESC, updated_at DESC LIMIT ?",
            (max(1, min(limit, 1000)),),
        ).fetchall()
        clusters: list[MemoryCluster] = []
        for row in rows:
            mem_rows = self.conn.execute(
                "SELECT memory_id FROM cluster_memories WHERE cluster_id = ?",
                (row["id"],),
            ).fetchall()
            clusters.append(
                MemoryCluster(
                    id=row["id"],
                    memory_ids=[mr["memory_id"] for mr in mem_rows],
                    centroid_embedding=_des_emb(row["centroid_embedding"]),
                    insight_id=row["insight_id"],
                    created_at=row["created_at"],
                    updated_at=row["updated_at"],
                )
            )
        return clusters

    def get_relationships_for_cluster(self, cluster_id: bytes) -> list[Relationship]:
        entity_rows = self.conn.execute(
            """
            SELECT DISTINCT me.entity_id
            FROM cluster_memories cm
            JOIN memory_entities me ON me.memory_id = cm.memory_id
            WHERE cm.cluster_id = ?
            """,
            (cluster_id,),
        ).fetchall()
        entity_ids = [row["entity_id"] for row in entity_rows]
        if not entity_ids:
            return []
        placeholders = ",".join("?" for _ in entity_ids)
        rows = self.conn.execute(
            f"""
            SELECT *
            FROM relationships
            WHERE source_entity_id IN ({placeholders})
              AND target_entity_id IN ({placeholders})
            ORDER BY strength DESC
            """,
            (*entity_ids, *entity_ids),
        ).fetchall()
        return [self._row_to_rel(row) for row in rows]

    def set_cluster_insight(self, cluster_id: bytes, insight_id: bytes) -> None:
        self.conn.execute(
            "UPDATE clusters SET insight_id = ?, updated_at = ? WHERE id = ?",
            (insight_id, int(time.time() * 1000), cluster_id),
        )
        self.conn.commit()

    # ================================================================
    # Tier & Decay Operations
    # ================================================================

    def get_memories_by_tier(self, tier: str) -> int:
        row = self.conn.execute(
            """SELECT COUNT(*) as cnt FROM memories
               WHERE is_active = 1
               AND json_extract(metadata, '$.tier') = ?""",
            (tier,),
        ).fetchone()
        return int(row["cnt"] if row else 0)

    def promote_tier(self, from_tier: str, to_tier: str, min_age_seconds: int) -> int:
        cutoff = int((time.time() - min_age_seconds) * 1000)
        cur = self.conn.execute(
            """UPDATE memories SET metadata = json_set(metadata, '$.tier', ?)
               WHERE is_active = 1
               AND json_extract(metadata, '$.tier') = ?
               AND created_at < ?""",
            (to_tier, from_tier, cutoff),
        )
        self.conn.commit()
        return int(cur.rowcount)

    def promote_high_value(self, from_tier: str, to_tier: str) -> int:
        cur = self.conn.execute(
            """UPDATE memories
               SET metadata = json_set(metadata, '$.tier', ?)
               WHERE is_active = 1
                 AND json_extract(metadata, '$.tier') = ?
                 AND (
                   importance >= 0.7
                   OR COALESCE(json_extract(metadata, '$.access_count'), 0) >= 3
                 )""",
            (to_tier, from_tier),
        )
        self.conn.commit()
        return int(cur.rowcount)

    def get_entity_count(self) -> int:
        row = self.conn.execute("SELECT COUNT(*) as cnt FROM entities").fetchone()
        return int(row["cnt"] if row else 0)

    def get_relationship_count(self) -> int:
        row = self.conn.execute("SELECT COUNT(*) as cnt FROM relationships").fetchone()
        return int(row["cnt"] if row else 0)

    def get_insight_count(self) -> int:
        row = self.conn.execute(
            """
            SELECT COUNT(*) AS cnt
            FROM memories
            WHERE is_active = 1
              AND source = 'consolidation'
              AND COALESCE(json_extract(metadata, '$.type'), '') = 'insight'
            """
        ).fetchone()
        return int(row["cnt"] if row else 0)

    # ================================================================
    # Memory/Audit Helpers
    # ================================================================

    def list_memory_records(
        self,
        since: int | None = None,
        limit: int = 5000,
        active_only: bool = False,
    ) -> list[sqlite3.Row]:
        query = "SELECT * FROM memories"
        clauses: list[str] = []
        params: list[object] = []

        if active_only:
            clauses.append("is_active = 1")
        if since is not None:
            clauses.append("created_at >= ?")
            params.append(since)

        if clauses:
            query += " WHERE " + " AND ".join(clauses)
        query += " ORDER BY created_at DESC LIMIT ?"
        params.append(max(1, min(limit, 100000)))

        return self.conn.execute(query, params).fetchall()

    def get_memory_count(self, since: int | None = None, active_only: bool = False) -> int:
        query = "SELECT COUNT(*) AS cnt FROM memories"
        clauses: list[str] = []
        params: list[object] = []

        if active_only:
            clauses.append("is_active = 1")
        if since is not None:
            clauses.append("created_at >= ?")
            params.append(since)

        if clauses:
            query += " WHERE " + " AND ".join(clauses)

        row = self.conn.execute(query, params).fetchone()
        return int(row["cnt"] if row else 0)

    def get_source_counts(self, since: int | None = None, active_only: bool = True) -> dict[str, int]:
        query = "SELECT source, COUNT(*) AS cnt FROM memories"
        clauses: list[str] = []
        params: list[object] = []

        if active_only:
            clauses.append("is_active = 1")
        if since is not None:
            clauses.append("created_at >= ?")
            params.append(since)

        if clauses:
            query += " WHERE " + " AND ".join(clauses)
        query += " GROUP BY source"

        rows = self.conn.execute(query, params).fetchall()
        return {str(r["source"]): int(r["cnt"]) for r in rows}

    def get_tier_counts(self, since: int | None = None, active_only: bool = True) -> dict[str, int]:
        query = (
            "SELECT COALESCE(json_extract(metadata, '$.tier'), 'unknown') AS tier, COUNT(*) AS cnt "
            "FROM memories"
        )
        clauses: list[str] = []
        params: list[object] = []

        if active_only:
            clauses.append("is_active = 1")
        if since is not None:
            clauses.append("created_at >= ?")
            params.append(since)

        if clauses:
            query += " WHERE " + " AND ".join(clauses)
        query += " GROUP BY tier"

        rows = self.conn.execute(query, params).fetchall()
        return {str(r["tier"]): int(r["cnt"]) for r in rows}

    def find_memory_ids_by_entity_name(self, entity_name: str, limit: int = 50) -> list[bytes]:
        rows = self.conn.execute(
            """
            SELECT me.memory_id
            FROM memory_entities me
            JOIN entities e ON me.entity_id = e.id
            WHERE e.canonical_name LIKE ?
            ORDER BY e.mention_count DESC
            LIMIT ?
            """,
            (f"%{entity_name.lower()}%", max(1, min(limit, 1000))),
        ).fetchall()
        return [r["memory_id"] for r in rows]

    # ================================================================
    # v0.3 Security / RBAC Helpers
    # ================================================================

    def insert_user(self, payload: dict[str, Any]) -> None:
        self.conn.execute(
            """
            INSERT OR REPLACE INTO users (username, role, token_hash, created_by, created_at, active)
            VALUES (?, ?, ?, ?, ?, ?)
            """,
            (
                str(payload.get("username", "")),
                str(payload.get("role", "reader")),
                str(payload.get("token_hash", "")),
                str(payload.get("created_by", "system")),
                int(payload.get("created_at", int(time.time() * 1000))),
                1 if bool(payload.get("active", True)) else 0,
            ),
        )
        self.conn.commit()

    def get_user_by_token_hash(self, token_hash: str) -> dict[str, Any] | None:
        row = self.conn.execute(
            """
            SELECT username, role, token_hash, created_by, created_at, active
            FROM users
            WHERE token_hash = ?
            LIMIT 1
            """,
            (token_hash,),
        ).fetchone()
        if not row:
            return None
        return {
            "username": str(row["username"]),
            "role": str(row["role"]),
            "token_hash": str(row["token_hash"]),
            "created_by": str(row["created_by"]),
            "created_at": int(row["created_at"]),
            "active": bool(row["active"]),
        }

    def deactivate_user(self, username: str) -> bool:
        cur = self.conn.execute("UPDATE users SET active = 0 WHERE username = ?", (username,))
        self.conn.commit()
        return int(cur.rowcount) > 0

    def get_all_users(self) -> list[dict[str, Any]]:
        rows = self.conn.execute(
            """
            SELECT username, role, created_by, created_at, active
            FROM users
            ORDER BY created_at DESC
            """
        ).fetchall()
        return [
            {
                "username": str(row["username"]),
                "role": str(row["role"]),
                "created_by": str(row["created_by"]),
                "created_at": int(row["created_at"]),
                "active": bool(row["active"]),
            }
            for row in rows
        ]

    def insert_device_token(self, payload: dict[str, Any]) -> None:
        perms = payload.get("permissions", [])
        if isinstance(perms, set):
            perms = sorted(list(perms))
        if not isinstance(perms, list):
            perms = [str(perms)]
        self.conn.execute(
            """
            INSERT INTO device_tokens (device_name, token_hash, permissions, created_at, active)
            VALUES (?, ?, ?, ?, ?)
            """,
            (
                str(payload.get("device_name", "")),
                str(payload.get("token_hash", "")),
                json.dumps([str(p) for p in perms]),
                int(payload.get("created_at", int(time.time() * 1000))),
                1 if bool(payload.get("active", True)) else 0,
            ),
        )
        self.conn.commit()

    def get_device_token_by_hash(self, token_hash: str) -> dict[str, Any] | None:
        row = self.conn.execute(
            """
            SELECT id, device_name, token_hash, permissions, created_at, active
            FROM device_tokens
            WHERE token_hash = ?
            LIMIT 1
            """,
            (token_hash,),
        ).fetchone()
        if not row:
            return None
        try:
            permissions = json.loads(row["permissions"]) if row["permissions"] else []
        except Exception:
            permissions = []
        return {
            "id": int(row["id"]),
            "device_name": str(row["device_name"]),
            "token_hash": str(row["token_hash"]),
            "permissions": permissions if isinstance(permissions, list) else [],
            "created_at": int(row["created_at"]),
            "active": bool(row["active"]),
        }

    def insert_audit_log(self, payload: dict[str, Any]) -> None:
        self.conn.execute(
            """
            INSERT INTO audit_log (timestamp, username, action, resource, success, details)
            VALUES (?, ?, ?, ?, ?, ?)
            """,
            (
                int(payload.get("timestamp", int(time.time() * 1000))),
                str(payload.get("username", "system")),
                str(payload.get("action", "")),
                str(payload.get("resource", "")),
                1 if bool(payload.get("success", True)) else 0,
                json.dumps(payload.get("details", {}) if isinstance(payload.get("details"), dict) else {}),
            ),
        )
        self.conn.commit()

    def query_audit_log(
        self,
        user: str | None = None,
        action: str | None = None,
        since: int | None = None,
        limit: int = 100,
    ) -> list[dict[str, Any]]:
        query = "SELECT timestamp, username, action, resource, success, details FROM audit_log"
        clauses: list[str] = []
        params: list[Any] = []
        if user:
            clauses.append("username = ?")
            params.append(user)
        if action:
            clauses.append("action = ?")
            params.append(action)
        if since is not None:
            clauses.append("timestamp >= ?")
            params.append(int(since))
        if clauses:
            query += " WHERE " + " AND ".join(clauses)
        query += " ORDER BY timestamp DESC LIMIT ?"
        params.append(max(1, min(limit, 1000)))

        rows = self.conn.execute(query, params).fetchall()
        out: list[dict[str, Any]] = []
        for row in rows:
            try:
                details = json.loads(row["details"]) if row["details"] else {}
            except Exception:
                details = {}
            out.append(
                {
                    "timestamp": int(row["timestamp"]),
                    "user": str(row["username"]),
                    "action": str(row["action"]),
                    "resource": str(row["resource"]),
                    "success": bool(row["success"]),
                    "details": details,
                }
            )
        out.reverse()
        return out

    # ================================================================
    # v0.3 Sync / Federation Helpers
    # ================================================================

    def count_memories(self) -> int:
        row = self.conn.execute("SELECT COUNT(*) AS cnt FROM memories WHERE is_active = 1").fetchone()
        return int(row["cnt"] if row else 0)

    def set_memory_embedding(self, memory_id: bytes, embedding: list[float] | Any) -> None:
        if np is None:
            return
        arr = np.array(embedding, dtype=np.float32)
        self.conn.execute("UPDATE memories SET embedding = ? WHERE id = ?", (_ser_emb(arr), memory_id))
        self.conn.commit()

    def get_memory(self, memory_id: bytes | str) -> dict[str, Any] | None:
        mid = self._coerce_id(memory_id)
        if mid is None:
            return None
        row = self.conn.execute("SELECT * FROM memories WHERE id = ?", (mid,)).fetchone()
        if not row:
            return None
        return self._row_to_memory_dict(row)

    def get_memory_by_content_hash(self, content_hash: str) -> dict[str, Any] | None:
        if not content_hash:
            return None
        try:
            row = self.conn.execute(
                "SELECT * FROM memories WHERE content_hash = ? ORDER BY created_at DESC LIMIT 1",
                (content_hash,),
            ).fetchone()
        except sqlite3.OperationalError:
            row = self.conn.execute(
                "SELECT * FROM memories WHERE content = ? ORDER BY created_at DESC LIMIT 1",
                (content_hash,),
            ).fetchone()
        if not row:
            return None
        return self._row_to_memory_dict(row)

    def find_memory_by_hash(self, content_hash: str) -> dict[str, Any] | None:
        return self.get_memory_by_content_hash(content_hash)

    def get_memories_since_checkpoint(self, since_checkpoint: str | None = None) -> list[dict[str, Any]]:
        cutoff = self._checkpoint_to_ms(since_checkpoint)
        rows = self.conn.execute(
            "SELECT * FROM memories WHERE created_at > ? ORDER BY created_at ASC",
            (cutoff,),
        ).fetchall()
        return [self._row_to_memory_dict(row) for row in rows]

    def get_entities_since_checkpoint(self, since_checkpoint: str | None = None) -> list[dict[str, Any]]:
        cutoff = self._checkpoint_to_ms(since_checkpoint)
        rows = self.conn.execute(
            """
            SELECT * FROM entities
            WHERE last_seen > ? OR first_seen > ?
            ORDER BY last_seen ASC
            """,
            (cutoff, cutoff),
        ).fetchall()
        return [self._entity_to_dict(self._row_to_entity(row)) for row in rows]

    def get_relationships_since_checkpoint(self, since_checkpoint: str | None = None) -> list[dict[str, Any]]:
        cutoff = self._checkpoint_to_ms(since_checkpoint)
        rows = self.conn.execute(
            "SELECT * FROM relationships WHERE last_seen > ? OR first_seen > ? ORDER BY last_seen ASC",
            (cutoff, cutoff),
        ).fetchall()
        return [self._relationship_to_dict(self._row_to_rel(row)) for row in rows]

    def get_concepts_since_checkpoint(self, since_checkpoint: str | None = None) -> list[dict[str, Any]]:
        cutoff = self._checkpoint_to_ms(since_checkpoint)
        rows = self.conn.execute(
            """
            SELECT id, child_entity_id, parent_entity_id, relation_type, confidence, source, created_at
            FROM concept_hierarchy
            WHERE created_at > ?
            ORDER BY created_at ASC
            """,
            (cutoff,),
        ).fetchall()
        return [
            {
                "id": row["id"].hex() if isinstance(row["id"], (bytes, bytearray)) else "",
                "child_entity_id": row["child_entity_id"].hex(),
                "parent_entity_id": row["parent_entity_id"].hex(),
                "relation_type": str(row["relation_type"]),
                "confidence": float(row["confidence"]),
                "source": str(row["source"]),
                "created_at": int(row["created_at"]),
            }
            for row in rows
        ]

    def insert_remote_memory(self, remote: dict[str, Any]) -> None:
        now = int(time.time() * 1000)
        memory_id = self._coerce_id(remote.get("id")) or hashlib.sha256(
            f"{remote.get('content', '')}\x00{remote.get('created_at', now)}".encode("utf-8")
        ).digest()

        persona_id = self._coerce_id(remote.get("persona_id")) or hashlib.sha256(b"default-persona").digest()
        content = str(remote.get("content", ""))
        created_at = int(remote.get("created_at", now))

        self.upsert_memory_snapshot(
            memory_id=memory_id,
            persona_id=persona_id,
            content=content,
            source=str(remote.get("source", "peer")),
            importance=float(remote.get("importance", 0.5)),
            metadata=remote.get("metadata", {}) if isinstance(remote.get("metadata"), dict) else {},
            created_at=created_at,
            is_active=bool(remote.get("is_active", True)),
            device_origin=str(remote.get("device_origin", remote.get("source_device_id", "peer"))),
        )

        emb = remote.get("embedding")
        if emb is not None:
            try:
                self.set_memory_embedding(memory_id, emb)
            except Exception as exc:
                logger.debug("Skipping invalid remote embedding for %s: %s", memory_id.hex(), exc)

    def update_memory_from_remote(self, remote: dict[str, Any]) -> None:
        self.insert_remote_memory(remote)

    def store_conflict(self, local_memory_id: bytes | str | None, remote_data: dict[str, Any]) -> None:
        local_payload = {}
        if local_memory_id is not None:
            local = self.get_memory(local_memory_id)
            if local:
                local_payload = local
        self._store_sync_conflict(local_payload, remote_data, "memory")

    def get_entity_by_name(self, name: str) -> Entity | None:
        canonical = name.lower().strip()
        row = self.conn.execute(
            "SELECT * FROM entities WHERE canonical_name = ? ORDER BY mention_count DESC LIMIT 1",
            (canonical,),
        ).fetchone()
        if row:
            return self._row_to_entity(row)
        row = self.conn.execute(
            "SELECT * FROM entities WHERE canonical_name LIKE ? ORDER BY mention_count DESC LIMIT 1",
            (f"%{canonical}%",),
        ).fetchone()
        return self._row_to_entity(row) if row else None

    def find_entity_by_name(self, name: str) -> Entity | None:
        return self.get_entity_by_name(name)

    def get_all_entities_as_dicts(self) -> list[dict[str, Any]]:
        rows = self.conn.execute("SELECT * FROM entities ORDER BY mention_count DESC").fetchall()
        return [self._entity_to_dict(self._row_to_entity(row)) for row in rows]

    def insert_remote_entity(self, remote: dict[str, Any]) -> Entity:
        entity = self.upsert_entity(
            name=str(remote.get("name", "")),
            entity_type=str(remote.get("entity_type", "CONCEPT")),
            confidence=float(remote.get("importance", remote.get("confidence", 0.7))),
            seen_at=int(remote.get("last_seen", remote.get("first_seen", int(time.time() * 1000)))),
        )

        row = self.conn.execute("SELECT * FROM entities WHERE id = ?", (entity.id,)).fetchone()
        if row:
            mention_count = int(max(1, remote.get("mention_count", row["mention_count"])))
            importance = float(max(row["importance"], float(remote.get("importance", row["importance"]))))
            first_seen = int(min(row["first_seen"], int(remote.get("first_seen", row["first_seen"]))))
            last_seen = int(max(row["last_seen"], int(remote.get("last_seen", row["last_seen"]))))
            metadata = remote.get("metadata", {})
            self.conn.execute(
                """
                UPDATE entities
                SET mention_count = ?, importance = ?, first_seen = ?, last_seen = ?, metadata = ?
                WHERE id = ?
                """,
                (
                    mention_count,
                    importance,
                    first_seen,
                    last_seen,
                    json.dumps(metadata if isinstance(metadata, dict) else {}),
                    entity.id,
                ),
            )
            self.conn.commit()
        return entity

    def update_entity_from_remote(self, remote: dict[str, Any]) -> None:
        entity = self.get_entity_by_name(str(remote.get("name", "")))
        if entity is None:
            self.insert_remote_entity(remote)
            return
        self.update_entity(
            entity_id=entity.id.hex(),
            name=str(remote.get("name", entity.name)),
            importance=max(entity.importance, float(remote.get("importance", entity.importance))),
            mention_count=max(entity.mention_count, int(remote.get("mention_count", entity.mention_count))),
        )

    def store_entity_conflict(self, local: dict[str, Any], remote: dict[str, Any]) -> None:
        self._store_sync_conflict(local, remote, "entity")

    def update_entity(
        self,
        entity_id: str | bytes,
        name: str | None = None,
        importance: float | None = None,
        mention_count: int | None = None,
    ) -> None:
        eid = self._coerce_id(entity_id)
        if eid is None:
            return
        row = self.conn.execute("SELECT * FROM entities WHERE id = ?", (eid,)).fetchone()
        if not row:
            return

        new_name = str(name if name is not None else row["name"])
        canonical = new_name.lower().strip()
        new_importance = float(importance if importance is not None else row["importance"])
        new_mentions = int(mention_count if mention_count is not None else row["mention_count"])
        self.conn.execute(
            """
            UPDATE entities
            SET name = ?, canonical_name = ?, importance = ?, mention_count = ?, last_seen = ?
            WHERE id = ?
            """,
            (
                new_name,
                canonical,
                max(0.0, min(new_importance, 1.0)),
                max(1, new_mentions),
                int(time.time() * 1000),
                eid,
            ),
        )
        self.conn.commit()

    def upsert_relationship_by_id(
        self,
        source_entity_id: str | bytes,
        target_entity_id: str | bytes,
        relation_type: str,
        strength: float = 0.5,
    ) -> None:
        sid = self._coerce_id(source_entity_id)
        tid = self._coerce_id(target_entity_id)
        if sid is None or tid is None:
            return
        now = int(time.time() * 1000)
        existing = self.conn.execute(
            """
            SELECT id, strength, evidence_count
            FROM relationships
            WHERE source_entity_id = ? AND target_entity_id = ? AND relation_type = ?
            """,
            (sid, tid, relation_type),
        ).fetchone()
        if existing:
            self.conn.execute(
                """
                UPDATE relationships
                SET strength = ?, evidence_count = ?, last_seen = ?
                WHERE id = ?
                """,
                (
                    max(float(existing["strength"]), float(strength)),
                    int(existing["evidence_count"]) + 1,
                    now,
                    existing["id"],
                ),
            )
        else:
            rid = hashlib.sha256(f"{sid.hex()}\x00{tid.hex()}\x00{relation_type}".encode("utf-8")).digest()
            self.conn.execute(
                """
                INSERT INTO relationships
                (id, source_entity_id, target_entity_id, relation_type, strength, evidence_count, first_seen, last_seen, metadata)
                VALUES (?, ?, ?, ?, ?, 1, ?, ?, '{}')
                """,
                (rid, sid, tid, relation_type, float(strength), now, now),
            )
        self.conn.commit()

    def upsert_remote_relationship(self, remote: dict[str, Any]) -> None:
        self.upsert_relationship_by_id(
            source_entity_id=remote.get("source_entity_id", ""),
            target_entity_id=remote.get("target_entity_id", ""),
            relation_type=str(remote.get("relation_type", "related-to")),
            strength=float(remote.get("strength", 0.5)),
        )

    def upsert_remote_concept(self, remote: dict[str, Any]) -> None:
        hid = self._coerce_id(remote.get("id")) or hashlib.sha256(
            f"{remote.get('child_entity_id', '')}:{remote.get('parent_entity_id', '')}:{remote.get('relation_type', 'IS-A')}".encode(
                "utf-8"
            )
        ).digest()
        child = self._coerce_id(remote.get("child_entity_id"))
        parent = self._coerce_id(remote.get("parent_entity_id"))
        if child is None or parent is None:
            return
        self.insert_concept_hierarchy(
            hid=hid,
            child_id=child,
            parent_id=parent,
            relation_type=str(remote.get("relation_type", "IS-A")),
            confidence=float(remote.get("confidence", 0.8)),
            source=str(remote.get("source", "peer")),
            created_at=int(remote.get("created_at", int(time.time() * 1000))),
        )

    def _store_sync_conflict(self, local: dict[str, Any], remote: dict[str, Any], conflict_type: str) -> None:
        try:
            self.conn.execute(
                """
                INSERT INTO sync_conflicts (local_data, remote_data, conflict_type, created_at, resolved, resolution)
                VALUES (?, ?, ?, ?, 0, '')
                """,
                (
                    json.dumps(local),
                    json.dumps(remote),
                    conflict_type,
                    int(time.time() * 1000),
                ),
            )
            self.conn.commit()
        except sqlite3.OperationalError:
            # Pre-v0.3 schema fallback: silently drop conflict persistence.
            pass

    def _checkpoint_to_ms(self, since_checkpoint: str | None) -> int:
        if not since_checkpoint:
            return 0
        if since_checkpoint.isdigit():
            return int(since_checkpoint)
        try:
            row = self.conn.execute(
                "SELECT created_at FROM checkpoints WHERE hex(id) LIKE ? || '%' ORDER BY created_at DESC LIMIT 1",
                (since_checkpoint.lower(),),
            ).fetchone()
            return int(row["created_at"] if row else 0)
        except Exception:
            return 0

    @staticmethod
    def _entity_to_dict(entity: Entity) -> dict[str, Any]:
        return {
            "id": entity.id.hex() if entity.id else "",
            "name": entity.name,
            "canonical_name": entity.canonical_name,
            "entity_type": entity.entity_type,
            "description": entity.description,
            "mention_count": entity.mention_count,
            "first_seen": entity.first_seen,
            "last_seen": entity.last_seen,
            "importance": entity.importance,
            "metadata": entity.metadata,
        }

    @staticmethod
    def _relationship_to_dict(rel: Relationship) -> dict[str, Any]:
        return {
            "id": rel.id.hex() if rel.id else "",
            "source_entity_id": rel.source_entity_id.hex(),
            "target_entity_id": rel.target_entity_id.hex(),
            "relation_type": rel.relation_type,
            "strength": rel.strength,
            "evidence_count": rel.evidence_count,
            "first_seen": rel.first_seen,
            "last_seen": rel.last_seen,
            "metadata": rel.metadata,
        }

    def _row_to_memory_dict(self, row: sqlite3.Row) -> dict[str, Any]:
        metadata = {}
        try:
            metadata = json.loads(row["metadata"]) if row["metadata"] else {}
        except Exception:
            metadata = {}

        embedding = _des_emb(row["embedding"])
        return {
            "id": row["id"].hex() if isinstance(row["id"], (bytes, bytearray)) else str(row["id"]),
            "persona_id": (
                row["persona_id"].hex()
                if isinstance(row["persona_id"], (bytes, bytearray))
                else str(row["persona_id"])
            ),
            "content": str(row["content"]),
            "source": str(row["source"]),
            "importance": float(row["importance"]),
            "metadata": metadata,
            "created_at": int(row["created_at"]),
            "is_active": bool(row["is_active"]),
            "content_hash": _content_hash(str(row["content"])),
            "device_origin": str(row["device_origin"]) if "device_origin" in row.keys() else "local",
            "embedding": embedding.tolist() if embedding is not None and hasattr(embedding, "tolist") else None,
        }

    @staticmethod
    def _coerce_id(value: Any) -> bytes | None:
        if isinstance(value, (bytes, bytearray)):
            return bytes(value)
        if isinstance(value, str):
            value = value.strip()
            if not value:
                return None
            try:
                return bytes.fromhex(value)
            except Exception:
                return None
        return None
