"""
db.py — OpenSoul MCP · Data Access Layer

Principle: INSERT-only. Every record is chained; history is never deleted or modified. Evolution is expressed through new entries.
"""

import json
import sqlite3
import sys
import threading
import uuid
from datetime import datetime
from pathlib import Path

DB_PATH = Path(__file__).parent / "soul.db"

SOUL_TABLES = (
    "souls", "states", "silences", "contradictions",
    "artifacts", "philosophy", "profile_history",
)

def _now():   return datetime.now().isoformat()
def _chain(): import chain as c; return c


def _escape_like(s: str) -> str:
    """Escape LIKE wildcards (% _) to prevent user input from affecting match scope."""
    return s.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")


def _open_db() -> sqlite3.Connection:
    """Unified database connection entry point.

    · row_factory=sqlite3.Row: supports column-name access (row["field"]),
      while maintaining integer-index compatibility (row[0]); no code changes needed after schema evolution.
    · journal_mode=WAL: allows concurrent reads and writes (daemon + api + MCP running simultaneously).
    · busy_timeout=5000: waits up to 5 seconds on write conflicts to avoid "database is locked".
    """
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=5000")
    return conn


# ── Schema ─────────────────────────────────────────────────────

def init_db():
    conn = _open_db()

    conn.executescript("""
    CREATE TABLE IF NOT EXISTS souls (
        id               INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp        TEXT NOT NULL,
        context          TEXT,
        ai_suggestion    TEXT,
        human_decision   TEXT NOT NULL,
        divergence_reason TEXT NOT NULL,
        tags             TEXT DEFAULT '[]',
        notes            TEXT,
        emotional_state  TEXT,
        energy           TEXT,
        body_state       TEXT,
        memory_triggers  TEXT DEFAULT '[]',
        decision_layer   TEXT,
        time_horizon     TEXT,
        contradicts_id   INTEGER,
        social_context   TEXT,
        source_device    TEXT
    );
    CREATE TABLE IF NOT EXISTS profile_history (
        id              INTEGER PRIMARY KEY AUTOINCREMENT,
        key             TEXT NOT NULL,
        value           TEXT NOT NULL,
        category        TEXT DEFAULT 'general',
        recorded_at     TEXT NOT NULL,
        emotional_state TEXT,
        notes           TEXT
    );
    CREATE TABLE IF NOT EXISTS philosophy (
        id        INTEGER PRIMARY KEY AUTOINCREMENT,
        principle TEXT NOT NULL,
        source    TEXT,
        examples  TEXT DEFAULT '[]',
        timestamp TEXT NOT NULL
    );
    CREATE TABLE IF NOT EXISTS states (
        id               INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp        TEXT NOT NULL,
        emotional_state  TEXT,
        energy           TEXT,
        body_state       TEXT,
        social_context   TEXT,
        environment      TEXT,
        sense_smell      TEXT,
        sense_touch      TEXT,
        sense_taste      TEXT,
        sense_sound_env  TEXT,
        sense_visual_env TEXT,
        biometric_note   TEXT,
        biometric_ref    TEXT,
        source_device    TEXT,
        notes            TEXT,
        tags             TEXT DEFAULT '[]'
    );
    CREATE TABLE IF NOT EXISTS silences (
        id           INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp    TEXT NOT NULL,
        topic        TEXT NOT NULL,
        silence_form TEXT NOT NULL,
        observation  TEXT,
        tags         TEXT DEFAULT '[]'
    );
    CREATE TABLE IF NOT EXISTS contradictions (
        id          INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp   TEXT NOT NULL,
        soul_id_a   INTEGER,
        soul_id_b   INTEGER,
        description TEXT NOT NULL,
        analysis    TEXT,
        resolution  TEXT,
        tags        TEXT DEFAULT '[]'
    );
    CREATE TABLE IF NOT EXISTS artifacts (
        id             INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp      TEXT NOT NULL,
        artifact_type  TEXT NOT NULL,
        path           TEXT,
        description    TEXT,
        linked_soul_id INTEGER,
        tags           TEXT DEFAULT '[]',
        content_hash   TEXT,
        binary_data    BLOB,
        is_primary     INTEGER DEFAULT 0
    );
    CREATE TABLE IF NOT EXISTS system_config (
        key        TEXT PRIMARY KEY,
        value      TEXT,
        updated_at TEXT
    );
    """)

    # Backward compatibility: add potentially missing columns
    for col, typ in [
        ("emotional_state", "TEXT"), ("energy", "TEXT"), ("body_state", "TEXT"),
        ("memory_triggers", "TEXT DEFAULT '[]'"), ("decision_layer", "TEXT"),
        ("time_horizon", "TEXT"), ("contradicts_id", "INTEGER"), ("social_context", "TEXT"),
        ("source_device", "TEXT"),
    ]:
        _add_col(conn, "souls", col, typ)

    for col in ("sense_smell", "sense_touch", "sense_taste", "sense_sound_env",
                "sense_visual_env", "biometric_note", "biometric_ref", "source_device"):
        _add_col(conn, "states", col, "TEXT")

    # ── Emotion Coordinate System (v4.0 Phase A) ──────────────────────────────────
    # Multi-dimensional emotion annotation: valence/intensity/duration/trigger_type/trigger_ref/label/secondary
    _EMOTION_COORD_COLS = [
        ("emotion_valence",      "REAL"),    # Valence -1.0 (negative) ~ 1.0 (positive)
        ("emotion_intensity",    "REAL"),    # Intensity 0.0 ~ 1.0
        ("emotion_duration",     "TEXT"),    # Duration: minutes/hours/days/weeks
        ("emotion_trigger_type", "TEXT"),    # Trigger type: person/event/thought/environment
        ("emotion_trigger_ref",  "TEXT"),    # Trigger reference (person name/event description/relation_id)
        ("emotion_label",        "TEXT"),    # Normalized emotion label (via EMOTION_LABELS mapping)
        ("emotion_secondary",    "TEXT"),    # Secondary emotion
    ]
    for table in ("souls", "states"):
        for col, typ in _EMOTION_COORD_COLS:
            _add_col(conn, table, col, typ)

    for col, typ in [
        ("content_hash", "TEXT"), ("binary_data", "BLOB"), ("is_primary", "INTEGER DEFAULT 0"),
    ]:
        _add_col(conn, "artifacts", col, typ)

    # entry_id: cross-device globally unique identifier (foundation for multi-device dedup)
    # Note: do not backfill old records to avoid breaking existing hash chains. New entries are injected by _write().
    for table in SOUL_TABLES:
        _add_col(conn, table, "entry_id", "TEXT")

    # ── Three-Dimensional Timeline (v3.0) ──────────────────────────────────────────
    # memory_time: original memory time text ("childhood"/"1995")  prospect_time: original prospect time text ("next year")
    # *_start/*_end: sortable ISO intervals  *_precision: exact/year/decade/period/vague
    # time_direction: past/present/future/timeless (primary quick-filter index)
    _TIMELINE_COLS = [
        ("memory_time",           "TEXT"),
        ("memory_time_start",     "TEXT"),
        ("memory_time_end",       "TEXT"),
        ("memory_time_precision", "TEXT"),
        ("prospect_time",         "TEXT"),
        ("prospect_time_start",   "TEXT"),
        ("prospect_time_end",     "TEXT"),
        ("prospect_time_precision","TEXT"),
        ("time_direction",        "TEXT"),
    ]
    for table in SOUL_TABLES:
        for col, typ in _TIMELINE_COLS:
            _add_col(conn, table, col, typ)

    # ── Scene Fingerprint (v3.0) ──────────────────────────────────────────
    # scene_fingerprint: JSON multi-axis scene {"social":"colleague","pressure":"high","role":"work"}
    for table in ("souls", "states"):
        _add_col(conn, table, "scene_fingerprint", "TEXT")
    # soul_chunks carry scene metadata (redundant storage, supports SQL index filtering)
    for col in ("scene_fingerprint", "scene_social", "scene_energy",
                "scene_pressure", "scene_role", "time_direction"):
        _add_col(conn, "soul_chunks", col, "TEXT")

    # Main table time-direction index (quick filter past/future records by time_direction)
    for table in SOUL_TABLES:
        conn.execute(f"CREATE INDEX IF NOT EXISTS idx_{table}_timedir ON {table}(time_direction)")

    # ── Long-form Materials Table (v3.0 Phase 3) ─────────────────────────────
    # soul_materials: diaries, articles, interviews, PDF summaries and other long-form text materials
    # Raw material stored here, chunks split and written to soul_chunks (anchor/satellite/paragraph)
    conn.execute("""
    CREATE TABLE IF NOT EXISTS soul_materials (
        id             INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp      TEXT NOT NULL,
        title          TEXT,
        material_type  TEXT NOT NULL,
        source         TEXT,
        raw_text       TEXT NOT NULL,
        char_count     INTEGER,
        chunk_count    INTEGER DEFAULT 0,
        linked_soul_id INTEGER,
        tags           TEXT DEFAULT '[]',
        memory_time           TEXT,
        memory_time_start     TEXT,
        memory_time_end       TEXT,
        memory_time_precision TEXT,
        prospect_time         TEXT,
        prospect_time_start   TEXT,
        prospect_time_end     TEXT,
        prospect_time_precision TEXT,
        time_direction        TEXT DEFAULT 'present',
        scene_fingerprint     TEXT,
        entry_id       TEXT
    )""")
    # Add material_id association column to soul_chunks
    _add_col(conn, "soul_chunks", "material_id", "INTEGER")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_soul_chunks_material ON soul_chunks(material_id)")

    # ── Relationship Network (v4.0 Phase B) ──────────────────────────────────
    conn.execute("""
    CREATE TABLE IF NOT EXISTS soul_relations (
        id             INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp      TEXT NOT NULL,
        alias          TEXT NOT NULL,
        real_name      TEXT,
        relation_type  TEXT,
        importance     REAL DEFAULT 0.5,
        emotional_tone TEXT,
        first_met TEXT,
        notes          TEXT,
        tags           TEXT DEFAULT '[]',
        entry_id       TEXT
    )""")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_sr_alias ON soul_relations(alias)")

    conn.execute("""
    CREATE TABLE IF NOT EXISTS soul_relation_events (
        id             INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp      TEXT NOT NULL,
        relation_id    INTEGER NOT NULL,
        event_type     TEXT NOT NULL,
        description    TEXT NOT NULL,
        emotional_state TEXT,
        emotion_label  TEXT,
        emotion_valence REAL,
        linked_soul_id INTEGER,
        tags           TEXT DEFAULT '[]',
        entry_id       TEXT,
        FOREIGN KEY (relation_id) REFERENCES soul_relations(id)
    )""")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_sre_rel ON soul_relation_events(relation_id)")

    # Add relation_id association column to soul_chunks
    _add_col(conn, "soul_chunks", "relation_id", "INTEGER")

    # ── Narrative Engine (v4.0 Phase D) ──────────────────────────────────
    conn.execute("""
    CREATE TABLE IF NOT EXISTS narrative_sessions (
        id             INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp      TEXT NOT NULL,
        topic          TEXT,
        media_id       INTEGER,
        status         TEXT DEFAULT 'active',
        turn_count     INTEGER DEFAULT 0,
        soul_chunks_generated INTEGER DEFAULT 0,
        notes          TEXT,
        entry_id       TEXT
    )""")
    conn.execute("""
    CREATE TABLE IF NOT EXISTS narrative_turns (
        id             INTEGER PRIMARY KEY AUTOINCREMENT,
        session_id     INTEGER NOT NULL,
        turn_number    INTEGER NOT NULL,
        timestamp      TEXT NOT NULL,
        question       TEXT NOT NULL,
        answer         TEXT,
        depth_level    INTEGER DEFAULT 0,
        chunks_generated INTEGER DEFAULT 0,
        entry_id       TEXT,
        FOREIGN KEY (session_id) REFERENCES narrative_sessions(id)
    )""")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_nt_session ON narrative_turns(session_id)")

    # ── Source Weight (v4.0 Phase C) ──────────────────────────────────
    # source_weight: source confidence (user direct=1.0, AI interpret=0.7, game inference=0.55)
    # source_engine: source engine identifier (human_direct/ai_interpret/game_inference/...)
    # verified_by_user: 0=unverified, 1=confirmed, -1=rejected
    _add_col(conn, "soul_chunks", "source_weight", "REAL DEFAULT 1.0")
    _add_col(conn, "soul_chunks", "source_engine", "TEXT DEFAULT 'human_direct'")
    _add_col(conn, "soul_chunks", "verified_by_user", "INTEGER DEFAULT 0")
    _add_col(conn, "soul_chunks", "weight_reason", "TEXT")
    _add_col(conn, "soul_chunks", "original_content", "TEXT")
    _add_col(conn, "soul_chunks", "verified_at", "TEXT")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_sc_engine ON soul_chunks(source_engine)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_sc_weight ON soul_chunks(source_weight)")

    # ── Vector Foundation: FTS Full-text Index + Vector Table ──────────────────────────
    # soul_fts  : full-text search, usable now, working foundation for recall
    # soul_vectors : vector slots, auto-filled when embedding model is connected
    conn.execute("""
    CREATE VIRTUAL TABLE IF NOT EXISTS soul_fts USING fts5(
        content,
        soul_id    UNINDEXED,
        table_name UNINDEXED
    )""")
    conn.execute("""
    CREATE TABLE IF NOT EXISTS soul_vectors (
        id         INTEGER PRIMARY KEY AUTOINCREMENT,
        soul_id    INTEGER NOT NULL,
        table_name TEXT NOT NULL,
        model      TEXT NOT NULL DEFAULT 'pending',
        vector     BLOB,
        created_at TEXT NOT NULL
    )""")
    # ── Chunk Vector Table: each record split into semantic field-level fragments, independently vectorized ──────────
    # chunk_type: context/decision/reason/emotion/note/topic/observation/
    #             conflict/analysis/principle/identity
    conn.execute("""
    CREATE TABLE IF NOT EXISTS soul_chunks (
        id         INTEGER PRIMARY KEY AUTOINCREMENT,
        soul_id    INTEGER NOT NULL,
        table_name TEXT NOT NULL,
        chunk_type TEXT NOT NULL,
        content    TEXT NOT NULL,
        model      TEXT DEFAULT 'pending',
        vector     BLOB,
        created_at TEXT NOT NULL
    )""")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_soul_chunks_soul ON soul_chunks(soul_id, table_name)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_soul_chunks_type ON soul_chunks(chunk_type)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_soul_chunks_scene ON soul_chunks(scene_social, scene_role)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_soul_chunks_timedir ON soul_chunks(time_direction)")
    # vector_status: track vectorization status for each chunk (pending/done/failed)
    _add_col(conn, "soul_chunks", "vector_status", "TEXT DEFAULT 'pending'")

    # ── Embedding Model Version Management ──────────────────────────────────────
    # Track historically used embedding models for batch re-vectorization during migration
    conn.execute("""
    CREATE TABLE IF NOT EXISTS embedding_model_versions (
        id          INTEGER PRIMARY KEY AUTOINCREMENT,
        model_name  TEXT NOT NULL UNIQUE,  -- e.g. bge-m3, nomic-embed-text
        dimensions  INTEGER,               -- vector dimensions
        introduced  TEXT NOT NULL,         -- first activation time
        deprecated  TEXT,                  -- deprecation time (NULL=currently in use)
        notes       TEXT
    )""")
    # Record current model (ON CONFLICT idempotent)
    conn.execute("""
    INSERT OR IGNORE INTO embedding_model_versions (model_name, introduced, notes)
    VALUES ('bge-m3', datetime('now'), 'Native Chinese multilingual, migrated from nomic-embed-text, 2026-03-04')
    """)

    # ── Dimension Seed Vector Library ──────────────────────────────────────────────
    # Representative text anchors for each soul dimension (8~15 per dimension), used for cold-start classification
    # Seed texts are batch-vectorized when Ollama is online; keyword-only classification when offline
    conn.execute("""
    CREATE TABLE IF NOT EXISTS dimension_seeds (
        id          INTEGER PRIMARY KEY AUTOINCREMENT,
        dimension   TEXT NOT NULL,   -- dimension name: identity/value/emotion/cognition/...
        seed_text   TEXT NOT NULL,   -- representative text
        model       TEXT DEFAULT 'pending',
        vector      BLOB,            -- embedding (filled when Ollama is online)
        weight      REAL DEFAULT 1.0, -- seed weight (higher-quality seeds have higher weight)
        created_at  TEXT NOT NULL DEFAULT (datetime('now'))
    )""")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_dim_seeds_dim ON dimension_seeds(dimension)")
    # Initialize seed texts (idempotent, deduped by dimension+seed_text)
    _init_dimension_seeds(conn)

    # ── LoRA Training Data Pre-accumulation ─────────────────────────────────────────
    # Fields embedded now; when soul entries reach >= 1000, training pairs can be extracted at zero cost
    # Format: instruction + input -> output (standard Alpaca training format)
    conn.execute("""
    CREATE TABLE IF NOT EXISTS soul_training_pairs (
        id           INTEGER PRIMARY KEY AUTOINCREMENT,
        instruction  TEXT NOT NULL,    -- task description (e.g. "answer this question from the soul owner's perspective")
        input        TEXT NOT NULL,    -- question/scenario
        output       TEXT NOT NULL,    -- the soul owner's actual answer (extracted from souls table)
        source_table TEXT NOT NULL,    -- source table (souls/philosophy/etc.)
        source_id    INTEGER NOT NULL, -- source record ID
        quality      REAL DEFAULT 1.0, -- quality score (0~1, low-quality pairs can be filtered)
        created_at   TEXT NOT NULL DEFAULT (datetime('now'))
    )""")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_stp_quality ON soul_training_pairs(quality DESC)")
    # v4.0 Phase E: DPO training pair extension
    _add_col(conn, "soul_training_pairs", "rejected_output", "TEXT")
    _add_col(conn, "soul_training_pairs", "pair_type", "TEXT DEFAULT 'sft'")
    _add_col(conn, "soul_training_pairs", "source_engine", "TEXT")

    # ── Game API (v4.0 Phase G) ──────────────────────────────────
    conn.execute("""
    CREATE TABLE IF NOT EXISTS game_registry (
        id             INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp      TEXT NOT NULL,
        game_id        TEXT NOT NULL UNIQUE,
        game_name      TEXT NOT NULL,
        description    TEXT,
        mappings       TEXT DEFAULT '{}',
        entry_id       TEXT
    )""")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gr_gid ON game_registry(game_id)")

    conn.execute("""
    CREATE TABLE IF NOT EXISTS game_sessions (
        id             INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp      TEXT NOT NULL,
        game_id        TEXT NOT NULL,
        status         TEXT DEFAULT 'active',
        decisions_count INTEGER DEFAULT 0,
        events_count   INTEGER DEFAULT 0,
        ended_at       TEXT,
        notes          TEXT,
        entry_id       TEXT
    )""")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gs_game ON game_sessions(game_id)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gs_status ON game_sessions(status)")

    # Add game_session_id association column to soul_chunks
    _add_col(conn, "soul_chunks", "game_session_id", "INTEGER")

    # ── Real-time Stream Ingestion Log ───────────────────────────────────────────
    conn.execute("""
    CREATE TABLE IF NOT EXISTS stream_log (
        id             INTEGER PRIMARY KEY AUTOINCREMENT,
        timestamp      TEXT NOT NULL,
        device_id      TEXT NOT NULL,
        source_type    TEXT NOT NULL,
        content_hash   TEXT,
        content_length INTEGER,
        relevant       INTEGER DEFAULT 0,
        recorded_count INTEGER DEFAULT 0,
        reason         TEXT
    )""")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_sl_device ON stream_log(device_id)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_sl_ts ON stream_log(timestamp)")

    # ── Entity Matching (v4.0 Phase I) ──────────────────────────────────
    # soul_entities: named entities extracted from soul records (person/place/thing/time), used as third recall channel
    conn.execute("""
    CREATE TABLE IF NOT EXISTS soul_entities (
        id         INTEGER PRIMARY KEY AUTOINCREMENT,
        soul_id    INTEGER NOT NULL,
        table_name TEXT NOT NULL,
        entity     TEXT NOT NULL,
        entity_type TEXT NOT NULL,
        created_at TEXT NOT NULL
    )""")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_se_entity ON soul_entities(entity)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_se_soul ON soul_entities(soul_id, table_name)")

    _chain().init_chain(conn)

    # sqlite-vec migration trigger check (log warning when soul_chunks > 5000)
    chunk_count = conn.execute(
        "SELECT COUNT(*) FROM soul_chunks WHERE vector IS NOT NULL"
    ).fetchone()[0]
    if chunk_count > 5000:
        import sys
        print(f"[soul-mcp] WARNING: soul_chunks vectorized count {chunk_count} > 5000, consider migrating to sqlite-vec.",
              file=sys.stderr)

    conn.commit()
    conn.close()

    _migrate_state_files()
    _rebuild_fts_index()  # Backfill historical records into FTS


def _add_col(conn, table, col, col_type):
    existing = {r[1] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()}
    if col not in existing:
        conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} {col_type}")


def _migrate_state_files():
    """One-time migration: move scattered state files into system_config, then delete files."""
    d = DB_PATH.parent
    conn = _open_db()

    def _set(key, val):
        conn.execute(
            "INSERT INTO system_config (key, value, updated_at) VALUES (?,?,?)"
            " ON CONFLICT(key) DO NOTHING",
            (key, val, _now())
        )

    if (d / "PROD_SEAL").exists():
        _set("mode", "prod")
    else:
        _set("mode", "dev")

    res = d / "RESURRECTION_SEAL"
    if res.exists():
        for line in res.read_text(encoding="utf-8").splitlines():
            if line.startswith("复活时间"):  # "Resurrection time" marker in legacy files
                _set("resurrected_at", line.split(":", 1)[1].strip())
                break

    conn.commit()
    conn.close()

    # Retire old state files (VOID_SEAL is preserved as it signals chain integrity)
    for f in ("PROD_SEAL", "RESURRECTION_SEAL", "DEV_MODE"):
        p = d / f
        if p.exists():
            p.unlink()


def _cfg(key):
    conn = _open_db()
    row = conn.execute("SELECT value FROM system_config WHERE key=?", (key,)).fetchone()
    conn.close()
    return row[0] if row else None


def set_mode(mode: str):
    conn = _open_db()
    conn.execute(
        "INSERT INTO system_config (key, value, updated_at) VALUES ('mode',?,?)"
        " ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at",
        (mode, _now())
    )
    conn.commit()
    conn.close()


# ── FTS Index ───────────────────────────────────────────────────

# Extract indexable natural language content from each table
def _fts_content(table: str, row) -> str:
    """Extract full-text indexable content. row is sqlite3.Row, accessed by column name, unaffected by column order."""
    def g(f):
        try:
            v = row[f]
            return v if v else ""
        except (IndexError, KeyError):
            return ""
    def _j(v): return " ".join(json.loads(v)) if v else ""

    if table == "souls":
        return " ".join(filter(None, [
            g("context"), g("human_decision"), g("divergence_reason"),
            g("notes"), g("emotional_state"), _j(g("memory_triggers")),
        ]))
    if table == "philosophy":
        return " ".join(filter(None, [g("principle"), g("source")]))
    if table == "profile_history":
        return f"{g('key')} {g('value')}"
    if table == "states":
        return " ".join(filter(None, [g("emotional_state"), g("body_state"), g("notes")]))
    if table == "silences":
        return " ".join(filter(None, [g("topic"), g("observation")]))
    if table == "contradictions":
        return " ".join(filter(None, [g("description"), g("analysis")]))
    return ""


def _fts_index(conn, table: str, rid: int, row: tuple):
    content = _fts_content(table, row)
    if content.strip():
        conn.execute(
            "INSERT INTO soul_fts (content, soul_id, table_name) VALUES (?,?,?)",
            (content, rid, table)
        )


def _rebuild_fts_index():
    """Sync FTS index at startup (compare counts first; clear and rebuild if inconsistent to avoid bloat)."""
    conn = _open_db()
    try:
        actual_count = sum(
            conn.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0]
            for t in SOUL_TABLES
        )
        fts_count = conn.execute("SELECT COUNT(*) FROM soul_fts").fetchone()[0]
        # Allow minor discrepancy (empty content rows won't be indexed); only rebuild on bloat or empty index
        if 0 < fts_count <= actual_count:
            return  # Count is normal, no rebuild needed
        # Count mismatch (bloat or missing), clear and full rebuild
        conn.execute("DELETE FROM soul_fts")
        for table in SOUL_TABLES:
            try:
                for row in conn.execute(f"SELECT * FROM {table}").fetchall():
                    _fts_index(conn, table, row[0], row)
            except Exception:
                pass
        conn.commit()
    finally:
        conn.close()


# ── Scene Fingerprint ──────────────────────────────────────────────────

SCENE_AXES = {
    "social":   ["独处", "亲密关系", "家庭", "朋友", "同事", "陌生人", "公众场合"],
    "energy":   ["高能量", "中等", "低能量", "疲惫"],
    "pressure": ["高压", "中等压力", "放松", "无压力"],
    "role":     ["工作", "家庭", "个人", "社交", "学习"],
}

def _build_scene_fingerprint(social_context=None, energy=None,
                              decision_layer=None) -> dict | None:
    """Infer structured scene fingerprint from free-text fields."""
    fp = {}
    if social_context:
        sc = social_context
        _social_map = {
            "独处": ["独处", "一个人", "自己"],
            "亲密关系": ["伴侣", "恋人", "对象", "老婆", "老公", "女朋友", "男朋友"],
            "家庭": ["家人", "父母", "爸", "妈", "家里", "儿子", "女儿", "孩子"],
            "朋友": ["朋友", "哥们", "闺蜜", "兄弟"],
            "同事": ["同事", "老板", "下属", "团队", "领导"],
            "陌生人": ["陌生人", "路人"],
            "公众场合": ["演讲", "公开", "会议", "台上"],
        }
        for label, keywords in _social_map.items():
            if any(k in sc for k in keywords):
                fp["social"] = label
                break
        if "social" not in fp and social_context.strip():
            fp["social"] = social_context.strip()
    if energy:
        en = energy
        if any(k in en for k in ["高", "充沛", "旺盛"]):
            fp["energy"] = "高能量"
        elif any(k in en for k in ["低", "疲", "累", "困"]):
            fp["energy"] = "低能量"
        else:
            fp["energy"] = energy.strip()
    if decision_layer:
        dl = decision_layer
        if any(k in dl for k in ["工作", "职业", "事业"]):
            fp["role"] = "工作"
        elif any(k in dl for k in ["家庭", "亲情"]):
            fp["role"] = "家庭"
    return fp if fp else None


# ── Emotion Label Normalization ─────────────────────────────────────────────
# Map user free-text emotions to standard labels, reducing synonym fragmentation

EMOTION_LABELS = {
    # Positive
    "开心": "快乐", "高兴": "快乐", "快乐": "快乐", "愉悦": "快乐", "喜悦": "快乐",
    "兴奋": "兴奋", "激动": "兴奋", "亢奋": "兴奋",
    "满足": "满足", "充实": "满足", "知足": "满足",
    "平静": "平静", "安宁": "平静", "淡定": "平静", "从容": "平静",
    "感动": "感动", "触动": "感动",
    "自信": "自信", "坚定": "坚定", "笃定": "坚定",
    "好奇": "好奇", "期待": "期待", "憧憬": "期待",
    "感恩": "感恩", "感激": "感恩",
    # Negative
    "焦虑": "焦虑", "紧张": "焦虑", "不安": "焦虑", "担忧": "焦虑", "忧虑": "焦虑",
    "愤怒": "愤怒", "生气": "愤怒", "气愤": "愤怒", "恼火": "愤怒",
    "悲伤": "悲伤", "难过": "悲伤", "伤心": "悲伤", "心痛": "悲伤",
    "疲惫": "疲惫", "疲劳": "疲惫", "倦怠": "疲惫", "累": "疲惫",
    "无聊": "无聊", "空虚": "空虚", "迷茫": "迷茫", "困惑": "迷茫",
    "孤独": "孤独", "寂寞": "孤独",
    "恐惧": "恐惧", "害怕": "恐惧",
    "内疚": "内疚", "愧疚": "内疚", "羞愧": "羞愧",
    "沮丧": "沮丧", "失望": "失望", "挫败": "挫败",
    "嫉妒": "嫉妒", "羡慕": "羡慕",
    # Compound
    "矛盾": "矛盾", "纠结": "矛盾", "犹豫": "犹豫",
    "怀旧": "怀旧", "思念": "思念",
}

def _normalize_emotion(text: str | None) -> str | None:
    """Map free-text emotion to a standard label."""
    if not text:
        return None
    text = text.strip()
    if text in EMOTION_LABELS:
        return EMOTION_LABELS[text]
    # Substring matching
    for key, label in EMOTION_LABELS.items():
        if key in text:
            return label
    return text  # Keep original text if normalization fails


def emotion_patterns(person: str = None, scene: str = None,
                     months: int = 6) -> dict:
    """Emotion pattern analysis: scene x emotion matrix / trigger distribution / valence timeline.

    Returns:
        scene_emotion_matrix: {scene: {label: count}}
        trigger_distribution: {trigger_type: count}
        valence_timeline: [{date, avg_valence, count}]
        top_emotions: [(label, count)]
        person_emotion_matrix: {person: {label: count}} (if any trigger_ref matches)
    """
    conn = _open_db()
    cutoff = None
    if months:
        from datetime import timedelta
        cutoff = (datetime.now() - timedelta(days=months * 30)).isoformat()

    # Collect emotion data (souls + states)
    emotion_rows = []
    for table in ("souls", "states"):
        sql = f"SELECT * FROM {table} WHERE emotional_state IS NOT NULL"
        params = []
        if cutoff:
            sql += " AND timestamp >= ?"
            params.append(cutoff)
        try:
            rows = conn.execute(sql, params).fetchall()
            for r in rows:
                emotion_rows.append(r)
        except Exception:
            pass

    # Scene x Emotion matrix
    scene_emotion = {}  # {scene: {label: count}}
    trigger_dist = {}   # {trigger_type: count}
    person_emotion = {} # {person: {label: count}}
    valence_data = []   # [(date, valence)]
    label_count = {}    # {label: count}

    for r in emotion_rows:
        label = _normalize_emotion(r["emotional_state"])
        if not label:
            continue
        label_count[label] = label_count.get(label, 0) + 1

        # Scene dimension
        try:
            fp = r["scene_fingerprint"]
            if fp:
                fp_dict = json.loads(fp) if isinstance(fp, str) else fp
                scene_key = fp_dict.get("social", "unknown")
            else:
                sc = r["social_context"] if "social_context" in r.keys() else None
                scene_key = sc or "unknown"
        except Exception:
            scene_key = "unknown"

        if scene and scene_key != scene:
            continue

        if scene_key not in scene_emotion:
            scene_emotion[scene_key] = {}
        scene_emotion[scene_key][label] = scene_emotion[scene_key].get(label, 0) + 1

        # Trigger type
        try:
            tt = r["emotion_trigger_type"]
            if tt:
                trigger_dist[tt] = trigger_dist.get(tt, 0) + 1
        except (KeyError, IndexError):
            pass

        # Person x Emotion
        try:
            tr = r["emotion_trigger_ref"]
            if tr:
                if person and person not in tr:
                    continue
                if tr not in person_emotion:
                    person_emotion[tr] = {}
                person_emotion[tr][label] = person_emotion[tr].get(label, 0) + 1
        except (KeyError, IndexError):
            pass

        # Valence timeline
        try:
            val = r["emotion_valence"]
            if val is not None:
                date = r["timestamp"][:10]
                valence_data.append((date, float(val)))
        except (KeyError, IndexError):
            pass

    # Aggregate valence by date
    valence_by_date = {}
    for date, val in valence_data:
        if date not in valence_by_date:
            valence_by_date[date] = []
        valence_by_date[date].append(val)
    valence_timeline = [
        {"date": d, "avg_valence": round(sum(vs) / len(vs), 2), "count": len(vs)}
        for d, vs in sorted(valence_by_date.items())
    ]

    top_emotions = sorted(label_count.items(), key=lambda x: -x[1])[:10]

    conn.close()
    return {
        "scene_emotion_matrix": scene_emotion,
        "trigger_distribution": trigger_dist,
        "valence_timeline": valence_timeline,
        "top_emotions": top_emotions,
        "person_emotion_matrix": person_emotion,
        "total_records": len(emotion_rows),
    }


# ── Chunk Extraction ───────────────────────────────────────────────────

def _extract_chunks(table: str, row) -> list:
    """Split a record into semantic field-level fragments, carrying scene and time metadata.
    row is sqlite3.Row, accessed by column name, unaffected by column order or future ALTER TABLE changes.
    Returns [(chunk_type, content, scene_fingerprint, time_direction), ...]
    """
    chunks = []
    scene_fp = None
    time_dir = None

    def g(f):
        try:
            return row[f]
        except (IndexError, KeyError):
            return None

    # Extract time direction
    time_dir = g("time_direction")

    def add(ctype, *fields):
        text = " ".join(str(f).strip() for f in fields if f and str(f).strip())
        if text and len(text) >= 8:
            chunks.append((ctype, text, scene_fp, time_dir))

    if table == "souls":
        scene_fp = _build_scene_fingerprint(
            social_context=g("social_context"),
            energy=g("energy"),
            decision_layer=g("decision_layer"),
        )
        # Also try reading from JSON field (structured storage from record time)
        if not scene_fp:
            try:
                fp_raw = g("scene_fingerprint")
                if fp_raw:
                    scene_fp = json.loads(fp_raw) if isinstance(fp_raw, str) else fp_raw
            except Exception:
                pass
        add("context",  g("context"))
        add("decision", g("human_decision"))
        add("reason",   g("divergence_reason"))
        # Scene-aware emotion: embed context so vectors can distinguish emotions across different scenes
        emo_parts = [p for p in [
            f"scene:{g('social_context')}" if g("social_context") else None,
            f"energy:{g('energy')}" if g("energy") else None,
            f"emotion:{g('emotional_state')}" if g("emotional_state") else None,
        ] if p]
        if emo_parts and len(" ".join(emo_parts)) >= 8:
            chunks.append(("emotion", " ".join(emo_parts), scene_fp, time_dir))
        elif g("emotional_state") and len(str(g("emotional_state"))) >= 8:
            chunks.append(("emotion", g("emotional_state"), scene_fp, time_dir))
        add("note",     g("notes"))

    elif table == "states":
        scene_fp = _build_scene_fingerprint(
            social_context=g("social_context"),
            energy=g("energy"),
        )
        add("emotion", g("emotional_state"), g("energy"), g("body_state"))
        add("note",    g("notes"))

    elif table == "silences":
        add("topic",       g("topic"))
        add("observation", g("observation"))

    elif table == "contradictions":
        add("conflict", g("description"))
        add("analysis", g("analysis"))

    elif table == "philosophy":
        ex_raw = g("examples")
        ex = " ".join(json.loads(ex_raw)) if ex_raw else ""
        add("principle", g("principle"))
        add("context",   g("source"), ex)

    elif table == "profile_history":
        add("identity", g("key"), g("value"))

    # Timeline chunks (common to all tables)
    mem = g("memory_time")
    if mem and len(str(mem)) >= 4:
        chunks.append(("memory_time", f"memory time: {mem}", scene_fp, time_dir))
    pro = g("prospect_time")
    if pro and len(str(pro)) >= 4:
        chunks.append(("prospect_time", f"prospect time: {pro}", scene_fp, time_dir))

    return chunks


# ── Automatic Training Data Accumulation ──────────────────────────────────────────

def _maybe_generate_training_pair(conn, table: str, rid: int, row):
    """Determine whether to auto-generate a training pair during data write.

    Trigger conditions (generate if any is met):
    1. souls table with context + human_decision both >= 20 chars
    2. philosophy table (all principles are high-quality training data)
    3. profile_history table (key+value >= 15 chars)
    """
    def g(f):
        try:
            return row[f] or ""
        except (IndexError, KeyError):
            return ""

    pairs = []
    if table == "souls":
        ctx = g("context")
        decision = g("human_decision")
        reason = g("divergence_reason")
        if len(ctx) >= 20 and len(decision) >= 20:
            output = f"I chose {decision}. Because {reason}" if reason else f"I chose {decision}."
            pairs.append({
                "instruction": "You are the soul owner. Based on your memories and values, answer the following question in first person.",
                "input": f"What would you choose in this situation? Context: {ctx}",
                "output": output,
                "quality": 1.0,
            })
    elif table == "philosophy":
        principle = g("principle")
        if len(principle) >= 10:
            pairs.append({
                "instruction": "You are the soul owner. Share one of your life principles.",
                "input": "What principles do you follow when making decisions?",
                "output": f"I have always believed: {principle}",
                "quality": 1.0,
            })
    elif table == "profile_history":
        key = g("key")
        value = g("value")
        if len(f"{key}: {value}") >= 15:
            pairs.append({
                "instruction": "You are the soul owner. Introduce yourself.",
                "input": f"What is your {key}?",
                "output": f"My {key} is {value}.",
                "quality": 0.9,
            })

    for p in pairs:
        conn.execute(
            """INSERT INTO soul_training_pairs
               (instruction, input, output, source_table, source_id, quality, created_at)
               VALUES (?,?,?,?,?,?,datetime('now'))""",
            (p["instruction"], p["input"], p["output"], table, rid, p["quality"]))


def export_training_pairs(pair_type: str = None, min_quality: float = 0.0) -> list:
    """Export training data pairs."""
    conn = _open_db()
    sql = "SELECT * FROM soul_training_pairs WHERE quality >= ?"
    params = [min_quality]
    if pair_type:
        sql += " AND pair_type = ?"
        params.append(pair_type)
    sql += " ORDER BY quality DESC, created_at DESC"
    rows = conn.execute(sql, params).fetchall()
    conn.close()
    return rows


# ── Source Weight Constants ──────────────────────────────────────────────

ENGINE_DEFAULT_WEIGHT = {
    "human_direct":   1.0,   # User actively entered directly
    "human_prompted": 0.9,   # User answer guided by questions
    "ai_augmented":   0.85,  # AI-generated but user-confirmed
    "ai_interpret":   0.7,   # AI auto-interpreted, unconfirmed
    "device_stream":  0.65,  # Wearable device real-time stream AI extraction
    "game_explicit":  0.7,   # Explicit choice in game
    "game_inference": 0.55,  # Game behavior inference
}

def _infer_source_engine(source_device: str | None) -> str:
    """Infer source_engine from source_device."""
    if not source_device:
        return "human_direct"
    sd = source_device.lower()
    if sd.startswith("stream:"):
        return "device_stream"
    if sd.startswith("engine:game"):
        return "game_inference"
    if sd.startswith("engine:question"):
        return "human_prompted"
    if sd.startswith("engine:"):
        return "human_direct"
    if sd in ("mcp", "api"):
        return "human_direct"
    if sd == "daemon":
        return "ai_interpret"
    return "human_direct"


# ── INSERT helper ──────────────────────────────────────────────

def _write(table, sql, params):
    conn = _open_db()
    cur  = conn.execute(sql, params)
    rid  = cur.lastrowid
    # Inject globally unique entry_id (for cross-device dedup)
    conn.execute(f"UPDATE {table} SET entry_id=? WHERE id=?",
                 (f"local-{uuid.uuid4().hex}", rid))
    row  = conn.execute(f"SELECT * FROM {table} WHERE id=?", (rid,)).fetchone()
    _chain().append(conn, table, rid, row)
    _fts_index(conn, table, rid, row)   # Sync into FTS

    # Infer source engine and weight
    try:
        sd = row["source_device"]
    except (IndexError, KeyError):
        sd = None
    src_engine = _infer_source_engine(sd)
    src_weight = ENGINE_DEFAULT_WEIGHT.get(src_engine, 1.0)

    # Synchronously write chunk records (content+scene+time+weight, vectors filled asynchronously later)
    chunks = _extract_chunks(table, row)
    for ctype, content, scene_fp, time_dir in chunks:
        fp_json = json.dumps(scene_fp, ensure_ascii=False) if scene_fp else None
        conn.execute(
            "INSERT INTO soul_chunks"
            " (soul_id, table_name, chunk_type, content, created_at, vector_status,"
            "  scene_fingerprint, scene_social, scene_energy, scene_pressure, scene_role,"
            "  time_direction, source_weight, source_engine)"
            " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
            (rid, table, ctype, content, _now(), "pending",
             fp_json,
             scene_fp.get("social") if scene_fp else None,
             scene_fp.get("energy") if scene_fp else None,
             scene_fp.get("pressure") if scene_fp else None,
             scene_fp.get("role") if scene_fp else None,
             time_dir, src_weight, src_engine)
        )

    # Auto-accumulate training data
    _maybe_generate_training_pair(conn, table, rid, row)

    # Entity extraction (Phase I)
    full_text = " ".join(content for _, content, _, _ in chunks)
    entities = _extract_entities(full_text)
    for entity, etype in entities:
        conn.execute(
            "INSERT INTO soul_entities (soul_id, table_name, entity, entity_type, created_at)"
            " VALUES (?,?,?,?,?)",
            (rid, table, entity, etype, _now()))

    conn.commit()
    conn.close()

    # Vectorization: async in background thread, does not block main recording flow
    def _vectorize():
        try:
            import vectors as _v
            if _v.ollama_alive():
                _v.store_chunk_vectors(rid, table, chunks)
        except Exception:
            pass

    threading.Thread(target=_vectorize, daemon=True).start()


# ── Souls ──────────────────────────────────────────────────────

def record(context, human_decision, divergence_reason,
           ai_suggestion="", tags=None, notes="",
           emotional_state=None, energy=None, body_state=None,
           memory_triggers=None, decision_layer=None,
           time_horizon=None, contradicts_id=None, social_context=None,
           source_device=None,
           memory_time=None, memory_time_start=None, memory_time_end=None,
           memory_time_precision=None,
           prospect_time=None, prospect_time_start=None, prospect_time_end=None,
           prospect_time_precision=None,
           time_direction="present",
           scene_fingerprint=None,
           emotion_valence=None, emotion_intensity=None,
           emotion_duration=None, emotion_trigger_type=None,
           emotion_trigger_ref=None, emotion_label=None,
           emotion_secondary=None):
    # Auto-infer scene fingerprint
    if scene_fingerprint is None:
        scene_fingerprint = _build_scene_fingerprint(
            social_context=social_context, energy=energy,
            decision_layer=decision_layer)
    if isinstance(scene_fingerprint, dict):
        scene_fingerprint = json.dumps(scene_fingerprint, ensure_ascii=False)
    # Auto-normalize emotion label
    if emotion_label is None and emotional_state:
        emotion_label = _normalize_emotion(emotional_state)
    _write("souls",
        """INSERT INTO souls
           (timestamp,context,ai_suggestion,human_decision,divergence_reason,
            tags,notes,emotional_state,energy,body_state,
            memory_triggers,decision_layer,time_horizon,contradicts_id,social_context,
            source_device,
            memory_time,memory_time_start,memory_time_end,memory_time_precision,
            prospect_time,prospect_time_start,prospect_time_end,prospect_time_precision,
            time_direction,scene_fingerprint,
            emotion_valence,emotion_intensity,emotion_duration,
            emotion_trigger_type,emotion_trigger_ref,emotion_label,emotion_secondary)
           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
        (_now(), context, ai_suggestion, human_decision, divergence_reason,
         json.dumps(tags or [], ensure_ascii=False), notes,
         emotional_state, energy, body_state,
         json.dumps(memory_triggers or [], ensure_ascii=False),
         decision_layer, time_horizon, contradicts_id, social_context,
         source_device,
         memory_time, memory_time_start, memory_time_end, memory_time_precision,
         prospect_time, prospect_time_start, prospect_time_end, prospect_time_precision,
         time_direction, scene_fingerprint,
         emotion_valence, emotion_intensity, emotion_duration,
         emotion_trigger_type, emotion_trigger_ref, emotion_label, emotion_secondary))


def list_recent(limit=10, tag=None):
    conn = _open_db()
    rows = conn.execute(
        "SELECT * FROM souls WHERE tags LIKE ? ESCAPE '\\' ORDER BY timestamp DESC LIMIT ?"
        if tag else "SELECT * FROM souls ORDER BY timestamp DESC LIMIT ?",
        (f"%{_escape_like(tag)}%", limit) if tag else (limit,)
    ).fetchall()
    conn.close()
    return rows


def search(keyword):
    conn = _open_db()
    q = f"%{_escape_like(keyword)}%"
    rows = conn.execute(
        """SELECT * FROM souls WHERE context LIKE ? ESCAPE '\\' OR human_decision LIKE ? ESCAPE '\\'
           OR divergence_reason LIKE ? ESCAPE '\\' OR notes LIKE ? ESCAPE '\\'
           OR memory_triggers LIKE ? ESCAPE '\\' OR emotional_state LIKE ? ESCAPE '\\'
           ORDER BY timestamp DESC""", (q,)*6
    ).fetchall()
    conn.close()
    return rows


def all_souls():
    conn = _open_db()
    rows = conn.execute("SELECT * FROM souls ORDER BY timestamp ASC").fetchall()
    conn.close()
    return rows


# ── Soft Delete (Redact) ────────────────────────────────────────────
# Does not break hash chain, does not physically delete rows. Content cleared, vectors nulled, AI no longer recalls.
# On-chain block_hash is based on write-time snapshot; chain integrity is unaffected after redact.

REDACTED_MARKER = "[REDACTED BY USER]"

def redact_entry(soul_id: int, reason: str = "") -> dict:
    """
    Soft-delete a soul event.

    Effects:
      - souls table: context/human_decision/divergence_reason/notes replaced with REDACTED_MARKER
      - soul_chunks table: content replaced with REDACTED_MARKER, vector set to NULL, vector_status='redacted'
      - soul_fts table: corresponding index deleted (excluded from full-text search)
      - soul_entities table: corresponding entities deleted (excluded from entity recall)
      - Hash chain: untouched (block_hash preserved, chain integrity unaffected)
      - soul_training_pairs: associated training pairs deleted

    Unaffected:
      - souls.id / timestamp / tags / entry_id (row exists, traceable as "once had this entry")
      - chain_log records (hashes permanently retained on chain)
    """
    conn = _open_db()

    # Check existence
    row = conn.execute("SELECT id, human_decision FROM souls WHERE id = ?", (soul_id,)).fetchone()
    if not row:
        conn.close()
        return {"ok": False, "message": f"Soul event #{soul_id} does not exist"}

    if row["human_decision"] == REDACTED_MARKER:
        conn.close()
        return {"ok": False, "message": f"Soul event #{soul_id} is already in redacted state"}

    redact_note = f"[Redacted at {_now()}] Reason: {reason}" if reason else f"[Redacted at {_now()}]"

    # 1. Clear core fields in souls table (preserve id/timestamp/tags/entry_id)
    conn.execute("""
        UPDATE souls SET
            context = ?,
            human_decision = ?,
            divergence_reason = ?,
            ai_suggestion = ?,
            notes = ?,
            emotional_state = NULL,
            energy = NULL,
            body_state = NULL,
            memory_triggers = '[]',
            sense_context = NULL
        WHERE id = ?
    """, (REDACTED_MARKER, REDACTED_MARKER, REDACTED_MARKER, "", redact_note, soul_id))

    # 2. Clear soul_chunks (preserve rows, clear content and vectors)
    conn.execute("""
        UPDATE soul_chunks SET
            content = ?,
            vector = NULL,
            vector_status = 'redacted',
            original_content = content
        WHERE soul_id = ? AND table_name = 'souls'
    """, (REDACTED_MARKER, soul_id))

    chunks_affected = conn.execute(
        "SELECT changes()").fetchone()[0]

    # 3. Delete FTS index
    conn.execute(
        "DELETE FROM soul_fts WHERE soul_id = ? AND table_name = 'souls'",
        (soul_id,))

    # 4. Delete entities
    conn.execute(
        "DELETE FROM soul_entities WHERE soul_id = ? AND table_name = 'souls'",
        (soul_id,))

    # 5. Delete training data pairs
    try:
        conn.execute(
            "DELETE FROM soul_training_pairs WHERE soul_id = ? AND table_name = 'souls'",
            (soul_id,))
    except Exception:
        pass  # Table may not have soul_id column

    conn.commit()
    conn.close()

    return {
        "ok": True,
        "message": f"Soul event #{soul_id} has been redacted (content cleared, {chunks_affected} chunks marked as redacted)",
        "soul_id": soul_id,
        "chunks_redacted": chunks_affected,
    }


def find_persona_souls(question_id: str = None) -> list:
    """
    Find persona card related soul events.
    question_id: e.g. 'P01', leave empty to return all persona card entries.
    """
    conn = _open_db()
    rows = conn.execute("SELECT * FROM souls ORDER BY timestamp ASC").fetchall()
    conn.close()

    results = []
    for r in rows:
        tags = json.loads(r["tags"]) if r["tags"] else []
        if "persona" not in tags:
            continue
        if question_id:
            if question_id not in tags:
                continue
        results.append(r)
    return results


# ── Profile ────────────────────────────────────────────────────

def set_profile(key, value, category="general", emotional_state=None, notes=None,
                memory_time=None, memory_time_start=None, memory_time_end=None,
                memory_time_precision=None,
                prospect_time=None, prospect_time_start=None, prospect_time_end=None,
                prospect_time_precision=None,
                time_direction="present"):
    _write("profile_history",
        """INSERT INTO profile_history
           (key,value,category,recorded_at,emotional_state,notes,
            memory_time,memory_time_start,memory_time_end,memory_time_precision,
            prospect_time,prospect_time_start,prospect_time_end,prospect_time_precision,
            time_direction)
           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
        (key, value, category, _now(), emotional_state, notes,
         memory_time, memory_time_start, memory_time_end, memory_time_precision,
         prospect_time, prospect_time_start, prospect_time_end, prospect_time_precision,
         time_direction))


def get_profile(category=None):
    conn = _open_db()
    if category:
        rows = conn.execute(
            """SELECT key,value,category,recorded_at FROM profile_history
               WHERE category=? AND (key,recorded_at) IN (
                   SELECT key,MAX(recorded_at) FROM profile_history WHERE category=? GROUP BY key
               ) ORDER BY key""", (category, category)).fetchall()
    else:
        rows = conn.execute(
            """SELECT key,value,category,recorded_at FROM profile_history
               WHERE (key,recorded_at) IN (
                   SELECT key,MAX(recorded_at) FROM profile_history GROUP BY key
               ) ORDER BY category,key""").fetchall()
    conn.close()
    return rows


# ── Philosophy ─────────────────────────────────────────────────

def add_philosophy(principle, source="", examples=None,
                   memory_time=None, memory_time_start=None, memory_time_end=None,
                   memory_time_precision=None,
                   prospect_time=None, prospect_time_start=None, prospect_time_end=None,
                   prospect_time_precision=None,
                   time_direction="present"):
    _write("philosophy",
        """INSERT INTO philosophy
           (principle,source,examples,timestamp,
            memory_time,memory_time_start,memory_time_end,memory_time_precision,
            prospect_time,prospect_time_start,prospect_time_end,prospect_time_precision,
            time_direction)
           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""",
        (principle, source, json.dumps(examples or [], ensure_ascii=False), _now(),
         memory_time, memory_time_start, memory_time_end, memory_time_precision,
         prospect_time, prospect_time_start, prospect_time_end, prospect_time_precision,
         time_direction))


def all_philosophy(limit=None):
    conn = _open_db()
    if limit:
        rows = conn.execute(
            "SELECT * FROM philosophy ORDER BY timestamp DESC LIMIT ?", (limit,)
        ).fetchall()
        rows = list(reversed(rows))  # Maintain chronological order
    else:
        rows = conn.execute("SELECT * FROM philosophy ORDER BY timestamp ASC").fetchall()
    conn.close()
    return rows


# ── Other tables ───────────────────────────────────────────────

def record_state(emotional_state=None, energy=None, body_state=None,
                 social_context=None, environment=None,
                 sense_smell=None, sense_touch=None, sense_taste=None,
                 sense_sound_env=None, sense_visual_env=None,
                 biometric_note=None, biometric_ref=None,
                 source_device=None, notes=None, tags=None,
                 memory_time=None, memory_time_start=None, memory_time_end=None,
                 memory_time_precision=None,
                 prospect_time=None, prospect_time_start=None, prospect_time_end=None,
                 prospect_time_precision=None,
                 time_direction="present",
                 scene_fingerprint=None,
                 emotion_valence=None, emotion_intensity=None,
                 emotion_duration=None, emotion_trigger_type=None,
                 emotion_trigger_ref=None, emotion_label=None,
                 emotion_secondary=None):
    # Auto-infer scene fingerprint
    if scene_fingerprint is None:
        scene_fingerprint = _build_scene_fingerprint(
            social_context=social_context, energy=energy)
    if isinstance(scene_fingerprint, dict):
        scene_fingerprint = json.dumps(scene_fingerprint, ensure_ascii=False)
    # Auto-normalize emotion label
    if emotion_label is None and emotional_state:
        emotion_label = _normalize_emotion(emotional_state)
    _write("states",
        """INSERT INTO states
           (timestamp,emotional_state,energy,body_state,social_context,environment,
            sense_smell,sense_touch,sense_taste,sense_sound_env,sense_visual_env,
            biometric_note,biometric_ref,source_device,notes,tags,
            memory_time,memory_time_start,memory_time_end,memory_time_precision,
            prospect_time,prospect_time_start,prospect_time_end,prospect_time_precision,
            time_direction,scene_fingerprint,
            emotion_valence,emotion_intensity,emotion_duration,
            emotion_trigger_type,emotion_trigger_ref,emotion_label,emotion_secondary)
           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
        (_now(), emotional_state, energy, body_state, social_context, environment,
         sense_smell, sense_touch, sense_taste, sense_sound_env, sense_visual_env,
         biometric_note, biometric_ref, source_device, notes,
         json.dumps(tags or [], ensure_ascii=False),
         memory_time, memory_time_start, memory_time_end, memory_time_precision,
         prospect_time, prospect_time_start, prospect_time_end, prospect_time_precision,
         time_direction, scene_fingerprint,
         emotion_valence, emotion_intensity, emotion_duration,
         emotion_trigger_type, emotion_trigger_ref, emotion_label, emotion_secondary))


def record_silence(topic, silence_form, observation="", tags=None,
                   memory_time=None, memory_time_start=None, memory_time_end=None,
                   memory_time_precision=None,
                   prospect_time=None, prospect_time_start=None, prospect_time_end=None,
                   prospect_time_precision=None,
                   time_direction="present"):
    _write("silences",
        """INSERT INTO silences
           (timestamp,topic,silence_form,observation,tags,
            memory_time,memory_time_start,memory_time_end,memory_time_precision,
            prospect_time,prospect_time_start,prospect_time_end,prospect_time_precision,
            time_direction)
           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
        (_now(), topic, silence_form, observation, json.dumps(tags or [], ensure_ascii=False),
         memory_time, memory_time_start, memory_time_end, memory_time_precision,
         prospect_time, prospect_time_start, prospect_time_end, prospect_time_precision,
         time_direction))


def record_contradiction(description, soul_id_a=None, soul_id_b=None,
                         analysis=None, resolution=None, tags=None,
                         memory_time=None, memory_time_start=None, memory_time_end=None,
                         memory_time_precision=None,
                         prospect_time=None, prospect_time_start=None, prospect_time_end=None,
                         prospect_time_precision=None,
                         time_direction="present"):
    _write("contradictions",
        """INSERT INTO contradictions
           (timestamp,soul_id_a,soul_id_b,description,analysis,resolution,tags,
            memory_time,memory_time_start,memory_time_end,memory_time_precision,
            prospect_time,prospect_time_start,prospect_time_end,prospect_time_precision,
            time_direction)
           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
        (_now(), soul_id_a, soul_id_b, description, analysis, resolution,
         json.dumps(tags or [], ensure_ascii=False),
         memory_time, memory_time_start, memory_time_end, memory_time_precision,
         prospect_time, prospect_time_start, prospect_time_end, prospect_time_precision,
         time_direction))


def record_artifact(artifact_type, description, path=None,
                    linked_soul_id=None, tags=None,
                    memory_time=None, memory_time_start=None, memory_time_end=None,
                    memory_time_precision=None,
                    prospect_time=None, prospect_time_start=None, prospect_time_end=None,
                    prospect_time_precision=None,
                    time_direction="present"):
    _write("artifacts",
        """INSERT INTO artifacts
           (timestamp,artifact_type,path,description,linked_soul_id,tags,
            memory_time,memory_time_start,memory_time_end,memory_time_precision,
            prospect_time,prospect_time_start,prospect_time_end,prospect_time_precision,
            time_direction)
           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
        (_now(), artifact_type, path, description, linked_soul_id,
         json.dumps(tags or [], ensure_ascii=False),
         memory_time, memory_time_start, memory_time_end, memory_time_precision,
         prospect_time, prospect_time_start, prospect_time_end, prospect_time_precision,
         time_direction))


def ingest_media_artifact(artifact_type, description, content_hash,
                          path=None, binary_data=None, is_primary=0,
                          linked_soul_id=None, tags=None,
                          memory_time=None, memory_time_start=None, memory_time_end=None,
                          memory_time_precision=None,
                          prospect_time=None, prospect_time_start=None, prospect_time_end=None,
                          prospect_time_precision=None,
                          time_direction="present"):
    """Ingest media artifact: hash + semantic description + optional binary (soul photo only)."""
    _write("artifacts",
        """INSERT INTO artifacts
           (timestamp,artifact_type,path,description,linked_soul_id,tags,
            content_hash,binary_data,is_primary,
            memory_time,memory_time_start,memory_time_end,memory_time_precision,
            prospect_time,prospect_time_start,prospect_time_end,prospect_time_precision,
            time_direction)
           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
        (_now(), artifact_type, path, description, linked_soul_id,
         json.dumps(tags or [], ensure_ascii=False),
         content_hash, binary_data, is_primary,
         memory_time, memory_time_start, memory_time_end, memory_time_precision,
         prospect_time, prospect_time_start, prospect_time_end, prospect_time_precision,
         time_direction))


# ── Materials (Long-form Text) ────────────────────────────────────

def _split_paragraphs(text: str, max_len: int = 300, min_len: int = 30) -> list[str]:
    """Split long text by paragraphs; merge short paragraphs, split long ones by sentence."""
    raw_parts = [p.strip() for p in text.split("\n") if p.strip()]
    paragraphs = []
    buf = ""
    for part in raw_parts:
        if len(buf) + len(part) < max_len:
            buf = f"{buf}\n{part}" if buf else part
        else:
            if buf:
                paragraphs.append(buf)
            buf = part
    if buf:
        paragraphs.append(buf)
    # Split overly long paragraphs by period/question mark/exclamation mark
    result = []
    for p in paragraphs:
        if len(p) <= max_len:
            if len(p) >= min_len:
                result.append(p)
        else:
            import re
            sents = re.split(r'(?<=[。？！.?!])', p)
            sub = ""
            for s in sents:
                if len(sub) + len(s) < max_len:
                    sub += s
                else:
                    if sub and len(sub) >= min_len:
                        result.append(sub)
                    sub = s
            if sub and len(sub) >= min_len:
                result.append(sub)
    return result


def _generate_satellites(text: str, max_count: int = 8) -> list[str]:
    """Extract satellite tag phrases from long text (10-50 char key semantic fragments).
    Simple strategy: first sentence of each paragraph + quoted/book-title content."""
    satellites = []
    import re
    # Quoted and book-title content
    for pattern in (r'「(.{4,50}?)」', r'"(.{4,50}?)"', r'《(.{4,50}?)》',
                    r'"(.{4,50}?)"', r'『(.{4,50}?)』'):
        satellites.extend(re.findall(pattern, text))
    # First sentence of each paragraph (before period)
    for para in text.split("\n"):
        para = para.strip()
        if len(para) < 10:
            continue
        first_sent = re.split(r'[。？！.?!]', para)[0].strip()
        if 10 <= len(first_sent) <= 50:
            satellites.append(first_sent)
    # Deduplicate and truncate
    seen = set()
    unique = []
    for s in satellites:
        if s not in seen:
            seen.add(s)
            unique.append(s)
    return unique[:max_count]


def ingest_material(title, material_type, raw_text, source="",
                    linked_soul_id=None, tags=None,
                    memory_time=None, memory_time_start=None, memory_time_end=None,
                    memory_time_precision=None,
                    prospect_time=None, prospect_time_start=None, prospect_time_end=None,
                    prospect_time_precision=None,
                    time_direction="present",
                    scene_fingerprint=None):
    """Ingest long-form material: auto-split into three-layer chunks (anchor + satellite + paragraph).

    material_type: diary/article/interview/letter/note/transcript/other
    Returns: {"material_id": int, "chunks": int, "paragraphs": int, "satellites": int}
    """
    if isinstance(scene_fingerprint, dict):
        scene_fingerprint = json.dumps(scene_fingerprint, ensure_ascii=False)

    conn = _open_db()
    cur = conn.execute(
        """INSERT INTO soul_materials
           (timestamp, title, material_type, source, raw_text, char_count,
            linked_soul_id, tags,
            memory_time, memory_time_start, memory_time_end, memory_time_precision,
            prospect_time, prospect_time_start, prospect_time_end, prospect_time_precision,
            time_direction, scene_fingerprint, entry_id)
           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
        (_now(), title, material_type, source, raw_text, len(raw_text),
         linked_soul_id, json.dumps(tags or [], ensure_ascii=False),
         memory_time, memory_time_start, memory_time_end, memory_time_precision,
         prospect_time, prospect_time_start, prospect_time_end, prospect_time_precision,
         time_direction, scene_fingerprint, f"local-{uuid.uuid4().hex}"))
    mid = cur.lastrowid

    # Parse scene fingerprint for chunk use
    fp = None
    if scene_fingerprint:
        try:
            fp = json.loads(scene_fingerprint) if isinstance(scene_fingerprint, str) else scene_fingerprint
        except Exception:
            pass

    fp_json = json.dumps(fp, ensure_ascii=False) if fp else None
    time_dir = time_direction or "present"

    def _insert_chunk(chunk_type, content):
        conn.execute(
            """INSERT INTO soul_chunks
               (soul_id, table_name, chunk_type, content, created_at, vector_status,
                material_id, scene_fingerprint, scene_social, scene_energy,
                scene_pressure, scene_role, time_direction)
               VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""",
            (mid, "soul_materials", chunk_type, content, _now(), "pending",
             mid, fp_json,
             fp.get("social") if fp else None,
             fp.get("energy") if fp else None,
             fp.get("pressure") if fp else None,
             fp.get("role") if fp else None,
             time_dir))

    # Layer 1: anchor_summary (full-text summary, first 200 chars + title)
    summary = f"{title}：{raw_text[:200]}" if len(raw_text) > 200 else f"{title}：{raw_text}"
    _insert_chunk("anchor_summary", summary)

    # Layer 2: satellite (semantic tag phrases)
    satellites = _generate_satellites(raw_text)
    for sat in satellites:
        _insert_chunk("satellite", sat)

    # Layer 3: paragraph (segmented)
    paragraphs = _split_paragraphs(raw_text)
    for para in paragraphs:
        _insert_chunk("paragraph", para)

    total_chunks = 1 + len(satellites) + len(paragraphs)
    conn.execute("UPDATE soul_materials SET chunk_count=? WHERE id=?", (total_chunks, mid))

    # FTS index (full text into FTS, using material_id as soul_id)
    fts_text = f"{title} {raw_text[:500]}"
    conn.execute(
        "INSERT INTO soul_fts (content, soul_id, table_name) VALUES (?,?,?)",
        (fts_text, mid, "soul_materials"))

    conn.commit()
    conn.close()

    # Async vectorization
    chunks_for_vec = ([("anchor_summary", summary)]
                      + [("satellite", s) for s in satellites]
                      + [("paragraph", p) for p in paragraphs])
    def _vectorize():
        try:
            import vectors as _v
            if _v.ollama_alive():
                _v.store_chunk_vectors(mid, "soul_materials", chunks_for_vec)
        except Exception:
            pass
    threading.Thread(target=_vectorize, daemon=True).start()

    return {
        "material_id": mid,
        "chunks": total_chunks,
        "paragraphs": len(paragraphs),
        "satellites": len(satellites),
    }


def list_materials(limit=10, material_type=None):
    conn = _open_db()
    if material_type:
        rows = conn.execute(
            "SELECT id, timestamp, title, material_type, source, char_count, chunk_count, tags"
            " FROM soul_materials WHERE material_type=? ORDER BY timestamp DESC LIMIT ?",
            (material_type, limit)).fetchall()
    else:
        rows = conn.execute(
            "SELECT id, timestamp, title, material_type, source, char_count, chunk_count, tags"
            " FROM soul_materials ORDER BY timestamp DESC LIMIT ?",
            (limit,)).fetchall()
    conn.close()
    return rows


def get_material(material_id: int):
    conn = _open_db()
    row = conn.execute("SELECT * FROM soul_materials WHERE id=?", (material_id,)).fetchone()
    conn.close()
    return row


def _init_dimension_seeds(conn):
    """Initialize representative seed texts for each dimension (idempotent, skips if exists)."""
    seeds = [
        # ── Identity ──
        ("identity", "我是谁？我的名字和内在有什么关联？"),
        ("identity", "我的出生、家庭和成长经历如何塑造了我？"),
        ("identity", "我在别人眼中是什么样的人？这和我自己的认知一致吗？"),
        ("identity", "如果用三个词描述我的本质，会是什么？"),
        ("identity", "我的身份感会在什么情况下动摇或强化？"),
        ("identity", "我认同哪些群体？这种认同给我带来了什么？"),
        ("identity", "有什么事情，即使没有外部认可，我也坚信自己在做？"),
        # ── Value ──
        ("value", "什么是我绝对不能妥协的底线？"),
        ("value", "如果金钱不是问题，我会如何安排我的生活？"),
        ("value", "我最尊敬的人有什么共同特质？"),
        ("value", "我做决定时最看重的是什么因素？"),
        ("value", "我愿意为什么付出时间、金钱和精力？"),
        ("value", "什么事情让我感到内疚或羞愧？这说明我看重什么？"),
        ("value", "我认为什么是真正的成功？"),
        ("value", "面对道德两难时，我的直觉反应是什么？"),
        # ── Emotion ──
        ("emotion", "什么最容易让我感到愤怒或受伤？"),
        ("emotion", "我如何处理负面情绪？倾向于压抑还是表达？"),
        ("emotion", "什么样的事情能让我感到真正的快乐和满足？"),
        ("emotion", "我在什么情况下会感到焦虑？"),
        ("emotion", "悲伤的时候，我最需要什么？"),
        ("emotion", "我对哪些情绪最难以接受或承认？"),
        ("emotion", "什么样的美会让我感动？"),
        # ── Cognition ──
        ("cognition", "我更倾向于直觉还是分析型思考？"),
        ("cognition", "我是如何学习新事物的？"),
        ("cognition", "我对待不确定性和模糊性的态度是什么？"),
        ("cognition", "我的思维更线性还是发散型？"),
        ("cognition", "什么类型的问题最能吸引我的注意力？"),
        ("cognition", "我在做决策时的典型过程是什么？"),
        # ── Relationship ──
        ("relationship", "我在亲密关系中的核心需求是什么？"),
        ("relationship", "我更倾向于深度少数关系还是广泛社交？"),
        ("relationship", "当有人伤害我时，我的典型反应是什么？"),
        ("relationship", "我如何表达对他人的关心和爱？"),
        ("relationship", "我在关系中最常见的冲突模式是什么？"),
        # ── Creation ──
        ("creation", "我通过什么方式表达自己？"),
        ("creation", "什么样的创作过程让我感到充实？"),
        ("creation", "我的审美偏好是什么？"),
        ("creation", "有什么作品（书/音乐/电影/艺术）深深影响了我？"),
        # ── Mission ──
        ("mission", "五年后，我希望成为什么样的人？"),
        ("mission", "如果我知道自己只剩一年，我会如何度过？"),
        ("mission", "我觉得自己在世界上的使命或意义是什么？"),
        ("mission", "什么样的遗产或影响是我希望留下的？"),
    ]
    for dim, text in seeds:
        conn.execute(
            "INSERT OR IGNORE INTO dimension_seeds (dimension, seed_text)"
            " SELECT ?, ? WHERE NOT EXISTS (SELECT 1 FROM dimension_seeds WHERE dimension=? AND seed_text=?)",
            (dim, text, dim, text)
        )


def format_profile(rows) -> dict:
    """Convert get_profile() results to {key: value} dict (for soul_api use)."""
    return {r["key"]: r["value"] for r in rows}


def get_soul_photo():
    """Return the latest soul photo (most recent entry with is_primary=1)."""
    conn = _open_db()
    row = conn.execute(
        """SELECT id,timestamp,description,content_hash,binary_data,path
           FROM artifacts WHERE is_primary=1
           ORDER BY timestamp DESC LIMIT 1"""
    ).fetchone()
    conn.close()
    return row


# ── Soul Recall (Heartbeat Layer) ─────────────────────────────────────────

def _extract_entities(text: str) -> list[tuple[str, str]]:
    """
    Extract named entities from text (lightweight regex, no external NLP dependency).
    Returns [(entity, entity_type), ...]

    entity_type: person/place/time/thing
    """
    import re
    entities = []
    seen = set()

    def _add(entity, etype):
        e = entity.strip()
        if e and len(e) >= 2 and e not in seen:
            seen.add(e)
            entities.append((e, etype))

    # Person name pattern: Chinese names (2-4 chars) in relationship context
    # Fetch known aliases from soul_relations
    try:
        conn = _open_db()
        aliases = [r[0] for r in conn.execute("SELECT alias FROM soul_relations").fetchall()]
        conn.close()
        for alias in aliases:
            if alias in text:
                _add(alias, "person")
    except Exception:
        pass

    # Time patterns
    for pat in (
        r'\d{4}年\d{1,2}月\d{1,2}日',
        r'\d{4}年\d{1,2}月',
        r'\d{4}年',
        r'[去前今明后]年',
        r'[上下这那]个?月',
        r'童年|少年|青年|成年|大学|高中|初中|小学|幼儿园',
        r'春天|夏天|秋天|冬天',
    ):
        for m in re.finditer(pat, text):
            _add(m.group(), "time")

    # Place name pattern (simple: words ending with city/province/district/county/town/village/road/street suffixes)
    for m in re.finditer(r'(?<![年月日\d])[\u4e00-\u9fff]{2,6}(?:市|省|区|县|镇|村|路|街|大学|学校|公司|医院)', text):
        _add(m.group(), "place")

    # Thing pattern (content within book title marks)
    for m in re.finditer(r'《(.{2,20}?)》', text):
        _add(m.group(1), "thing")

    return entities


def _rrf_score(rank: int, k: int = 60) -> float:
    """Reciprocal Rank Fusion weight: 1 / (k + rank)"""
    return 1.0 / (k + rank)


def _weighted_rrf_score(rank: int, weight: float = 1.0, k: int = 60) -> float:
    """Weighted RRF: base RRF + source weight correction term.

    Correction = alpha * (weight - 0.5), alpha = base * 0.3
    Weight affects at most ~30% of RRF score; semantic relevance remains the dominant factor.
    """
    base = 1.0 / (k + rank)
    alpha = base * 0.3
    return base + alpha * (weight - 0.5)


def recall_soul(query: str, limit: int = 6) -> list:
    """
    Recall most relevant soul memories using natural language.
    Strategy: RRF hybrid retrieval = Dense (chunk-level vectors) + Sparse (FTS5) fusion ranking
    Fallback: FTS5 only -> keyword LIKE
    """
    conn = _open_db()
    scores: dict[tuple, float] = {}   # (soul_id, table_name) -> RRF cumulative score

    # ── Channel A: Chunk-level semantic vectors (Dense + source weight weighting) ──
    try:
        import vectors as _v
        chunk_count = conn.execute(
            "SELECT COUNT(*) FROM soul_chunks WHERE vector IS NOT NULL"
        ).fetchone()[0]
        if chunk_count > 0 and _v.ollama_alive():
            chunk_hits = _v.search_chunks(query, limit=limit * 3)
            # Rank by best chunk per record, carrying source_weight
            best: dict[tuple, tuple] = {}  # key → (score, weight)
            for h in chunk_hits:
                key = (h["soul_id"], h["table_name"])
                w = h.get("source_weight", 1.0) or 1.0
                if key not in best or h["score"] > best[key][0]:
                    best[key] = (h["score"], w)
            for rank, (key, (_, w)) in enumerate(
                sorted(best.items(), key=lambda x: -x[1][0])
            ):
                scores[key] = scores.get(key, 0.0) + _weighted_rrf_score(rank, w)
    except Exception:
        pass

    # ── Channel B: FTS5 Full-text Search (Sparse) ──
    try:
        fts_hits = conn.execute(
            "SELECT soul_id, table_name FROM soul_fts WHERE soul_fts MATCH ? LIMIT ?",
            (query, limit * 3)
        ).fetchall()
        for rank, (sid, tbl) in enumerate(fts_hits):
            key = (sid, tbl)
            scores[key] = scores.get(key, 0.0) + _rrf_score(rank)
    except Exception:
        pass

    # ── Channel C: Entity Matching ──
    try:
        # Extract entities from query
        query_entities = _extract_entities(query)
        # Also directly match known entities with query keywords
        entity_hits = conn.execute(
            "SELECT DISTINCT soul_id, table_name FROM soul_entities"
            " WHERE entity LIKE ? ESCAPE '\\' LIMIT ?",
            (f"%{_escape_like(query)}%", limit * 2)
        ).fetchall()
        for e, etype in query_entities:
            more = conn.execute(
                "SELECT DISTINCT soul_id, table_name FROM soul_entities WHERE entity=?",
                (e,)).fetchall()
            entity_hits = list(entity_hits) + list(more)
        # Sort by occurrence count after dedup
        entity_counts: dict[tuple, int] = {}
        for row in entity_hits:
            key = (row[0], row[1])
            entity_counts[key] = entity_counts.get(key, 0) + 1
        for rank, (key, _) in enumerate(
            sorted(entity_counts.items(), key=lambda x: -x[1])
        ):
            scores[key] = scores.get(key, 0.0) + _rrf_score(rank, k=30)
    except Exception:
        pass

    # ── Sort and take top-N ──
    if scores:
        hits = [k for k, _ in sorted(scores.items(), key=lambda x: -x[1])][:limit]
    else:
        # Final fallback: keyword LIKE
        q = f"%{_escape_like(query)}%"
        hits = [(r[0], "souls") for r in conn.execute(
            "SELECT id FROM souls WHERE context LIKE ? ESCAPE '\\' OR human_decision LIKE ? ESCAPE '\\'"
            " OR divergence_reason LIKE ? ESCAPE '\\' ORDER BY timestamp DESC LIMIT ?",
            (q, q, q, limit)).fetchall()]

    def _tl_info(row):
        """Extract timeline metadata (only include non-empty values)."""
        tl = {}
        for k in ("memory_time", "prospect_time", "time_direction"):
            try:
                v = row[k]
                if v:
                    tl[k] = v
            except (IndexError, KeyError):
                pass
        return tl

    memories = []
    for rid, table in hits:
        try:
            if table not in SOUL_TABLES and table != "soul_materials":
                continue
            row = conn.execute(f"SELECT * FROM {table} WHERE id=?", (rid,)).fetchone()
            if not row:
                continue
            if table == "souls":
                memories.append({
                    "type": "decision", "id": row["id"],
                    "timestamp": row["timestamp"][:10],
                    "context": row["context"],
                    "decision": row["human_decision"],
                    "reason": row["divergence_reason"],
                    "emotion": row["emotional_state"],
                    "layer": row["decision_layer"],
                    **_tl_info(row),
                })
            elif table == "philosophy":
                memories.append({
                    "type": "philosophy", "id": row["id"],
                    "timestamp": row["timestamp"][:10],
                    "principle": row["principle"],
                    **_tl_info(row),
                })
            elif table == "profile_history":
                memories.append({
                    "type": "identity", "id": row["id"],
                    "timestamp": row["recorded_at"][:10],
                    "key": row["key"], "value": row["value"],
                    **_tl_info(row),
                })
            elif table == "states":
                memories.append({
                    "type": "state", "id": row["id"],
                    "timestamp": row["timestamp"][:10],
                    "emotion": row["emotional_state"],
                    "energy": row["energy"],
                    "body": row["body_state"],
                    "notes": row["notes"],
                    **_tl_info(row),
                })
            elif table == "silences":
                memories.append({
                    "type": "silence", "id": row["id"],
                    "timestamp": row["timestamp"][:10],
                    "topic": row["topic"],
                    "form": row["silence_form"],
                    "observation": row["observation"],
                    **_tl_info(row),
                })
            elif table == "contradictions":
                memories.append({
                    "type": "contradiction", "id": row["id"],
                    "timestamp": row["timestamp"][:10],
                    "description": row["description"],
                    "resolution": row["resolution"],
                    **_tl_info(row),
                })
            elif table == "artifacts":
                memories.append({
                    "type": "artifact", "id": row["id"],
                    "timestamp": row["timestamp"][:10],
                    "artifact_type": row["artifact_type"],
                    "description": row["description"],
                    **_tl_info(row),
                })
            elif table == "soul_materials":
                memories.append({
                    "type": "material", "id": row["id"],
                    "timestamp": row["timestamp"][:10],
                    "title": row["title"],
                    "material_type": row["material_type"],
                    "excerpt": row["raw_text"][:200],
                    **_tl_info(row),
                })
        except Exception:
            pass

    conn.close()
    return memories


def soul_prompt(query: str) -> str:
    """
    Generate complete recall context for soul inquiry.
    Claude uses this to answer in first person from the soul owner's perspective.
    The more fragments, the closer the answer is to the real you.
    """
    memories = recall_soul(query, limit=8)
    profile  = get_profile()
    philos   = all_philosophy()

    lines = [f"=== Soul Recall: {query} ===\n"]

    if profile:
        lines.append("-- Personality Foundation --")
        for r in profile:
            lines.append(f"  {r['key']}: {r['value']}")

    if philos:
        lines.append("\n-- Decision Philosophy --")
        for r in philos:
            lines.append(f"  · {r['principle']}")

    def _tl_tag(m):
        """Generate timeline annotation (e.g. 'memory:childhood' 'prospect:next year')."""
        parts = []
        if m.get("memory_time"):   parts.append(f"memory:{m['memory_time']}")
        if m.get("prospect_time"): parts.append(f"prospect:{m['prospect_time']}")
        return f" 〈{'｜'.join(parts)}〉" if parts else ""

    if memories:
        lines.append(f"\n-- Related Soul Fragments ({len(memories)} entries) --")
        for m in memories:
            tl = _tl_tag(m)
            if m["type"] == "decision":
                lines.append(f"  [{m['timestamp']}] {m['decision']}{tl}")
                if m.get("reason"): lines.append(f"    -> {m['reason']}")
                if m.get("emotion"): lines.append(f"    emotion: {m['emotion']}")
            elif m["type"] == "philosophy":
                lines.append(f"  [{m['timestamp']}] principle: {m['principle']}{tl}")
            elif m["type"] == "identity":
                lines.append(f"  [{m['timestamp']}] {m['key']}: {m['value']}{tl}")
            elif m["type"] == "state":
                parts = [p for p in [m.get("emotion"), m.get("energy"), m.get("body")] if p]
                lines.append(f"  [{m['timestamp']}] state: {' / '.join(parts) if parts else m.get('notes', '')}{tl}")
            elif m["type"] == "silence":
                lines.append(f"  [{m['timestamp']}] silence: {m.get('topic', '')} ({m.get('form', '')}){tl}")
                if m.get("observation"): lines.append(f"    -> {m['observation']}")
            elif m["type"] == "contradiction":
                lines.append(f"  [{m['timestamp']}] contradiction: {m.get('description', '')}{tl}")
                if m.get("resolution"): lines.append(f"    -> resolution: {m['resolution']}")
            elif m["type"] == "artifact":
                lines.append(f"  [{m['timestamp']}] artifact[{m.get('artifact_type','')}]: {m.get('description', '')}{tl}")
            elif m["type"] == "material":
                lines.append(f"  [{m['timestamp']}] material[{m.get('material_type','')}]: {m.get('title', '')}{tl}")
                if m.get("excerpt"): lines.append(f"    -> {m['excerpt'][:100]}...")
    else:
        lines.append("\n(Insufficient soul fragments to answer this question. Continue recording to enrich the soul.)")

    lines += [
        "\n=== Recall Complete ===",
        "Based on the above memories, answer in first person from the soul owner's perspective.",
        "This is not imitation, but inference based on real records. State honestly when fragments are insufficient.",
        f"\nQuestion: {query}",
    ]

    return "\n".join(lines)


def cold_start_sketch() -> dict:
    """
    Cold-start instant personality sketch: provide a basic portrait when entries < 10.
    Extract keywords from existing entries, output a displayable sketch instead of "insufficient data".

    Returns: {
        "level": "embryo"|"seed"|"sprout",
        "sketch": "...",        # A personality description
        "top_tags": [...],      # High-frequency tags
        "emotion_pattern": ..., # Emotion tendency
        "earliest": "...",      # Earliest entry time
        "total": N,
    }
    """
    conn = _open_db()
    total = conn.execute("SELECT COUNT(*) FROM souls").fetchone()[0]

    # Extract high-frequency tags
    all_tags: dict[str, int] = {}
    for (tags_json,) in conn.execute("SELECT tags FROM souls WHERE tags IS NOT NULL").fetchall():
        try:
            for t in json.loads(tags_json):
                all_tags[t] = all_tags.get(t, 0) + 1
        except Exception:
            pass
    top_tags = sorted(all_tags.items(), key=lambda x: -x[1])[:8]

    # Emotion distribution
    emotions = [r[0] for r in conn.execute(
        "SELECT emotional_state FROM souls WHERE emotional_state IS NOT NULL"
    ).fetchall()]
    emotion_counter: dict[str, int] = {}
    for e in emotions:
        emotion_counter[e] = emotion_counter.get(e, 0) + 1
    top_emotion = max(emotion_counter, key=emotion_counter.get) if emotion_counter else None

    # Earliest entry
    earliest = conn.execute(
        "SELECT timestamp FROM souls ORDER BY timestamp ASC LIMIT 1"
    ).fetchone()
    conn.close()

    if total == 0:
        level = "embryo"
        sketch = "Soul recording has not yet begun. The first entry will ignite your digital soul."
    elif total < 5:
        level = "embryo"
        tags_str = ", ".join(t for t, _ in top_tags[:4]) if top_tags else "not yet tagged"
        sketch = (
            f"Digital soul emerging, {total} entries recorded. "
            f"Faintly visible keywords: {tags_str}. "
            f"The more you record, the clearer the portrait."
        )
    elif total < 20:
        level = "seed"
        tags_str = ", ".join(t for t, _ in top_tags[:6]) if top_tags else "not yet tagged"
        emotion_str = f"Emotion tendency: {top_emotion}" if top_emotion else ""
        sketch = (
            f"Soul outline emerging ({total} entries). "
            f"Core keywords: {tags_str}. {emotion_str}. "
            f"Continue recording to unlock a more complete soul map."
        )
    else:
        level = "sprout"
        tags_str = ", ".join(t for t, _ in top_tags[:8]) if top_tags else "not yet tagged"
        sketch = (
            f"Soul is growing ({total} entries). "
            f"High-frequency dimensions: {tags_str}. "
            f"Suggest using soul recall for deep exploration."
        )

    return {
        "level": level,
        "sketch": sketch,
        "top_tags": [t for t, _ in top_tags],
        "emotion_pattern": top_emotion,
        "earliest": earliest[0][:10] if earliest else None,
        "total": total,
    }


# ── Table Readers ────────────────────────────────────────────────────

def list_states(limit=10):
    """Retrieve recent state snapshots."""
    conn = _open_db()
    rows = conn.execute(
        "SELECT * FROM states ORDER BY timestamp DESC LIMIT ?", (limit,)
    ).fetchall()
    conn.close()
    return rows


def list_silences(limit=10):
    """Retrieve recent silence records."""
    conn = _open_db()
    rows = conn.execute(
        "SELECT * FROM silences ORDER BY timestamp DESC LIMIT ?", (limit,)
    ).fetchall()
    conn.close()
    return rows


def list_contradictions(limit=10):
    """Retrieve recent contradiction records."""
    conn = _open_db()
    rows = conn.execute(
        "SELECT * FROM contradictions ORDER BY timestamp DESC LIMIT ?", (limit,)
    ).fetchall()
    conn.close()
    return rows


def list_artifacts(limit=10, artifact_type=None):
    """Retrieve recent multimodal artifacts. Can filter by type."""
    conn = _open_db()
    if artifact_type:
        rows = conn.execute(
            "SELECT * FROM artifacts WHERE artifact_type=? ORDER BY timestamp DESC LIMIT ?",
            (artifact_type, limit)
        ).fetchall()
    else:
        rows = conn.execute(
            "SELECT * FROM artifacts ORDER BY timestamp DESC LIMIT ?", (limit,)
        ).fetchall()
    conn.close()
    return rows


# ── Narrative Engine ─────────────────────────────────────────────────────

GUIDED_DEPTH_LEVELS = {
    0: "raw_input",           # User's original text
    1: "fact_supplement",     # who/what/when/where
    2: "emotion_mining",      # feelings/emotions/body reactions
    3: "meaning_exploration", # why it matters/relation to values
    4: "cross_time_link",     # similar past experiences/future impact
}

def start_narrative(topic: str = None, media_id: int = None,
                    notes: str = None) -> dict:
    """Start a narrative session."""
    conn = _open_db()
    cur = conn.execute(
        """INSERT INTO narrative_sessions
           (timestamp, topic, media_id, notes, entry_id)
           VALUES (?,?,?,?,?)""",
        (_now(), topic, media_id, notes, f"local-{uuid.uuid4().hex}"))
    sid = cur.lastrowid
    conn.commit()
    conn.close()
    return {"session_id": sid, "topic": topic}


def narrative_turn(session_id: int, question: str, answer: str = None,
                   depth_level: int = 0) -> dict:
    """Record a narrative Q&A turn. Auto-extracts chunks when answer is non-empty."""
    conn = _open_db()

    # Get current turn number
    sess = conn.execute("SELECT * FROM narrative_sessions WHERE id=?",
                        (session_id,)).fetchone()
    if not sess:
        conn.close()
        return {"ok": False, "message": f"Session #{session_id} does not exist"}

    turn_num = (sess["turn_count"] or 0) + 1

    cur = conn.execute(
        """INSERT INTO narrative_turns
           (session_id, turn_number, timestamp, question, answer,
            depth_level, entry_id)
           VALUES (?,?,?,?,?,?,?)""",
        (session_id, turn_num, _now(), question, answer,
         depth_level, f"local-{uuid.uuid4().hex}"))
    tid = cur.lastrowid

    chunks_gen = 0
    if answer and len(answer) >= 8:
        # Extract chunks, mark as narrative engine source
        content = answer
        chunk_type = "note"  # Default type
        if depth_level >= 3:
            chunk_type = "reason"  # Deep exploration is closer to reason/meaning
        elif depth_level >= 2:
            chunk_type = "emotion"

        conn.execute(
            """INSERT INTO soul_chunks
               (soul_id, table_name, chunk_type, content, created_at,
                vector_status, source_weight, source_engine)
               VALUES (?,?,?,?,?,?,?,?)""",
            (tid, "narrative_turns", chunk_type, content,
             _now(), "pending", 0.9, "human_prompted"))
        chunks_gen = 1

        # Async vectorization
        def _vectorize():
            try:
                import vectors as _v
                if _v.ollama_alive():
                    _v.store_chunk_vectors(tid, "narrative_turns",
                                           [(chunk_type, content)])
            except Exception:
                pass
        threading.Thread(target=_vectorize, daemon=True).start()

    # Update session statistics
    conn.execute(
        "UPDATE narrative_sessions SET turn_count=?, soul_chunks_generated="
        "soul_chunks_generated+? WHERE id=?",
        (turn_num, chunks_gen, session_id))
    conn.execute("UPDATE narrative_turns SET chunks_generated=? WHERE id=?",
                 (chunks_gen, tid))
    conn.commit()
    conn.close()

    return {
        "ok": True,
        "turn_id": tid,
        "turn_number": turn_num,
        "depth_level": depth_level,
        "chunks_generated": chunks_gen,
        "next_depth": min(depth_level + 1, 4),
        "next_depth_name": GUIDED_DEPTH_LEVELS.get(min(depth_level + 1, 4), ""),
    }


def end_narrative(session_id: int) -> dict:
    """End a narrative session."""
    conn = _open_db()
    sess = conn.execute("SELECT * FROM narrative_sessions WHERE id=?",
                        (session_id,)).fetchone()
    if not sess:
        conn.close()
        return {"ok": False, "message": f"Session #{session_id} does not exist"}
    conn.execute("UPDATE narrative_sessions SET status='completed' WHERE id=?",
                 (session_id,))
    conn.commit()
    conn.close()
    return {
        "ok": True,
        "session_id": session_id,
        "turns": sess["turn_count"],
        "chunks": sess["soul_chunks_generated"],
    }


def list_narratives(limit: int = 10) -> list:
    """List narrative sessions."""
    conn = _open_db()
    rows = conn.execute(
        "SELECT * FROM narrative_sessions ORDER BY timestamp DESC LIMIT ?",
        (limit,)).fetchall()
    conn.close()
    return rows


# ── Source Weight Verification ─────────────────────────────────────────────────

def verify_chunk(chunk_id: int, correct: bool, corrected_content: str = None) -> dict:
    """User verifies an AI-generated chunk.

    correct=True -> verified_by_user=1, weight may be upgraded
    correct=False -> verified_by_user=-1, original content saved, DPO training pair generated
    """
    conn = _open_db()
    chunk = conn.execute("SELECT * FROM soul_chunks WHERE id=?", (chunk_id,)).fetchone()
    if not chunk:
        conn.close()
        return {"ok": False, "message": f"chunk #{chunk_id} does not exist"}

    if correct:
        new_weight = chunk["source_weight"] or 1.0
        # AI interpretation confirmed -> weight upgrade
        if chunk["source_engine"] in ("ai_interpret",):
            new_weight = 0.85
        conn.execute(
            "UPDATE soul_chunks SET verified_by_user=1, verified_at=?, "
            "source_weight=?, source_engine=CASE WHEN source_engine='ai_interpret' "
            "THEN 'ai_augmented' ELSE source_engine END WHERE id=?",
            (_now(), new_weight, chunk_id))
        conn.commit()
        conn.close()
        return {"ok": True, "message": f"chunk #{chunk_id} confirmed, weight={new_weight}"}
    else:
        old_content = chunk["content"]
        if corrected_content:
            conn.execute(
                "UPDATE soul_chunks SET verified_by_user=-1, verified_at=?, "
                "original_content=content, content=?, source_weight=1.0, "
                "source_engine='human_direct' WHERE id=?",
                (_now(), corrected_content, chunk_id))
            # Generate DPO training pair
            conn.execute(
                """INSERT INTO soul_training_pairs
                   (instruction, input, output, source_table, source_id, quality, created_at)
                   VALUES (?,?,?,?,?,?,datetime('now'))""",
                ("You are the soul owner. Describe in first person based on the material.",
                 old_content[:200],
                 corrected_content,
                 chunk["table_name"], chunk["soul_id"], 1.0))
        else:
            conn.execute(
                "UPDATE soul_chunks SET verified_by_user=-1, verified_at=?, "
                "source_weight=0.3 WHERE id=?",
                (_now(), chunk_id))
        conn.commit()
        conn.close()
        return {"ok": True, "message": f"chunk #{chunk_id} marked as inaccurate"}


# ── Relationship Network ────────────────────────────────────────────────────

def add_relation(alias, real_name=None, relation_type=None,
                 importance=0.5, emotional_tone=None,
                 first_met=None, notes=None, tags=None):
    """Record a relationship. INSERT-only; same-name relationships can be recorded multiple times to track evolution."""
    conn = _open_db()
    cur = conn.execute(
        """INSERT INTO soul_relations
           (timestamp, alias, real_name, relation_type, importance,
            emotional_tone, first_met, notes, tags, entry_id)
           VALUES (?,?,?,?,?,?,?,?,?,?)""",
        (_now(), alias, real_name, relation_type, importance,
         emotional_tone, first_met, notes,
         json.dumps(tags or [], ensure_ascii=False),
         f"local-{uuid.uuid4().hex}"))
    rid = cur.lastrowid
    conn.commit()
    conn.close()
    return rid


def list_relations(limit=20):
    """List relationship network (sorted by importance)."""
    conn = _open_db()
    rows = conn.execute(
        """SELECT r.*, (SELECT COUNT(*) FROM soul_relation_events WHERE relation_id=r.id) as event_count
           FROM soul_relations r ORDER BY r.importance DESC, r.timestamp DESC LIMIT ?""",
        (limit,)).fetchall()
    conn.close()
    return rows


def record_relation_event(relation_id, event_type, description,
                          emotional_state=None, emotion_valence=None,
                          linked_soul_id=None, tags=None):
    """Record a relationship event."""
    emotion_label = _normalize_emotion(emotional_state)
    conn = _open_db()
    cur = conn.execute(
        """INSERT INTO soul_relation_events
           (timestamp, relation_id, event_type, description,
            emotional_state, emotion_label, emotion_valence,
            linked_soul_id, tags, entry_id)
           VALUES (?,?,?,?,?,?,?,?,?,?)""",
        (_now(), relation_id, event_type, description,
         emotional_state, emotion_label, emotion_valence,
         linked_soul_id, json.dumps(tags or [], ensure_ascii=False),
         f"local-{uuid.uuid4().hex}"))
    eid = cur.lastrowid

    # Write to soul_chunks for semantic retrieval
    content = f"Relationship event [{event_type}]: {description}"
    if emotional_state:
        content += f" emotion:{emotional_state}"
    if len(content) >= 8:
        conn.execute(
            """INSERT INTO soul_chunks
               (soul_id, table_name, chunk_type, content, created_at,
                vector_status, relation_id)
               VALUES (?,?,?,?,?,?,?)""",
            (eid, "soul_relation_events", "relation_event", content,
             _now(), "pending", relation_id))

    conn.commit()
    conn.close()

    # Async vectorization
    def _vectorize():
        try:
            import vectors as _v
            if _v.ollama_alive():
                _v.store_chunk_vectors(eid, "soul_relation_events",
                                       [("relation_event", content)])
        except Exception:
            pass
    threading.Thread(target=_vectorize, daemon=True).start()

    return eid


def get_relation(relation_id: int):
    """Get a relationship and all its events."""
    conn = _open_db()
    rel = conn.execute("SELECT * FROM soul_relations WHERE id=?", (relation_id,)).fetchone()
    events = conn.execute(
        "SELECT * FROM soul_relation_events WHERE relation_id=? ORDER BY timestamp ASC",
        (relation_id,)).fetchall() if rel else []
    conn.close()
    return rel, events


# ── Game API ──────────────────────────────────────────────────

def register_game(game_id: str, game_name: str, description: str = "",
                  mappings: dict = None) -> dict:
    """Register a game and its dimension mapping rules."""
    import game_mapper
    if mappings:
        errors = game_mapper.validate_mappings(mappings)
        if errors:
            return {"ok": False, "errors": errors}
    conn = _open_db()
    existing = conn.execute("SELECT id FROM game_registry WHERE game_id=?", (game_id,)).fetchone()
    if existing:
        conn.execute(
            "UPDATE game_registry SET game_name=?, description=?, mappings=?, timestamp=? WHERE game_id=?",
            (game_name, description, json.dumps(mappings or {}, ensure_ascii=False), _now(), game_id))
        conn.commit()
        conn.close()
        return {"ok": True, "game_id": game_id, "updated": True}
    conn.execute(
        """INSERT INTO game_registry (timestamp, game_id, game_name, description, mappings, entry_id)
           VALUES (?,?,?,?,?,?)""",
        (_now(), game_id, game_name, description,
         json.dumps(mappings or {}, ensure_ascii=False),
         f"local-{uuid.uuid4().hex}"))
    conn.commit()
    conn.close()
    return {"ok": True, "game_id": game_id, "updated": False}


def start_game_session(game_id: str, notes: str = None) -> dict:
    """Start a game session."""
    conn = _open_db()
    # Check if game is registered
    game = conn.execute("SELECT * FROM game_registry WHERE game_id=?", (game_id,)).fetchone()
    if not game:
        conn.close()
        return {"ok": False, "message": f"Game '{game_id}' not registered. Please call register_game first"}
    cur = conn.execute(
        """INSERT INTO game_sessions (timestamp, game_id, notes, entry_id)
           VALUES (?,?,?,?)""",
        (_now(), game_id, notes, f"local-{uuid.uuid4().hex}"))
    sid = cur.lastrowid
    conn.commit()
    conn.close()
    return {"ok": True, "session_id": sid, "game_name": game["game_name"]}


def end_game_session(session_id: int) -> dict:
    """End a game session."""
    conn = _open_db()
    session = conn.execute("SELECT * FROM game_sessions WHERE id=?", (session_id,)).fetchone()
    if not session:
        conn.close()
        return {"ok": False, "message": f"Session #{session_id} does not exist"}
    if session["status"] != "active":
        conn.close()
        return {"ok": False, "message": f"Session #{session_id} has already ended"}
    conn.execute(
        "UPDATE game_sessions SET status='ended', ended_at=? WHERE id=?",
        (_now(), session_id))
    conn.commit()
    conn.close()
    return {
        "ok": True, "session_id": session_id,
        "decisions": session["decisions_count"],
        "events": session["events_count"],
    }


def record_game_decision(session_id: int, action: str, choice: str = "",
                          context: str = "", emotional_state: str = None,
                          tags: list = None) -> dict:
    """
    Record a game decision, automatically mapped to a soul event.

    1. Use game_mapper to map action+choice to dimension/tags/weight
    2. Write to souls table (source_device='engine:game:{game_id}')
    3. Update session counters
    """
    import game_mapper

    conn = _open_db()
    session = conn.execute("SELECT * FROM game_sessions WHERE id=?", (session_id,)).fetchone()
    if not session:
        conn.close()
        return {"ok": False, "message": f"Session #{session_id} does not exist"}
    if session["status"] != "active":
        conn.close()
        return {"ok": False, "message": f"Session #{session_id} has already ended"}

    # Get game custom mappings
    game = conn.execute("SELECT * FROM game_registry WHERE game_id=?",
                        (session["game_id"],)).fetchone()
    custom_mappings = json.loads(game["mappings"]) if game and game["mappings"] else {}
    conn.close()

    # Map decision
    mapped = game_mapper.map_decision(action, choice, context, custom_mappings)

    # Merge emotions
    emo = emotional_state
    emo_valence = None
    if not emo and mapped["emotion_hint"]:
        emo = mapped["emotion_hint"]["emotion"]
        emo_valence = mapped["emotion_hint"]["valence"]

    # Merge tags
    all_tags = list(set((tags or []) + mapped["tags"]))

    # Write to souls table
    source_device = f"engine:game:{session['game_id']}"
    record(
        context=f"[Game:{game['game_name'] if game else session['game_id']}] {context}",
        human_decision=mapped["decision_text"],
        divergence_reason=mapped["reason_text"],
        tags=all_tags,
        emotional_state=emo,
        decision_layer="values" if mapped["dimension"] == "value" else "identity",
        source_device=source_device,
        emotion_valence=emo_valence,
    )

    # Update session counters
    conn2 = _open_db()
    conn2.execute(
        "UPDATE game_sessions SET decisions_count = decisions_count + 1 WHERE id=?",
        (session_id,))
    conn2.commit()
    conn2.close()

    return {
        "ok": True,
        "mapped_dimension": mapped["dimension"],
        "source_engine": mapped["source_engine"],
        "weight": mapped["weight"],
        "emotion_inferred": emo,
    }


def record_game_event(session_id: int, event_type: str, description: str,
                       emotional_state: str = None, tags: list = None) -> dict:
    """
    Record a game event (non-decisional, e.g. scene change/NPC dialogue/environment), mapped to a state snapshot.
    """
    conn = _open_db()
    session = conn.execute("SELECT * FROM game_sessions WHERE id=?", (session_id,)).fetchone()
    if not session:
        conn.close()
        return {"ok": False, "message": f"Session #{session_id} does not exist"}

    game = conn.execute("SELECT * FROM game_registry WHERE game_id=?",
                        (session["game_id"],)).fetchone()
    conn.close()

    game_name = game["game_name"] if game else session["game_id"]
    source_device = f"engine:game:{session['game_id']}"

    record_state(
        emotional_state=emotional_state,
        notes=f"[Game:{game_name}] [{event_type}] {description}",
        tags=list(set((tags or []) + ["game", event_type])),
        source_device=source_device,
    )

    # Update session counters
    conn2 = _open_db()
    conn2.execute(
        "UPDATE game_sessions SET events_count = events_count + 1 WHERE id=?",
        (session_id,))
    conn2.commit()
    conn2.close()

    return {"ok": True, "event_type": event_type}


def game_stats(game_id: str = None) -> dict:
    """Get game statistics."""
    conn = _open_db()
    if game_id:
        games = conn.execute(
            "SELECT * FROM game_registry WHERE game_id=?", (game_id,)).fetchall()
    else:
        games = conn.execute("SELECT * FROM game_registry ORDER BY timestamp DESC").fetchall()

    stats = []
    for g in games:
        sessions = conn.execute(
            """SELECT COUNT(*) as total,
                      SUM(CASE WHEN status='active' THEN 1 ELSE 0 END) as active,
                      SUM(decisions_count) as decisions,
                      SUM(events_count) as events
               FROM game_sessions WHERE game_id=?""",
            (g["game_id"],)).fetchone()
        stats.append({
            "game_id": g["game_id"],
            "game_name": g["game_name"],
            "description": g["description"],
            "sessions_total": sessions["total"] or 0,
            "sessions_active": sessions["active"] or 0,
            "total_decisions": sessions["decisions"] or 0,
            "total_events": sessions["events"] or 0,
            "mappings_count": len(json.loads(g["mappings"])) if g["mappings"] else 0,
        })
    conn.close()
    return {"games": stats}


def list_game_sessions(game_id: str = None, limit: int = 10) -> list:
    """List game sessions."""
    conn = _open_db()
    if game_id:
        rows = conn.execute(
            "SELECT * FROM game_sessions WHERE game_id=? ORDER BY timestamp DESC LIMIT ?",
            (game_id, limit)).fetchall()
    else:
        rows = conn.execute(
            "SELECT * FROM game_sessions ORDER BY timestamp DESC LIMIT ?",
            (limit,)).fetchall()
    conn.close()
    return rows


# ── Dimension Coverage (Question Engine Dependency) ─────────────────────────────────

SOUL_DIMENSIONS = ("identity", "value", "emotion", "cognition",
                   "relationship", "creation", "mission")

def soul_coverage() -> dict:
    """
    Soul data coverage analysis v2 -- core input for the question engine.
    Returns: dimension coverage / dimension x scene matrix / time direction distribution / scene distribution /
    table freshness / emotion distribution / blank zones / weak zones / suggested question directions.
    """
    conn = _open_db()

    # ── chunk_type statistics ──
    chunk_stats = {}
    for row in conn.execute(
        "SELECT chunk_type, COUNT(*) as cnt, MAX(created_at) as latest"
        " FROM soul_chunks GROUP BY chunk_type"
    ).fetchall():
        chunk_stats[row["chunk_type"]] = {
            "count": row["cnt"],
            "latest": row["latest"][:10] if row["latest"] else None,
        }

    # ── Table time freshness ──
    table_freshness = {}
    ts_col_map = {
        "souls": "timestamp", "states": "timestamp", "silences": "timestamp",
        "contradictions": "timestamp", "artifacts": "timestamp",
        "philosophy": "timestamp", "profile_history": "recorded_at",
    }
    for table in SOUL_TABLES:
        ts_col = ts_col_map.get(table, "timestamp")
        row = conn.execute(
            f"SELECT COUNT(*) as cnt, MIN({ts_col}) as earliest, MAX({ts_col}) as latest"
            f" FROM {table}"
        ).fetchone()
        table_freshness[table] = {
            "count": row["cnt"],
            "earliest": row["earliest"][:10] if row["earliest"] else None,
            "latest": row["latest"][:10] if row["latest"] else None,
        }

    # ── Emotion distribution (states + souls) ──
    emotion_dist = {}
    for (e,) in conn.execute(
        "SELECT emotional_state FROM souls WHERE emotional_state IS NOT NULL"
    ).fetchall():
        emotion_dist[e] = emotion_dist.get(e, 0) + 1
    for (e,) in conn.execute(
        "SELECT emotional_state FROM states WHERE emotional_state IS NOT NULL"
    ).fetchall():
        emotion_dist[e] = emotion_dist.get(e, 0) + 1

    # ── Profile coverage ──
    profile_keys = [r["key"] for r in conn.execute(
        "SELECT DISTINCT key FROM profile_history"
    ).fetchall()]

    # ── Dimension seed coverage ──
    dimension_coverage = {}
    for dim in SOUL_DIMENSIONS:
        seed_count = conn.execute(
            "SELECT COUNT(*) FROM dimension_seeds WHERE dimension=?", (dim,)
        ).fetchone()[0]
        related_chunks = chunk_stats.get(dim, {"count": 0, "latest": None})
        dimension_coverage[dim] = {
            "seeds": seed_count,
            "chunks": related_chunks["count"],
            "latest": related_chunks["latest"],
        }

    sparse_dimensions = [d for d, v in dimension_coverage.items() if v["chunks"] == 0]
    weak_dimensions = sorted(
        [(d, v["chunks"]) for d, v in dimension_coverage.items() if v["chunks"] > 0],
        key=lambda x: x[1]
    )[:3]

    # ── v3.0: Dimension x Scene Matrix ──
    # chunk_type to dimension mapping (not one-to-one, approximate)
    _type_to_dim = {
        "emotion": "emotion", "decision": "value", "reason": "cognition",
        "context": "relationship", "note": "identity", "topic": "emotion",
        "observation": "cognition", "conflict": "relationship",
        "analysis": "cognition", "principle": "value", "identity": "identity",
    }
    # Reverse mapping: dimension -> primary chunk_type
    _type_to_dim_reverse = {
        "emotion": "emotion", "value": "decision", "cognition": "reason",
        "relationship": "context", "identity": "identity",
        "creation": "note", "mission": "principle",
    }
    # Query all chunks with scene data
    scene_rows = conn.execute(
        "SELECT chunk_type, scene_social, scene_role, COUNT(*) as cnt"
        " FROM soul_chunks"
        " WHERE scene_social IS NOT NULL OR scene_role IS NOT NULL"
        " GROUP BY chunk_type, scene_social, scene_role"
    ).fetchall()

    dim_scene_matrix = {}  # {dimension: {scene_key: count}}
    for row in scene_rows:
        dim = _type_to_dim.get(row["chunk_type"])
        if not dim:
            continue
        scene_key = f"{row['scene_social'] or '?'}×{row['scene_role'] or '?'}"
        if dim not in dim_scene_matrix:
            dim_scene_matrix[dim] = {}
        dim_scene_matrix[dim][scene_key] = dim_scene_matrix[dim].get(scene_key, 0) + row["cnt"]

    # Scene gaps: has dimension data but some scenes completely missing
    all_scenes = set()
    for scenes in dim_scene_matrix.values():
        all_scenes.update(scenes.keys())
    scene_gaps = []
    for dim in SOUL_DIMENSIONS:
        if dim in dim_scene_matrix:
            covered = set(dim_scene_matrix[dim].keys())
            missing = all_scenes - covered
            for s in missing:
                scene_gaps.append({"dimension": dim, "scene": s})

    # Scene skew: a dimension's data concentrated in a single scene
    scene_skews = []
    for dim, scenes in dim_scene_matrix.items():
        total = sum(scenes.values())
        if total < 3:
            continue
        for scene, cnt in scenes.items():
            if cnt / total > 0.7:
                scene_skews.append({
                    "dimension": dim, "dominant_scene": scene,
                    "ratio": round(cnt / total, 2), "total": total,
                })

    # ── v3.0: Time direction distribution ──
    time_dir_dist = {}
    for row in conn.execute(
        "SELECT time_direction, COUNT(*) as cnt FROM soul_chunks"
        " WHERE time_direction IS NOT NULL GROUP BY time_direction"
    ).fetchall():
        time_dir_dist[row["time_direction"]] = row["cnt"]

    # ── v3.0: Overall scene distribution ──
    scene_social_dist = {}
    for row in conn.execute(
        "SELECT scene_social, COUNT(*) as cnt FROM soul_chunks"
        " WHERE scene_social IS NOT NULL GROUP BY scene_social"
    ).fetchall():
        scene_social_dist[row["scene_social"]] = row["cnt"]

    scene_role_dist = {}
    for row in conn.execute(
        "SELECT scene_role, COUNT(*) as cnt FROM soul_chunks"
        " WHERE scene_role IS NOT NULL GROUP BY scene_role"
    ).fetchall():
        scene_role_dist[row["scene_role"]] = row["cnt"]

    # ── v3.0: Material statistics ──
    material_count = conn.execute("SELECT COUNT(*) FROM soul_materials").fetchone()[0]

    # ── v4.0: Engine contribution distribution ──
    engine_distribution = {}
    for row in conn.execute(
        "SELECT COALESCE(source_engine, 'human_direct') as engine,"
        " COUNT(*) as cnt, SUM(COALESCE(source_weight, 1.0)) as weighted"
        " FROM soul_chunks GROUP BY engine"
    ).fetchall():
        engine_distribution[row["engine"]] = {
            "chunks": row["cnt"],
            "weighted_chunks": round(row["weighted"], 1),
        }

    # ── v4.0: Engine blind spots (dimension with only game/AI data, lacking real-life data) ──
    engine_blind_spots = []
    for dim in SOUL_DIMENSIONS:
        ct = _type_to_dim_reverse.get(dim)
        if not ct:
            continue
        real_life = conn.execute(
            "SELECT COUNT(*) FROM soul_chunks"
            " WHERE chunk_type IN (SELECT chunk_type FROM soul_chunks WHERE chunk_type=?)"
            " AND COALESCE(source_engine, 'human_direct') IN ('human_direct', 'human_prompted')"
            " AND chunk_type=?",
            (ct, ct)).fetchone()[0]
        game_only = conn.execute(
            "SELECT COUNT(*) FROM soul_chunks"
            " WHERE chunk_type=?"
            " AND COALESCE(source_engine, 'human_direct') IN ('game_inference', 'game_explicit')",
            (ct,)).fetchone()[0]
        if game_only > 0 and real_life == 0:
            engine_blind_spots.append({
                "dimension": dim,
                "game_only_chunks": game_only,
                "real_life_chunks": real_life,
            })

    conn.close()
    return {
        "dimensions": dimension_coverage,
        "sparse_dimensions": sparse_dimensions,
        "weak_dimensions": [{"dimension": d, "chunks": c} for d, c in weak_dimensions],
        "chunk_types": chunk_stats,
        "table_freshness": table_freshness,
        "emotion_distribution": emotion_dist,
        "profile_keys": profile_keys,
        # v3.0 additions
        "dim_scene_matrix": dim_scene_matrix,
        "scene_gaps": scene_gaps[:10],
        "scene_skews": scene_skews,
        "time_direction_distribution": time_dir_dist,
        "scene_social_distribution": scene_social_dist,
        "scene_role_distribution": scene_role_dist,
        "material_count": material_count,
        # v4.0 additions
        "engine_distribution": engine_distribution,
        "engine_blind_spots": engine_blind_spots,
    }


def suggest_questions(limit: int = 5) -> list[dict]:
    """
    Generate question suggestions based on dimension x scene coverage gaps.
    Returns [{dimension, scene, reason, sample_question}, ...], sorted by priority.

    Priority strategy:
    1. Completely blank dimensions (highest priority)
    2. Has dimension data but missing certain scenes (scene_gaps)
    3. Severely skewed dimensions (scene_skews)
    4. Dimensions with least data (weak_dimensions)
    5. Missing time directions (all present, lacking past/future)
    """
    cov = soul_coverage()
    suggestions = []

    # Dimension -> display name mapping
    _dim_cn = {
        "identity": "Identity", "value": "Values", "emotion": "Emotion Patterns",
        "cognition": "Cognitive Style", "relationship": "Relationship Patterns",
        "creation": "Creation & Expression", "mission": "Mission & Meaning",
    }
    # Scene -> sample question mapping
    _scene_questions = {
        "独处": "What do you do most when alone? Enjoy it or feel uneasy?",
        "家庭": "What role do you usually play in your family?",
        "同事": "When dealing with colleagues, do you tend to be proactive or passive?",
        "朋友": "What's your first reaction when a friend is in trouble?",
        "亲密关系": "What do you need most in an intimate relationship?",
        "工作": "What gives you the greatest sense of achievement at work?",
        "个人": "What have you done for yourself recently?",
        "学习": "What's your preferred way of learning new things?",
    }
    # Dimension -> generic question template
    _dim_questions = {
        "identity": "If you described your core traits in three words, what would they be?",
        "value": "What unconventional choice did you recently make that felt right?",
        "emotion": "What recently triggered your strongest emotional reaction?",
        "cognition": "When facing a complex problem, what's usually your first step?",
        "relationship": "What qualities do you value most in friends/partners?",
        "creation": "What have you wanted to create or express but haven't acted on?",
        "mission": "If you had only one year left, what would you most want to accomplish?",
    }

    # Strategy 1: Blank dimensions
    for dim in cov["sparse_dimensions"]:
        suggestions.append({
            "dimension": dim,
            "dimension_cn": _dim_cn.get(dim, dim),
            "scene": None,
            "priority": "P0",
            "reason": f"Dimension '{_dim_cn.get(dim, dim)}' is completely blank",
            "sample_question": _dim_questions.get(dim, ""),
        })

    # Strategy 2: Scene gaps
    for gap in cov.get("scene_gaps", []):
        dim, scene = gap["dimension"], gap["scene"]
        # Parse scene_key e.g. "alone x work"
        social, role = scene.split("×") if "×" in scene else (scene, "?")
        q = _scene_questions.get(social, _scene_questions.get(role, ""))
        suggestions.append({
            "dimension": dim,
            "dimension_cn": _dim_cn.get(dim, dim),
            "scene": scene,
            "priority": "P1",
            "reason": f"'{_dim_cn.get(dim, dim)}' has no data in scene {scene}",
            "sample_question": q or _dim_questions.get(dim, ""),
        })

    # Strategy 3: Scene skew
    for skew in cov.get("scene_skews", []):
        dim = skew["dimension"]
        suggestions.append({
            "dimension": dim,
            "dimension_cn": _dim_cn.get(dim, dim),
            "scene": f"non-{skew['dominant_scene']}",
            "priority": "P2",
            "reason": (f"'{_dim_cn.get(dim, dim)}' has {int(skew['ratio']*100)}%"
                       f" data from {skew['dominant_scene']}, needs data from other scenes"),
            "sample_question": _dim_questions.get(dim, ""),
        })

    # Strategy 4: Weak dimensions
    for w in cov.get("weak_dimensions", []):
        dim = w["dimension"]
        if dim not in [s["dimension"] for s in suggestions]:  # Deduplicate
            suggestions.append({
                "dimension": dim,
                "dimension_cn": _dim_cn.get(dim, dim),
                "scene": None,
                "priority": "P2",
                "reason": f"'{_dim_cn.get(dim, dim)}' has limited data ({w['chunks']} entries)",
                "sample_question": _dim_questions.get(dim, ""),
            })

    # Strategy 5: Missing time directions
    td = cov.get("time_direction_distribution", {})
    total_td = sum(td.values())
    if total_td > 10:
        if td.get("past", 0) == 0:
            suggestions.append({
                "dimension": "all", "dimension_cn": "Global",
                "scene": "past",
                "priority": "P1",
                "reason": "No past memory records found. Consider recording childhood/growth experiences",
                "sample_question": "What is your most vivid childhood memory?",
            })
        if td.get("future", 0) == 0:
            suggestions.append({
                "dimension": "all", "dimension_cn": "Global",
                "scene": "future",
                "priority": "P2",
                "reason": "No future prospect records found",
                "sample_question": "What do you hope to be doing five years from now?",
            })

    # Strategy 6: Engine blind spots -- dimension with only game data
    for blind in cov.get("engine_blind_spots", []):
        dim = blind["dimension"]
        suggestions.append({
            "dimension": dim,
            "dimension_cn": _dim_cn.get(dim, dim),
            "scene": "real_life",
            "priority": "P1",
            "reason": (f"'{_dim_cn.get(dim, dim)}' has only game engine data"
                       f" ({blind['game_only_chunks']} entries), needs real-life scene supplement"),
            "sample_question": _dim_questions.get(dim, ""),
        })

    # Strategy 7: Relationship gaps -- important relationships with no event records
    try:
        conn = _open_db()
        important_rels = conn.execute(
            "SELECT r.alias, r.importance, "
            "(SELECT COUNT(*) FROM soul_relation_events WHERE relation_id=r.id) as ec "
            "FROM soul_relations r WHERE r.importance >= 0.7 ORDER BY r.importance DESC"
        ).fetchall()
        for rel in important_rels:
            if rel["ec"] == 0:
                suggestions.append({
                    "dimension": "relationship",
                    "dimension_cn": "Relationship Patterns",
                    "scene": rel["alias"],
                    "priority": "P1",
                    "reason": f"Important relationship '{rel['alias']}' has no interaction event records",
                    "sample_question": f"When was your last deep conversation with {rel['alias']}?",
                })
        conn.close()
    except Exception:
        pass

    # Strategy 8: Emotion monotony -- a scene with only one type of emotion
    try:
        ep = emotion_patterns(months=6)
        for scene, emotions in ep.get("scene_emotion_matrix", {}).items():
            if scene == "unknown":
                continue
            total = sum(emotions.values())
            if total >= 3 and len(emotions) == 1:
                emo = list(emotions.keys())[0]
                suggestions.append({
                    "dimension": "emotion",
                    "dimension_cn": "Emotion Patterns",
                    "scene": scene,
                    "priority": "P2",
                    "reason": f"Scene '{scene}' only has '{emo}' emotion recorded, needs more diverse records",
                    "sample_question": f"In the {scene} context, have you ever felt emotions other than {emo}?",
                })
    except Exception:
        pass

    return suggestions[:limit]


# ── Multimodal Question Engine (v4.0 Phase H) ─────────────────────────────

# Question modality templates: generate different styles of questions based on dimension and modality
_QUESTION_MODALITY = {
    "text": {
        "identity":     ["What do you see as the biggest difference between you and others?", "If you were an animal, what would you be? Why?",
                         "What is the biggest misconception others have about you?"],
        "value":        ["When did you last stick to your principles despite it being painful?", "What will you absolutely never compromise on?",
                         "What do you believe is 'right' that most people disagree with?"],
        "emotion":      ["When was the last time you couldn't hold back tears?", "What can make you instantly furious?",
                         "What are you most afraid of losing?"],
        "cognition":    ["When facing a dilemma, what's your first step?", "Do you trust intuition or data more?",
                         "How do you judge whether someone is trustworthy?"],
        "relationship": ["Whose relationship with you has subtly changed recently?", "Who do you most want to say something unsaid to?",
                         "What do you think is the most fundamental difference between friendship and love?"],
        "creation":     ["What creation have you been wanting to do but keep putting off?", "If ability weren't a constraint, what would you most want to do?",
                         "What do you consider your best work/achievement?"],
        "mission":      ["If you died tomorrow, what would you most regret today?", "What do you want to leave for the world?",
                         "What would the you in ten years thank the current you for doing?"],
    },
    "visual": {
        "identity":     ["Take a photo that you feel most represents you (doesn't have to be a selfie)", "What does your desk/workspace look like?",
                         "What's the one photo in your phone you can't bear to delete?"],
        "value":        ["Photograph something you find most valuable but not expensive", "What does your bookshelf/collection corner look like?"],
        "emotion":      ["What does the view outside your window look like right now?", "Photograph a corner that makes you feel at ease"],
        "creation":     ["What does your recent creation/drawing/writing look like?", "What do you look like when working?"],
        "relationship": ["Your most recent photo with your most important person", "Where do you most often gather with friends?"],
    },
    "sensory": {
        "identity":     ["What do you smell right now? What does that scent remind you of?", "What's your favorite type of tactile sensation?"],
        "emotion":      ["Describe how your body feels right now: where is tense? Where is relaxed?",
                         "What's the most memorable taste you've had recently?", "What sound can instantly calm you?"],
        "cognition":    ["What posture and environment do you prefer when thinking?"],
        "creation":     ["What background music and scent do you like when creating?"],
    },
    "film": {
        "identity":     ["If your life were a movie, what would it be called? What genre?",
                         "Which movie/show character is most like you?"],
        "value":        ["Which movie's ending do you most disagree with? How would you change it?",
                         "Which fictional character's choices best represent your values?"],
        "emotion":      ["Which song makes you cry every time? Why?", "Which movie scene have you watched many times?"],
        "cognition":    ["What documentary/book recently changed your mind?"],
        "mission":      ["If you were a game protagonist, what would your main quest be?"],
    },
}


def generate_question(dimension: str = None, modality: str = "text",
                      avoid_recent: bool = True) -> dict:
    """
    Generate a multimodal question.

    Args:
        dimension: target dimension (auto-selects weakest coverage if empty)
        modality: modality text/visual/sensory/film
        avoid_recent: whether to avoid recently asked questions

    Returns:
        {dimension, modality, question, reason, priority}
    """
    import random

    _dim_cn = {
        "identity": "Identity", "value": "Values", "emotion": "Emotion Patterns",
        "cognition": "Cognitive Style", "relationship": "Relationship Patterns",
        "creation": "Creation & Expression", "mission": "Mission & Meaning",
    }

    # Auto-select dimension
    if not dimension:
        suggestions = suggest_questions(limit=3)
        if suggestions:
            dimension = suggestions[0]["dimension"]
            if dimension == "all":
                dimension = random.choice(list(SOUL_DIMENSIONS))
        else:
            dimension = random.choice(list(SOUL_DIMENSIONS))

    if dimension not in SOUL_DIMENSIONS:
        return {"ok": False, "message": f"Invalid dimension: {dimension}"}

    if modality not in _QUESTION_MODALITY:
        return {"ok": False, "message": f"Invalid modality: {modality} (options: text/visual/sensory/film)"}

    # Get candidate questions
    candidates = _QUESTION_MODALITY.get(modality, {}).get(dimension, [])
    if not candidates:
        # Fall back to text modality
        candidates = _QUESTION_MODALITY["text"].get(dimension, [])
        modality = "text"

    if not candidates:
        return {"ok": False, "message": f"No available questions for dimension {dimension}"}

    # Avoid recently asked questions (check last 20 from narrative_turns)
    if avoid_recent:
        try:
            conn = _open_db()
            recent = conn.execute(
                "SELECT question FROM narrative_turns ORDER BY timestamp DESC LIMIT 20"
            ).fetchall()
            conn.close()
            recent_set = {r["question"] for r in recent}
            filtered = [q for q in candidates if q not in recent_set]
            if filtered:
                candidates = filtered
        except Exception:
            pass

    question = random.choice(candidates)

    return {
        "ok": True,
        "dimension": dimension,
        "dimension_cn": _dim_cn.get(dimension, dimension),
        "modality": modality,
        "question": question,
        "reason": f"A {modality}-modality question targeting the '{_dim_cn.get(dimension, dimension)}' dimension",
    }


# ── System Health ────────────────────────────────────────────────────

def soul_health() -> dict:
    """System health snapshot: record counts per table / chain length / vector coverage / DB size."""
    conn = _open_db()
    counts = {t: conn.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0] for t in SOUL_TABLES}
    chain_len    = conn.execute("SELECT COUNT(*) FROM chain_log").fetchone()[0]
    chunk_total  = conn.execute("SELECT COUNT(*) FROM soul_chunks").fetchone()[0]
    chunk_done   = conn.execute("SELECT COUNT(*) FROM soul_chunks WHERE vector_status='done'").fetchone()[0]
    chunk_pending= conn.execute("SELECT COUNT(*) FROM soul_chunks WHERE vector_status='pending'").fetchone()[0]
    chunk_failed = conn.execute("SELECT COUNT(*) FROM soul_chunks WHERE vector_status='failed'").fetchone()[0]
    conn.close()
    db_size_mb = round(DB_PATH.stat().st_size / (1024 * 1024), 2) if DB_PATH.exists() else 0
    coverage = f"{chunk_done / chunk_total * 100:.1f}%" if chunk_total > 0 else "0%"
    return {
        "counts": counts,
        "chain_length": chain_len,
        "db_size_mb": db_size_mb,
        "chunks": {
            "total": chunk_total, "done": chunk_done,
            "pending": chunk_pending, "failed": chunk_failed,
            "vector_coverage": coverage,
        },
    }


# ── Cross-table Search ────────────────────────────────────────────────────

def search_all(keyword: str, limit: int = 20) -> list:
    """FTS5 full-text search across all tables, falls back to souls LIKE search.
    Returns [{"table": ..., "row": sqlite3.Row}, ...]
    """
    conn = _open_db()
    results = []
    try:
        hits = conn.execute(
            "SELECT soul_id, table_name FROM soul_fts WHERE soul_fts MATCH ? LIMIT ?",
            (keyword, limit)
        ).fetchall()
        for h in hits:
            tname = h["table_name"]
            if tname not in SOUL_TABLES:
                continue
            row = conn.execute(
                f"SELECT * FROM {tname} WHERE id=?", (h["soul_id"],)
            ).fetchone()
            if row:
                results.append({"table": tname, "row": row})
    except Exception:
        # Fall back to LIKE when FTS fails
        q = f"%{_escape_like(keyword)}%"
        for row in conn.execute(
            "SELECT * FROM souls WHERE context LIKE ? ESCAPE '\\' OR human_decision LIKE ? ESCAPE '\\'"
            " OR divergence_reason LIKE ? ESCAPE '\\' ORDER BY timestamp DESC LIMIT ?",
            (q, q, q, limit)
        ).fetchall():
            results.append({"table": "souls", "row": row})
    conn.close()
    return results
