# Copyright (c) 2026 PULSE Contributors. Proprietary. All rights reserved.
"""Dedup Daemon — synaptic consolidation (merge similar, absorb duplicates).

P6-1: Reads from SQLite (authoritative), not JSONL (stale).
P6-2: Cross-table dedup (intents<->evidence overlap).
P6-4: O(n) fingerprint + content-hash dedup replaces O(n^2) similarity.
"""
import hashlib, json, os, re, sys, time
from datetime import datetime, timezone
from pathlib import Path

if os.name == "nt" and __name__ == "__main__":
    if hasattr(sys.stdout, 'reconfigure'):
        sys.stdout.reconfigure(encoding='utf-8', errors='replace')
    if hasattr(sys.stderr, 'reconfigure'):
        sys.stderr.reconfigure(encoding='utf-8', errors='replace')

SCRIPT_DIR = Path(__file__).resolve().parent
ROOT_DIR = SCRIPT_DIR.parent.parent.parent.parent
if str(ROOT_DIR) not in sys.path:
    sys.path.insert(0, str(ROOT_DIR))

from pulse.core.brain_utils import (
    EVIDENCE_DIR, PRIVATE_EVIDENCE_DIR, METADATA_DIR,
    load_json, save_json, text_similarity,
    now_iso as _now_iso,
)

EXACT_THRESHOLD = 0.95
MERGE_THRESHOLD = 0.75
MIN_DESCRIPTION_LEN = 10

_CONSOL_COUNT_RE = re.compile(r'\((\d+) consolidations?\)')


def _get_text(item: dict) -> str:
    """Extract comparable text from an evidence item."""
    parts = []
    for key in ("description", "text", "pattern", "summary", "title"):
        v = item.get(key, "")
        if v:
            parts.append(str(v))
    return " ".join(parts).strip()


def _merge_items(keeper: dict, absorbed: dict) -> dict:
    """Merge absorbed item INTO keeper, strengthening the keeper."""
    keeper["consolidation_count"] = keeper.get("consolidation_count", 1) + 1
    keeper_text = _get_text(keeper)
    absorbed_text = _get_text(absorbed)
    if len(absorbed_text) > len(keeper_text):
        for key in ("description", "text", "summary"):
            if key in absorbed and absorbed[key]:
                keeper[key] = absorbed[key]
    for list_key in ("tags", "capabilities", "required_capabilities"):
        if list_key in absorbed and isinstance(absorbed[list_key], list):
            existing = set(keeper.get(list_key, []))
            existing.update(absorbed[list_key])
            keeper[list_key] = sorted(existing)
    if "confidence" in absorbed:
        keeper["confidence"] = max(
            keeper.get("confidence", 0),
            absorbed.get("confidence", 0)
        )
    keeper["_last_consolidated"] = _now_iso()
    return keeper


def dedup_json_file(path: Path, dry_run=False, verbose=False) -> dict:
    """Deduplicate a single JSON evidence file. Returns stats."""
    items = load_json(path)
    if not isinstance(items, list) or len(items) < 2:
        return {"file": path.name, "original": len(items) if isinstance(items, list) else 0,
                "merged": 0, "removed": 0}

    texts = []
    for item in items:
        texts.append(_get_text(item))

    absorbed_indices = set()
    merge_count = 0

    # Size gate: skip O(n^2) similarity for large files, use fingerprint-only
    _SIMILARITY_SIZE_LIMIT = 5000
    run_similarity = len(items) <= _SIMILARITY_SIZE_LIMIT

    if not run_similarity and verbose:
        print(f"  [DEDUP] Skipping O(n^2) similarity for large file ({len(items)} items)")

    if run_similarity:
        for i in range(len(items)):
            if i in absorbed_indices:
                continue
            if len(texts[i]) < MIN_DESCRIPTION_LEN:
                continue
            for j in range(i + 1, len(items)):
                if j in absorbed_indices:
                    continue
                if len(texts[j]) < MIN_DESCRIPTION_LEN:
                    continue
                sim = text_similarity(texts[i], texts[j])
                if sim >= MERGE_THRESHOLD:
                    items[i] = _merge_items(items[i], items[j])
                    absorbed_indices.add(j)
                    merge_count += 1

    # Fingerprint-based dedup (O(n)) — always runs
    seen_fps: dict = {}
    for idx, item in enumerate(items):
        if idx in absorbed_indices:
            continue
        fp = item.get("fingerprint", "")
        if fp and len(fp) > 4:
            if fp in seen_fps:
                keeper_idx = seen_fps[fp]
                items[keeper_idx] = _merge_items(items[keeper_idx], item)
                absorbed_indices.add(idx)
                merge_count += 1
            else:
                seen_fps[fp] = idx

    if merge_count > 0 and not dry_run:
        consolidated = [item for idx, item in enumerate(items) if idx not in absorbed_indices]
        save_json(path, consolidated)

    return {
        "file": path.name,
        "original": len(items),
        "merged": merge_count,
        "remaining": len(items) - len(absorbed_indices),
    }


def _cross_table_dedup(conn, verbose=False):
    """Phase 3: Remove rows that exist in multiple tables (keep higher-value table)."""
    try:
        # intents<->evidence: keep intent, drop evidence
        _pairs = [
            ("evidence", "intents", "SELECT e.id FROM evidence e INNER JOIN intents i ON SUBSTR(e.content,1,80)=SUBSTR(i.content,1,80) LIMIT 500"),
            ("intents", "lessons", "SELECT i.id FROM intents i INNER JOIN lessons l ON SUBSTR(i.content,1,80)=SUBSTR(l.content,1,80) LIMIT 500"),
            ("intents", "patterns", "SELECT i.id FROM intents i INNER JOIN patterns p ON SUBSTR(i.content,1,80)=SUBSTR(p.content,1,80) LIMIT 500"),
            ("intents", "failures", "SELECT i.id FROM intents i INNER JOIN failures f ON SUBSTR(i.content,1,80)=SUBSTR(f.content,1,80) LIMIT 500"),
        ]
        for drop_table, keep_table, query in _pairs:
            rows = conn.execute(query).fetchall()
            if rows:
                with conn:
                    for r in rows:
                        conn.execute(f"DELETE FROM {drop_table} WHERE id = ?", (r["id"],))
                print(f"  CROSS-TABLE: {len(rows)} {drop_table} rows duplicated in {keep_table} -- removed")
    except Exception as e:
        if verbose:
            print(f"  Cross-table dedup error: {e}")


def run_full_dedup(dry_run=False, verbose=False):
    """Run deduplication across all brain chambers."""
    print(f"[DEDUP] {'DRY RUN — ' if dry_run else ''}Synaptic Consolidation starting...")
    start = time.time()

    # === Episodic Evidence (JSON files) ===
    print("[DEDUP] Scanning Episodic evidence...")
    json_stats = []
    for evidence_dir in [EVIDENCE_DIR, PRIVATE_EVIDENCE_DIR]:
        if not evidence_dir.exists():
            continue
        for json_file in sorted(evidence_dir.rglob("*.json")):
            if json_file.name.startswith("_"):
                continue
            stats = dedup_json_file(json_file, dry_run=dry_run, verbose=verbose)
            if stats["merged"] > 0:
                json_stats.append(stats)
                if verbose:
                    print(f"  {stats['file']}: {stats['original']} -> {stats['remaining']} "
                          f"({stats['merged']} merged)")

    # === SQLite Knowledge Tables (P6-1: reads from SQLite, not JSONL) ===
    print("[DEDUP] Scanning SQLite knowledge tables...")
    pattern_stats = []
    lesson_stats = []
    _DEDUP_TABLES = [("lessons", "lessons"), ("failures", "failures"), ("patterns", "patterns"),
                      ("intents", "intents"), ("facts", "facts"), ("evidence", "evidence")]
    try:
        from pulse.core.brain_db import get_conn
        conn = get_conn()
    except Exception as _db_err:
        print(f"  [DEDUP] SQLite unavailable: {_db_err} — skipping brain dedup")
        conn = None

    if conn:
        for table, label in _DEDUP_TABLES:
            try:
                rows = conn.execute(
                    f"SELECT id, content, title, fingerprint, confidence, "
                    f"consolidation_count FROM {table}"
                ).fetchall()
            except Exception:
                continue
            if len(rows) < 2:
                continue

            # Phase 1: Fingerprint-based exact dedup (O(n))
            fp_groups: dict[str, list] = {}
            for r in rows:
                fp = r["fingerprint"] or ""
                if fp and len(fp) > 4:
                    fp_groups.setdefault(fp, []).append(dict(r))
            absorbed_ids = set()
            for fp, group in fp_groups.items():
                if len(group) < 2:
                    continue
                group.sort(key=lambda x: -(x["confidence"] or 0))
                for dup in group[1:]:
                    absorbed_ids.add(dup["id"])
                    if verbose:
                        print(f"  FP-DEDUP [{table}]: absorb id={dup['id']} fp={fp[:12]}")

            # Phase 2: Content-hash dedup (O(n)) — first 80 chars normalized
            content_groups: dict[str, list] = {}
            for r in rows:
                rid = r["id"]
                if rid in absorbed_ids:
                    continue
                text = (r["content"] or r["title"] or "")[:80].lower().strip()
                if len(text) < MIN_DESCRIPTION_LEN:
                    continue
                ch = hashlib.md5(text.encode("utf-8")).hexdigest()[:12]
                content_groups.setdefault(ch, []).append(dict(r))
            for ch, group in content_groups.items():
                if len(group) < 2:
                    continue
                group.sort(key=lambda x: -(x["confidence"] or 0))
                for dup in group[1:]:
                    absorbed_ids.add(dup["id"])
                    if verbose:
                        print(f"  CONTENT-DEDUP [{table}]: absorb id={dup['id']}")

            merge_count = len(absorbed_ids)
            if merge_count > 0 and not dry_run:
                try:
                    with conn:
                        for aid in absorbed_ids:
                            conn.execute(
                                "DELETE FROM knowledge_fts WHERE table_name=? AND item_id=?",
                                (table, aid),
                            )
                            conn.execute(f"DELETE FROM {table} WHERE id = ?", (aid,))
                except Exception as _del_err:
                    print(f"  [DEDUP] Delete failed for {table}: {_del_err}")
                    merge_count = 0

            stat = {"file": table, "sections": len(rows), "merged": merge_count}
            if label == "patterns":
                pattern_stats.append(stat)
            else:
                lesson_stats.append(stat)

        # Phase 3: Cross-table dedup (P6-2)
        if not dry_run:
            _cross_table_dedup(conn, verbose)


    total_json_merged = sum(s["merged"] for s in json_stats)
    total_md_merged = sum(s["merged"] for s in pattern_stats + lesson_stats)
    elapsed = time.time() - start
    print(f"\n[DEDUP] Done in {elapsed:.1f}s — {total_json_merged} evidence + {total_md_merged} sections merged")
    log = {
        "timestamp": _now_iso(), "dry_run": dry_run,
        "json_stats": json_stats,
        "pattern_stats": [s for s in pattern_stats if s["merged"] > 0],
        "lesson_stats": [s for s in lesson_stats if s["merged"] > 0],
        "totals": {"evidence_merged": total_json_merged, "sections_merged": total_md_merged,
                   "elapsed_seconds": round(elapsed, 2)},
    }
    return log


if __name__ == "__main__":
    run_full_dedup(dry_run="--dry-run" in sys.argv, verbose="--verbose" in sys.argv or "-v" in sys.argv)
