# Copyright (c) 2026 PULSE Contributors. Proprietary. All rights reserved.
"""Dedup Daemon — synaptic consolidation (merge similar, absorb duplicates)."""
import io, json, os, re, sys, time
from datetime import datetime, timezone
from pathlib import Path

if os.name == "nt" and __name__ == "__main__":
    if hasattr(sys.stdout, 'reconfigure'):
        sys.stdout.reconfigure(encoding='utf-8', errors='replace')
    if hasattr(sys.stderr, 'reconfigure'):
        sys.stderr.reconfigure(encoding='utf-8', errors='replace')

SCRIPT_DIR = Path(__file__).resolve().parent
ROOT_DIR = SCRIPT_DIR.parent.parent.parent.parent
if str(ROOT_DIR) not in sys.path:
    sys.path.insert(0, str(ROOT_DIR))

from pulse.core.brain_utils import (
    EVIDENCE_DIR, PRIVATE_EVIDENCE_DIR, PATTERNS_DIR, LESSONS_DIR,
    METADATA_DIR, load_json, save_json, text_similarity, atomic_update_json,
    now_iso as _now_iso,
)

EXACT_THRESHOLD = 0.95
MERGE_THRESHOLD = 0.75
MIN_DESCRIPTION_LEN = 10

_CONSOL_COUNT_RE = re.compile(r'\((\d+) consolidations?\)')

def _get_text(item: dict) -> str:
    """Extract comparable text from an evidence item."""
    parts = []
    for key in ("description", "text", "pattern", "summary", "title"):
        v = item.get(key, "")
        if v:
            parts.append(str(v))
    return " ".join(parts).strip()


def _merge_items(keeper: dict, absorbed: dict) -> dict:
    """Merge absorbed item INTO keeper, strengthening the keeper."""
    # Bump consolidation count
    keeper["consolidation_count"] = keeper.get("consolidation_count", 1) + 1

    # Keep the longer/richer description
    keeper_text = _get_text(keeper)
    absorbed_text = _get_text(absorbed)
    if len(absorbed_text) > len(keeper_text):
        for key in ("description", "text", "summary"):
            if key in absorbed and absorbed[key]:
                keeper[key] = absorbed[key]

    # Merge tags/capabilities
    for list_key in ("tags", "capabilities", "required_capabilities"):
        if list_key in absorbed and isinstance(absorbed[list_key], list):
            existing = set(keeper.get(list_key, []))
            existing.update(absorbed[list_key])
            keeper[list_key] = sorted(existing)

    # Keep higher confidence
    if "confidence" in absorbed:
        keeper["confidence"] = max(
            keeper.get("confidence", 0),
            absorbed.get("confidence", 0)
        )

    # Track merge history
    keeper["_last_consolidated"] = _now_iso()

    return keeper


def dedup_json_file(path: Path, dry_run=False, verbose=False) -> dict:
    """Deduplicate a single JSON evidence file. Returns stats."""
    items = load_json(path)
    if not isinstance(items, list) or len(items) < 2:
        return {"file": path.name, "original": len(items) if isinstance(items, list) else 0,
                "merged": 0, "removed": 0}

    # Build text index
    texts = []
    for item in items:
        texts.append(_get_text(item))

    # Find merge pairs (greedy: first match wins)
    absorbed_indices = set()
    merge_count = 0

    # Size gate: files > 5000 items would cause O(n²) explosion (e.g. 10,684 items = 114M comparisons).
    # For large files, skip similarity dedup and rely on fingerprint-based dedup (O(n)) only.
    _SIMILARITY_SIZE_LIMIT = 5000
    run_similarity = len(items) <= _SIMILARITY_SIZE_LIMIT

    if not run_similarity and verbose:
        print(f"  [DEDUP] Skipping O(n²) similarity dedup for large file ({len(items)} items > {_SIMILARITY_SIZE_LIMIT} limit) — fingerprint dedup only")

    if run_similarity:
        for i in range(len(items)):
            if i in absorbed_indices:
                continue
            if len(texts[i]) < MIN_DESCRIPTION_LEN:
                continue

            for j in range(i + 1, len(items)):
                if j in absorbed_indices:
                    continue
                if len(texts[j]) < MIN_DESCRIPTION_LEN:
                    continue

                sim = text_similarity(texts[i], texts[j])
                if sim >= MERGE_THRESHOLD:
                    if verbose:
                        label = "ABSORB" if sim >= EXACT_THRESHOLD else "MERGE"
                        print(f"  {label} [{sim:.2f}]: '{texts[i][:40]}' + '{texts[j][:40]}'")
                    items[i] = _merge_items(items[i], items[j])
                    absorbed_indices.add(j)
                    merge_count += 1

    # Fingerprint-based dedup (O(n)) — always runs regardless of file size
    seen_fps: dict = {}
    for idx, item in enumerate(items):
        if idx in absorbed_indices:
            continue
        fp = item.get("fingerprint", "")
        if fp and len(fp) > 4:
            if fp in seen_fps:
                keeper_idx = seen_fps[fp]
                items[keeper_idx] = _merge_items(items[keeper_idx], item)
                absorbed_indices.add(idx)
                merge_count += 1
            else:
                seen_fps[fp] = idx

    if merge_count > 0 and not dry_run:
        consolidated = [item for idx, item in enumerate(items) if idx not in absorbed_indices]
        save_json(path, consolidated)

    return {
        "file": path.name,
        "original": len(items),
        "merged": merge_count,
        "remaining": len(items) - len(absorbed_indices),
    }


def _bump_markdown_consolidation_count(lines: list[str]) -> list[str]:
    """Increment **Consolidation Count:** N in a list of markdown lines."""
    for i, line in enumerate(lines):
        m = _CONSOL_COUNT_RE.search(line)
        if m:
            new_count = int(m.group(2)) + 1
            lines[i] = _CONSOL_COUNT_RE.sub(rf"\g<1>{new_count}", line)
            return lines
    return lines


def run_full_dedup(dry_run=False, verbose=False):
    """Run deduplication across all brain chambers."""
    print(f"[DEDUP] {'DRY RUN — ' if dry_run else ''}Synaptic Consolidation starting...")
    start = time.time()

    # === Episodic Evidence (JSON) ===
    print("[DEDUP] Scanning Episodic evidence...")
    json_stats = []
    for evidence_dir in [EVIDENCE_DIR, PRIVATE_EVIDENCE_DIR]:
        if not evidence_dir.exists():
            continue
        for json_file in sorted(evidence_dir.rglob("*.json")):
            if json_file.name.startswith("_"):
                continue
            stats = dedup_json_file(json_file, dry_run=dry_run, verbose=verbose)
            if stats["merged"] > 0:
                json_stats.append(stats)
                if verbose:
                    print(f"  {stats['file']}: {stats['original']} -> {stats['remaining']} "
                          f"({stats['merged']} merged)")

    # === JSONL Brain Files (replaces MD dedup) ===
    print("[DEDUP] Scanning JSONL brain files...")
    from pulse.core.brain_utils import LESSONS_JSONL, FAILS_JSONL, PATTERNS_JSONL, read_jsonl_entries
    from pulse.brain.save_writers import rewrite_jsonl
    pattern_stats = []
    lesson_stats = []
    for jsonl_path, label in [(LESSONS_JSONL, "lessons"), (FAILS_JSONL, "failures"), (PATTERNS_JSONL, "patterns")]:
        rows = read_jsonl_entries(jsonl_path)
        if len(rows) < 2:
            continue
        absorbed = set()
        for i in range(len(rows)):
            if i in absorbed:
                continue
            ti = (rows[i].get("content") or rows[i].get("title", ""))[:200]
            if len(ti) < MIN_DESCRIPTION_LEN:
                continue
            for j in range(i + 1, len(rows)):
                if j in absorbed:
                    continue
                tj = (rows[j].get("content") or rows[j].get("title", ""))[:200]
                if len(tj) < MIN_DESCRIPTION_LEN:
                    continue
                sim = text_similarity(ti[:80], tj[:80])
                if sim >= MERGE_THRESHOLD:
                    rows[i] = _merge_items(rows[i], rows[j])
                    absorbed.add(j)
                    if verbose:
                        print(f"  JSONL MERGE [{sim:.2f}]: '{ti[:60]}...'")
        if absorbed and not dry_run:
            surviving = [r for idx, r in enumerate(rows) if idx not in absorbed]
            # SQLite-first: delete absorbed + update winners, JSONL fallback
            try:
                from pulse.brain.save_writers.jsonl_store import _jsonl_to_table
                from pulse.core.brain_db import get_conn
                table = _jsonl_to_table(jsonl_path)
                if table:
                    conn = get_conn()
                    absorbed_fps = [rows[idx].get("fingerprint", "") for idx in absorbed if rows[idx].get("fingerprint")]
                    with conn:
                        # Clean FTS BEFORE deleting source rows (subquery needs them)
                        if absorbed_fps:
                            ph = ",".join("?" for _ in absorbed_fps)
                            conn.execute(
                                f"DELETE FROM knowledge_fts WHERE table_name=? AND item_id IN "
                                f"(SELECT id FROM {table} WHERE fingerprint IN ({ph}))",
                                [table] + absorbed_fps,
                            )
                        for fp in absorbed_fps:
                            conn.execute(f"DELETE FROM {table} WHERE fingerprint = ?", (fp,))
                        for idx, r in enumerate(rows):
                            if idx not in absorbed and r.get("fingerprint"):
                                conn.execute(
                                    f"UPDATE {table} SET consolidation_count = ?, confidence = ?, "
                                    f"metadata = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%SZ','now') "
                                    f"WHERE fingerprint = ?",
                                    (int(r.get("consolidation_count", 1)),
                                     float(r.get("confidence", 0.5)),
                                     json.dumps({k: v for k, v in r.items() if k.startswith("_")}),
                                     r["fingerprint"]),
                                )
                _dedup_sqlite_ok = True
            except Exception:
                _dedup_sqlite_ok = False  # SQLite unavailable — JSONL rewrite below is the fallback
            # P47 FIX: Only fall back to JSONL rewrite if SQLite failed
            # (unconditional rewrite_jsonl was clobbering backfilled confidence values)
            if not _dedup_sqlite_ok:
                rewrite_jsonl(jsonl_path, surviving)
        stat = {"file": jsonl_path.name, "sections": len(rows), "merged": len(absorbed)}
        if label == "patterns":
            pattern_stats.append(stat)
        else:
            lesson_stats.append(stat)

    total_json_merged = sum(s["merged"] for s in json_stats)
    total_md_merged = sum(s["merged"] for s in pattern_stats + lesson_stats)
    elapsed = time.time() - start
    print(f"\n[DEDUP] Done in {elapsed:.1f}s — {total_json_merged} evidence + {total_md_merged} sections merged")
    log = {
        "timestamp": _now_iso(), "dry_run": dry_run,
        "json_stats": json_stats,
        "pattern_stats": [s for s in pattern_stats if s["merged"] > 0],
        "lesson_stats": [s for s in lesson_stats if s["merged"] > 0],
        "totals": {"evidence_merged": total_json_merged, "sections_merged": total_md_merged,
                   "elapsed_seconds": round(elapsed, 2)},
    }
    save_json(METADATA_DIR / "dedup_log.json", log)
    return log

if __name__ == "__main__":
    run_full_dedup(dry_run="--dry-run" in sys.argv, verbose="--verbose" in sys.argv or "-v" in sys.argv)
