#!/usr/bin/env python3
# Copyright (c) 2026 PULSE Contributors. Proprietary. All rights reserved.
# See LICENSE file for details.
"""Dedup Daemon — synaptic consolidation (merge similar, absorb duplicates).

Covers: Episodic JSON, Semantic patterns .md, Implicit lessons .md.
Usage: python pulse_dedup.py [--verbose] [--dry-run]
"""
import io
import json
import os
import re
import sys
import time
from datetime import datetime, timezone
from pathlib import Path

# Windows Unicode fix (cp1252 can't handle emoji in evidence text)
if os.name == "nt" and __name__ == "__main__":
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace")

SCRIPT_DIR = Path(__file__).resolve().parent
ROOT_DIR = SCRIPT_DIR.parent.parent.parent

# When run directly, ensure pulse package is importable
if str(ROOT_DIR) not in sys.path:
    sys.path.insert(0, str(ROOT_DIR))

from pulse.core.brain_utils import (
    EVIDENCE_DIR, PRIVATE_EVIDENCE_DIR, PATTERNS_DIR, LESSONS_DIR,
    METADATA_DIR, load_json, save_json, text_similarity, atomic_update_json,
    now_iso as _now_iso,
)

EXACT_THRESHOLD = 0.95
MERGE_THRESHOLD = 0.75
MIN_DESCRIPTION_LEN = 10

def _get_text(item: dict) -> str:
    """Extract comparable text from an evidence item."""
    parts = []
    for key in ("description", "text", "pattern", "summary", "title"):
        v = item.get(key, "")
        if v:
            parts.append(str(v))
    return " ".join(parts).strip()


def _merge_items(keeper: dict, absorbed: dict) -> dict:
    """Merge absorbed item INTO keeper, strengthening the keeper."""
    # Bump consolidation count
    keeper["consolidation_count"] = keeper.get("consolidation_count", 1) + 1

    # Keep the longer/richer description
    keeper_text = _get_text(keeper)
    absorbed_text = _get_text(absorbed)
    if len(absorbed_text) > len(keeper_text):
        for key in ("description", "text", "summary"):
            if key in absorbed and absorbed[key]:
                keeper[key] = absorbed[key]

    # Merge tags/capabilities
    for list_key in ("tags", "capabilities", "required_capabilities"):
        if list_key in absorbed and isinstance(absorbed[list_key], list):
            existing = set(keeper.get(list_key, []))
            existing.update(absorbed[list_key])
            keeper[list_key] = sorted(existing)

    # Keep higher confidence
    if "confidence" in absorbed:
        keeper["confidence"] = max(
            keeper.get("confidence", 0),
            absorbed.get("confidence", 0)
        )

    # Track merge history
    keeper["_last_consolidated"] = _now_iso()

    return keeper


def dedup_json_file(path: Path, dry_run=False, verbose=False) -> dict:
    """Deduplicate a single JSON evidence file. Returns stats."""
    items = load_json(path)
    if not isinstance(items, list) or len(items) < 2:
        return {"file": path.name, "original": len(items) if isinstance(items, list) else 0,
                "merged": 0, "removed": 0}

    # Build text index
    texts = []
    for item in items:
        texts.append(_get_text(item))

    # Find merge pairs (greedy: first match wins)
    absorbed_indices = set()
    merge_count = 0

    for i in range(len(items)):
        if i in absorbed_indices:
            continue
        if len(texts[i]) < MIN_DESCRIPTION_LEN:
            continue

        for j in range(i + 1, len(items)):
            if j in absorbed_indices:
                continue
            if len(texts[j]) < MIN_DESCRIPTION_LEN:
                continue

            sim = text_similarity(texts[i], texts[j])

            if sim >= EXACT_THRESHOLD:
                # Exact duplicate — absorb silently
                if verbose:
                    print(f"  ABSORB [{sim:.2f}]: '{texts[j][:60]}...'")
                items[i] = _merge_items(items[i], items[j])
                absorbed_indices.add(j)
                merge_count += 1

            elif sim >= MERGE_THRESHOLD:
                # Similar — merge and strengthen
                if verbose:
                    print(f"  MERGE  [{sim:.2f}]: '{texts[i][:40]}' + '{texts[j][:40]}'")
                items[i] = _merge_items(items[i], items[j])
                absorbed_indices.add(j)
                merge_count += 1

    if merge_count > 0 and not dry_run:
        # Write consolidated list (only non-absorbed items)
        consolidated = [item for idx, item in enumerate(items) if idx not in absorbed_indices]
        save_json(path, consolidated)

    return {
        "file": path.name,
        "original": len(items),
        "merged": merge_count,
        "remaining": len(items) - len(absorbed_indices),
    }

_CONSOL_COUNT_RE = re.compile(r"(\*\*Consolidation Count:\*\*\s*)(\d+)")

def _bump_markdown_consolidation_count(lines: list[str]) -> list[str]:
    """Increment **Consolidation Count:** N in a list of markdown lines."""
    for i, line in enumerate(lines):
        m = _CONSOL_COUNT_RE.search(line)
        if m:
            new_count = int(m.group(2)) + 1
            lines[i] = _CONSOL_COUNT_RE.sub(rf"\g<1>{new_count}", line)
            return lines
    return lines


def dedup_markdown_dir(base_dir: Path, dry_run=False, verbose=False) -> list[dict]:
    """Deduplicate markdown sections across files in a directory.
    Finds ## sections with similar titles and merges their content."""
    if not base_dir.exists():
        return []

    stats = []
    # Collect all sections across all files
    sections_by_file = {}

    for md_file in sorted(base_dir.rglob("*.md")):
        if md_file.name.startswith("_"):
            continue
        try:
            text = md_file.read_text(encoding="utf-8", errors="replace")
        except OSError:
            continue

        sections = []
        current = None
        for line in text.splitlines():
            if line.startswith("## "):
                if current:
                    sections.append(current)
                current = {"title": line[3:].strip(), "lines": [line], "file": md_file}
            elif current is not None:
                current["lines"].append(line)
            # Lines before first section are preamble — keep as-is

        if current:
            sections.append(current)
        if sections:
            sections_by_file[md_file] = sections

    # Find duplicate sections WITHIN each file
    total_merged = 0
    for md_file, sections in sections_by_file.items():
        absorbed = set()
        for i in range(len(sections)):
            if i in absorbed:
                continue
            for j in range(i + 1, len(sections)):
                if j in absorbed:
                    continue
                sim = text_similarity(sections[i]["title"], sections[j]["title"])
                if sim >= MERGE_THRESHOLD:
                    if verbose:
                        print(f"  MD MERGE [{sim:.2f}]: '{sections[i]['title']}' + '{sections[j]['title']}' in {md_file.name}")
                    # Append j's content to i
                    sections[i]["lines"].extend(sections[j]["lines"][1:])  # Skip duplicate ## header
                    _bump_markdown_consolidation_count(sections[i]["lines"])
                    absorbed.add(j)
                    total_merged += 1

        if absorbed and not dry_run:
            # Rebuild file
            try:
                text = md_file.read_text(encoding="utf-8", errors="replace")
            except OSError:
                continue

            # Extract preamble (everything before first ##)
            preamble_lines = []
            for line in text.splitlines():
                if line.startswith("## "):
                    break
                preamble_lines.append(line)

            # Write preamble + consolidated sections
            new_lines = preamble_lines
            for idx, sec in enumerate(sections):
                if idx not in absorbed:
                    new_lines.extend(sec["lines"])
                    new_lines.append("")  # Blank line between sections

            md_file.write_text("\n".join(new_lines), encoding="utf-8")

        stats.append({
            "file": md_file.name,
            "sections": len(sections),
            "merged": len(absorbed),
        })

    return stats


def run_full_dedup(dry_run=False, verbose=False):
    """Run deduplication across all brain chambers."""
    print(f"[DEDUP] {'DRY RUN — ' if dry_run else ''}Synaptic Consolidation starting...")
    start = time.time()

    # === Episodic Evidence (JSON) ===
    print("[DEDUP] Scanning Episodic evidence...")
    json_stats = []
    for evidence_dir in [EVIDENCE_DIR, PRIVATE_EVIDENCE_DIR]:
        if not evidence_dir.exists():
            continue
        for json_file in sorted(evidence_dir.rglob("*.json")):
            if json_file.name.startswith("_"):
                continue
            stats = dedup_json_file(json_file, dry_run=dry_run, verbose=verbose)
            if stats["merged"] > 0:
                json_stats.append(stats)
                if verbose:
                    print(f"  {stats['file']}: {stats['original']} -> {stats['remaining']} "
                          f"({stats['merged']} merged)")

    # === Semantic Patterns (Markdown) ===
    print("[DEDUP] Scanning Semantic patterns...")
    pattern_stats = dedup_markdown_dir(PATTERNS_DIR, dry_run=dry_run, verbose=verbose)

    # === Implicit Lessons (Markdown) ===
    print("[DEDUP] Scanning Implicit lessons...")
    lesson_stats = dedup_markdown_dir(LESSONS_DIR, dry_run=dry_run, verbose=verbose)

    # Summary
    total_json_merged = sum(s["merged"] for s in json_stats)
    total_md_merged = sum(s["merged"] for s in pattern_stats + lesson_stats)
    elapsed = time.time() - start

    print(f"\n[DEDUP] Consolidation complete in {elapsed:.1f}s")
    print(f"  Evidence items merged: {total_json_merged}")
    print(f"  Markdown sections merged: {total_md_merged}")

    # Write consolidation log
    log = {
        "timestamp": _now_iso(),
        "dry_run": dry_run,
        "json_stats": json_stats,
        "pattern_stats": [s for s in pattern_stats if s["merged"] > 0],
        "lesson_stats": [s for s in lesson_stats if s["merged"] > 0],
        "totals": {
            "evidence_merged": total_json_merged,
            "markdown_merged": total_md_merged,
            "elapsed_seconds": round(elapsed, 2),
        },
    }
    log_path = METADATA_DIR / "dedup_log.json"
    save_json(log_path, log)

    return log


if __name__ == "__main__":
    dry = "--dry-run" in sys.argv
    verb = "--verbose" in sys.argv or "-v" in sys.argv
    run_full_dedup(dry_run=dry, verbose=verb)
