#!/usr/bin/env python3
# Copyright (c) 2026 PULSE Contributors. Proprietary. All rights reserved.
"""Consolidation Stages (extracted per Law 7 from consolidation_daemon.py)."""
import hashlib
import subprocess
import sys
from datetime import datetime, timezone, timedelta
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
ROOT_DIR = SCRIPT_DIR.parent.parent.parent
STATE_DIR = ROOT_DIR / "state"


import json
import logging
import re
from pulse.core.brain_utils import (
    HIPPOCAMPUS_DIR, PATTERNS_DIR, LESSONS_DIR, FAILS_DIR, EVIDENCE_METRICS_DIR,
    load_json, load_json_dict, save_json, text_similarity,
    now_iso as _now_iso,
)
from pulse.core.brain_io import _acquire

# === Bounded hash tracking to prevent re-processing across cycles ===

PROCESSED_HASHES_FILE = STATE_DIR / "consolidation_hashes.json"
MAX_TRACKED_HASHES = 5000  # Bounded to prevent infinite growth (~200KB max)


def _load_processed_hashes() -> dict[str, None]:
    """Load processed hashes as an insertion-ordered dict."""
    if PROCESSED_HASHES_FILE.exists():
        try:
            data = json.loads(PROCESSED_HASHES_FILE.read_text(encoding="utf-8"))
            return {h: None for h in data.get("hashes", [])}
        except Exception:
            pass
    return {}


def _save_processed_hashes(hashes: dict[str, None]):
    """Save bounded hashes (keeps most recent MAX_TRACKED_HASHES via LRU eviction)."""
    # Evict oldest if over limit (dict preserves insertion order in Python 3.7+)
    while len(hashes) > MAX_TRACKED_HASHES:
        oldest = next(iter(hashes))
        del hashes[oldest]
    STATE_DIR.mkdir(parents=True, exist_ok=True)
    with _acquire(PROCESSED_HASHES_FILE):
        PROCESSED_HASHES_FILE.write_text(
            json.dumps({"hashes": list(hashes.keys()), "count": len(hashes),
                         "updated": datetime.now(timezone.utc).isoformat()}),
            encoding="utf-8"
        )


def _entry_hash(text: str) -> str:
    """Hash first 80 chars for cross-cycle tracking."""
    return hashlib.md5(text[:80].lower().strip().encode()).hexdigest()
from .consolidation_helpers import (
    extract_event_sequences, mine_cross_session_patterns,
    register_discovered_procedures,
    promote_cross_patterns_to_schemas,
    promote_sequences_to_schemas,
)

DEDUP_SCRIPT = SCRIPT_DIR.parent / "synthesis" / "pulse_dedup.py"

def stage_dedup(verbose=False):
    """Run existing dedup logic as subprocess."""
    if not DEDUP_SCRIPT.exists():
        return {"status": "skipped", "reason": "dedup script missing"}
    try:
        result = subprocess.run(
            [sys.executable, str(DEDUP_SCRIPT)],
            capture_output=True, text=True, timeout=120,
        )
        lines = result.stdout.strip().splitlines()
        if verbose:
            for line in lines:
                print(f"  [DEDUP] {line}")
        merged = 0
        for line in lines:
            if "items merged:" in line.lower():
                try:
                    merged = int(line.split(":")[-1].strip())
                except (ValueError, IndexError):
                    logging.debug("Suppressed exception in consolidation_stages", exc_info=True)
        return {"status": "ok", "merged": merged}
    except Exception as e:
        return {"status": "error", "error": str(e)}

def stage_pattern_mining(verbose=False):
    """Stage 2: Mine patterns from past 7 days."""
    sequences = extract_event_sequences(days=7, verbose=verbose)
    cross_patterns = mine_cross_session_patterns(days=7, verbose=verbose)
    return {
        "status": "ok",
        "sequences": sequences,
        "cross_patterns": cross_patterns,
        "total_found": len(sequences) + len(cross_patterns),
    }

def stage_schema_promotion(cross_patterns, verbose=False):
    """Promote high-confidence cross-session patterns to schemas.json."""
    return promote_cross_patterns_to_schemas(cross_patterns, verbose=verbose)

def stage_schema_promotion_from_sequences(sequences, verbose=False):
    """Promote event sequences with 5+ occurrences to schemas.json."""
    return promote_sequences_to_schemas(sequences, verbose=verbose)

def stage_ltd_decay(verbose=False):
    """Flag items never retrieved in 30+ days (Long-Term Depression)."""
    from pulse.core.brain_utils import RETRIEVAL_METRICS, EVIDENCE_SESSIONS_DIR
    import hashlib
    metrics = load_json_dict(RETRIEVAL_METRICS)
    total_queries = metrics.get("total_queries", 0)
    if total_queries < 10:
        if verbose:
            print(f"  [LTD] Skipped — only {total_queries} queries (need 10+)")
        return {"status": "skipped", "reason": f"only {total_queries} queries", "flagged": 0}
    retrieved_keys = set(metrics.get("items", {}).keys())
    cutoff = datetime.now(timezone.utc) - timedelta(days=30)
    session_file = EVIDENCE_SESSIONS_DIR / "current_session.json"
    items = load_json(session_file)
    if not isinstance(items, list):
        return {"status": "skipped", "reason": "no session data", "flagged": 0}
    flagged = 0
    for item in items:
        if item.get("_ltd_flagged"):
            continue
        ts_str = item.get("timestamp", "")
        if not ts_str:
            continue
        try:
            ts = datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
        except (ValueError, TypeError):
            continue
        if ts > cutoff:
            continue
        desc = item.get("description", "")
        if not desc or desc == "[historical - no source text]":
            continue
        key = hashlib.md5(desc[:60].lower().encode("utf-8")).hexdigest()[:12]
        if key not in retrieved_keys:
            item["_ltd_flagged"] = True
            item["_ltd_flagged_at"] = _now_iso()
            flagged += 1
    if flagged > 0:
        # Layer 3: Re-sign modified items after LTD flagging
        from pulse.core.pulse_crypto import sign_item
        items = [sign_item(it, it.get("_agent_id", "consolidation"))
                 if isinstance(it, dict) else it for it in items]
        save_json(session_file, items)
    if verbose:
        print(f"  [LTD] Flagged {flagged} unretrieved items (30+ days old)")
    return {"status": "ok", "flagged": flagged}

def stage_graph_update(verbose=False):
    """Stage 4: Rebuild knowledge graph from current brain state."""
    try:
        from pulse.graph.knowledge_graph import incremental_update
        result = incremental_update(verbose=verbose)
        node_count = result.get("metadata", {}).get("node_count", 0)
        edge_count = result.get("metadata", {}).get("edge_count", 0)
        if verbose:
            print(f"  [GRAPH] Updated: {node_count} nodes, {edge_count} edges")
        return {"status": "ok", "nodes": node_count, "edges": edge_count}
    except ImportError:
        if verbose:
            print("  [GRAPH] knowledge_graph.py not available")
        return {"status": "skipped", "reason": "knowledge_graph not available"}
    except Exception as e:
        if verbose:
            print(f"  [GRAPH] Error: {e}")
        return {"status": "error", "error": str(e)}


def stage_reconsolidation(verbose=False):
    """Stage 2d: Auto-reconsolidation — merge similar entries (similarity > 0.6).

    Uses bounded hash tracking to skip entries already processed in previous cycles.
    """
    processed = _load_processed_hashes()
    merged_total = 0
    new_hashes = 0
    for md_file in [LESSONS_DIR / "auto_lessons.md", PATTERNS_DIR / "auto_patterns.md",
                    FAILS_DIR / "auto_fails.md"]:
        if not md_file.exists():
            continue
        text = md_file.read_text(encoding="utf-8", errors="replace")
        sections = text.split("\n## ")
        if len(sections) < 3:  # Header + at least 2 entries needed
            continue
        entries = []
        for section in sections[1:]:
            lines = section.strip().split("\n")
            title = lines[0].strip()
            body = "\n".join(lines[1:])
            conf_match = re.search(r"\*\*Confidence:\*\*\s*([\d.]+)", body)
            conf = float(conf_match.group(1)) if conf_match else 0.5
            ehash = _entry_hash(title)
            already_seen = ehash in processed
            entries.append({"title": title, "body": body, "conf": conf,
                            "merged": False, "hash": ehash, "seen": already_seen})
        # Find similar pairs and merge (only process entries not yet seen)
        for i in range(len(entries)):
            if entries[i]["merged"] or entries[i]["seen"]:
                continue
            for j in range(i + 1, len(entries)):
                if entries[j]["merged"]:
                    continue
                sim = text_similarity(entries[i]["title"][:80], entries[j]["title"][:80])
                if sim >= 0.6:
                    # Keep higher confidence, merge info
                    if entries[j]["conf"] > entries[i]["conf"]:
                        i, j = j, i  # Swap so i is always the keeper
                    entries[i]["conf"] = min(1.0, max(entries[i]["conf"], entries[j]["conf"]) + 0.05)
                    # Bump consolidation count in keeper's body
                    cc_match = re.search(r"(\*\*Consolidation Count:\*\*\s*)(\d+)", entries[i]["body"])
                    if cc_match:
                        new_cc = int(cc_match.group(2)) + 1
                        entries[i]["body"] = re.sub(
                            r"\*\*Consolidation Count:\*\*\s*\d+",
                            f"**Consolidation Count:** {new_cc}",
                            entries[i]["body"],
                        )
                    entries[j]["merged"] = True
                    merged_total += 1
        # Mark surviving entries as processed
        for e in entries:
            if not e["merged"] and not e["seen"]:
                processed[e["hash"]] = None
                new_hashes += 1
        # Rebuild file without merged entries
        if any(e["merged"] for e in entries):
            new_sections = [sections[0]]
            for e in entries:
                if not e["merged"]:
                    new_sections.append(f"{e['title']}\n{e['body']}")
            with _acquire(md_file):
                md_file.write_text("\n## ".join(new_sections), encoding="utf-8")
    _save_processed_hashes(processed)
    if verbose:
        print(f"  [RECONSOLIDATION] Merged {merged_total} similar entries, tracked {new_hashes} new hashes")
    return {"status": "ok", "merged": merged_total, "new_hashes": new_hashes}


def stage_metacognitive_audit(verbose=False):
    """Stage 5: Brain quality audit — flag vague/unused/contradicted items, then auto-fix."""
    _ACTIONABLE = re.compile(
        r"(?:use|fix|add|remove|change|avoid|always|never|check|run|set|switch|"
        r"replace|install|update|create|delete|move|ensure|implement|deploy)", re.IGNORECASE
    )
    report = {"vague": 0, "no_verb": 0, "unused_14d": 0, "contradicted": 0,
              "total_audited": 0, "timestamp": _now_iso()}

    # Load retrieval metrics
    retrieval_file = EVIDENCE_METRICS_DIR / "retrieval_metrics.json"
    retrieval_data = load_json_dict(retrieval_file)
    retrieved_keys = set(retrieval_data.get("items", {}).keys())
    cutoff_14d = datetime.now(timezone.utc) - timedelta(days=14)

    # Load contradiction log
    contradiction_file = Path(HIPPOCAMPUS_DIR).parent / "state" / "contradiction_log.json"
    contradiction_data = load_json_dict(contradiction_file)
    repeat_counts = contradiction_data.get("repeat_counts", {})

    for md_file in [LESSONS_DIR / "auto_lessons.md", PATTERNS_DIR / "auto_patterns.md",
                    FAILS_DIR / "auto_fails.md"]:
        if not md_file.exists():
            continue
        text = md_file.read_text(encoding="utf-8", errors="replace")
        sections = text.split("\n## ")
        for section in sections[1:]:
            title = section.strip().split("\n")[0].strip()
            if not title:
                continue
            report["total_audited"] += 1
            # Check: too short/vague
            if len(title) < 30:
                report["vague"] += 1
            # Check: no actionable verb
            if not _ACTIONABLE.search(title):
                report["no_verb"] += 1
            # Check: never retrieved after 14 days
            import hashlib
            key = hashlib.md5(title[:60].lower().encode("utf-8")).hexdigest()[:12]
            date_match = re.search(r"\*\*Date:\*\*\s*(\d{4}-\d{2}-\d{2})", section)
            if date_match and key not in retrieved_keys:
                try:
                    item_date = datetime.fromisoformat(date_match.group(1) + "T00:00:00+00:00")
                    if item_date < cutoff_14d:
                        report["unused_14d"] += 1
                except (ValueError, TypeError):
                    logging.debug("Suppressed exception in consolidation_stages", exc_info=True)
            # Check: contradicted
            title_key = title[:60].lower()
            if repeat_counts.get(title_key, 0) >= 3:
                report["contradicted"] += 1

    # Write report
    report_file = EVIDENCE_METRICS_DIR / "metacognition_report.json"
    report_file.parent.mkdir(parents=True, exist_ok=True)

    # Auto-fix: reduce confidence on flagged items, archive below 0.2
    fixes = _apply_metacognitive_fixes(retrieved_keys, repeat_counts, cutoff_14d, verbose)
    report["fixes_applied"] = fixes
    save_json(report_file, report)

    if verbose:
        print(f"  [METACOGNITION] Audited {report['total_audited']} items: "
              f"{report['vague']} vague, {report['no_verb']} no-verb, "
              f"{report['unused_14d']} unused, {report['contradicted']} contradicted")
    return report


def stage_hierarchical_prune(verbose=False):
    """Stage 6: Miller's Law enforcement — max 5 items per section, 200 entries per file.

    Two-level hierarchy:
      Level 1 (file): Keep top 200 entries by quality (handled by pulse_prune.py)
      Level 2 (section): Keep top 5 list items per ## section (Miller's Law 4±1)
    """
    from pulse.core.brain_utils import load_config
    config = load_config()
    max_chunk = config.get("validation", {}).get("max_chunk_size", 5)
    min_conf = config.get("pruning", {}).get("min_confidence_to_keep", 0.3)

    # Level 1: File-level pruning (reuse existing pulse_prune logic)
    from pulse.admin.brain_ops.pulse_prune import prune_file, BRAIN_FILES
    for bf in BRAIN_FILES:
        try:
            prune_file(bf)
        except Exception as e:
            if verbose:
                print(f"  [PRUNE-L1] Error pruning {bf.name}: {e}")

    # Level 2: Section-level chunking (Miller's Law)
    trimmed_total = 0
    for md_file in [LESSONS_DIR / "auto_lessons.md", PATTERNS_DIR / "auto_patterns.md",
                    FAILS_DIR / "auto_fails.md"]:
        if not md_file.exists():
            continue
        text = md_file.read_text(encoding="utf-8", errors="replace")
        sections = text.split("\n## ")
        if len(sections) < 2:
            continue

        rebuilt = [sections[0]]
        file_trimmed = 0
        for section in sections[1:]:
            lines = section.split("\n")
            title = lines[0]
            body_lines = lines[1:]

            # Count list items in this section
            item_indices = [i for i, ln in enumerate(body_lines) if ln.startswith("- ") or ln.startswith("* ")]
            if len(item_indices) <= max_chunk:
                rebuilt.append(section)
                continue

            # Over limit — keep only the last N items (most recent at bottom)
            keep_indices = set(item_indices[-max_chunk:])
            new_body = []
            for i, ln in enumerate(body_lines):
                if i in set(item_indices) and i not in keep_indices:
                    file_trimmed += 1
                    continue
                new_body.append(ln)
            rebuilt.append(title + "\n" + "\n".join(new_body))

        if file_trimmed > 0:
            with _acquire(md_file):
                md_file.write_text("\n## ".join(rebuilt), encoding="utf-8")
            trimmed_total += file_trimmed

    if verbose:
        print(f"  [PRUNE] Hierarchical: {trimmed_total} over-limit items trimmed")
    return {"status": "ok", "trimmed": trimmed_total}


def stage_cross_domain_transfer(verbose=False):
    """Stage 2e: Detect patterns appearing across 2+ evidence domains."""
    from pulse.core.brain_analysis import detect_cross_domain_patterns
    evidence_dir = HIPPOCAMPUS_DIR / "Evidence"
    transfers = detect_cross_domain_patterns(evidence_dir)
    if not transfers:
        if verbose:
            print("  [TRANSFER] No cross-domain patterns found")
        return {"status": "ok", "transfers": 0, "promoted": 0}
    # Promote universal patterns (3+ domains) to schemas
    schemas_file = PATTERNS_DIR / "schemas.json"
    schemas_data = load_json_dict(schemas_file)
    schemas = schemas_data.get("schemas", [])
    existing = {s.get("name", "").lower() for s in schemas}
    promoted = 0
    for t in transfers:
        if not t.get("universal"):
            continue
        name = f"Cross-domain: {t['pattern'][:50]}"
        if name.lower() in existing:
            continue
        schemas.append({
            "name": name, "confidence": 0.65, "evidence_count": t["count"],
            "domains": t["domains"], "type": "cross_domain_transfer",
            "created": _now_iso(),
        })
        existing.add(name.lower())
        promoted += 1
        if verbose:
            print(f"  [TRANSFER] {name} ({t['count']} domains)")
    if promoted:
        schemas_data["schemas"] = schemas
        schemas_data["last_updated"] = _now_iso()
        save_json(schemas_file, schemas_data)
    if verbose:
        print(f"  [TRANSFER] {len(transfers)} cross-domain, {promoted} promoted")
    return {"status": "ok", "transfers": len(transfers), "promoted": promoted}


def _apply_metacognitive_fixes(retrieved_keys, repeat_counts, cutoff_14d, verbose=False):
    """Reduce confidence on vague/unused/contradicted items. Archive items below 0.2."""
    import hashlib
    fixes = {"confidence_reduced": 0, "archived": 0}
    for md_file in [LESSONS_DIR / "auto_lessons.md", PATTERNS_DIR / "auto_patterns.md",
                    FAILS_DIR / "auto_fails.md"]:
        if not md_file.exists():
            continue
        text = md_file.read_text(encoding="utf-8", errors="replace")
        sections = text.split("\n## ")
        if len(sections) < 2:
            continue
        rebuilt = [sections[0]]
        for section in sections[1:]:
            lines = section.strip().split("\n")
            title = lines[0].strip()
            if not title:
                rebuilt.append(section)
                continue
            body = "\n".join(lines[1:])
            conf_match = re.search(r"\*\*Confidence:\*\*\s*([\d.]+)", body)
            conf = float(conf_match.group(1)) if conf_match else 0.5
            penalty = 0.0
            if len(title) < 30:
                penalty += 0.1
            key = hashlib.md5(title[:60].lower().encode("utf-8")).hexdigest()[:12]
            date_match = re.search(r"\*\*Date:\*\*\s*(\d{4}-\d{2}-\d{2})", body)
            if date_match and key not in retrieved_keys:
                try:
                    if datetime.fromisoformat(date_match.group(1) + "T00:00:00+00:00") < cutoff_14d:
                        penalty += 0.15
                except (ValueError, TypeError):
                    logging.debug("Suppressed exception in consolidation_stages", exc_info=True)
            if repeat_counts.get(title[:60].lower(), 0) >= 3:
                penalty += 0.2
            if penalty > 0:
                new_conf = max(0.0, round(conf - penalty, 2))
                if conf_match:
                    body = body.replace(conf_match.group(0), f"**Confidence:** {new_conf}")
                fixes["confidence_reduced"] += 1
                if new_conf < 0.2:
                    fixes["archived"] += 1
                    continue  # Drop section (archived)
            rebuilt.append(f"{title}\n{body}")
        with _acquire(md_file):
            md_file.write_text("\n## ".join(rebuilt), encoding="utf-8")
    if verbose:
        print(f"  [METACOGNITION] Fixes: {fixes['confidence_reduced']} reduced, {fixes['archived']} archived")
    return fixes