#!/usr/bin/env python3
# Copyright (c) 2026 PULSE Contributors. Proprietary. All rights reserved.
"""Retroactive LLM Extractor — Brain V2. Semantic sweep of capture logs via LLM Router."""

import importlib
import io, json, os, re, shutil, sys, time
from datetime import datetime, timezone
from pathlib import Path

import logging
log = logging.getLogger("pulse.admin.retroactive.retroactive_llm_extractor")

SCRIPT_DIR = Path(__file__).resolve().parent
_ALLOWED_REBUILD_MODULES = frozenset({"pulse.brain.index.search_index", "pulse.graph.knowledge_graph_build"})
ROOT_DIR = SCRIPT_DIR.parent.parent.parent

if str(ROOT_DIR) not in sys.path:
    sys.path.insert(0, str(ROOT_DIR))

if os.name == "nt" and __name__ == "__main__":
    if hasattr(sys.stdout, 'reconfigure'):
        sys.stdout.reconfigure(encoding='utf-8', errors='replace')
    if hasattr(sys.stderr, 'reconfigure'):
        sys.stderr.reconfigure(encoding='utf-8', errors='replace')

from pulse.core.brain_utils import HIPPOCAMPUS_DIR, truncate_at_sentence
from pulse.core.llm_router import dispatch, extract_json as router_extract_json
from pulse.brain.save_writers import append_knowledge_md, append_decision
from pulse.core.brain_analysis import compute_item_salience, compute_item_confidence

STATE_DIR = ROOT_DIR / "state"
ARCHIVE_DIR = HIPPOCAMPUS_DIR / "Archive"

CAPTURE_LOGS = [STATE_DIR / f"pulse_captured_{a}.log" for a in ("claude", "gemini", "codex", "grok")]
BRAIN_FILES = {
    "lessons":  HIPPOCAMPUS_DIR / "Lessons"  / "auto_lessons.md",
    "failures": HIPPOCAMPUS_DIR / "Failures" / "auto_fails.md",   # auto_fails NOT auto_failures (matches _MD_TO_TYPE)
    "patterns": HIPPOCAMPUS_DIR / "Patterns" / "auto_patterns.md",
}

BATCH_SIZE = 15
MAX_BATCH_TOKENS = 12000

SCRIBE_PROMPT = """You are the PULSE Memory Recovery Agent. Extract engineering knowledge from chat transcripts into exactly 5 categories.

CATEGORIES:
- "lessons": Reusable rules or best practices. Something an engineer should ALWAYS or NEVER do.
  YES: "Always add encoding='utf-8' to subprocess.run with text=True on Windows to avoid cp1252 crashes."
  YES: "Use filelock with 60s timeout when 24+ concurrent agents write to the same file."
  NO: "The config migration was completed successfully." (status update)
  NO: "Bridge quality check failed because contamination passed threshold." (failure, not lesson)

- "failures": Bugs, crashes, regressions, or design mistakes. Something BROKE or WENT WRONG.
  YES: "Module-level sys.stdout wrapping crashed all imports because it ran before __main__ guard."
  YES: "Headless workers marked 14/15 tasks done despite producing no file changes — LLM stopped calling tools."
  NO: "Always verify model IDs before deploying swarm." (lesson, not failure)

- "patterns": Recurring workflows or debugging strategies. A HOW-TO or WHEN-THEN approach.
  YES: "When translation latency spikes, check for multiple bot instances and sequential paragraph processing."
  NO: "The bridge was rewritten to use absolute imports." (status update)

- "facts": Declarative statements about infrastructure, services, tools, accounts, or configurations.
  YES: "Website deployed on Vercel"
  YES: "Using Stripe for payments"
  YES: "Domain registered on Porkbun"
  YES: "Package name is pulse-os on PyPI"
  NO: "We should deploy on Vercel" (want, not fact)
  NO: "Vercel is amazing" (opinion, not fact)

- "decisions": Committed architectural or technical choices — what approach was SELECTED and why.
  YES: "Use SQLite over JSONL for all brain writes — atomic, concurrent-safe, no manual file locking needed."
  YES: "Deploy on Vercel instead of self-hosting — zero-ops, auto-deploys from GitHub."
  YES: "Adopt FSRS-6 for spaced repetition instead of SM-2 — better retention curve for long-term memory."
  YES: "Switch capture logs to suffixed naming (pulse_captured_claude.log) to fix 8-day data gap."
  NO: "We should use SQLite" (want/hypothetical, not committed)
  NO: "SQLite migration completed" (status update)
  NO: "Should we use Vercel?" (question)

RULES:
1. Each item: complete sentence, 30-200 chars, actionable, specific.
2. Facts: short declarative statements (10-150 chars) about WHAT is used/deployed/configured WHERE.
3. Decisions: include both the choice AND the reason (if stated). Format: "Use X over Y because Z" or "Use X — reason."
4. Status updates or task summaries are NEVER knowledge — skip them.
5. If ambiguous between categories, pick the STRONGEST match.
6. If no knowledge: return empty arrays.

OUTPUT — strict JSON, no markdown fences:
{"lessons": ["..."], "failures": ["..."], "patterns": ["..."], "facts": ["..."], "decisions": ["..."]}

Transcript:
"""

extract_json = router_extract_json

from pulse.admin.retroactive.prompt_echo_filter import build_echo_filter
_is_prompt_echo, _filter_prompt_echoes = build_echo_filter(SCRIBE_PROMPT)


_LOG_LINE = re.compile(
    r"^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]\s+"
    r"(USER|CLAUDE|GEMINI|CODEX|GROK):\s+"
    r"(.+?)(?:\s*\|\s*SIG:\s*\S+)?\s*$"
)


def _parse_interactions(log_path: Path, start_offset: int = 0):
    """Yield (timestamp, role, message) from a capture log.

    start_offset: byte offset to seek before reading — avoids re-dispatching
    already-processed content (stale-batch bug fix).
    """
    if not log_path.exists():
        return
    current_ts, current_role, lines = None, None, []
    with open(log_path, "r", encoding="utf-8", errors="replace") as f:
        if start_offset > 0:
            f.seek(start_offset)
        for line in f:
            m = _LOG_LINE.match(line)
            if m:
                if current_role:
                    yield current_ts, current_role, " ".join(lines)
                current_ts, current_role, lines = m.group(1), m.group(2), [m.group(3)]
            elif line.rstrip() and current_role:
                lines.append(line.rstrip())
    if current_role:
        yield current_ts, current_role, " ".join(lines)


def batch_interactions(log_path: Path, batch_size: int = BATCH_SIZE,
                       start_offset: int = 0):
    """Group interactions into batches. Yields (batch_text, count, first_ts).

    start_offset: passed to _parse_interactions — skips already-dispatched bytes.
    """
    batch_lines, pair_count, char_count, first_ts = [], 0, 0, None
    for ts, role, msg in _parse_interactions(log_path, start_offset=start_offset):
        if first_ts is None:    
            first_ts = ts       
        if len(msg) > 2000:     
            msg = truncate_at_sentence(msg, 2000) + "... [truncated]"
        batch_lines.append(f"[{ts}] {role}: {msg}")

        char_count += len(msg)
        if role != "USER":
            pair_count += 1
        if pair_count >= batch_size or char_count > MAX_BATCH_TOKENS * 4:
            yield "\n".join(batch_lines), pair_count, first_ts
            batch_lines, pair_count, char_count, first_ts = [], 0, 0, None
    if batch_lines:
        yield "\n".join(batch_lines), pair_count, first_ts


_FAILURE_SIGNALS = re.compile(
    r"\bfailed\b|\bbroke\b|\bcrashed\b|\bbug\b|\bregression\b|\bdata loss\b|\bcorrupted\b"
    r"|did not work|went wrong|caused\s+(?:\w+\s+){0,3}error|\bwas destroyed\b"
    r"|\bviolated\b|\bleaked\b|\btimed out\b|\bdeadlocked\b|\bwas missing\b"
    r"|\brace condition (?:occurred|caused|destroyed|corrupted)\b"
    r"|produced\s+(?:\w+\s+){0,2}wrong|\bfails? silently\b", re.IGNORECASE)
_LESSON_SIGNALS = re.compile(
    r"\balways \b|\bnever \b|\bshould \b|\bmust \b|\bdo not \b|\bdon'?t "
    r"|\b(?:use|ensure|avoid)\s+\w|\breplace\s+\w.*?\bwith\b|\bswitch\s+to\b"
    r"|\badd\s+\w.*?\bto prevent\b|\bbest practice\b|\brule of thumb\b"
    r"|\bgoing forward\b|\bremember to \b|\bmake sure ", re.IGNORECASE)
_STATUS_SIGNALS = re.compile(
    r"\bwas completed\s*$|\bhas been (?:deployed|shipped|merged|released)\b"
    r"|\bbatch \d+\s+(?:wants|needs)\s+cleaning\b|\bsuccessfully (?:migrated|updated|fixed)\b"
    r"|\bis now (?:live|deployed)\b|\bgrew from \d+ to \d+\b", re.IGNORECASE)


def _reclassify(items: dict) -> dict:
    """Reclassify misplacements. Drop status updates. lesson wins when co-occurring."""
    result = {"lessons": [], "failures": [], "patterns": []}
    # Facts pass through unchanged (different category, no reclassification)
    if items.get("facts"):
        result["facts"] = [f for f in items["facts"] if isinstance(f, str)]
    for category in ("lessons", "failures", "patterns"):
        for item in items.get(category, []):
            if not isinstance(item, str):
                continue
            has_fail = bool(_FAILURE_SIGNALS.search(item))
            has_lesson = bool(_LESSON_SIGNALS.search(item))
            has_status = bool(_STATUS_SIGNALS.search(item))
            # Multi-signal: lesson always wins when co-occurring
            if has_lesson and (has_fail or has_status):
                result["lessons"].append(item)
                continue
            # Pure status with no lesson/failure signal -> drop
            if has_status and not has_fail and not has_lesson:
                continue
            # Single-signal reclassification
            if category == "lessons" and has_fail and not has_lesson:
                result["failures"].append(item)
            elif category == "failures" and has_lesson and not has_fail:
                result["lessons"].append(item)
            else:
                result[category].append(item)
    return result


def write_to_brain(items: dict, source_agent: str, timestamp: str, dry_run: bool = False):
    """Write extracted items to brain files via append_knowledge_md + fact_store + decision_store."""
    items = _filter_prompt_echoes(items)  # Purge prompt example echoes FIRST
    items = _reclassify(items)
    written = {"lessons": 0, "failures": 0, "patterns": 0, "facts": 0, "decisions": 0}
    for item_type, filepath in BRAIN_FILES.items():
        for desc in items.get(item_type, []):
            if not isinstance(desc, str):
                continue
            desc = desc.strip()
            if len(desc) < 25 or len(desc) > 500 or len(desc.split()) < 5:
                continue
            sal = compute_item_salience(desc, priority=1.0)
            conf = compute_item_confidence(desc, source_type="scribe")
            try:
                from pulse.brain.extraction.wants_classify import _classify_domain as _cd
                _domain = _cd(desc)
            except Exception:
                _domain = "general"
            append_knowledge_md(filepath=filepath, description=desc,
                                source=f"scribe:{source_agent}", timestamp=timestamp,
                                domain=_domain, confidence=conf, salience=sal, dry_run=dry_run)
            written[item_type] += 1

    # Facts: LLM returns plain strings → parse through fact_parser for structure
    raw_facts = items.get("facts", [])
    if raw_facts and not dry_run:
        from pulse.brain.extraction.fact_parser import parse_facts
        from pulse.brain.persistence.fact_store import store_facts
        structured = []
        for fact_str in raw_facts[:10]:
            if not isinstance(fact_str, str) or len(fact_str.strip()) < 10:
                continue
            parsed = parse_facts(fact_str.strip())
            if parsed:
                for f in parsed:
                    f["confidence"] = min(f.get("confidence", 0.7), 0.5)
                structured.extend(parsed)
        if structured:
            written["facts"] = store_facts(structured, source=f"scribe:{source_agent}")

    # Decisions: LLM returns plain "Use X over Y because Z" strings → store_decision
    raw_decisions = items.get("decisions", [])
    if raw_decisions and not dry_run:
        try:
            from pulse.brain.persistence.decision_store import store_decision
            for dec_str in raw_decisions[:8]:
                if not isinstance(dec_str, str) or len(dec_str.strip()) < 20:
                    continue
                dec_str = dec_str.strip()
                try:
                    from pulse.brain.extraction.wants_classify import _classify_domain as _cd2
                    _ddomain = _cd2(dec_str)
                except Exception:
                    _ddomain = "general"
                dec = {
                    "choice": dec_str[:120],
                    "reasoning": "",
                    "scope": "general",
                    "confidence": 0.7,
                    "domain": _ddomain,
                    "participants": [f"scribe:{source_agent}"],
                    "timestamp": timestamp,
                    "extraction_method": "scribe_llm",
                    "status": "active",
                    "reversibility": "moderate",
                    "alternatives_rejected": [],
                    "tags": [],
                }
                if store_decision(dec):
                    written["decisions"] += 1
        except Exception as _exc:
            pass  # QR14: silent — consider log.debug("%s", _exc)

    return written

def _archive_brain():
    """Archive current brain files before re-extraction."""
    ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
    for name, path in BRAIN_FILES.items():
        if path.exists() and path.stat().st_size > 200:
            shutil.copy2(path, ARCHIVE_DIR / f"{path.stem}_{ts}.md")
            print(f"  Archived: {path.name} ({path.stat().st_size // 1024}KB)")
            header = f"---\ntype: auto-extracted\nlast_updated: {ts[:8]}\n---\n\n"
            header += f"# {path.stem.replace('auto_', '').replace('_', ' ').title()}\n\n---\n"
            path.write_text(header, encoding="utf-8")
    from pulse.brain.save_writers import _DEDUP_CACHE
    _DEDUP_CACHE.clear()

if __name__ == "__main__":
    from pulse.admin.retroactive.retroactive_llm_extractor_main import main
    from pulse.brain.extraction.wants_classify import _classify_domain
    main()
