#!/usr/bin/env python3
# Copyright (c) 2026 PULSE Contributors. Proprietary. All rights reserved.
"""Retroactive LLM Extractor — Brain V2. Semantic sweep of capture logs via LLM Router."""

import io, json, os, re, shutil, sys, time
from datetime import datetime, timezone
from pathlib import Path

SCRIPT_DIR = Path(__file__).resolve().parent
ROOT_DIR = SCRIPT_DIR.parent.parent.parent

if str(ROOT_DIR) not in sys.path:
    sys.path.insert(0, str(ROOT_DIR))

if os.name == "nt" and __name__ == "__main__":
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace")

from pulse.core.brain_utils import HIPPOCAMPUS_DIR
from pulse.core.llm_router import dispatch, extract_json as router_extract_json
from pulse.brain.save_writers import append_knowledge_md, append_decision

STATE_DIR = ROOT_DIR / "state"
ARCHIVE_DIR = HIPPOCAMPUS_DIR / "Archive"

CAPTURE_LOGS = [
    STATE_DIR / "pulse_captured_claude.log",
    STATE_DIR / "pulse_captured_gemini.log",
    STATE_DIR / "pulse_captured_codex.log",
    STATE_DIR / "pulse_captured_grok.log",
]

BRAIN_FILES = {
    "lessons": HIPPOCAMPUS_DIR / "Lessons" / "auto_lessons.md",
    "failures": HIPPOCAMPUS_DIR / "Failures" / "auto_fails.md",
    "patterns": HIPPOCAMPUS_DIR / "Patterns" / "auto_patterns.md",
}

BATCH_SIZE = 15
MAX_BATCH_TOKENS = 12000

SCRIBE_PROMPT = """You are the PULSE Memory Recovery Agent. Extract engineering knowledge from chat transcripts into exactly 3 categories.

CATEGORIES:
- "lessons": Reusable rules or best practices. Something an engineer should ALWAYS or NEVER do.
  YES: "Always add encoding='utf-8' to subprocess.run with text=True on Windows to avoid cp1252 crashes."
  YES: "Use filelock with 60s timeout when 24+ concurrent agents write to the same file."
  NO: "The config migration was completed successfully." (status update)
  NO: "Bridge quality check failed because contamination passed threshold." (failure, not lesson)

- "failures": Bugs, crashes, regressions, or design mistakes. Something BROKE or WENT WRONG.
  YES: "Module-level sys.stdout wrapping crashed all imports because it ran before __main__ guard."
  YES: "Headless workers marked 14/15 tasks done despite producing no file changes — LLM stopped calling tools."
  NO: "Always verify model IDs before deploying swarm." (lesson, not failure)

- "patterns": Recurring workflows or debugging strategies. A HOW-TO or WHEN-THEN approach.
  YES: "When translation latency spikes, check for multiple bot instances and sequential paragraph processing."
  NO: "The bridge was rewritten to use absolute imports." (status update)

RULES:
1. Each item: complete sentence, 30-200 chars, actionable, specific.
2. Status updates or task summaries are NEVER knowledge — skip them.
3. If ambiguous between categories, pick the STRONGEST match.
4. If no knowledge: return empty arrays.

OUTPUT — strict JSON, no markdown fences:
{"lessons": ["..."], "failures": ["..."], "patterns": ["..."]}

Transcript:
"""

extract_json = router_extract_json

# --- Prompt Echo Detection (extracted to prompt_echo_filter.py for Law 7) ---
from pulse.admin.retroactive.prompt_echo_filter import build_echo_filter
_is_prompt_echo, _filter_prompt_echoes = build_echo_filter(SCRIBE_PROMPT)


_LOG_LINE = re.compile(
    r"^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]\s+"
    r"(USER|CLAUDE|GEMINI|CODEX|GROK):\s+"
    r"(.+?)(?:\s*\|\s*SIG:\s*\S+)?\s*$"
)


def _parse_interactions(log_path: Path):
    """Yield (timestamp, role, message) from a capture log."""
    if not log_path.exists():
        return
    current_ts, current_role, lines = None, None, []
    with open(log_path, "r", encoding="utf-8", errors="replace") as f:
        for line in f:
            m = _LOG_LINE.match(line)
            if m:
                if current_role:
                    yield current_ts, current_role, " ".join(lines)
                current_ts, current_role, lines = m.group(1), m.group(2), [m.group(3)]
            elif line.rstrip() and current_role:
                lines.append(line.rstrip())
    if current_role:
        yield current_ts, current_role, " ".join(lines)


def batch_interactions(log_path: Path, batch_size: int = BATCH_SIZE):
    """Group interactions into batches. Yields (batch_text, count, first_ts)."""
    batch_lines, pair_count, char_count, first_ts = [], 0, 0, None
    for ts, role, msg in _parse_interactions(log_path):
        if first_ts is None:
            first_ts = ts
        if len(msg) > 2000:
            msg = msg[:2000] + "... [truncated]"
        batch_lines.append(f"[{ts}] {role}: {msg}")
        char_count += len(msg)
        if role != "USER":
            pair_count += 1
        if pair_count >= batch_size or char_count > MAX_BATCH_TOKENS * 4:
            yield "\n".join(batch_lines), pair_count, first_ts
            batch_lines, pair_count, char_count, first_ts = [], 0, 0, None
    if batch_lines:
        yield "\n".join(batch_lines), pair_count, first_ts


_FAILURE_SIGNALS = re.compile(
    r"\bfailed\b|\bbroke\b|\bcrashed\b|\bbug\b|\bregression\b|\bdata loss\b|\bcorrupted\b"
    r"|did not work|went wrong|caused\s+(?:\w+\s+){0,3}error|\bwas destroyed\b"
    r"|\bviolated\b|\bleaked\b|\btimed out\b|\bdeadlocked\b|\bwas missing\b"
    r"|\brace condition (?:occurred|caused|destroyed|corrupted)\b"
    r"|produced\s+(?:\w+\s+){0,2}wrong|\bfails? silently\b", re.IGNORECASE)
_LESSON_SIGNALS = re.compile(
    r"\balways \b|\bnever \b|\bshould \b|\bmust \b|\bdo not \b|\bdon'?t "
    r"|\b(?:use|ensure|avoid)\s+\w|\breplace\s+\w.*?\bwith\b|\bswitch\s+to\b"
    r"|\badd\s+\w.*?\bto prevent\b|\bbest practice\b|\brule of thumb\b"
    r"|\bgoing forward\b|\bremember to \b|\bmake sure ", re.IGNORECASE)
_STATUS_SIGNALS = re.compile(
    r"\bwas completed\s*$|\bhas been (?:deployed|shipped|merged|released)\b"
    r"|\bbatch \d+\s+(?:wants|needs)\s+cleaning\b|\bsuccessfully (?:migrated|updated|fixed)\b"
    r"|\bis now (?:live|deployed)\b|\bgrew from \d+ to \d+\b", re.IGNORECASE)


def _reclassify(items: dict) -> dict:
    """Reclassify misplacements. Drop status updates. lesson wins when co-occurring."""
    result = {"lessons": [], "failures": [], "patterns": []}
    for category in ("lessons", "failures", "patterns"):
        for item in items.get(category, []):
            if not isinstance(item, str):
                continue
            has_fail = bool(_FAILURE_SIGNALS.search(item))
            has_lesson = bool(_LESSON_SIGNALS.search(item))
            has_status = bool(_STATUS_SIGNALS.search(item))
            # Multi-signal: lesson always wins when co-occurring
            if has_lesson and (has_fail or has_status):
                result["lessons"].append(item)
                continue
            # Pure status with no lesson/failure signal -> drop
            if has_status and not has_fail and not has_lesson:
                continue
            # Single-signal reclassification
            if category == "lessons" and has_fail and not has_lesson:
                result["failures"].append(item)
            elif category == "failures" and has_lesson and not has_fail:
                result["lessons"].append(item)
            else:
                result[category].append(item)
    return result


def write_to_brain(items: dict, source_agent: str, timestamp: str, dry_run: bool = False):
    """Write extracted items to brain files via append_knowledge_md."""
    items = _filter_prompt_echoes(items)  # Purge prompt example echoes FIRST
    items = _reclassify(items)
    written = {"lessons": 0, "failures": 0, "patterns": 0}
    for item_type, filepath in BRAIN_FILES.items():
        for desc in items.get(item_type, []):
            if not isinstance(desc, str):
                continue
            desc = desc.strip()
            if len(desc) < 25 or len(desc) > 500 or len(desc.split()) < 5:
                continue
            append_knowledge_md(filepath=filepath, description=desc,
                                source=f"scribe:{source_agent}", timestamp=timestamp,
                                domain="general", confidence=0.8, salience=2.0, dry_run=dry_run)
            written[item_type] += 1
    return written


def _archive_brain():
    """Archive current brain files before re-extraction."""
    ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
    for name, path in BRAIN_FILES.items():
        if path.exists() and path.stat().st_size > 200:
            shutil.copy2(path, ARCHIVE_DIR / f"{path.stem}_{ts}.md")
            print(f"  Archived: {path.name} ({path.stat().st_size // 1024}KB)")
            header = f"---\ntype: auto-extracted\nlast_updated: {ts[:8]}\n---\n\n"
            header += f"# {path.stem.replace('auto_', '').replace('_', ' ').title()}\n\n---\n"
            path.write_text(header, encoding="utf-8")
    from pulse.brain.save_writers import _DEDUP_CACHE
    _DEDUP_CACHE.clear()


def main():
    import argparse
    parser = argparse.ArgumentParser(description="Brain V2 Retroactive LLM Extraction")
    parser.add_argument("--dry-run", action="store_true")
    parser.add_argument("--no-archive", action="store_true")
    parser.add_argument("--batch-size", type=int, default=BATCH_SIZE)
    args = parser.parse_args()

    print(f"{'='*60}\nBRAIN V2 — Retroactive LLM Extraction\n{'='*60}")
    print(f"Batch size: {args.batch_size}" + (" | DRY RUN" if args.dry_run else ""))

    if not args.no_archive and not args.dry_run:
        print("\n[1/4] Archiving brain files...")
        _archive_brain()

    print("\n[2/4] Processing capture logs...")
    total_batches, total_errors = 0, 0
    total_items = {"lessons": 0, "failures": 0, "patterns": 0}
    start = time.time()

    for log_path in CAPTURE_LOGS:
        if not log_path.exists():
            continue
        agent = log_path.stem.replace("pulse_captured_", "")
        print(f"\n  [{agent}] {log_path.stat().st_size / 1048576:.1f}MB...")
        batch_num, log_items = 0, {"lessons": 0, "failures": 0, "patterns": 0}
        est = int(log_path.stat().st_size / (args.batch_size * 800)) or 1

        for batch_text, count, first_ts in batch_interactions(log_path, args.batch_size):
            batch_num += 1
            total_batches += 1
            if args.dry_run:
                print(f"    Batch {batch_num}: {count} interactions, ~{len(batch_text)//4} tokens")
                continue
            try:
                raw = dispatch(SCRIBE_PROMPT + batch_text, task_type="fast")
                written = write_to_brain(extract_json(raw), agent,
                                         first_ts or datetime.now(timezone.utc).isoformat())
                for k in log_items:
                    log_items[k] += written[k]
                    total_items[k] += written[k]
                grand = sum(total_items.values())
                rate = total_batches / (time.time() - start) if time.time() > start else 0
                if batch_num % 10 == 0:
                    print(f"    [{agent}] {batch_num}/~{est} ({grand} total) [{rate:.1f} b/s]")
            except Exception as e:
                total_errors += 1
                print(f"\n    [{agent}] Batch {batch_num} ERROR: {e}")
        print(f"  [{agent}] {batch_num} batches -> {sum(log_items.values())} items")

    if not args.dry_run:
        print("\n[3/4] Updating bridge state...")
        try:
            from pulse.core.brain_utils import atomic_update_json
            def _reset(data):
                data = data if isinstance(data, dict) else {}
                for lp in CAPTURE_LOGS:
                    if lp.exists():
                        data[str(lp.name)] = lp.stat().st_size
                return data
            atomic_update_json(STATE_DIR / "pulse_bridge_state.json", _reset, default={})
        except Exception as e:
            print(f"  Bridge state: {e}")
        print("\n[4/4] Rebuilding index + graph...")
        try:
            from pulse.brain.search_index import build_index
            build_index(verbose=True)
        except Exception as e:
            print(f"  Index: {e}")
        try:
            from pulse.graph.knowledge_graph_build import build_graph
            build_graph(verbose=True)
        except Exception as e:
            print(f"  Graph: {e}")

    elapsed = time.time() - start
    grand = sum(total_items.values())
    print(f"\n{'='*60}")
    print(f"DONE: {total_batches} batches -> {grand} items "
          f"(L:{total_items['lessons']} F:{total_items['failures']} P:{total_items['patterns']}) "
          f"| {total_errors} errors | {elapsed:.0f}s")
    print(f"{'='*60}")


if __name__ == "__main__":
    main()
