#!/usr/bin/env python3
# Copyright (c) 2026 PULSE Contributors. Proprietary. All rights reserved.
"""Retroactive LLM Extractor — Brain V2. Semantic sweep of capture logs via LLM Router."""

import importlib
import io, json, os, re, shutil, sys, time
from datetime import datetime, timezone
from pathlib import Path

SCRIPT_DIR = Path(__file__).resolve().parent
_ALLOWED_REBUILD_MODULES = frozenset({"pulse.brain.index.search_index", "pulse.graph.knowledge_graph_build"})
ROOT_DIR = SCRIPT_DIR.parent.parent.parent

if str(ROOT_DIR) not in sys.path:
    sys.path.insert(0, str(ROOT_DIR))

if os.name == "nt" and __name__ == "__main__":
    if hasattr(sys.stdout, 'reconfigure'):
        sys.stdout.reconfigure(encoding='utf-8', errors='replace')
    if hasattr(sys.stderr, 'reconfigure'):
        sys.stderr.reconfigure(encoding='utf-8', errors='replace')

from pulse.core.brain_utils import HIPPOCAMPUS_DIR
from pulse.core.llm_router import dispatch, extract_json as router_extract_json
from pulse.brain.save_writers import append_knowledge_md, append_decision
from pulse.core.brain_analysis import compute_item_salience, compute_item_confidence

STATE_DIR = ROOT_DIR / "state"
ARCHIVE_DIR = HIPPOCAMPUS_DIR / "Archive"

CAPTURE_LOGS = [STATE_DIR / f"pulse_captured_{a}.log" for a in ("claude", "gemini", "codex", "grok")]
BRAIN_FILES = {
    "lessons":  HIPPOCAMPUS_DIR / "Lessons"  / "auto_lessons.md",
    "failures": HIPPOCAMPUS_DIR / "Failures" / "auto_fails.md",   # auto_fails NOT auto_failures (matches _MD_TO_TYPE)
    "patterns": HIPPOCAMPUS_DIR / "Patterns" / "auto_patterns.md",
}

BATCH_SIZE = 15
MAX_BATCH_TOKENS = 12000

SCRIBE_PROMPT = """You are the PULSE Memory Recovery Agent. Extract engineering knowledge from chat transcripts into exactly 4 categories.

CATEGORIES:
- "lessons": Reusable rules or best practices. Something an engineer should ALWAYS or NEVER do.
  YES: "Always add encoding='utf-8' to subprocess.run with text=True on Windows to avoid cp1252 crashes."
  YES: "Use filelock with 60s timeout when 24+ concurrent agents write to the same file."
  NO: "The config migration was completed successfully." (status update)
  NO: "Bridge quality check failed because contamination passed threshold." (failure, not lesson)

- "failures": Bugs, crashes, regressions, or design mistakes. Something BROKE or WENT WRONG.
  YES: "Module-level sys.stdout wrapping crashed all imports because it ran before __main__ guard."
  YES: "Headless workers marked 14/15 tasks done despite producing no file changes — LLM stopped calling tools."
  NO: "Always verify model IDs before deploying swarm." (lesson, not failure)

- "patterns": Recurring workflows or debugging strategies. A HOW-TO or WHEN-THEN approach.
  YES: "When translation latency spikes, check for multiple bot instances and sequential paragraph processing."
  NO: "The bridge was rewritten to use absolute imports." (status update)

- "facts": Declarative statements about infrastructure, services, tools, accounts, or configurations.
  YES: "Website deployed on Vercel"
  YES: "Using Stripe for payments"
  YES: "Domain registered on Porkbun"
  YES: "Package name is pulse-os on PyPI"
  NO: "We should deploy on Vercel" (want, not fact)
  NO: "Vercel is amazing" (opinion, not fact)

RULES:
1. Each item: complete sentence, 30-200 chars, actionable, specific.
2. Facts: short declarative statements (10-150 chars) about WHAT is used/deployed/configured WHERE.
3. Status updates or task summaries are NEVER knowledge — skip them.
4. If ambiguous between categories, pick the STRONGEST match.
5. If no knowledge: return empty arrays.

OUTPUT — strict JSON, no markdown fences:
{"lessons": ["..."], "failures": ["..."], "patterns": ["..."], "facts": ["..."]}

Transcript:
"""

extract_json = router_extract_json

from pulse.admin.retroactive.prompt_echo_filter import build_echo_filter
_is_prompt_echo, _filter_prompt_echoes = build_echo_filter(SCRIBE_PROMPT)


_LOG_LINE = re.compile(
    r"^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]\s+"
    r"(USER|CLAUDE|GEMINI|CODEX|GROK):\s+"
    r"(.+?)(?:\s*\|\s*SIG:\s*\S+)?\s*$"
)


def _parse_interactions(log_path: Path, start_offset: int = 0):
    """Yield (timestamp, role, message) from a capture log.

    start_offset: byte offset to seek before reading — avoids re-dispatching
    already-processed content (stale-batch bug fix).
    """
    if not log_path.exists():
        return
    current_ts, current_role, lines = None, None, []
    with open(log_path, "r", encoding="utf-8", errors="replace") as f:
        if start_offset > 0:
            f.seek(start_offset)
        for line in f:
            m = _LOG_LINE.match(line)
            if m:
                if current_role:
                    yield current_ts, current_role, " ".join(lines)
                current_ts, current_role, lines = m.group(1), m.group(2), [m.group(3)]
            elif line.rstrip() and current_role:
                lines.append(line.rstrip())
    if current_role:
        yield current_ts, current_role, " ".join(lines)


def batch_interactions(log_path: Path, batch_size: int = BATCH_SIZE,
                       start_offset: int = 0):
    """Group interactions into batches. Yields (batch_text, count, first_ts).

    start_offset: passed to _parse_interactions — skips already-dispatched bytes.
    """
    batch_lines, pair_count, char_count, first_ts = [], 0, 0, None
    for ts, role, msg in _parse_interactions(log_path, start_offset=start_offset):
        if first_ts is None:
            first_ts = ts
        if len(msg) > 2000:
            msg = msg[:2000] + "... [truncated]"
        batch_lines.append(f"[{ts}] {role}: {msg}")
        char_count += len(msg)
        if role != "USER":
            pair_count += 1
        if pair_count >= batch_size or char_count > MAX_BATCH_TOKENS * 4:
            yield "\n".join(batch_lines), pair_count, first_ts
            batch_lines, pair_count, char_count, first_ts = [], 0, 0, None
    if batch_lines:
        yield "\n".join(batch_lines), pair_count, first_ts


_FAILURE_SIGNALS = re.compile(
    r"\bfailed\b|\bbroke\b|\bcrashed\b|\bbug\b|\bregression\b|\bdata loss\b|\bcorrupted\b"
    r"|did not work|went wrong|caused\s+(?:\w+\s+){0,3}error|\bwas destroyed\b"
    r"|\bviolated\b|\bleaked\b|\btimed out\b|\bdeadlocked\b|\bwas missing\b"
    r"|\brace condition (?:occurred|caused|destroyed|corrupted)\b"
    r"|produced\s+(?:\w+\s+){0,2}wrong|\bfails? silently\b", re.IGNORECASE)
_LESSON_SIGNALS = re.compile(
    r"\balways \b|\bnever \b|\bshould \b|\bmust \b|\bdo not \b|\bdon'?t "
    r"|\b(?:use|ensure|avoid)\s+\w|\breplace\s+\w.*?\bwith\b|\bswitch\s+to\b"
    r"|\badd\s+\w.*?\bto prevent\b|\bbest practice\b|\brule of thumb\b"
    r"|\bgoing forward\b|\bremember to \b|\bmake sure ", re.IGNORECASE)
_STATUS_SIGNALS = re.compile(
    r"\bwas completed\s*$|\bhas been (?:deployed|shipped|merged|released)\b"
    r"|\bbatch \d+\s+(?:wants|needs)\s+cleaning\b|\bsuccessfully (?:migrated|updated|fixed)\b"
    r"|\bis now (?:live|deployed)\b|\bgrew from \d+ to \d+\b", re.IGNORECASE)


def _reclassify(items: dict) -> dict:
    """Reclassify misplacements. Drop status updates. lesson wins when co-occurring."""
    result = {"lessons": [], "failures": [], "patterns": []}
    # Facts pass through unchanged (different category, no reclassification)
    if items.get("facts"):
        result["facts"] = [f for f in items["facts"] if isinstance(f, str)]
    for category in ("lessons", "failures", "patterns"):
        for item in items.get(category, []):
            if not isinstance(item, str):
                continue
            has_fail = bool(_FAILURE_SIGNALS.search(item))
            has_lesson = bool(_LESSON_SIGNALS.search(item))
            has_status = bool(_STATUS_SIGNALS.search(item))
            # Multi-signal: lesson always wins when co-occurring
            if has_lesson and (has_fail or has_status):
                result["lessons"].append(item)
                continue
            # Pure status with no lesson/failure signal -> drop
            if has_status and not has_fail and not has_lesson:
                continue
            # Single-signal reclassification
            if category == "lessons" and has_fail and not has_lesson:
                result["failures"].append(item)
            elif category == "failures" and has_lesson and not has_fail:
                result["lessons"].append(item)
            else:
                result[category].append(item)
    return result


def write_to_brain(items: dict, source_agent: str, timestamp: str, dry_run: bool = False):
    """Write extracted items to brain files via append_knowledge_md + fact_store."""
    items = _filter_prompt_echoes(items)  # Purge prompt example echoes FIRST
    items = _reclassify(items)
    written = {"lessons": 0, "failures": 0, "patterns": 0, "facts": 0}
    for item_type, filepath in BRAIN_FILES.items():
        for desc in items.get(item_type, []):
            if not isinstance(desc, str):
                continue
            desc = desc.strip()
            if len(desc) < 25 or len(desc) > 500 or len(desc.split()) < 5:
                continue
            sal = compute_item_salience(desc, priority=1.0)
            conf = compute_item_confidence(desc, source_type="scribe")
            append_knowledge_md(filepath=filepath, description=desc,
                                source=f"scribe:{source_agent}", timestamp=timestamp,
                                domain=_classify_domain(desc), confidence=conf, salience=sal, dry_run=dry_run)
            written[item_type] += 1

    # Facts: LLM returns plain strings → parse through fact_parser for structure
    raw_facts = items.get("facts", [])
    if raw_facts and not dry_run:
        from pulse.brain.extraction.fact_parser import parse_facts
        from pulse.brain.persistence.fact_store import store_facts
        structured = []
        for fact_str in raw_facts[:10]:
            if not isinstance(fact_str, str) or len(fact_str.strip()) < 10:
                continue
            parsed = parse_facts(fact_str.strip())
            if parsed:
                for f in parsed:
                    f["confidence"] = min(f.get("confidence", 0.7), 0.5)
                structured.extend(parsed)
        if structured:
            written["facts"] = store_facts(structured, source=f"scribe:{source_agent}")

    return written

def _archive_brain():
    """Archive current brain files before re-extraction."""
    ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
    for name, path in BRAIN_FILES.items():
        if path.exists() and path.stat().st_size > 200:
            shutil.copy2(path, ARCHIVE_DIR / f"{path.stem}_{ts}.md")
            print(f"  Archived: {path.name} ({path.stat().st_size // 1024}KB)")
            header = f"---\ntype: auto-extracted\nlast_updated: {ts[:8]}\n---\n\n"
            header += f"# {path.stem.replace('auto_', '').replace('_', ' ').title()}\n\n---\n"
            path.write_text(header, encoding="utf-8")
    from pulse.brain.save_writers import _DEDUP_CACHE
    _DEDUP_CACHE.clear()

if __name__ == "__main__":
    from pulse.admin.retroactive.retroactive_llm_extractor_main import main
    from pulse.brain.extraction.wants_classify import _classify_domain
    main()
