#!/usr/bin/env python3
# Copyright (c) 2026 PULSE Contributors. Proprietary. All rights reserved.
"""Retroactive LLM Extractor — Brain V2. Semantic sweep of capture logs via LLM Router."""

import importlib
import io, json, os, re, shutil, sys, time
from datetime import datetime, timezone
from pathlib import Path

import logging
log = logging.getLogger("pulse.admin.retroactive.retroactive_llm_extractor")

SCRIPT_DIR = Path(__file__).resolve().parent
_ALLOWED_REBUILD_MODULES = frozenset({"pulse.brain.index.search_index", "pulse.graph.knowledge_graph_build"})
ROOT_DIR = SCRIPT_DIR.parent.parent.parent

if str(ROOT_DIR) not in sys.path:
    sys.path.insert(0, str(ROOT_DIR))

if os.name == "nt" and __name__ == "__main__":
    if hasattr(sys.stdout, 'reconfigure'):
        sys.stdout.reconfigure(encoding='utf-8', errors='replace')
    if hasattr(sys.stderr, 'reconfigure'):
        sys.stderr.reconfigure(encoding='utf-8', errors='replace')

from pulse.core.brain_utils import HIPPOCAMPUS_DIR, truncate_at_sentence
from pulse.core.llm_router import dispatch, extract_json as router_extract_json
from pulse.core.brain_analysis import compute_item_salience, compute_item_confidence

STATE_DIR = ROOT_DIR / "state"
ARCHIVE_DIR = HIPPOCAMPUS_DIR / "Archive"

CAPTURE_LOGS = [STATE_DIR / f"pulse_captured_{a}.log" for a in ("claude", "gemini", "codex", "grok")]

BATCH_SIZE = 15
MAX_BATCH_TOKENS = 12000

SCRIBE_PROMPT = """You are the PULSE Memory Recovery Agent. Extract engineering knowledge from chat transcripts into exactly 5 categories.

CATEGORIES:
- "lessons": Reusable rules or best practices. Something an engineer should ALWAYS or NEVER do.
- "failures": Bugs, crashes, regressions, or design mistakes. Something BROKE or WENT WRONG.
- "patterns": Recurring workflows or debugging strategies. A HOW-TO or WHEN-THEN approach.
- "facts": Declarative statements about infrastructure, services, tools, accounts, or configurations.
- "decisions": Committed architectural or technical choices — what approach was SELECTED and why.

RULES:
1. Each item: complete sentence, 30-200 chars, actionable, specific.
2. Facts: short declarative statements (10-150 chars) about WHAT is used/deployed/configured WHERE.
3. Decisions: include both the choice AND the reason (if stated). Format: "Use X over Y because Z".
4. Status updates or task summaries are NEVER knowledge — skip them.
5. If ambiguous between categories, pick the STRONGEST match.

OUTPUT — strict JSON:
{"lessons": ["..."], "failures": ["..."], "patterns": ["..."], "facts": ["..."], "decisions": ["..."]}

Transcript:
"""

extract_json = router_extract_json

from pulse.admin.retroactive.prompt_echo_filter import build_echo_filter
_is_prompt_echo, _filter_prompt_echoes = build_echo_filter(SCRIBE_PROMPT)


_LOG_LINE = re.compile(
    r"^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]\s+"
    r"(USER|CLAUDE|GEMINI|CODEX|GROK):\s+"
    r"(.+?)(?:\s*\|\s*SIG:\s*\S+)?\s*$"
)


def _parse_interactions(log_path: Path, start_offset: int = 0):
    if not log_path.exists():
        return
    current_ts, current_role, lines = None, None, []
    with open(log_path, "r", encoding="utf-8", errors="replace") as f:
        if start_offset > 0:
            f.seek(start_offset)
        for line in f:
            m = _LOG_LINE.match(line)
            if m:
                if current_role:
                    yield current_ts, current_role, " ".join(lines)
                current_ts, current_role, lines = m.group(1), m.group(2), [m.group(3)]
            elif line.rstrip() and current_role:
                lines.append(line.rstrip())
    if current_role:
        yield current_ts, current_role, " ".join(lines)


def batch_interactions(log_path: Path, batch_size: int = BATCH_SIZE,
                       start_offset: int = 0):
    batch_lines, pair_count, char_count, first_ts = [], 0, 0, None
    for ts, role, msg in _parse_interactions(log_path, start_offset=start_offset):
        if first_ts is None:    
            first_ts = ts       
        if len(msg) > 2000:     
            msg = truncate_at_sentence(msg, 2000) + "... [truncated]"
        batch_lines.append(f"[{ts}] {role}: {msg}")

        char_count += len(msg)
        if role != "USER":
            pair_count += 1
        if pair_count >= batch_size or char_count > MAX_BATCH_TOKENS * 4:
            yield "\n".join(batch_lines), pair_count, first_ts
            batch_lines, pair_count, char_count, first_ts = [], 0, 0, None
    if batch_lines:
        yield "\n".join(batch_lines), pair_count, first_ts


_FAILURE_SIGNALS = re.compile(r"\bfailed\b|\bbroke\b|\bcrashed\b|\bbug\b|\bregression\b|\bdata loss\b|\bcorrupted\b", re.IGNORECASE)
_LESSON_SIGNALS = re.compile(r"\balways \b|\bnever \b|\bshould \b|\bmust \b|\bdo not \b", re.IGNORECASE)
_STATUS_SIGNALS = re.compile(r"\bwas completed\s*$|\bhas been (?:deployed|shipped|merged|released)\b", re.IGNORECASE)


def _reclassify(items: dict) -> dict:
    result = {"lessons": [], "failures": [], "patterns": []}
    if items.get("facts"):
        result["facts"] = [f for f in items["facts"] if isinstance(f, str)]
    for category in ("lessons", "failures", "patterns"):
        for item in items.get(category, []):
            if not isinstance(item, str): continue
            has_fail, has_lesson, has_status = bool(_FAILURE_SIGNALS.search(item)), bool(_LESSON_SIGNALS.search(item)), bool(_STATUS_SIGNALS.search(item))
            if has_lesson and (has_fail or has_status):
                result["lessons"].append(item)
                continue
            if has_status and not has_fail and not has_lesson: continue
            if category == "lessons" and has_fail and not has_lesson:
                result["failures"].append(item)
            elif category == "failures" and has_lesson and not has_fail:
                result["lessons"].append(item)
            else:
                result[category].append(item)
    return result


def write_to_brain(items: dict, source_agent: str, timestamp: str, dry_run: bool = False):
    """Write extracted items directly to SQLite (authoritative)."""
    items = _filter_prompt_echoes(items)
    items = _reclassify(items)
    written = {"lessons": 0, "failures": 0, "patterns": 0, "facts": 0, "decisions": 0}
    
    from pulse.core.brain_db import insert_knowledge
    from pulse.brain.extraction.wants_classify import _classify_domain as _cd
    
    for item_type in ("lessons", "failures", "patterns"):
        for desc in items.get(item_type, []):
            if not isinstance(desc, str) or len(desc) < 25 or len(desc.split()) < 5:
                continue
            _domain = _cd(desc) if _cd else "general"
            entry = {
                "content": desc, "title": desc[:80], "domain": _domain,
                "agent": f"scribe:{source_agent}", "timestamp": timestamp,
                "salience": compute_item_salience(desc, priority=1.0),
                "confidence": compute_item_confidence(desc, source_type="scribe"),
                "type": item_type.rstrip("s")
            }
            if insert_knowledge(item_type, entry):
                written[item_type] += 1

    # Facts: LLM returns plain strings
    raw_facts = items.get("facts", [])
    if raw_facts and not dry_run:
        from pulse.brain.persistence.fact_store import store_facts
        from pulse.brain.extraction.fact_parser import parse_facts
        structured = []
        for fact_str in raw_facts[:10]:
            parsed = parse_facts(fact_str.strip())
            if parsed:
                for f in parsed: f["confidence"] = min(f.get("confidence", 0.7), 0.5)
                structured.extend(parsed)
        if structured:
            written["facts"] = store_facts(structured, source=f"scribe:{source_agent}")

    # Decisions: store_decision
    raw_decisions = items.get("decisions", [])
    if raw_decisions and not dry_run:
        from pulse.brain.persistence.decision_store import store_decision
        for dec_str in raw_decisions[:8]:
            dec = {
                "choice": dec_str[:120], "reasoning": "", "scope": "general",
                "confidence": 0.7, "domain": _cd(dec_str) if _cd else "general",
                "participants": [f"scribe:{source_agent}"], "timestamp": timestamp,
                "extraction_method": "scribe_llm", "status": "active"
            }
            if store_decision(dec): written["decisions"] += 1

    return written

def _archive_brain():
    """Archive functionality removed: SQLite is authoritative, no MD files to truncate."""
    pass

if __name__ == "__main__":
    from pulse.admin.retroactive.retroactive_llm_extractor_main import main
    main()
