#!/usr/bin/env python3
# Copyright (c) 2026 PULSE Contributors. Proprietary. All rights reserved.
"""D16 Pipeline: Data loading, profiling, search, synthesis, and storage stages.

Extracted from agent_procedure_synthesizer.py per Law 7 (files <= 300 lines).
"""
from __future__ import annotations
import json, re
from datetime import datetime, timedelta, timezone

from pulse.core.brain_utils import HIPPOCAMPUS_DIR, now_iso
from pulse.core.brain_io import _acquire
from pulse.core.paths import get_root_dir

ROOT = get_root_dir()
HEBBIAN_FILE = ROOT / "state" / "hebbian_matrix.json"
PROCEDURES_FILE = HIPPOCAMPUS_DIR / "Evidence" / "Metrics" / "procedural_log.json"

# --- Thresholds ---
MIN_OBSERVATIONS = 10
MIN_DOMAIN_OBS = 3
WEAK_WEIGHT = 0.4
WEAK_RATE = 0.5
MAX_PROCS_PER_RUN = 5
MAX_PROCS_PER_MODEL = 8
TTL_DAYS = 30
_GARBAGE_RE = re.compile(r"[(){}\[\]\"']|python\s|import\s|\.py")

SYNTH_PROMPT = """Analyze a model's weak domains and create defensive procedures.

MODEL: {model_id}
WEAK DOMAINS:
{weak_summary}

BRAIN KNOWLEDGE:
{brain_knowledge}

Create {max_procs} step-by-step defensive procedures to help this model avoid failures in its weak domains.

Output ONLY valid JSON:
{{"procedures": [{{"name": "Name", "description": "When to use", "steps": ["Step 1", "Step 2", "Step 3"], "domain": "category", "confidence": 0.65}}]}}

Rules: non-empty "steps" (2-6 items), confidence 0.5-0.8, focus on WEAK domains, be specific not vague. JSON only."""


# --- Data Loading ---

def _load_hebbian() -> dict:
    """Load hebbian_matrix.json safely."""
    if not HEBBIAN_FILE.exists():
        return {}
    try:
        with _acquire(HEBBIAN_FILE):
            data = json.loads(HEBBIAN_FILE.read_text(encoding="utf-8"))
        return data.get("matrix", {})
    except (json.JSONDecodeError, OSError):
        return {}


def _sanitize_domain(name: str) -> str | None:
    """Reject garbage domain names. Returns cleaned name or None."""
    name = name.strip().lower()
    if not name or len(name) > 40:
        return None
    if " " in name and len(name) > 20:
        return None
    if _GARBAGE_RE.search(name):
        return None
    if name.count(",") > 1:
        return None
    return name


def _load_procedures() -> dict:
    """Load procedural_log.json."""
    if not PROCEDURES_FILE.exists():
        return {"procedures": [], "last_updated": now_iso()}
    try:
        data = json.loads(PROCEDURES_FILE.read_text(encoding="utf-8"))
        if isinstance(data, list):
            return {"procedures": data, "last_updated": now_iso()}
        return data if isinstance(data, dict) else {"procedures": [], "last_updated": now_iso()}
    except (json.JSONDecodeError, OSError):
        return {"procedures": [], "last_updated": now_iso()}


# --- Pipeline Stages ---

def _prune_expired(verbose_fn) -> int:
    """Remove expired agent-synthesized procedures."""
    now = datetime.now(timezone.utc)
    pruned = 0
    with _acquire(PROCEDURES_FILE):
        current = _load_procedures()
        procs = current.get("procedures", [])
        kept = []
        for p in procs:
            if p.get("source") != "agent_synthesized":
                kept.append(p)
                continue
            expires = p.get("expires_at", "")
            if not expires:
                kept.append(p)
                continue
            try:
                exp_dt = datetime.fromisoformat(expires)
                if exp_dt.tzinfo is None:
                    exp_dt = exp_dt.replace(tzinfo=timezone.utc)
                if exp_dt < now:
                    pruned += 1
                    verbose_fn(f"[D16]   Pruned expired: {p.get('name', '?')}")
                    continue
            except (ValueError, TypeError):
                pass
            kept.append(p)
        if pruned:
            current["procedures"] = kept
            current["last_updated"] = now_iso()
            PROCEDURES_FILE.parent.mkdir(parents=True, exist_ok=True)
            # P47: SQLite-first — sync surviving procedures after prune
            try:
                from pulse.brain.sqlite_writers import bulk_upsert_procedures
                bulk_upsert_procedures(kept, source="agent_synth_pipeline")
            except Exception as _e:
                import logging as _logging
                _logging.getLogger("pulse.agent_synth_pipeline").warning("bulk_upsert_procedures failed: %s", _e)
        PROCEDURES_FILE.write_text(json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8")
    verbose_fn(f"[D16] Pruned {pruned} expired procedure(s)")
    return pruned


def _find_weak_models(matrix: dict) -> dict:
    """Find models with weak domains from hebbian matrix.

    Returns: {model_id: [{"domain", "weight", "success_rate", "observations"}]}
    """
    weak_models = {}
    for model_id, domains in matrix.items():
        total_obs = 0
        weak_domains = []
        for raw_domain, stats in domains.items():
            domain = _sanitize_domain(raw_domain)
            if not domain:
                continue
            s = stats.get("successes", 0)
            f = stats.get("failures", 0)
            obs = s + f
            total_obs += obs
            w = stats.get("weight", 0.5)
            rate = s / obs if obs > 0 else 0.0
            if obs >= MIN_DOMAIN_OBS and (w < WEAK_WEIGHT or rate < WEAK_RATE):
                weak_domains.append({
                    "domain": domain, "weight": round(w, 3),
                    "success_rate": round(rate, 3), "observations": obs,
                })
        if total_obs >= MIN_OBSERVATIONS and weak_domains:
            weak_domains.sort(key=lambda d: d["success_rate"])
            weak_models[model_id] = weak_domains
    return weak_models


def _count_existing(model_id: str) -> int:
    """Count existing agent-synthesized procedures for a model."""
    current = _load_procedures()
    return sum(1 for p in current.get("procedures", [])
               if p.get("source") == "agent_synthesized"
               and p.get("synthesized_for") == model_id)


def _search_domain_knowledge(domain: str) -> dict:
    """Use brain_search for domain-specific knowledge."""
    try:
        from pulse.brain.brain_search import brain_search
        result = brain_search(domain, max_per_source=3)
        hits = result.get("results", {})
        out = {"lessons": [], "failures": [], "anti_patterns": []}
        for key, target in [("wins", "lessons"), ("fails", "failures"), ("anti_patterns", "anti_patterns")]:
            for h in hits.get(key, [])[:5]:
                t = (h.get("text") or h.get("title", ""))[:80].strip()
                if t and len(t) > 10:
                    out[target].append(t)
        return out
    except Exception:
        return {"lessons": [], "failures": [], "anti_patterns": []}


def _build_context(model_id: str, weak_domains: list, knowledge: dict) -> str:
    """Build LLM prompt context from model profile + brain hits."""
    weak_lines = []
    for wd in weak_domains[:5]:
        weak_lines.append(
            f"  {wd['domain']}: weight={wd['weight']}, "
            f"success_rate={wd['success_rate']:.0%} ({wd['observations']} obs)")
    brain_lines = []
    for label, key in [("Lessons", "lessons"), ("Failures", "failures"), ("Anti-patterns", "anti_patterns")]:
        if knowledge.get(key):
            brain_lines.append(f"{label}:\n" + "\n".join(f"  - {item}" for item in knowledge[key]))
    n_procs = min(MAX_PROCS_PER_RUN, max(3, len(weak_domains) * 2))
    return SYNTH_PROMPT.format(
        model_id=model_id,
        weak_summary="\n".join(weak_lines) or "(none detected)",
        brain_knowledge="\n".join(brain_lines) or "(no brain data for these domains)",
        max_procs=n_procs)


def _synthesize_procedures(model_id: str, context: str) -> list[dict]:
    """LLM-synthesize defensive procedures."""
    from pulse.core.llm_router import dispatch, extract_json
    try:
        result = extract_json(dispatch(context, task_type="fast"))
    except Exception:
        return []
    valid = []
    for proc in result.get("procedures", []):
        name = (proc.get("name") or "").strip()
        steps = [s for s in proc.get("steps", []) if isinstance(s, str) and s.strip()]
        if not name or not steps:
            continue
        proc["steps"] = steps
        proc.setdefault("confidence", 0.65)
        proc.setdefault("domain", "general")
        proc.setdefault("description", "")
        valid.append(proc)
    return valid[:MAX_PROCS_PER_RUN]


def _fallback_procedures(model_id: str, weak_domains: list) -> list[dict]:
    """Rule-based fallback when LLM is unavailable."""
    procs = []
    for wd in weak_domains[:3]:
        procs.append({
            "name": f"{wd['domain'].title()} Recovery for {model_id.split('-')[0]}",
            "description": f"Defensive checklist for {wd['domain']} domain (success rate: {wd['success_rate']:.0%})",
            "steps": [
                f"Check anti-patterns for {wd['domain']} before starting",
                "Review relevant brain lessons with brain_query.py",
                "Test changes before committing",
                "Save failure learnings explicitly after completion",
            ],
            "domain": wd["domain"],
            "confidence": 0.55,
        })
    return procs


def _store_procedures(model_id: str, raw_procs: list[dict], verbose_fn) -> int:
    """Append agent-synthesized procedures to procedural_log.json. Returns count stored."""
    if not raw_procs:
        return 0
    now_str = now_iso()
    expires = (datetime.now(timezone.utc) + timedelta(days=TTL_DAYS)).isoformat()

    with _acquire(PROCEDURES_FILE):
        current = _load_procedures()
        procs = current.get("procedures", [])
        existing_names = {(p.get("name") or "").lower() for p in procs}
        added = 0
        for proc in raw_procs:
            name = proc["name"]
            if name.lower() in existing_names:
                verbose_fn(f"[D16]     Skipped duplicate: {name}")
                continue
            procs.append({
                "name": name, "description": proc.get("description", ""),
                "steps": proc["steps"], "domain": proc.get("domain", "general"),
                "confidence": proc.get("confidence", 0.65), "created": now_str,
                "source": "agent_synthesized",
                "synthesized_for": model_id, "synthesized_at": now_str,
                "expires_at": expires,
                "executions": 0, "successes": 0, "failures": 0,
            })
            existing_names.add(name.lower())
            added += 1
        current["procedures"] = procs
        current["last_updated"] = now_str
        PROCEDURES_FILE.parent.mkdir(parents=True, exist_ok=True)
        # P47: SQLite-first — write new entries to SQLite before JSON export
        try:
            from pulse.brain.sqlite_writers import bulk_upsert_procedures
            new_entries = [p for p in procs if p.get("source") == "agent_synthesized"
                           and p.get("synthesized_for") == model_id]
            bulk_upsert_procedures(new_entries, source="agent_synth_pipeline")
        except Exception as _e:
            import logging as _logging
            _logging.getLogger("pulse.agent_synth_pipeline").warning("bulk_upsert_procedures failed: %s", _e)
        PROCEDURES_FILE.write_text(json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8")
    return added
