#!/usr/bin/env python3
# Copyright (c) 2026 PULSE Contributors. Proprietary. All rights reserved.
"""
Brain Search Index: Inverted keyword index for fast pre-filtering.

Built during consolidation (Stage 6b). Queried as fast-path in brain_search.py.
Hybrid design: index handles static sources, live parallel handles dynamic ones.

Index structure (state/search_index.json):
  {
    "version": 1,
    "built_at": "...",
    "item_count": N,
    "items": [
      {"source": "lessons", "title": "...", "text": "...", "salience": 0.8, "date": "..."},
      ...
    ],
    "index": {
      "docker": [0, 5, 12],
      "memory": [0, 3, 8],
      ...
    }
  }
"""
import re
import time
from pathlib import Path

from pulse.core.brain_utils import (
    HIPPOCAMPUS_DIR, LESSONS_DIR, FAILS_DIR, PATTERNS_DIR,
    EVIDENCE_DIR, STATE_DIR, load_json, load_json_dict, save_json,
    search_relevance, now_iso,
)
from pulse.brain.search_utils import compute_salience, days_since, get_retrieval_count

INDEX_FILE = STATE_DIR / "search_index.json"

# Stop words — too common to be useful in index
_STOP_WORDS = frozenset({
    "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
    "have", "has", "had", "do", "does", "did", "will", "would", "could",
    "should", "may", "might", "shall", "can", "need", "must", "dare",
    "to", "of", "in", "for", "on", "with", "at", "by", "from", "as",
    "into", "through", "during", "before", "after", "above", "below",
    "between", "out", "off", "over", "under", "again", "further", "then",
    "once", "here", "there", "when", "where", "why", "how", "all", "each",
    "every", "both", "few", "more", "most", "other", "some", "such", "no",
    "nor", "not", "only", "own", "same", "so", "than", "too", "very",
    "and", "but", "or", "if", "while", "that", "this", "these", "those",
    "it", "its", "we", "they", "them", "their", "what", "which", "who",
    "whom", "use", "used", "using", "file", "also", "just",
})

_TOKEN_RE = re.compile(r"[a-z0-9_]{3,}")


def _tokenize(text: str) -> set[str]:
    """Extract meaningful keywords from text."""
    return {w for w in _TOKEN_RE.findall(text.lower()) if w not in _STOP_WORDS}


def build_index(verbose: bool = False) -> dict:
    """Build inverted keyword index from all static brain sources.

    Static sources: lessons, failures, patterns, schemas, procedures,
    domain knowledge, evidence (sessions/decisions/intents).
    Dynamic sources (predictions, active anti-patterns) are NOT indexed.
    """
    start = time.time()
    items = []

    # 1. Markdown brain files (lessons, failures, patterns)
    md_sources = {
        "lessons": LESSONS_DIR / "auto_lessons.md",
        "failures": FAILS_DIR / "auto_fails.md",
        "patterns": PATTERNS_DIR / "auto_patterns.md",
    }
    for source_name, md_file in md_sources.items():
        if not md_file.exists():
            continue
        text = md_file.read_text(encoding="utf-8", errors="replace")
        for section in text.split("\n## ")[1:]:
            lines = section.strip().split("\n")
            title = lines[0].strip()
            body = "\n".join(lines[1:])[:300]
            date_match = re.search(r"\*\*Date:\*\*\s*(\d{4}-\d{2}-\d{2})", body)
            date = date_match.group(1) if date_match else ""
            items.append({
                "source": source_name,
                "title": title,
                "text": body[:200],
                "date": date,
            })

    # 2. Domain knowledge (curated .md files in Knowledge/)
    knowledge_dir = HIPPOCAMPUS_DIR / "Knowledge"
    if knowledge_dir.exists():
        for md_file in knowledge_dir.glob("*.md"):
            text = md_file.read_text(encoding="utf-8", errors="replace")
            for section in text.split("\n## ")[1:]:
                lines = section.strip().split("\n")
                title = lines[0].strip()
                body = "\n".join(lines[1:])[:300]
                items.append({
                    "source": "knowledge",
                    "title": f"{md_file.stem}: {title}",
                    "text": body[:200],
                    "date": "",
                })

    # 3. Schemas (JSON)
    schemas_file = HIPPOCAMPUS_DIR / "Evidence" / "Metrics" / "schemas.json"
    if schemas_file.exists():
        schemas = load_json(schemas_file)
        for s in schemas if isinstance(schemas, list) else []:
            items.append({
                "source": "schemas",
                "title": s.get("name", ""),
                "text": s.get("description", "")[:200],
                "date": s.get("promoted_at", ""),
            })

    # 4. Procedures (JSON)
    proc_file = HIPPOCAMPUS_DIR / "Evidence" / "Metrics" / "procedural_log.json"
    if proc_file.exists():
        procs = load_json(proc_file)
        for p in procs if isinstance(procs, list) else []:
            items.append({
                "source": "procedures",
                "title": p.get("procedure", ""),
                "text": f"success: {p.get('success_rate', 0)}, runs: {p.get('total_runs', 0)}",
                "date": p.get("last_run", ""),
            })

    # 5. Evidence sessions
    if EVIDENCE_DIR.exists():
        for ef in sorted(EVIDENCE_DIR.rglob("*.json"))[:50]:  # Cap to prevent bloat
            ev_items = load_json(ef)
            if not isinstance(ev_items, list):
                continue
            for item in ev_items[:20]:  # Cap per file
                text = (item.get("evidence", "")
                        or item.get("description", "")
                        or item.get("want", "")
                        or item.get("dont_want", "")
                        or item.get("lesson", "")
                        or "")
                if not text or len(text) < 10:
                    continue
                items.append({
                    "source": "evidence",
                    "title": ef.stem,
                    "text": text[:200],
                    "date": item.get("date", "") or item.get("timestamp", ""),
                })

    # Build inverted index
    index = {}
    for i, item in enumerate(items):
        searchable = f"{item['title']} {item['text']}"
        keywords = _tokenize(searchable)
        for kw in keywords:
            index.setdefault(kw, []).append(i)

    elapsed_ms = int((time.time() - start) * 1000)
    result = {
        "version": 1,
        "built_at": now_iso(),
        "build_ms": elapsed_ms,
        "item_count": len(items),
        "term_count": len(index),
        "items": items,
        "index": index,
    }

    save_json(INDEX_FILE, result)
    if verbose:
        print(f"  [INDEX] Built: {len(items)} items, {len(index)} terms in {elapsed_ms}ms")
    return {"status": "ok", "items": len(items), "terms": len(index), "build_ms": elapsed_ms}


def query_index(query: str, config: dict, max_results: int = 10) -> list:
    """Fast-path search using the inverted index.

    Returns scored results matching query keywords.
    Falls back to empty list if index doesn't exist (live search takes over).
    """
    if not INDEX_FILE.exists():
        return []

    data = load_json_dict(INDEX_FILE)
    if not data or data.get("version") != 1:
        return []

    items = data.get("items", [])
    index = data.get("index", {})
    if not items or not index:
        return []

    # Tokenize query and find matching item IDs
    query_tokens = _tokenize(query)
    if not query_tokens:
        return []

    # Score items by number of matching keywords
    item_scores: dict[int, int] = {}
    for token in query_tokens:
        for item_id in index.get(token, []):
            item_scores[item_id] = item_scores.get(item_id, 0) + 1

    if not item_scores:
        return []

    # Rank by keyword overlap, then compute full salience for top candidates
    candidates = sorted(item_scores.items(), key=lambda x: -x[1])[:max_results * 3]

    results = []
    for item_id, keyword_hits in candidates:
        item = items[item_id]
        searchable = f"{item['title']} {item['text']}"
        relevance = search_relevance(query, searchable)
        if relevance < 0.15:
            continue
        days = days_since(item.get("date", ""))
        salience = compute_salience(
            relevance, days, config,
            retrieval_count=get_retrieval_count(searchable[:200])
        )
        results.append({
            "source": item["source"],
            "title": item.get("title", ""),
            "text": item.get("text", ""),
            "relevance": round(relevance, 3),
            "salience": salience,
            "from_index": True,
        })

    results.sort(key=lambda x: -x["salience"])
    return results[:max_results]
