#!/usr/bin/env python3
# Copyright (c) 2026 PULSE Contributors. Proprietary. All rights reserved.
# See LICENSE file for details.
"""
Brain Search Sources (Extended): Patterns, schemas, procedures, graph, etc. (V43.14)

Extended search functions for specialized brain regions.
All functions have the same signature: (query: str, config: dict) -> list

Dependencies: Python stdlib + brain_utils + search_utils
"""

import hashlib
import json
import re
from pathlib import Path

from pulse.core.brain_utils import (load_json, load_json_dict, search_relevance,
                         PATTERNS_DIR, DOMAINS_DIR, HIPPOCAMPUS_DIR,
                         LESSONS_JSONL, FAILS_JSONL, PATTERNS_JSONL, PRIVATE_JSONL)
from pulse.brain.index.search_utils import compute_salience, days_since, get_retrieval_count, SOURCE_WEIGHT

def _search_jsonl(jsonl_path: Path, query: str, config: dict,
                  source_name: str, result_type: str) -> list:
    """Generic JSONL search: stream lines, match query, return scored results.

    Returns empty list if JSONL doesn't exist or is empty, allowing MD fallback.
    """
    if not jsonl_path.exists() or jsonl_path.stat().st_size == 0:
        return []
    results = []
    with open(jsonl_path, "r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                entry = json.loads(line)
            except json.JSONDecodeError:
                continue
            content = entry.get("content", "")
            title = entry.get("title", "")
            search_text = f"{title} {content}"
            sim = search_relevance(query, search_text)
            if sim < 0.3:
                continue
            days = days_since(entry.get("timestamp", ""))
            item_salience = entry.get("salience", 1.0)
            results.append({
                "source": source_name,
                "file": jsonl_path.name,
                "title": title,
                "text": content[:200],
                "relevance": round(sim, 3),
                "salience": compute_salience(sim, days, config, item_salience,
                                             retrieval_count=get_retrieval_count(content[:200])),
                "confidence": entry.get("confidence", 0.5),
                "type": result_type,
                "consolidation_count": entry.get("consolidation_count", 1),
                "id": entry.get("id", ""),
                "_timestamp": entry.get("timestamp", ""),
            })
    return sorted(results, key=lambda x: -x["salience"])

def _sanitize_fts_query(query: str) -> str:
    """Sanitize a free-text query for FTS5 MATCH safety.

    Always uses OR for broad recall — precision is handled downstream by
    search_relevance() scoring + 0.3 relevance gate in _row_to_hit().
    AND was killing 62K entries: a 3+ word AND query matches almost nothing.
    """
    _STOP_WORDS = {
        'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
        'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
        'should', 'may', 'might', 'shall', 'can', 'to', 'of', 'in', 'for',
        'on', 'with', 'at', 'by', 'from', 'as', 'into', 'about', 'it', 'its',
        'this', 'that', 'these', 'those', 'i', 'we', 'you', 'he', 'she', 'they',
        'me', 'us', 'him', 'her', 'them', 'my', 'our', 'your', 'his', 'their',
        'what', 'which', 'who', 'when', 'where', 'how', 'not', 'no', 'nor',
        'but', 'or', 'and', 'if', 'then', 'so', 'up', 'out', 'just', 'also',
    }
    words = re.findall(r'[a-zA-Z0-9_]+', query)
    words = [w for w in words if len(w) >= 3 and w.lower() not in _STOP_WORDS]
    if not words:
        return ""
    # Always OR — FTS is the recall layer, Python scoring handles precision.
    # Words are unquoted so porter stemmer activates ("deploy" matches "deployment").
    # Quoting disables stemming — QR8 Panel 12 found this killed morphological matching.
    return " OR ".join(words[:10])
def _row_to_hit(row, source_name: str, result_type: str,
                query: str, config: dict) -> dict:
    """Convert a SQLite Row to the same hit dict format as _search_jsonl.

    Applies search_relevance() and compute_salience() scoring so results
    are ranked identically to the JSONL path. Scales final salience by
    SOURCE_WEIGHT[source_type] so CONFIRMED items rank above CONVERSATIONAL.
    """
    content = row["content"] or ""
    title = row["title"] or ""
    search_text = f"{title} {content}"
    sim = search_relevance(query, search_text)
    days = days_since(row["timestamp"] or "")
    item_salience = float(row["salience"]) if row["salience"] else 1.0
    tags_raw = row["tags"] if row["tags"] else "[]"
    try:
        tags = json.loads(tags_raw) if isinstance(tags_raw, str) else tags_raw
    except (json.JSONDecodeError, TypeError):
        tags = []

    # Extract source_type — real column (added via migration); default CONVERSATIONAL
    source_type = row["source_type"] if row["source_type"] else "CONVERSATIONAL"
    ground_weight = SOURCE_WEIGHT.get(source_type, 0.5)

    return {
        "source": source_name,
        "file": f"sqlite:{source_name}",
        "title": title,
        "text": content[:200],
        "relevance": round(sim, 3),
        "salience": compute_salience(sim, days, config, item_salience,
                                     retrieval_count=get_retrieval_count(content[:200]),
                                     ground_weight=ground_weight),
        "confidence": float(row["confidence"]) if row["confidence"] else 0.5,
        "type": result_type,
        "consolidation_count": int(row["consolidation_count"]) if row["consolidation_count"] else 1,
        "id": row["id"] or "",
        "source_type": source_type,
        "_timestamp": row["timestamp"] or "",
        "access_count": int(row["access_count"]) if row["access_count"] else 0,
        "validity": row["validity"] or "valid",
        "fingerprint": row["fingerprint"] or "",
        "last_accessed": row["last_accessed"] or "",
    }

def _search_sqlite(table_name: str, source_name: str, result_type: str,
                   query: str, config: dict) -> list:
    """Search a knowledge table via FTS5. Returns scored hits or empty list on failure."""
    fts_query = _sanitize_fts_query(query)
    if not fts_query:
        return []
    try:
        from pulse.core.brain_db import get_conn
        conn = get_conn()
        max_results = config.get("max_per_source", 50) if config else 50
        # Fetch 4x candidates — OR recall is broad, Python scoring handles precision.
        # ORDER BY rank uses FTS5 BM25 so items matching MORE query words come first.
        candidate_limit = max_results * 4
        rows = conn.execute("""
            SELECT t.id, t.content, t.title, t.domain, t.agent, t.salience,
                   t.confidence, t.consolidation_count, t.evidence_count,
                   t.tags, t.timestamp, t.source_type,
                   t.access_count, t.validity, t.fingerprint, t.last_accessed
            FROM {table} t
            JOIN knowledge_fts fts ON fts.item_id = t.id AND fts.table_name = ?
            WHERE fts.knowledge_fts MATCH ?
            ORDER BY fts.rank
            LIMIT ?
        """.format(table=table_name), (table_name, fts_query, candidate_limit)).fetchall()
        if not rows:
            return []
        hits = []
        for r in rows:
            hit = _row_to_hit(r, source_name, result_type, query, config)
            # Apply same relevance gate as JSONL path
            if hit["relevance"] >= 0.3:
                hits.append(hit)
        return sorted(hits, key=lambda x: -x["salience"])[:max_results]
    except Exception:
        return []

def search_wins(query: str, config: dict) -> list:
    """Search lessons — proven approaches (consolidation >= 3) + recent (< 7 days).

    Fresh lessons bypass the consolidation gate because new knowledge shouldn't
    be invisible until consolidated 3 times. After 7 days, consolidation proves value.
    SQLite FTS5 fast-path with JSONL fallback.
    """
    min_cc = config.get("wins", {}).get("min_consolidation_count", 3)
    recency_days = config.get("wins", {}).get("recency_bypass_days", 7)

    def _passes_gate(hit: dict) -> bool:
        """Proven (high consolidation) OR recent (< recency_days)."""
        if hit.get("consolidation_count", 1) >= min_cc:
            return True
        return days_since(hit.get("_timestamp", "")) < recency_days

    # Try SQLite FTS5 first
    results = _search_sqlite("lessons", "wins", "proven_approach", query, config)
    if results:
        return [r for r in results if _passes_gate(r)]

    # Fallback to JSONL scan
    jsonl_results = _search_jsonl(LESSONS_JSONL, query, config, "wins", "proven_approach")
    return [r for r in jsonl_results if _passes_gate(r)]

def search_fails(query: str, config: dict) -> list:
    """Search FAIL history for past mistakes. SQLite FTS5 with JSONL fallback."""
    results = _search_sqlite("failures", "fails", "past_mistake", query, config)
    if results:
        return results
    return _search_jsonl(FAILS_JSONL, query, config, "fails", "past_mistake")

def search_domains(query: str, config: dict) -> list:
    """Search domain knowledge files (JSON only — MDs removed from Hippocampus)."""
    results = []
    if not DOMAINS_DIR.exists():
        return results
    for df in sorted(DOMAINS_DIR.rglob("*.json")):
        if df.name == "knowledge_graph.json":
            continue  # Searched by search_graph()
        items = load_json(df)
        if isinstance(items, dict):
            items = items.get("facts", items.get("items", [items]))
        if not isinstance(items, list):
            continue
        for item in items:
            text = item.get("description", "") or item.get("text", "") or item.get("claim", "")
            sim = search_relevance(query, text)
            if sim >= 0.3:
                days = days_since(item.get("timestamp", ""))
                results.append({
                    "source": "domains", "file": df.name,
                    "text": text[:200], "relevance": round(sim, 3),
                    "salience": compute_salience(sim, days, config,
                                                 retrieval_count=get_retrieval_count(text[:200])),
                    "type": "domain_context",
                    "confidence": item.get("confidence", 0.5),
                })
    return sorted(results, key=lambda x: -x["salience"])

def search_private(query: str, config: dict) -> list:
    """Search private brain files. SQLite FTS5 with JSONL + JSON fallback.

    SQLite fast-path activates once the private table is populated.
    Falls back to JSONL + JSON flat-file scan otherwise.
    """
    # --- SQLite fast-path ---
    sqlite_results = _search_sqlite("private", "private", "private_knowledge", query, config)
    if sqlite_results:
        return sqlite_results

    # --- Flat-file fallback (primary path until migration) ---
    # JSONL fast-path
    results = _search_jsonl(PRIVATE_JSONL, query, config, "private", "private_knowledge")

    # Also search private JSON files (evidence)
    private_dir = HIPPOCAMPUS_DIR / "Private"
    if private_dir.exists():
        for pf in sorted(private_dir.rglob("*.json")):
            items = load_json(pf)
            if not isinstance(items, list):
                continue
            for item in items:
                text = item.get("description", "") or item.get("text", "")
                sim = search_relevance(query, text)
                if sim >= 0.3:
                    days = days_since(item.get("timestamp") or item.get("date"))
                    results.append({
                        "source": "private", "file": pf.name,
                        "text": text[:200], "relevance": round(sim, 3),
                        "salience": compute_salience(sim, days, config,
                                                     retrieval_count=get_retrieval_count(text[:200])),
                        "type": "private_knowledge"
                    })
    return sorted(results, key=lambda x: -x["salience"])


def search_patterns(query: str, config: dict) -> list:
    """Search Patterns files. SQLite FTS5 with JSONL fallback."""
    results = _search_sqlite("patterns", "patterns", "pattern", query, config)
    if results:
        return results
    return _search_jsonl(PATTERNS_JSONL, query, config, "patterns", "pattern")


# Re-export extended functions so existing callers don't break
from .search_sources_ext2 import (  # noqa: F401
    search_schemas,
    search_graph,
    search_procedures,
    search_meta_schemas,
)
