"""Unified Knowledge Graph for GreyMatter.

Entry types cover both technical and personal knowledge:
  Technical: pattern, correction, expertise, gotcha, reference, observation
  Personal:  mood, habit, goal, reflection, learning, values, relationship

Scopes: cross-cutting, domain-specific.

Knowledge categories control TTL-based freshness decay:
  infrastructure (7d), architecture (14d), configuration (14d),
  correction (30d), process (30d), pattern (60d), domain_expertise (90d).
"""
import logging
import re
from typing import Any, Optional, TYPE_CHECKING

from .db import Database
from .ids import generate_knowledge_id
from .knowledge_categorizer import (
    classify_entry as _classify_entry,
    get_ttl_for_category,
    VALID_CATEGORIES as KNOWLEDGE_CATEGORIES,
)

_log = logging.getLogger(__name__)

if TYPE_CHECKING:
    from .lineage import LineageTracker

# Optional SecureLLM knowledge scanner -- available when solo package is installed
try:
    from solo.knowledge_scanner import scan_knowledge_entry, KnowledgeScanConfig
    _SCANNER_AVAILABLE = True
except ImportError:
    _SCANNER_AVAILABLE = False


ENTRY_TYPES = {
    # Technical / work-focused
    "pattern", "correction", "expertise", "gotcha", "reference", "observation",
    # Personal / whole-person (classification_level=4, Cognitive — never syncs)
    "mood",          # emotional state tracking (daily energy, meeting impact)
    "habit",         # behavioral patterns (positive or negative)
    "goal",          # personal/professional objectives with progress
    "reflection",    # journal-style insights about self
    "learning",      # knowledge gaps identified, scaffolding progress
    "values",        # core beliefs and principles
    "relationship",  # how interactions with specific people/tools affect the user
}

# Personal entry types — classification_level=4 (Cognitive, never syncs)
PERSONAL_ENTRY_TYPES = {
    "mood", "habit", "goal", "reflection", "learning", "values", "relationship",
}

# Classification levels for knowledge entries
# 0=Open, 1=Team, 2=Restricted, 3=Personal (default), 4=Cognitive
CLASSIFICATION_OPEN = 0        # Open — shareable with anyone, no restrictions
CLASSIFICATION_TEAM = 1        # Team — shareable within the team
CLASSIFICATION_RESTRICTED = 2  # Restricted — limited distribution
CLASSIFICATION_PERSONAL = 3    # Personal — syncs to personal devices only (default)
CLASSIFICATION_COGNITIVE = 4   # Cognitive — never syncs, stays on originating device

# Backward compatibility alias
CLASSIFICATION_PUBLIC = CLASSIFICATION_OPEN

# Valid range for classification_level values
CLASSIFICATION_MIN = CLASSIFICATION_OPEN
CLASSIFICATION_MAX = CLASSIFICATION_COGNITIVE

DOMAINS = {
    # Work-focused
    "teammate", "gmnet", "security", "neuralpulse", "salesos",
    "infrastructure", "knowledge",
    "dashboard",      # GM dashboard product surface
    "mcp",            # MCP servers (YouTube, etc.)
    "devops",         # deployment pipelines, CI/CD, monitoring
    # Business / product
    "business",       # pricing, positioning, revenue, strategy
    "product",        # product discovery, user feedback, market signals
    # Personal / whole-person
    "personal",       # self-improvement, EQ, habits
    "learning",       # skill development, knowledge gaps
    "health",         # energy, mood, physical patterns
    "relationships",  # people interactions, team dynamics
    "career",         # professional growth, goals, achievements
    "creativity",     # ideas, inspiration, creative output
    "finance",        # business finances, pricing strategy, revenue targets
    "productivity",   # tool workflows, time management, solo dev efficiency
    "music",          # creative consumption, playlists, inspiration sources
}

SCOPES = {"cross-cutting", "domain-specific"}


class KnowledgeStore:
    """CRUD operations for the unified knowledge graph.

    Provides create, read, update, and prune operations for knowledge entries,
    plus health monitoring (staleness, near-duplicates, contradictions) and
    lineage tracking from source observations/patterns.

    Args:
        db: Database instance for SQLite access.
        lineage: Optional LineageTracker for recording promotion provenance.
    """

    def __init__(
        self,
        db: Database,
        lineage: "LineageTracker | None" = None,
        pii_scan: bool = False,
        pii_scan_config: "KnowledgeScanConfig | None" = None,
    ):
        self.db = db
        self._lineage = lineage
        self._pii_scan = pii_scan and _SCANNER_AVAILABLE
        self._pii_scan_config = pii_scan_config
        self._ensure_schema()

    def create(
        self,
        entry_type: str,
        title: str,
        content: str,
        domain: str | None = None,
        scope: str = "cross-cutting",
        valence: str | None = None,
        tags: str | None = None,
        classification_level: int | None = None,
        category: str | None = None,
    ) -> dict[str, Any]:
        """Create a new knowledge entry.

        Args:
            entry_type: One of ENTRY_TYPES (technical or personal).
            title: Short descriptive title for the entry.
            content: Full content of the knowledge entry.
            domain: Optional domain scope (e.g., 'auth', 'database').
            scope: 'cross-cutting' or 'domain-specific'.
            valence: Sentiment classification (e.g., 'positive', 'negative', 'neutral').
            tags: Comma-separated tags for categorization.
            classification_level: Privacy level (0=Open, 1=Team, 2=Restricted, 3=Personal, 4=Cognitive).
                Auto-set to 4 for personal types if not specified. Default is 3.
            category: Knowledge freshness category (e.g., 'infrastructure', 'domain_expertise').
                Auto-classified from content if not specified.

        Returns:
            The created knowledge entry dict.

        Raises:
            ValueError: If entry_type is not in ENTRY_TYPES.
        """
        if entry_type not in ENTRY_TYPES:
            raise ValueError(f"Invalid entry_type: {entry_type}. Must be one of {ENTRY_TYPES}")
        if classification_level is not None and not (
            CLASSIFICATION_MIN <= classification_level <= CLASSIFICATION_MAX
        ):
            raise ValueError(
                f"Invalid classification_level: {classification_level}. "
                f"Must be {CLASSIFICATION_MIN}–{CLASSIFICATION_MAX}"
            )
        if classification_level is None:
            classification_level = (
                CLASSIFICATION_COGNITIVE if entry_type in PERSONAL_ENTRY_TYPES
                else CLASSIFICATION_PERSONAL
            )
        # -- SecureLLM PII scan --
        if self._pii_scan:
            scan_result = scan_knowledge_entry(
                title=title, content=content, config=self._pii_scan_config,
            )
            if scan_result.blocked:
                raise ValueError(
                    f"Knowledge entry blocked by SecureLLM: PII detected "
                    f"({', '.join(t.value for t in scan_result.pii_types_found)})"
                )
            if scan_result.redacted_content is not None:
                title = scan_result.redacted_title or title
                content = scan_result.redacted_content

        entry_id = generate_knowledge_id()
        try:
            # Legacy schema: has NOT NULL on 'type', 'source', 'valence' columns
            self.db.execute(
                """INSERT INTO knowledge_entries
                   (id, entry_type, type, source, scope, domain, title, content,
                    valence, tags, classification_level)
                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (entry_id, entry_type, entry_type, "import", scope, domain, title, content,
                 valence or "neutral", tags, classification_level),
            )
        except Exception:
            # New schema (no type/source columns)
            self.db.execute(
                """INSERT INTO knowledge_entries
                   (id, entry_type, scope, domain, title, content, valence, tags,
                    classification_level)
                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (entry_id, entry_type, scope, domain, title, content, valence, tags,
                 classification_level),
            )
        # Auto-classify into freshness category
        self._apply_category(entry_id, entry_type, domain, title, content, category)
        return self.get(entry_id)

    def get(self, entry_id: str) -> dict[str, Any] | None:
        """Retrieve a single knowledge entry by ID.

        Args:
            entry_id: The unique identifier of the entry.

        Returns:
            The entry dict, or None if not found.
        """
        return self.db.execute_one("SELECT * FROM knowledge_entries WHERE id = ?", (entry_id,))

    def search(
        self,
        entry_type: str | None = None,
        domain: str | None = None,
        text: str | None = None,
        include_pruned: bool = False,
        include_invalid: bool = False,
        limit: int = 50,
    ) -> list[dict[str, Any]]:
        """Search knowledge entries with optional filters.

        Args:
            entry_type: Filter by entry type (pattern, correction, etc.).
            domain: Filter by domain.
            text: Full-text search across title and content (LIKE match).
            include_pruned: If True, include archived/pruned entries.
            include_invalid: If True, include entries with validation_status
                'stale' or 'invalid'. Default False: stale entries still appear
                but are prefixed with [STALE] for visibility.
            limit: Maximum number of results to return.

        Returns:
            List of matching entry dicts, ordered by created_at descending.
            Entries with validation_status='stale' have title prefixed with [STALE].
        """
        query = "SELECT * FROM knowledge_entries WHERE 1=1"
        params: list = []

        if not include_pruned:
            query += " AND pruned_at IS NULL"
        if not include_invalid:
            query += " AND (validation_status IS NULL OR validation_status != 'invalid')"
        if entry_type:
            query += " AND entry_type = ?"
            params.append(entry_type)
        if domain:
            query += " AND domain = ?"
            params.append(domain)
        if text:
            query += " AND (title LIKE ? OR content LIKE ?)"
            params.extend([f"%{text}%", f"%{text}%"])

        query += " ORDER BY created_at DESC LIMIT ?"
        params.append(limit)
        results = self.db.execute(query, tuple(params))
        return self._annotate_freshness(results)

    def search_summary(
        self,
        entry_type: str | None = None,
        domain: str | None = None,
        text: str | None = None,
        include_invalid: bool = False,
        limit: int = 25,
    ) -> dict[str, Any]:
        """Search returning compact summaries without full content.

        Returns a paginated result with truncated content previews (first
        200 characters) suitable for listing UIs and budget-constrained
        responses. Includes validation_status and last_validated_at for
        freshness visibility. Stale entries are prefixed with [STALE].

        Args:
            entry_type: Filter by entry type (pattern, correction, etc.).
            domain: Filter by domain.
            text: Full-text search across title and content (LIKE match).
            include_invalid: If True, include entries with validation_status
                'invalid'. Default False: invalid entries are excluded.
            limit: Maximum number of results to return.

        Returns:
            Dict with ``total`` (full match count), ``returned`` (page size),
            and ``entries`` (list of summary dicts with ``content_preview``).
        """
        where = "WHERE pruned_at IS NULL"
        params: list = []
        if not include_invalid:
            where += " AND (validation_status IS NULL OR validation_status != 'invalid')"
        if entry_type:
            where += " AND entry_type = ?"
            params.append(entry_type)
        if domain:
            where += " AND domain = ?"
            params.append(domain)
        if text:
            where += " AND (title LIKE ? OR content LIKE ?)"
            params.extend([f"%{text}%", f"%{text}%"])

        count_row = self.db.execute_one(
            f"SELECT COUNT(*) as total FROM knowledge_entries {where}", tuple(params)
        )
        total = count_row["total"] if count_row else 0
        results = self.db.execute(
            f"""SELECT id, entry_type, title, substr(content, 1, 200) as content_preview,
                       domain, scope, tags, created_at,
                       validation_status, last_validated_at
                FROM knowledge_entries {where}
                ORDER BY created_at DESC LIMIT ?""",
            tuple(params + [limit]),
        )
        results = self._annotate_freshness(results)
        return {"total": total, "returned": len(results), "entries": results}

    @staticmethod
    def _annotate_freshness(entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
        """Add [STALE] prefix to titles of stale entries for visual flagging.

        Modifies entries in-place and returns the same list.
        """
        for entry in entries:
            status = entry.get("validation_status")
            if status in ("stale", "invalid"):
                title = entry.get("title", "")
                if not title.startswith("[STALE]"):
                    entry["title"] = f"[STALE] {title}"
        return entries

    def update(self, entry_id: str, **kwargs) -> dict[str, Any] | None:
        """Update fields on an existing knowledge entry.

        Only allows updating: title, content, domain, scope, valence, tags.
        Other fields are silently ignored.

        Args:
            entry_id: The unique identifier of the entry to update.
            **kwargs: Field names and their new values.

        Returns:
            The updated entry dict, or None if the entry doesn't exist.

        Raises:
            ValueError: If a column name contains unsafe characters.
        """
        if self.get(entry_id) is None:
            return None
        allowed = {"title", "content", "domain", "scope", "valence", "tags"}
        updates = {k: v for k, v in kwargs.items() if k in allowed}
        if not updates:
            return self.get(entry_id)
        # Validate column names are safe identifiers
        for k in updates:
            if not re.match(r'^[a-z_]+$', k):
                raise ValueError(f"Invalid column name: {k}")
        set_clause = ", ".join(f"{k} = ?" for k in updates)
        params = list(updates.values()) + [entry_id]
        self.db.execute(
            f"UPDATE knowledge_entries SET {set_clause}, updated_at = datetime('now') WHERE id = ?",
            tuple(params),
        )
        return self.get(entry_id)

    def prune(self, entry_id: str) -> bool:
        """Soft-delete a knowledge entry by setting pruned_at.

        Args:
            entry_id: The unique identifier of the entry to prune.

        Returns:
            True if the entry was pruned, False if it doesn't exist.
        """
        if self.get(entry_id) is None:
            return False
        self.db.execute(
            "UPDATE knowledge_entries SET pruned_at = datetime('now') WHERE id = ?",
            (entry_id,),
        )
        return True


    def count(self, entry_type: str | None = None, domain: str | None = None) -> int:
        """Count active (non-pruned) knowledge entries, optionally filtered.

        Args:
            entry_type: Filter by entry type (pattern, correction, etc.).
            domain: Filter by domain.

        Returns:
            Number of matching active entries.
        """
        where = "WHERE pruned_at IS NULL"
        params: list = []
        if entry_type:
            where += " AND entry_type = ?"
            params.append(entry_type)
        if domain:
            where += " AND domain = ?"
            params.append(domain)
        row = self.db.execute_one(
            f"SELECT COUNT(*) as total FROM knowledge_entries {where}", tuple(params)
        )
        return row["total"] if row else 0

    def create_with_lineage(
        self,
        entry_type: str,
        title: str,
        content: str,
        source_observation_ids: list[str] | None = None,
        source_pattern_ids: list[str] | None = None,
        domain: str | None = None,
        scope: str = "cross-cutting",
        valence: str | None = None,
        tags: str | None = None,
        session_id: str = "",
    ) -> dict[str, Any]:
        """Create a knowledge entry and record lineage from source observations/patterns.

        Args:
            entry_type: Knowledge entry type.
            title: Entry title.
            content: Entry content.
            source_observation_ids: IDs of source observations.
            source_pattern_ids: IDs of source patterns.
            domain: Domain scope.
            scope: cross-cutting or domain-specific.
            valence: Sentiment classification.
            tags: Comma-separated tags.
            session_id: Session where promotion occurred.

        Returns:
            The created knowledge entry dict.
        """
        entry = self.create(
            entry_type=entry_type, title=title, content=content,
            domain=domain, scope=scope, valence=valence, tags=tags,
        )

        if self._lineage:
            for obs_id in (source_observation_ids or []):
                self._lineage.record_event(
                    source_id=obs_id,
                    source_type="observation",
                    target_id=entry["id"],
                    target_type="knowledge",
                    event_type="promoted_to_knowledge",
                    session_id=session_id,
                )
                # Update the observation's knowledge_id foreign key
                self.db.execute(
                    "UPDATE observations SET knowledge_id = ? WHERE id = ?",
                    (entry["id"], obs_id),
                )

            for pat_id in (source_pattern_ids or []):
                self._lineage.record_event(
                    source_id=pat_id,
                    source_type="pattern",
                    target_id=entry["id"],
                    target_type="knowledge",
                    event_type="promoted_to_knowledge",
                    session_id=session_id,
                )

        return entry

    def record_application(self, entry_id: str) -> None:
        """Record that a knowledge entry was applied, resetting staleness.

        Increments the entry's ``applied_count``, updates ``updated_at``,
        clears any stale flag, and resets validation_status so the entry
        re-enters context injection.  Call this whenever an agent or tool
        uses a knowledge entry so the staleness tracker knows it is still active.

        Args:
            entry_id: The unique identifier of the entry.
        """
        self.db.execute(
            """UPDATE knowledge_entries
               SET applied_count = applied_count + 1,
                   updated_at = datetime('now'),
                   stale_flagged_at = NULL,
                   validation_status = NULL
               WHERE id = ?""",
            (entry_id,),
        )

    def confirm(self, entry_id: str) -> bool:
        """Confirm a knowledge entry is still relevant, resetting staleness.

        Updates last_confirmed_at, clears stale_flagged_at, and resets
        validation_status so the entry re-enters context injection.
        """
        if self.get(entry_id) is None:
            return False
        self.db.execute(
            """UPDATE knowledge_entries
               SET last_confirmed_at = datetime('now'),
                   updated_at = datetime('now'),
                   stale_flagged_at = NULL,
                   validation_status = NULL
               WHERE id = ?""",
            (entry_id,),
        )
        return True

    def flag_stale(self, stale_days: int = 90) -> list[dict[str, Any]]:
        """Flag entries older than stale_days without confirmation or application.

        An entry is stale if:
        - It is not already pruned
        - It is not already flagged
        - It has never been confirmed (last_confirmed_at IS NULL) or was last
          confirmed more than stale_days ago
        - It has never been updated in the last stale_days

        Args:
            stale_days: Number of days without activity before flagging.

        Returns:
            List of newly flagged entries.
        """
        self.db.execute(
            """UPDATE knowledge_entries
               SET stale_flagged_at = datetime('now')
               WHERE pruned_at IS NULL
                 AND stale_flagged_at IS NULL
                 AND (
                     COALESCE(last_confirmed_at, updated_at, created_at)
                     < datetime('now', ?)
                 )""",
            (f"-{stale_days} days",),
        )
        return self.get_stale()

    def get_stale(self, limit: int = 100) -> list[dict[str, Any]]:
        """Return entries flagged as stale but not yet pruned.

        Args:
            limit: Maximum number of stale entries to return.

        Returns:
            List of entry dicts ordered by ``stale_flagged_at`` ascending
            (oldest flags first).
        """
        return self.db.execute(
            """SELECT * FROM knowledge_entries
               WHERE stale_flagged_at IS NOT NULL
                 AND pruned_at IS NULL
               ORDER BY stale_flagged_at ASC
               LIMIT ?""",
            (limit,),
        )

    def unflag_stale(self, entry_id: str) -> bool:
        """Remove the stale flag from an entry (manual override).

        Use this when a flagged entry is still valuable but hasn't been
        applied or confirmed recently.  Clears ``stale_flagged_at`` and
        ``validation_status``, and bumps ``updated_at`` so the entry
        re-enters context injection.

        Args:
            entry_id: The unique identifier of the entry to unflag.

        Returns:
            True if the entry exists and was unflagged, False otherwise.
        """
        if self.get(entry_id) is None:
            return False
        self.db.execute(
            """UPDATE knowledge_entries
               SET stale_flagged_at = NULL,
                   validation_status = NULL,
                   updated_at = datetime('now')
               WHERE id = ?""",
            (entry_id,),
        )
        return True

    def auto_archive_stale(
        self, archive_days: int = 180, confidence_threshold: float = 0.5,
    ) -> list[dict[str, Any]]:
        """Auto-archive entries older than archive_days with low confidence.

        An entry is auto-archived if:
        - It is not already pruned
        - Its confidence is at or below confidence_threshold
        - Its last activity (confirmed, updated, or created) is older than archive_days

        Args:
            archive_days: Days without activity before auto-archiving.
            confidence_threshold: Maximum confidence for auto-archive (inclusive).

        Returns:
            List of entries that were archived.
        """
        # First, collect the entries that will be archived
        candidates = self.db.execute(
            """SELECT * FROM knowledge_entries
               WHERE pruned_at IS NULL
                 AND confidence <= ?
                 AND (
                     COALESCE(last_confirmed_at, updated_at, created_at)
                     < datetime('now', ?)
                 )""",
            (confidence_threshold, f"-{archive_days} days"),
        )
        if not candidates:
            return []

        # Archive them
        self.db.execute(
            """UPDATE knowledge_entries
               SET pruned_at = datetime('now'),
                   stale_flagged_at = COALESCE(stale_flagged_at, datetime('now'))
               WHERE pruned_at IS NULL
                 AND confidence <= ?
                 AND (
                     COALESCE(last_confirmed_at, updated_at, created_at)
                     < datetime('now', ?)
                 )""",
            (confidence_threshold, f"-{archive_days} days"),
        )
        return candidates

    def pruning_summary(self) -> dict[str, Any]:
        """Return a summary of knowledge graph health for pruning decisions.

        Returns:
            Dict with counts: ``total_active``, ``stale_flagged``,
            ``archived``, ``low_confidence`` (confidence <= 0.5),
            ``never_applied`` (applied_count == 0).
        """
        total = self.db.execute_one(
            "SELECT COUNT(*) as c FROM knowledge_entries WHERE pruned_at IS NULL"
        )
        stale = self.db.execute_one(
            """SELECT COUNT(*) as c FROM knowledge_entries
               WHERE stale_flagged_at IS NOT NULL AND pruned_at IS NULL"""
        )
        archived = self.db.execute_one(
            "SELECT COUNT(*) as c FROM knowledge_entries WHERE pruned_at IS NOT NULL"
        )
        low_confidence = self.db.execute_one(
            """SELECT COUNT(*) as c FROM knowledge_entries
               WHERE pruned_at IS NULL AND confidence <= 0.5"""
        )
        never_applied = self.db.execute_one(
            """SELECT COUNT(*) as c FROM knowledge_entries
               WHERE pruned_at IS NULL AND applied_count = 0"""
        )
        return {
            "total_active": total["c"] if total else 0,
            "stale_flagged": stale["c"] if stale else 0,
            "archived": archived["c"] if archived else 0,
            "low_confidence": low_confidence["c"] if low_confidence else 0,
            "never_applied": never_applied["c"] if never_applied else 0,
        }

    # --- Entry validation ---

    @staticmethod
    def _tokenize_title(title: str) -> set[str]:
        """Normalize and tokenize a title into a set of lowercase words.

        Strips punctuation and filters out short stopwords to improve
        duplicate matching quality.
        """
        stopwords = {"a", "an", "the", "is", "of", "to", "in", "for", "and", "or", "on", "at", "by"}
        words = re.findall(r"[a-z0-9]+", title.lower())
        return {w for w in words if w not in stopwords}

    def find_near_duplicates(self, similarity_threshold: float = 0.6) -> list[dict[str, Any]]:
        """Find pairs of active entries with near-duplicate titles.

        Uses word-level Jaccard similarity: |intersection| / |union|.
        Pairs scoring above *similarity_threshold* are reported.

        Args:
            similarity_threshold: Minimum Jaccard similarity to flag (0.0–1.0).

        Returns:
            List of dicts, each with ``entry_a_id``, ``entry_b_id``,
            ``title_a``, ``title_b``, and ``similarity`` score.
        """
        entries = self.db.execute(
            "SELECT id, title FROM knowledge_entries WHERE pruned_at IS NULL ORDER BY id",
        )
        duplicates: list[dict[str, Any]] = []
        tokenized = [(e["id"], e["title"], self._tokenize_title(e["title"])) for e in entries]

        for i, (id_a, title_a, tokens_a) in enumerate(tokenized):
            if not tokens_a:
                continue
            for j in range(i + 1, len(tokenized)):
                id_b, title_b, tokens_b = tokenized[j]
                if not tokens_b:
                    continue
                intersection = tokens_a & tokens_b
                union = tokens_a | tokens_b
                similarity = len(intersection) / len(union)
                if similarity >= similarity_threshold:
                    duplicates.append({
                        "entry_a_id": id_a,
                        "entry_b_id": id_b,
                        "title_a": title_a,
                        "title_b": title_b,
                        "similarity": round(similarity, 3),
                    })
        return duplicates

    def find_contradictions(self) -> list[dict[str, Any]]:
        """Detect potential contradictions among active knowledge entries.

        Heuristic: two entries share the same domain (non-NULL) and have
        high title similarity, but one contains negation signals in its
        content relative to the other (e.g., "always use" vs "never use").

        Returns:
            List of dicts with ``entry_a_id``, ``entry_b_id``, ``title_a``,
            ``title_b``, ``domain``, and ``reason`` describing the signal.
        """
        entries = self.db.execute(
            """SELECT id, title, content, domain
               FROM knowledge_entries
               WHERE pruned_at IS NULL AND domain IS NOT NULL
               ORDER BY domain, id""",
        )
        # Group by domain for pairwise comparison
        by_domain: dict[str, list[dict[str, Any]]] = {}
        for e in entries:
            by_domain.setdefault(e["domain"], []).append(e)

        contradictions: list[dict[str, Any]] = []
        for domain, group in by_domain.items():
            for i, a in enumerate(group):
                tokens_a = self._tokenize_title(a["title"])
                if not tokens_a:
                    continue
                for j in range(i + 1, len(group)):
                    b = group[j]
                    tokens_b = self._tokenize_title(b["title"])
                    if not tokens_b:
                        continue
                    # Require some title overlap to compare content
                    intersection = tokens_a & tokens_b
                    union = tokens_a | tokens_b
                    if len(intersection) / len(union) < 0.3:
                        continue
                    reason = self._detect_negation_signal(a["content"], b["content"])
                    if reason:
                        contradictions.append({
                            "entry_a_id": a["id"],
                            "entry_b_id": b["id"],
                            "title_a": a["title"],
                            "title_b": b["title"],
                            "domain": domain,
                            "reason": reason,
                        })
        return contradictions

    @staticmethod
    def _detect_negation_signal(content_a: str, content_b: str) -> str | None:
        """Check whether two content strings contain opposing directives.

        Returns a short reason string if a contradiction signal is found,
        or ``None`` if no signal is detected.
        """
        positive_patterns = [
            r"\balways\b", r"\bmust\b", r"\bshould\b", r"\buse\b",
            r"\benable\b", r"\brequired\b", r"\brecommended\b",
        ]
        negative_patterns = [
            r"\bnever\b", r"\bavoid\b", r"\bdon'?t\b", r"\bdo not\b",
            r"\bdisable\b", r"\bprohibited\b", r"\bnot recommended\b",
            r"\bshould not\b", r"\bmust not\b",
        ]

        a_lower = content_a.lower()
        b_lower = content_b.lower()

        a_has_positive = any(re.search(p, a_lower) for p in positive_patterns)
        a_has_negative = any(re.search(p, a_lower) for p in negative_patterns)
        b_has_positive = any(re.search(p, b_lower) for p in positive_patterns)
        b_has_negative = any(re.search(p, b_lower) for p in negative_patterns)

        if a_has_positive and b_has_negative and not a_has_negative:
            return "entry A uses affirmative language, entry B uses negation"
        if a_has_negative and b_has_positive and not b_has_negative:
            return "entry A uses negation, entry B uses affirmative language"
        return None

    # --- Knowledge freshness categorization ---

    def _apply_category(
        self,
        entry_id: str,
        entry_type: str,
        domain: str | None,
        title: str,
        content: str,
        manual_category: str | None = None,
    ) -> None:
        """Classify an entry and persist its category + TTL.

        Called after INSERT so the entry exists. Gracefully handles
        databases that lack the knowledge_category/ttl_days columns.
        """
        entry_dict = {
            "entry_type": entry_type,
            "domain": domain,
            "title": title,
            "content": content,
        }
        cat, ttl = _classify_entry(entry_dict, manual_override=manual_category)
        try:
            self.db.execute(
                "UPDATE knowledge_entries "
                "SET knowledge_category = ?, ttl_days = ? "
                "WHERE id = ?",
                (cat, ttl, entry_id),
            )
        except Exception:
            pass  # Columns not yet added by migration 009

    def check_ttl(self) -> list[dict[str, Any]]:
        """Flag entries whose TTL has expired as stale.

        An entry's TTL is measured from ``COALESCE(last_confirmed_at, created_at)``.
        Entries already flagged stale or pruned are skipped.

        Returns:
            List of entries that were newly flagged stale.
        """
        try:
            # Find entries past their TTL
            expired = self.db.execute(
                """SELECT id, title, knowledge_category, ttl_days,
                          last_confirmed_at, created_at
                   FROM knowledge_entries
                   WHERE pruned_at IS NULL
                     AND stale_flagged_at IS NULL
                     AND knowledge_category IS NOT NULL
                     AND ttl_days IS NOT NULL
                     AND COALESCE(last_confirmed_at, created_at)
                         < datetime('now', '-' || ttl_days || ' days')"""
            )
        except Exception:
            return []  # Columns not present yet

        if not expired:
            return []

        ids = [e["id"] for e in expired]
        for entry_id in ids:
            self.db.execute(
                """UPDATE knowledge_entries
                   SET stale_flagged_at = datetime('now')
                   WHERE id = ? AND stale_flagged_at IS NULL""",
                (entry_id,),
            )
        _log.info(
            "[knowledge] TTL check flagged %d entries as stale", len(ids),
        )
        return expired

    def freshness_report(self) -> dict[str, Any]:
        """Generate a freshness report grouped by knowledge category.

        Each entry is classified as:
          GREEN  — confirmed within TTL
          YELLOW — >75% of TTL elapsed, needs confirmation soon
          RED    — TTL expired, auto-flagged or should be flagged stale

        Returns:
            Dict with ``by_category`` (per-category breakdown),
            ``by_status`` (aggregate GREEN/YELLOW/RED counts),
            ``entries`` (individual entry statuses, limited to 100).
        """
        try:
            entries = self.db.execute(
                """SELECT id, title, entry_type, domain,
                          knowledge_category, ttl_days,
                          last_confirmed_at, created_at,
                          stale_flagged_at, pruned_at
                   FROM knowledge_entries
                   WHERE pruned_at IS NULL
                     AND knowledge_category IS NOT NULL
                   ORDER BY knowledge_category, created_at DESC"""
            )
        except Exception:
            return {
                "by_category": {},
                "by_status": {"green": 0, "yellow": 0, "red": 0},
                "entries": [],
            }

        by_cat: dict[str, dict[str, int]] = {}
        totals = {"green": 0, "yellow": 0, "red": 0}
        entry_details: list[dict[str, Any]] = []

        for e in entries:
            cat = e.get("knowledge_category") or "domain_expertise"
            ttl = e.get("ttl_days") or 90
            base_ts = e.get("last_confirmed_at") or e.get("created_at")

            status = _compute_freshness_status(base_ts, ttl, e.get("stale_flagged_at"))

            if cat not in by_cat:
                by_cat[cat] = {"green": 0, "yellow": 0, "red": 0, "total": 0, "ttl_days": ttl}
            by_cat[cat][status] += 1
            by_cat[cat]["total"] += 1
            totals[status] += 1

            if len(entry_details) < 100:
                entry_details.append({
                    "id": e["id"],
                    "title": e["title"],
                    "domain": e.get("domain"),
                    "knowledge_category": cat,
                    "ttl_days": ttl,
                    "status": status.upper(),
                    "last_confirmed_at": e.get("last_confirmed_at"),
                })

        return {
            "by_category": by_cat,
            "by_status": totals,
            "entries": entry_details,
        }

    def classify_all(self) -> dict[str, int]:
        """Re-classify all unclassified active entries.

        Entries with a non-NULL knowledge_category that was NOT set to the
        default 'domain_expertise' are assumed to have been manually
        overridden and are skipped.

        Returns:
            Dict mapping category names to count of entries classified.
        """
        entries = self.db.execute(
            """SELECT id, entry_type, domain, title, content
               FROM knowledge_entries
               WHERE pruned_at IS NULL
                 AND (knowledge_category IS NULL
                      OR knowledge_category = 'domain_expertise')"""
        )
        counts: dict[str, int] = {}
        for entry in entries:
            cat, ttl = _classify_entry(entry)
            try:
                self.db.execute(
                    "UPDATE knowledge_entries "
                    "SET knowledge_category = ?, ttl_days = ? "
                    "WHERE id = ?",
                    (cat, ttl, entry["id"]),
                )
            except Exception:
                continue
            counts[cat] = counts.get(cat, 0) + 1
        return counts

    # --- Schema migration helpers ---

    def _ensure_schema(self) -> None:
        """Add columns that may be missing from older databases."""
        try:
            self.db.execute(
                "ALTER TABLE knowledge_entries ADD COLUMN superseded_by TEXT"
            )
        except Exception:
            pass  # Column already exists or table not yet created
        # Ensure freshness category columns exist
        for col, default in [
            ("knowledge_category", "TEXT DEFAULT 'domain_expertise'"),
            ("ttl_days", "INTEGER DEFAULT 90"),
        ]:
            try:
                self.db.execute(
                    f"ALTER TABLE knowledge_entries ADD COLUMN {col} {default}"
                )
            except Exception:
                pass  # Column already exists

    # --- Contradiction detection on create ---

    def detect_and_flag_contradictions(self, new_entry: dict) -> list[dict[str, Any]]:
        """Detect contradictions and flag old entries as stale.

        Called after creating a new entry.  Searches for existing entries
        that the new entry contradicts, flags them as stale, and records
        the ``superseded_by`` link.

        Args:
            new_entry: The newly created entry dict.

        Returns:
            List of conflict dicts for entries that were flagged.
        """
        from .contradiction import ContradictionDetector

        detector = ContradictionDetector()
        candidates = self._get_contradiction_candidates(new_entry)
        conflicts = detector.find_conflicts(new_entry, candidates)

        for conflict in conflicts:
            old_id = conflict["old_entry_id"]
            new_id = new_entry["id"]
            try:
                self.db.execute(
                    """UPDATE knowledge_entries
                       SET stale_flagged_at = datetime('now'),
                           superseded_by = ?
                       WHERE id = ? AND pruned_at IS NULL
                         AND stale_flagged_at IS NULL""",
                    (new_id, old_id),
                )
            except Exception:
                # Fallback if superseded_by column missing on legacy DB
                self.db.execute(
                    """UPDATE knowledge_entries
                       SET stale_flagged_at = datetime('now')
                       WHERE id = ? AND pruned_at IS NULL
                         AND stale_flagged_at IS NULL""",
                    (old_id,),
                )
            _log.info(
                "[knowledge] Flagged %s as stale — contradicted by %s: %s",
                old_id, new_id, conflict["reason"],
            )

        return conflicts

    def _get_contradiction_candidates(self, entry: dict) -> list[dict[str, Any]]:
        """Return active entries that could contradict *entry*."""
        domain = entry.get("domain")
        entry_id = entry.get("id", "")
        if domain:
            return self.db.execute(
                """SELECT * FROM knowledge_entries
                   WHERE pruned_at IS NULL
                     AND stale_flagged_at IS NULL
                     AND (domain = ? OR domain IS NULL)
                     AND id != ?
                   ORDER BY created_at DESC LIMIT 200""",
                (domain, entry_id),
            )
        return self.db.execute(
            """SELECT * FROM knowledge_entries
               WHERE pruned_at IS NULL
                 AND stale_flagged_at IS NULL
                 AND id != ?
               ORDER BY created_at DESC LIMIT 200""",
            (entry_id,),
        )

    def batch_detect_contradictions(self) -> dict[str, Any]:
        """Run contradiction detection across all active entries.

        Returns:
            Report dict with total_entries_scanned, contradictions_found,
            and conflicts list.
        """
        from .contradiction import ContradictionDetector

        detector = ContradictionDetector()
        entries = self.db.execute(
            "SELECT * FROM knowledge_entries WHERE pruned_at IS NULL ORDER BY created_at",
        )
        conflicts = detector.find_all_conflicts(entries)
        return {
            "total_entries_scanned": len(entries),
            "contradictions_found": len(conflicts),
            "conflicts": conflicts,
        }

    def health(self, stale_days: int = 90) -> dict[str, Any]:
        """Run all validation checks and return a unified health report.

        Combines staleness detection, near-duplicate detection, and
        contradiction detection into a single report suitable for
        ``/api/knowledge/health``.

        Args:
            stale_days: Days without activity before an entry is stale.

        Returns:
            Dict with keys: ``summary`` (counts), ``stale_entries`` (list),
            ``near_duplicates`` (list of pairs), ``contradictions`` (list).
        """
        # Run staleness scan (flags entries as side effect)
        self.flag_stale(stale_days=stale_days)
        stale_entries = self.get_stale()

        near_duplicates = self.find_near_duplicates()
        contradictions = self.find_contradictions()

        summary = self.pruning_summary()
        summary["near_duplicate_pairs"] = len(near_duplicates)
        summary["contradiction_pairs"] = len(contradictions)

        return {
            "summary": summary,
            "stale_entries": stale_entries,
            "near_duplicates": near_duplicates,
            "contradictions": contradictions,
        }


def _compute_freshness_status(
    base_timestamp: str | None,
    ttl_days: int,
    stale_flagged_at: str | None,
) -> str:
    """Compute GREEN/YELLOW/RED freshness status for an entry.

    Args:
        base_timestamp: ISO timestamp of last confirmation or creation.
        ttl_days: TTL in days for this entry's category.
        stale_flagged_at: If set, entry is already flagged stale.

    Returns:
        'red', 'yellow', or 'green'.
    """
    if stale_flagged_at:
        return "red"
    if not base_timestamp:
        return "yellow"  # No timestamp at all — treat as warning

    from datetime import datetime, timezone

    try:
        dt = datetime.fromisoformat(base_timestamp)
        if dt.tzinfo is None:
            dt = dt.replace(tzinfo=timezone.utc)
        now = datetime.now(timezone.utc)
        elapsed_days = (now - dt).total_seconds() / 86400.0
    except (ValueError, TypeError):
        return "yellow"

    if elapsed_days >= ttl_days:
        return "red"
    elif elapsed_days >= ttl_days * 0.75:
        return "yellow"
    else:
        return "green"
