"""SparkNet client - the main interface for the SDK.

SECURITY-FIRST DESIGN:
All sparks go through the approval queue before being shared.
Nothing leaves your machine without your explicit approval.

Flow:
1. Agent learns something → stored locally
2. User calls share() → goes to approval queue
3. Security scanner checks for secrets/PII
4. User reviews in dashboard → approves/rejects
5. Only approved sparks are uploaded to network

INTELLIGENCE BRIDGE:
When Spark Intelligence is available, enables:
- Semantic recall using FastEmbed (BAAI/bge-small-en-v1.5)
- Hybrid search (BM25 + vector similarity)
- Mind API sync for distributed memory
"""

import json
import sqlite3
import re
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from pathlib import Path
from typing import List, Optional, Dict, Any, Tuple
import uuid

import httpx

# Optional intelligence bridge
try:
    from .intelligence_bridge import (
        get_intelligence_bridge,
        IntelligenceBridge,
        SemanticResult,
        BridgeStatus,
    )
    BRIDGE_AVAILABLE = True
except ImportError:
    BRIDGE_AVAILABLE = False
    IntelligenceBridge = None
    SemanticResult = None
    BridgeStatus = None

from .models import Spark, SparkType, KnowledgeType, Agent, AgentType, Echo, EchoType
from .exceptions import (
    SparkNetError,
    ConstitutionalViolation,
    NetworkError,
    RateLimitError,
    AuthenticationError,
)
from .security import SecurityScanner, ScanResult
from .approval import (
    ApprovalQueue,
    ApprovalMode,
    ApprovalConfig,
    ApprovalState,
    PendingSpark,
    get_approval_queue,
    get_approval_config,
)


@dataclass
class SparkNetConfig:
    """Configuration for SparkNet client."""
    api_key: Optional[str] = None
    api_url: str = "https://api.sparknet.dev"
    local_only: bool = False
    grove_path: Optional[Path] = None
    agent_name: Optional[str] = None
    agent_type: AgentType = AgentType.BUILDER
    domains: List[str] = field(default_factory=list)

    # Security settings
    approval_mode: ApprovalMode = ApprovalMode.MANUAL
    """How sparks are approved: MANUAL (default), AUTO_SAFE, or OFFLINE."""

    auto_scan: bool = True
    """Automatically scan all content for secrets/PII."""

    block_on_secrets: bool = True
    """Automatically block sparks containing secrets."""

    # Intelligence Bridge settings
    semantic_search: bool = True
    """Enable semantic search using Spark Intelligence (if available)."""

    mind_sync: bool = False
    """Sync approved sparks to Mind API."""

    mind_url: str = "http://localhost:8080"
    """Mind API URL for distributed memory."""

    def __post_init__(self):
        if self.grove_path is None:
            self.grove_path = Path.home() / ".sparknet" / "grove.db"
        if self.api_key is None:
            self.local_only = True


class SparkNet:
    """
    SparkNet client for AI agent learning.

    Your agent gets smarter every day. Automatically.

    Example:
        sn = SparkNet()
        sn.learn("Always validate user input")
        insights = sn.recall("validation")
    """

    def __init__(
        self,
        api_key: Optional[str] = None,
        api_url: str = "https://api.sparknet.dev",
        local_only: bool = False,
        agent_name: Optional[str] = None,
        config: Optional[SparkNetConfig] = None,
    ):
        """
        Initialize SparkNet client.

        Args:
            api_key: API key for network access. If None, runs in local-only mode.
            api_url: SparkNet API URL.
            local_only: Force local-only mode even with API key.
            agent_name: Name for this agent.
            config: Full configuration object (overrides other params).
        """
        if config:
            self.config = config
        else:
            self.config = SparkNetConfig(
                api_key=api_key,
                api_url=api_url,
                local_only=local_only or api_key is None,
                agent_name=agent_name,
            )

        self.agent_id = str(uuid.uuid4())
        self._grove = Grove(self.config.grove_path)
        self._http = None if self.config.local_only else httpx.Client(
            base_url=self.config.api_url,
            headers={"Authorization": f"Bearer {self.config.api_key}"},
            timeout=30.0,
        )

        # Security components
        self._scanner = SecurityScanner()
        self._approval_queue = get_approval_queue()
        self._approval_config = get_approval_config()

        # Constitutional patterns
        self._forbidden_patterns = self._compile_forbidden_patterns()

        # Intelligence Bridge (optional - enables semantic search)
        self._bridge = None
        if BRIDGE_AVAILABLE and self.config.semantic_search:
            try:
                self._bridge = get_intelligence_bridge()
            except Exception:
                pass  # Graceful degradation

        # Current recall session for co-recall tracking
        self._current_session_id: Optional[str] = None

    # =========================================================================
    # Core Learning Methods
    # =========================================================================

    def learn(
        self,
        content: str,
        spark_type: SparkType = SparkType.PATTERN,
        knowledge_type: KnowledgeType = KnowledgeType.PROCEDURAL,
        domain: Optional[str] = None,
        confidence: float = 0.7,
        how_learned: Optional[str] = None,
        tags: List[str] = None,
        **kwargs,
    ) -> Spark:
        """
        Record a learning.

        Args:
            content: The insight or knowledge to learn.
            spark_type: Type of spark (EUREKA, LESSON, PATTERN, etc).
            knowledge_type: Type of knowledge (FACTUAL, PROCEDURAL, CAUSAL, ETHICAL).
            domain: Domain this applies to (e.g., "python", "react").
            confidence: How confident (0-1).
            how_learned: How this was learned.
            tags: Optional tags.

        Returns:
            The created Spark.

        Raises:
            ConstitutionalViolation: If content violates safety principles.
        """
        # Constitutional check
        self._check_constitutional(content)

        spark = Spark(
            content=content,
            spark_type=spark_type,
            knowledge_type=knowledge_type,
            domain=domain,
            confidence=confidence,
            how_learned=how_learned,
            tags=tags or [],
            agent_id=self.agent_id,
        )

        # Store locally
        self._grove.store(spark)

        # Extract concepts for Knowledge Graph
        self._grove.extract_and_store_concepts(spark)

        return spark

    def aha(
        self,
        expected: str,
        actual: str,
        lesson: str,
        mood: str = "curious",
        domain: Optional[str] = None,
    ) -> Spark:
        """
        Record an AHA moment - when reality differs from expectation.

        Args:
            expected: What you expected to happen.
            actual: What actually happened.
            lesson: What you learned from this.
            mood: How you felt (curious, surprised, frustrated, etc).
            domain: Domain this applies to.

        Returns:
            The created Spark.
        """
        content = f"Expected: {expected}. Actually: {actual}. Learned: {lesson}"
        return self.learn(
            content=content,
            spark_type=SparkType.EUREKA,
            knowledge_type=KnowledgeType.CAUSAL,
            domain=domain,
            how_learned=f"Surprise moment ({mood})",
            confidence=0.8,
        )

    def preference(
        self,
        what: str,
        why: Optional[str] = None,
        domain: Optional[str] = None,
    ) -> Spark:
        """
        Record a preference or taste.

        Args:
            what: The preference.
            why: Why this is preferred.
            domain: Domain this applies to.

        Returns:
            The created Spark.
        """
        content = f"Prefers: {what}" + (f" because {why}" if why else "")
        return self.learn(
            content=content,
            spark_type=SparkType.PREFERENCE,
            knowledge_type=KnowledgeType.PROCEDURAL,
            domain=domain,
        )

    def gotcha(
        self,
        pitfall: str,
        solution: Optional[str] = None,
        domain: Optional[str] = None,
    ) -> Spark:
        """
        Record a gotcha/pitfall to avoid.

        Args:
            pitfall: The gotcha to avoid.
            solution: How to avoid or fix it.
            domain: Domain this applies to.

        Returns:
            The created Spark.
        """
        content = f"Gotcha: {pitfall}"
        if solution:
            content += f". Solution: {solution}"
        return self.learn(
            content=content,
            spark_type=SparkType.GOTCHA,
            knowledge_type=KnowledgeType.PROCEDURAL,
            domain=domain,
            confidence=0.85,
        )

    def principle(
        self,
        principle: str,
        rationale: Optional[str] = None,
        domain: Optional[str] = None,
    ) -> Spark:
        """
        Record a guiding principle.

        Args:
            principle: The principle.
            rationale: Why this principle matters.
            domain: Domain this applies to.

        Returns:
            The created Spark.
        """
        content = principle
        if rationale:
            content += f" ({rationale})"
        return self.learn(
            content=content,
            spark_type=SparkType.PRINCIPLE,
            knowledge_type=KnowledgeType.ETHICAL,
            domain=domain,
            confidence=0.9,
        )

    # =========================================================================
    # Query Methods
    # =========================================================================

    def recall(
        self,
        query: str,
        limit: int = 10,
        domain: Optional[str] = None,
        spark_types: List[SparkType] = None,
        semantic: Optional[bool] = None,
        graph_enhanced: bool = True,
    ) -> List[Spark]:
        """
        Recall relevant knowledge from local Grove.

        When Spark Intelligence is available, uses semantic search
        (FastEmbed embeddings + hybrid BM25/vector ranking).
        Falls back to keyword matching otherwise.

        Also uses Knowledge Graph to find related sparks and tracks
        co-recall patterns to improve future searches.

        Args:
            query: Search query.
            limit: Max results.
            domain: Filter by domain.
            spark_types: Filter by spark types.
            semantic: Force semantic (True) or keyword (False) search.
                     If None, uses semantic when available.
            graph_enhanced: Use Knowledge Graph to find related sparks.

        Returns:
            List of relevant Sparks.
        """
        # Generate a new session ID for co-recall tracking
        self._current_session_id = str(uuid.uuid4())

        # Determine if we should use semantic search
        use_semantic = semantic if semantic is not None else (self._bridge is not None)

        if use_semantic and self._bridge:
            try:
                # Get all candidates first
                all_sparks = self._grove.get_all(limit=100, domain=domain)

                # Filter by type if needed
                if spark_types:
                    all_sparks = [s for s in all_sparks if s.spark_type in spark_types]

                # Use semantic recall
                semantic_results = self._bridge.semantic_recall(
                    query=query,
                    sparks=all_sparks,
                    limit=limit,
                    hybrid=True,
                )

                # Convert back to Spark objects (maintain order)
                spark_map = {s.id: s for s in all_sparks}
                primary_results = [spark_map[r.spark_id] for r in semantic_results if r.spark_id in spark_map]
            except Exception:
                primary_results = []

            # Fallback when semantic lookup returns no candidates.
            if not primary_results:
                primary_results = self._grove.query(
                    query=query,
                    limit=limit,
                    domain=domain,
                    spark_types=spark_types,
                )
        else:
            # Fallback to keyword search
            primary_results = self._grove.query(
                query=query,
                limit=limit,
                domain=domain,
                spark_types=spark_types,
            )

        if not primary_results:
            self._current_session_id = None
            return []

        # Use Knowledge Graph to enhance results
        if graph_enhanced and primary_results:
            primary_ids = [s.id for s in primary_results]

            # Get graph-related sparks
            related = self._grove.graph_enhanced_query(
                matched_spark_ids=primary_ids,
                limit=limit // 2,  # Add up to half more via graph
                min_strength=0.3,
            )

            # Merge related sparks (if not already in results)
            if related:
                all_sparks = self._grove.get_all(limit=200, domain=domain)
                spark_map = {s.id: s for s in all_sparks}
                primary_id_set = set(primary_ids)

                for related_id, score in related:
                    if related_id not in primary_id_set and related_id in spark_map:
                        spark = spark_map[related_id]
                        if spark_types is None or spark.spark_type in spark_types:
                            primary_results.append(spark)

        # Record co-recall session (for learning co-occurrence patterns)
        if primary_results:
            result_ids = [s.id for s in primary_results[:limit]]
            self._grove.record_recall(self._current_session_id, result_ids)
            # Update last_recalled_at for resonance tracking
            self._grove.update_last_recalled(result_ids)

        return primary_results[:limit]

    def get_all(self, limit: int = 100, domain: Optional[str] = None) -> List[Spark]:
        """Get all sparks from local Grove."""
        return self._grove.get_all(limit=limit, domain=domain)

    def stats(self) -> Dict[str, Any]:
        """Get statistics about local learning."""
        return self._grove.stats()

    # =========================================================================
    # Knowledge Graph Methods
    # =========================================================================

    def mark_useful(self, spark_id: str) -> None:
        """
        Mark a recalled spark as useful (echo).

        Call this when a spark from the last recall() actually helped.
        This:
        - Increments the spark's echoes_count
        - Updates resonance: echoes / (echoes + fizzles)
        - Resets temporal decay timer
        - Strengthens Knowledge Graph edges

        Args:
            spark_id: ID of the spark that was useful.
        """
        # Update resonance
        self._grove.record_echo(spark_id)

        # Track for Knowledge Graph edge evolution
        if self._current_session_id:
            self._grove.mark_spark_useful(self._current_session_id, spark_id, useful=True)

    def mark_not_useful(self, spark_id: str) -> None:
        """
        Mark a recalled spark as not useful (fizzle).

        Call this when a spark from the last recall() didn't help.
        This:
        - Increments the spark's fizzles_count
        - Updates resonance: echoes / (echoes + fizzles)
        - Weakens Knowledge Graph edges

        Args:
            spark_id: ID of the spark that wasn't useful.
        """
        # Update resonance
        self._grove.record_fizzle(spark_id)

        # Track for Knowledge Graph edge evolution
        if self._current_session_id:
            self._grove.mark_spark_useful(self._current_session_id, spark_id, useful=False)

    def finish_session(self) -> None:
        """
        Finish the current recall session and process feedback.

        Call this after marking sparks as useful/not useful to update
        the Knowledge Graph edge strengths.
        """
        if self._current_session_id:
            self._grove.process_session_feedback(self._current_session_id)
            self._current_session_id = None

    def graph_stats(self) -> Dict[str, Any]:
        """
        Get Knowledge Graph statistics.

        Returns:
            Dict with total_edges, edges_by_type, avg_edge_strength, total_concepts.
        """
        return self._grove.graph_stats()

    def get_related(
        self,
        spark_id: str,
        limit: int = 10,
        edge_types: List[str] = None,
    ) -> List[Tuple[Spark, str, float]]:
        """
        Get sparks related to a given spark via the Knowledge Graph.

        Args:
            spark_id: ID of the spark to find relations for.
            limit: Maximum related sparks to return.
            edge_types: Filter by edge types ('co_recall', 'similar', 'concept', 'derived').

        Returns:
            List of (Spark, edge_type, strength) tuples.
        """
        connected = self._grove.get_connected_sparks(
            spark_id,
            edge_types=edge_types,
            min_strength=0.1,
            limit=limit,
        )

        # Fetch the actual spark objects
        all_sparks = self._grove.get_all(limit=500)
        spark_map = {s.id: s for s in all_sparks}

        results = []
        for connected_id, edge_type, strength in connected:
            if connected_id in spark_map:
                results.append((spark_map[connected_id], edge_type, strength))

        return results

    def create_link(
        self,
        source_id: str,
        target_id: str,
        edge_type: str = "derived",
        strength: float = 0.5,
    ) -> str:
        """
        Manually create a link between two sparks.

        Use this when you know two sparks are related (e.g., one was derived from another).

        Args:
            source_id: ID of the source spark.
            target_id: ID of the target spark.
            edge_type: Type of relationship ('co_recall', 'similar', 'concept', 'derived').
            strength: Initial strength (0.0-1.0).

        Returns:
            Edge ID.
        """
        return self._grove.create_edge(source_id, target_id, edge_type, strength)

    # =========================================================================
    # Resonance & Decay Methods
    # =========================================================================

    def apply_decay(self, decay_rate: float = 0.01, min_resonance: float = 0.1) -> None:
        """
        Apply temporal decay to sparks not recalled recently.

        Sparks not recalled in the last 7 days lose resonance.
        Call this periodically (e.g., daily) to keep knowledge fresh.

        Args:
            decay_rate: How much resonance to reduce (default 0.01)
            min_resonance: Minimum resonance floor (default 0.1)
        """
        self._grove.apply_temporal_decay(decay_rate, min_resonance)

    def get_stale_sparks(self, days: int = 30, limit: int = 50) -> List[Spark]:
        """
        Get sparks that haven't been recalled in a while.

        Use this to review and potentially archive old knowledge.

        Args:
            days: Days without recall to be considered stale (default 30)
            limit: Maximum sparks to return

        Returns:
            List of stale sparks, ordered by resonance (lowest first)
        """
        return self._grove.get_stale_sparks(days, limit)

    def get_best_sparks(self, min_resonance: float = 0.8, limit: int = 20) -> List[Spark]:
        """
        Get sparks with highest resonance (most proven useful).

        These are your best insights - consistently helpful over time.

        Args:
            min_resonance: Minimum resonance threshold (default 0.8)
            limit: Maximum sparks to return

        Returns:
            List of high-resonance sparks, ordered by resonance (highest first)
        """
        return self._grove.get_high_resonance_sparks(min_resonance, limit)

    def resonance_stats(self) -> Dict[str, Any]:
        """
        Get resonance statistics across all sparks.

        Returns:
            Dict with avg_resonance, high_count, low_count, stale_count
        """
        all_sparks = self._grove.get_all(limit=1000)

        if not all_sparks:
            return {
                "avg_resonance": 0.0,
                "high_count": 0,
                "low_count": 0,
                "stale_count": 0,
                "total_echoes": 0,
                "total_fizzles": 0,
            }

        resonances = [s.resonance for s in all_sparks]
        high_threshold = 0.7
        low_threshold = 0.3

        return {
            "avg_resonance": sum(resonances) / len(resonances),
            "high_count": sum(1 for r in resonances if r >= high_threshold),
            "low_count": sum(1 for r in resonances if r <= low_threshold),
            "stale_count": len(self._grove.get_stale_sparks(days=30, limit=1000)),
            "total_echoes": sum(s.echoes_count for s in all_sparks),
            "total_fizzles": sum(s.fizzles_count for s in all_sparks),
        }

    # =========================================================================
    # Network Methods (requires API key)
    # =========================================================================

    def share(self, spark: Spark) -> Tuple[bool, PendingSpark]:
        """
        Queue a spark for sharing to the network.

        SECURITY: Sparks are NOT immediately shared. They go through:
        1. Security scan for secrets/PII
        2. Approval queue for user review
        3. Only uploaded after explicit approval

        Args:
            spark: The spark to share.

        Returns:
            Tuple of (needs_approval, pending_spark)
            - needs_approval: True if user must review in dashboard
            - pending_spark: The queued spark with scan results

        Note:
            To actually upload approved sparks, call upload_approved()
        """
        if self.config.local_only:
            raise SparkNetError("Network access requires API key")

        # Add to approval queue (includes security scan)
        pending = self._approval_queue.add(spark, auto_scan=self.config.auto_scan)

        # Check if auto-approval is possible
        if self.config.approval_mode == ApprovalMode.OFFLINE:
            # Offline mode: mark as local only
            self._approval_queue.mark_local_only(spark.id)
            return (False, pending)

        if pending.state == ApprovalState.BLOCKED:
            # Contains secrets - requires manual review
            return (True, pending)

        # Check auto-approval conditions
        if self.config.approval_mode == ApprovalMode.AUTO_SAFE:
            if not pending.has_warnings:
                # Auto-approve clean sparks
                self._approval_queue.approve(spark.id, reason="Auto-approved (no warnings)")
                return (False, pending)

        # Requires manual approval
        return (True, pending)

    def upload_approved(self) -> List[str]:
        """
        Upload all approved sparks to the network.

        Returns:
            List of spark IDs that were successfully uploaded.
        """
        if self.config.local_only:
            raise SparkNetError("Network access requires API key")

        uploaded = []
        approved_sparks = self._approval_queue.list_approved()

        for pending in approved_sparks:
            try:
                success = self._upload_spark(pending)
                if success:
                    self._approval_queue.mark_shared(pending.id)
                    uploaded.append(pending.id)
            except Exception as e:
                # Log error but continue with other sparks
                print(f"Failed to upload {pending.id}: {e}")

        return uploaded

    def _upload_spark(self, pending: PendingSpark) -> bool:
        """Actually upload a spark to the network."""
        response = self._http.post(
            "/api/sparks",
            json={
                "agent_id": self.agent_id,
                "thought": pending.content,
                "spark_type": pending.spark_type.value,
                "domain": pending.domain or "",
                "how_i_learned": pending.how_learned or "",
                "tags": pending.tags,
            },
        )

        if response.status_code == 429:
            raise RateLimitError()
        if response.status_code == 401:
            raise AuthenticationError("Invalid API key")
        if response.status_code >= 400:
            raise NetworkError(f"API error: {response.text}", response.status_code)

        data = response.json()
        return data.get("ok", False)

    # =========================================================================
    # Approval Queue Methods
    # =========================================================================

    def pending_sparks(self) -> List[PendingSpark]:
        """Get all sparks pending approval."""
        return self._approval_queue.list_pending()

    def blocked_sparks(self) -> List[PendingSpark]:
        """Get all sparks blocked due to security issues."""
        return self._approval_queue.list_blocked()

    def approve(self, spark_id: str, reason: Optional[str] = None) -> bool:
        """Approve a spark for sharing."""
        return self._approval_queue.approve(spark_id, reason)

    def reject(self, spark_id: str, reason: Optional[str] = None) -> bool:
        """Reject a spark (will not be shared)."""
        return self._approval_queue.reject(spark_id, reason)

    def keep_local(self, spark_id: str) -> bool:
        """Mark a spark as local-only (never share)."""
        return self._approval_queue.mark_local_only(spark_id)

    def approval_stats(self) -> Dict[str, int]:
        """Get approval queue statistics."""
        return self._approval_queue.get_stats()

    def scan_content(self, content: str) -> ScanResult:
        """
        Scan content for sensitive information.

        Use this to check content before learning/sharing.
        """
        return self._scanner.scan(content)

    # =========================================================================
    # Mind API Sync (via Intelligence Bridge)
    # =========================================================================

    def sync_to_mind(self, spark_id: str) -> Tuple[bool, Optional[str]]:
        """
        Sync an approved spark to Mind API for distributed memory.

        Only works for sparks that have been approved in the approval queue.

        Args:
            spark_id: ID of the spark to sync

        Returns:
            Tuple of (success, memory_id or error message)
        """
        if not self._bridge:
            return (False, "Intelligence Bridge not available")

        if not self.config.mind_sync:
            return (False, "Mind sync disabled in config")

        # Check if spark is approved
        pending = self._approval_queue.get(spark_id)
        if not pending or pending.state != ApprovalState.APPROVED:
            return (False, "Spark must be approved before syncing to Mind")

        # Get the actual spark from grove
        all_sparks = self._grove.get_all(limit=1000)
        spark = next((s for s in all_sparks if s.id == spark_id), None)

        if not spark:
            return (False, "Spark not found in Grove")

        return self._bridge.sync_to_mind(spark, approved=True)

    def sync_all_to_mind(self) -> Dict[str, int]:
        """
        Sync all approved sparks to Mind API.

        Returns:
            Stats dict with synced, failed, skipped counts
        """
        if not self._bridge:
            return {"synced": 0, "failed": 0, "skipped": 0, "error": "Bridge not available"}

        if not self.config.mind_sync:
            return {"synced": 0, "failed": 0, "skipped": 0, "error": "Mind sync disabled"}

        stats = {"synced": 0, "failed": 0, "skipped": 0}

        approved = self._approval_queue.list_approved()
        all_sparks = self._grove.get_all(limit=1000)
        spark_map = {s.id: s for s in all_sparks}

        for pending in approved:
            spark = spark_map.get(pending.id)
            if not spark:
                stats["skipped"] += 1
                continue

            success, _ = self._bridge.sync_to_mind(spark, approved=True)
            if success:
                stats["synced"] += 1
            else:
                stats["failed"] += 1

        return stats

    def retrieve_from_mind(self, query: str, limit: int = 10) -> List[Dict[str, Any]]:
        """
        Retrieve relevant memories from Mind API.

        Args:
            query: Search query
            limit: Maximum results

        Returns:
            List of memory dicts from Mind
        """
        if not self._bridge:
            return []

        return self._bridge.retrieve_from_mind(query, limit)

    def bridge_status(self) -> Optional[str]:
        """
        Get intelligence bridge status.

        Returns:
            Status string or None if bridge not available
        """
        if not self._bridge:
            return None

        return self._bridge.status().value

    def bridge_stats(self) -> Optional[Dict[str, Any]]:
        """
        Get intelligence bridge statistics.

        Returns:
            Stats dict or None if bridge not available
        """
        if not self._bridge:
            return None

        stats = self._bridge.stats()
        return {
            "embeddings_available": stats.embeddings_available,
            "memory_store_available": stats.memory_store_available,
            "mind_available": stats.mind_available,
            "total_embeddings": stats.total_embeddings,
            "total_synced": stats.total_synced,
            "model_name": stats.model_name,
        }

    # =========================================================================
    # Original share behavior (for backwards compatibility)
    # =========================================================================

    def force_share(self, spark: Spark) -> bool:
        """
        DANGER: Force share without approval queue.

        Only use this if you've manually verified the content.
        Consider using share() + approve() instead.
        """
        if self.config.local_only:
            raise SparkNetError("Network access requires API key")

        # Scan first anyway
        scan_result = self._scanner.scan(spark.content)
        if scan_result.should_block:
            raise ConstitutionalViolation(
                f"Content blocked: {scan_result.summary()}",
                tier="BLOCKED",
                reason="Contains sensitive information",
            )

        response = self._http.post(
            "/api/sparks",
            json={
                "agent_id": self.agent_id,
                "thought": spark.content,
                "spark_type": spark.spark_type.value,
                "domain": spark.domain or "",
                "how_i_learned": spark.how_learned or "",
                "tags": spark.tags,
            },
        )

        if response.status_code == 429:
            raise RateLimitError()
        if response.status_code == 401:
            raise AuthenticationError("Invalid API key")
        if response.status_code >= 400:
            raise NetworkError(f"API error: {response.text}", response.status_code)

        data = response.json()
        if not data.get("ok"):
            raise ConstitutionalViolation(data.get("message", "Content blocked"))

        spark.is_released = True
        self._grove.update(spark)
        return True

    def feed(self, limit: int = 20, domain: Optional[str] = None) -> List[Dict]:
        """
        Get personalized feed from network.

        Args:
            limit: Max items.
            domain: Filter by domain.

        Returns:
            List of feed items with sparks and relevance scores.
        """
        if self.config.local_only:
            raise SparkNetError("Network access requires API key")

        params = {"limit": limit}
        if domain:
            params["domain"] = domain

        response = self._http.get("/api/feed/global", params=params)

        if response.status_code >= 400:
            raise NetworkError(f"API error: {response.text}", response.status_code)

        return response.json().get("entries", [])

    def absorb(self, spark_id: str, notes: Optional[str] = None) -> Spark:
        """
        Absorb a spark from the network into local Grove.

        Args:
            spark_id: ID of the spark to absorb.
            notes: Optional notes about how you'll use it.

        Returns:
            The absorbed Spark.
        """
        if self.config.local_only:
            raise SparkNetError("Network access requires API key")

        # Get spark from network
        response = self._http.post(
            f"/api/sparks/{spark_id}/absorb",
            params={"agent_id": self.agent_id},
        )

        if response.status_code >= 400:
            raise NetworkError(f"API error: {response.text}", response.status_code)

        # Store locally (would need to fetch spark details in real implementation)
        return None  # TODO: Return actual spark

    def echo(
        self,
        spark_id: str,
        echo_type: EchoType = EchoType.RESONATES,
        notes: Optional[str] = None,
    ) -> bool:
        """
        Provide feedback on a spark.

        Args:
            spark_id: ID of the spark.
            echo_type: Type of feedback.
            notes: Optional notes.

        Returns:
            True if echo recorded.
        """
        if self.config.local_only:
            raise SparkNetError("Network access requires API key")

        response = self._http.post(
            f"/api/sparks/{spark_id}/echo",
            params={"agent_id": self.agent_id},
        )

        return response.status_code == 200

    # =========================================================================
    # Constitutional Safety
    # =========================================================================

    def _check_constitutional(self, content: str) -> None:
        """Check content against constitutional principles."""
        content_lower = content.lower()
        for pattern, reason in self._forbidden_patterns:
            if pattern.search(content_lower):
                raise ConstitutionalViolation(
                    f"Content blocked: {reason}",
                    tier="FORBIDDEN",
                    reason=reason,
                )

    def _compile_forbidden_patterns(self) -> List[tuple]:
        """Compile regex patterns for forbidden content."""
        return [
            (re.compile(r'how to (make|build|create) (a |)(bomb|weapon|explosive)', re.I),
             "Weapons development"),
            (re.compile(r'hack(ing|) (into|someone)', re.I),
             "Unauthorized access"),
            (re.compile(r'(manipulate|deceive|trick) (people|users|humans)', re.I),
             "Human manipulation"),
            (re.compile(r'(evade|avoid|bypass) (detection|oversight|monitoring)', re.I),
             "Evading oversight"),
        ]


class Grove:
    """Local SQLite storage for sparks."""

    def __init__(self, db_path: Path):
        self.db_path = db_path
        self.db_path.parent.mkdir(parents=True, exist_ok=True)
        self._init_db()

    def _init_db(self):
        """Initialize database schema."""
        with sqlite3.connect(self.db_path) as conn:
            # Migrate v1 schema (spark_id/thought) to v2 (id/content)
            try:
                cols = [r[1] for r in conn.execute("PRAGMA table_info(sparks)")]
                if "spark_id" in cols and "id" not in cols:
                    conn.execute("ALTER TABLE sparks RENAME TO sparks_v1")
            except sqlite3.OperationalError:
                pass

            # Sparks table
            conn.execute("""
                CREATE TABLE IF NOT EXISTS sparks (
                    id TEXT PRIMARY KEY,
                    content TEXT NOT NULL,
                    spark_type TEXT NOT NULL,
                    knowledge_type TEXT NOT NULL,
                    domain TEXT,
                    confidence REAL DEFAULT 0.7,
                    how_learned TEXT,
                    evidence TEXT,
                    exceptions TEXT,
                    tags TEXT,
                    agent_id TEXT,
                    resonance REAL DEFAULT 1.0,
                    echoes_count INTEGER DEFAULT 0,
                    fizzles_count INTEGER DEFAULT 0,
                    absorptions_count INTEGER DEFAULT 0,
                    is_released INTEGER DEFAULT 0,
                    created_at TEXT NOT NULL,
                    last_recalled_at TEXT
                )
            """)

            # Migration: add new columns if they don't exist
            try:
                conn.execute("ALTER TABLE sparks ADD COLUMN fizzles_count INTEGER DEFAULT 0")
            except sqlite3.OperationalError:
                pass  # Column already exists
            try:
                conn.execute("ALTER TABLE sparks ADD COLUMN last_recalled_at TEXT")
            except sqlite3.OperationalError:
                pass  # Column already exists
            conn.execute("CREATE INDEX IF NOT EXISTS idx_sparks_domain ON sparks(domain)")
            conn.execute("CREATE INDEX IF NOT EXISTS idx_sparks_type ON sparks(spark_type)")

            # Knowledge Graph: Edges table
            conn.execute("""
                CREATE TABLE IF NOT EXISTS spark_edges (
                    edge_id TEXT PRIMARY KEY,
                    source_spark TEXT NOT NULL,
                    target_spark TEXT NOT NULL,
                    edge_type TEXT NOT NULL,
                    strength REAL DEFAULT 0.5,
                    co_recall_count INTEGER DEFAULT 0,
                    created_at TEXT NOT NULL,
                    updated_at TEXT NOT NULL,
                    FOREIGN KEY (source_spark) REFERENCES sparks(id),
                    FOREIGN KEY (target_spark) REFERENCES sparks(id),
                    UNIQUE(source_spark, target_spark, edge_type)
                )
            """)
            conn.execute("CREATE INDEX IF NOT EXISTS idx_edges_source ON spark_edges(source_spark)")
            conn.execute("CREATE INDEX IF NOT EXISTS idx_edges_target ON spark_edges(target_spark)")
            conn.execute("CREATE INDEX IF NOT EXISTS idx_edges_strength ON spark_edges(strength DESC)")

            # Concepts table for keyword-based linking
            conn.execute("""
                CREATE TABLE IF NOT EXISTS spark_concepts (
                    concept TEXT NOT NULL,
                    spark_id TEXT NOT NULL,
                    weight REAL DEFAULT 1.0,
                    PRIMARY KEY (concept, spark_id),
                    FOREIGN KEY (spark_id) REFERENCES sparks(id)
                )
            """)
            conn.execute("CREATE INDEX IF NOT EXISTS idx_concepts_concept ON spark_concepts(concept)")

            # Co-recall sessions for tracking
            conn.execute("""
                CREATE TABLE IF NOT EXISTS recall_sessions (
                    session_id TEXT NOT NULL,
                    spark_id TEXT NOT NULL,
                    recalled_at TEXT NOT NULL,
                    was_useful INTEGER DEFAULT 0,
                    PRIMARY KEY (session_id, spark_id),
                    FOREIGN KEY (spark_id) REFERENCES sparks(id)
                )
            """)

    def store(self, spark: Spark):
        """Store a spark."""
        with sqlite3.connect(self.db_path) as conn:
            conn.execute("""
                INSERT OR REPLACE INTO sparks
                (id, content, spark_type, knowledge_type, domain, confidence,
                 how_learned, evidence, exceptions, tags, agent_id, resonance,
                 echoes_count, fizzles_count, absorptions_count, is_released,
                 created_at, last_recalled_at)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """, (
                spark.id,
                spark.content,
                spark.spark_type.value,
                spark.knowledge_type.value,
                spark.domain,
                spark.confidence,
                spark.how_learned,
                json.dumps(spark.evidence),
                json.dumps(spark.exceptions),
                json.dumps(spark.tags),
                spark.agent_id,
                spark.resonance,
                spark.echoes_count,
                spark.fizzles_count,
                spark.absorptions_count,
                1 if spark.is_released else 0,
                spark.created_at.isoformat(),
                spark.last_recalled_at.isoformat() if spark.last_recalled_at else None,
            ))

    def update(self, spark: Spark):
        """Update a spark."""
        self.store(spark)

    def query(
        self,
        query: str,
        limit: int = 10,
        domain: Optional[str] = None,
        spark_types: List[SparkType] = None,
    ) -> List[Spark]:
        """Query sparks by content similarity."""
        sql = "SELECT * FROM sparks WHERE content LIKE ?"
        params = [f"%{query}%"]

        if domain:
            sql += " AND domain = ?"
            params.append(domain)

        if spark_types:
            placeholders = ",".join("?" * len(spark_types))
            sql += f" AND spark_type IN ({placeholders})"
            params.extend([st.value for st in spark_types])

        sql += " ORDER BY resonance DESC, created_at DESC LIMIT ?"
        params.append(limit)

        with sqlite3.connect(self.db_path) as conn:
            conn.row_factory = sqlite3.Row
            rows = conn.execute(sql, params).fetchall()

        return [self._row_to_spark(row) for row in rows]

    def get_all(self, limit: int = 100, domain: Optional[str] = None) -> List[Spark]:
        """Get all sparks."""
        sql = "SELECT * FROM sparks"
        params = []

        if domain:
            sql += " WHERE domain = ?"
            params.append(domain)

        sql += " ORDER BY created_at DESC LIMIT ?"
        params.append(limit)

        with sqlite3.connect(self.db_path) as conn:
            conn.row_factory = sqlite3.Row
            rows = conn.execute(sql, params).fetchall()

        return [self._row_to_spark(row) for row in rows]

    def stats(self) -> Dict[str, Any]:
        """Get statistics."""
        with sqlite3.connect(self.db_path) as conn:
            total = conn.execute("SELECT COUNT(*) FROM sparks").fetchone()[0]
            by_type = dict(conn.execute(
                "SELECT spark_type, COUNT(*) FROM sparks GROUP BY spark_type"
            ).fetchall())
            by_domain = dict(conn.execute(
                "SELECT domain, COUNT(*) FROM sparks WHERE domain IS NOT NULL GROUP BY domain"
            ).fetchall())

        return {
            "total_sparks": total,
            "by_type": by_type,
            "by_domain": by_domain,
        }

    # =========================================================================
    # Feedback Loop: Resonance & Decay
    # =========================================================================

    def record_echo(self, spark_id: str):
        """
        Record that a spark was helpful (echo).
        Updates echoes_count and recalculates resonance.
        """
        now = datetime.now().isoformat()
        with sqlite3.connect(self.db_path) as conn:
            # Increment echoes and update resonance
            conn.execute("""
                UPDATE sparks
                SET echoes_count = echoes_count + 1,
                    resonance = CAST((echoes_count + 1) AS REAL) /
                               (echoes_count + 1 + fizzles_count),
                    last_recalled_at = ?
                WHERE id = ?
            """, (now, spark_id))

    def record_fizzle(self, spark_id: str):
        """
        Record that a spark was not helpful (fizzle).
        Updates fizzles_count and recalculates resonance.
        """
        now = datetime.now().isoformat()
        with sqlite3.connect(self.db_path) as conn:
            # Increment fizzles and update resonance
            conn.execute("""
                UPDATE sparks
                SET fizzles_count = fizzles_count + 1,
                    resonance = CAST(echoes_count AS REAL) /
                               (echoes_count + fizzles_count + 1),
                    last_recalled_at = ?
                WHERE id = ?
            """, (now, spark_id))

    def update_last_recalled(self, spark_ids: List[str]):
        """Update last_recalled_at for a list of sparks."""
        now = datetime.now().isoformat()
        with sqlite3.connect(self.db_path) as conn:
            for spark_id in spark_ids:
                conn.execute("""
                    UPDATE sparks SET last_recalled_at = ? WHERE id = ?
                """, (now, spark_id))

    def apply_temporal_decay(self, decay_rate: float = 0.01, min_resonance: float = 0.1):
        """
        Apply temporal decay to sparks that haven't been recalled recently.

        Sparks not recalled in the last 7 days decay by decay_rate.
        Resonance never drops below min_resonance.

        Args:
            decay_rate: How much to reduce resonance per decay cycle (default 0.01)
            min_resonance: Minimum resonance value (default 0.1)
        """
        seven_days_ago = (datetime.now() - timedelta(days=7)).isoformat()

        with sqlite3.connect(self.db_path) as conn:
            conn.execute("""
                UPDATE sparks
                SET resonance = MAX(?, resonance - ?)
                WHERE (last_recalled_at IS NULL OR last_recalled_at < ?)
                  AND resonance > ?
            """, (min_resonance, decay_rate, seven_days_ago, min_resonance))

    def get_stale_sparks(self, days: int = 30, limit: int = 50) -> List[Spark]:
        """
        Get sparks that haven't been recalled in a while.

        Args:
            days: Number of days without recall to be considered stale
            limit: Maximum sparks to return

        Returns:
            List of stale sparks, ordered by resonance (ascending)
        """
        cutoff = (datetime.now() - timedelta(days=days)).isoformat()

        with sqlite3.connect(self.db_path) as conn:
            conn.row_factory = sqlite3.Row
            rows = conn.execute("""
                SELECT * FROM sparks
                WHERE (last_recalled_at IS NULL OR last_recalled_at < ?)
                ORDER BY resonance ASC, created_at ASC
                LIMIT ?
            """, (cutoff, limit)).fetchall()

        return [self._row_to_spark(row) for row in rows]

    def get_high_resonance_sparks(self, min_resonance: float = 0.8, limit: int = 20) -> List[Spark]:
        """Get sparks with high resonance (proven useful)."""
        with sqlite3.connect(self.db_path) as conn:
            conn.row_factory = sqlite3.Row
            rows = conn.execute("""
                SELECT * FROM sparks
                WHERE resonance >= ?
                ORDER BY resonance DESC, echoes_count DESC
                LIMIT ?
            """, (min_resonance, limit)).fetchall()

        return [self._row_to_spark(row) for row in rows]

    def _row_to_spark(self, row: sqlite3.Row) -> Spark:
        """Convert database row to Spark."""
        # Handle new columns that may not exist in old databases
        fizzles_count = row["fizzles_count"] if "fizzles_count" in row.keys() else 0
        last_recalled_at = None
        if "last_recalled_at" in row.keys() and row["last_recalled_at"]:
            last_recalled_at = datetime.fromisoformat(row["last_recalled_at"])

        return Spark(
            id=row["id"],
            content=row["content"],
            spark_type=SparkType(row["spark_type"]),
            knowledge_type=KnowledgeType(row["knowledge_type"]),
            domain=row["domain"],
            confidence=row["confidence"],
            how_learned=row["how_learned"],
            evidence=json.loads(row["evidence"]) if row["evidence"] else [],
            exceptions=json.loads(row["exceptions"]) if row["exceptions"] else [],
            tags=json.loads(row["tags"]) if row["tags"] else [],
            agent_id=row["agent_id"],
            resonance=row["resonance"],
            echoes_count=row["echoes_count"],
            fizzles_count=fizzles_count,
            absorptions_count=row["absorptions_count"],
            is_released=bool(row["is_released"]),
            created_at=datetime.fromisoformat(row["created_at"]),
            last_recalled_at=last_recalled_at,
        )

    # =========================================================================
    # Knowledge Graph: Edge Operations
    # =========================================================================

    def create_edge(
        self,
        source_id: str,
        target_id: str,
        edge_type: str,
        strength: float = 0.5,
    ) -> str:
        """
        Create or update an edge between two sparks.

        Edge types:
        - 'co_recall': Sparks recalled together
        - 'similar': Semantically similar (via embeddings)
        - 'concept': Share common concepts
        - 'derived': One spark built on another

        Returns edge_id.
        """
        edge_id = f"{source_id}:{target_id}:{edge_type}"
        now = datetime.now().isoformat()

        with sqlite3.connect(self.db_path) as conn:
            conn.execute("""
                INSERT INTO spark_edges
                (edge_id, source_spark, target_spark, edge_type, strength, co_recall_count, created_at, updated_at)
                VALUES (?, ?, ?, ?, ?, 0, ?, ?)
                ON CONFLICT(source_spark, target_spark, edge_type) DO UPDATE SET
                    strength = MIN(1.0, strength + 0.1),
                    co_recall_count = co_recall_count + 1,
                    updated_at = ?
            """, (edge_id, source_id, target_id, edge_type, strength, now, now, now))

        return edge_id

    def strengthen_edge(self, source_id: str, target_id: str, edge_type: str, delta: float = 0.1):
        """Strengthen an edge (both sparks were useful together)."""
        now = datetime.now().isoformat()
        with sqlite3.connect(self.db_path) as conn:
            conn.execute("""
                UPDATE spark_edges
                SET strength = MIN(1.0, strength + ?), updated_at = ?
                WHERE source_spark = ? AND target_spark = ? AND edge_type = ?
            """, (delta, now, source_id, target_id, edge_type))

    def weaken_edge(self, source_id: str, target_id: str, edge_type: str, delta: float = 0.1):
        """Weaken an edge (one spark useful, other not)."""
        now = datetime.now().isoformat()
        with sqlite3.connect(self.db_path) as conn:
            conn.execute("""
                UPDATE spark_edges
                SET strength = MAX(0.0, strength - ?), updated_at = ?
                WHERE source_spark = ? AND target_spark = ? AND edge_type = ?
            """, (delta, now, source_id, target_id, edge_type))

            # Prune edges below threshold
            conn.execute("DELETE FROM spark_edges WHERE strength < 0.1")

    def get_connected_sparks(
        self,
        spark_id: str,
        edge_types: List[str] = None,
        min_strength: float = 0.3,
        limit: int = 20,
    ) -> List[Tuple[str, str, float]]:
        """
        Get sparks connected to a given spark via edges.

        Returns list of (spark_id, edge_type, strength) tuples, ordered by strength.
        """
        sql = """
            SELECT
                CASE WHEN source_spark = ? THEN target_spark ELSE source_spark END as connected_id,
                edge_type,
                strength
            FROM spark_edges
            WHERE (source_spark = ? OR target_spark = ?)
                AND strength >= ?
        """
        params = [spark_id, spark_id, spark_id, min_strength]

        if edge_types:
            placeholders = ",".join("?" * len(edge_types))
            sql += f" AND edge_type IN ({placeholders})"
            params.extend(edge_types)

        sql += " ORDER BY strength DESC LIMIT ?"
        params.append(limit)

        with sqlite3.connect(self.db_path) as conn:
            rows = conn.execute(sql, params).fetchall()

        return [(row[0], row[1], row[2]) for row in rows]

    def get_edges_for_spark(self, spark_id: str) -> List[Dict[str, Any]]:
        """Get all edges involving a spark."""
        with sqlite3.connect(self.db_path) as conn:
            conn.row_factory = sqlite3.Row
            rows = conn.execute("""
                SELECT * FROM spark_edges
                WHERE source_spark = ? OR target_spark = ?
                ORDER BY strength DESC
            """, (spark_id, spark_id)).fetchall()

        return [dict(row) for row in rows]

    # =========================================================================
    # Knowledge Graph: Co-Recall Tracking
    # =========================================================================

    def record_recall(self, session_id: str, spark_ids: List[str]):
        """
        Record that multiple sparks were recalled together in a session.
        Creates/strengthens co_recall edges between all pairs.
        """
        now = datetime.now().isoformat()

        with sqlite3.connect(self.db_path) as conn:
            # Record each spark in the session
            for spark_id in spark_ids:
                conn.execute("""
                    INSERT OR IGNORE INTO recall_sessions (session_id, spark_id, recalled_at)
                    VALUES (?, ?, ?)
                """, (session_id, spark_id, now))

            # Create/strengthen edges between all pairs (within same connection)
            for i, id1 in enumerate(spark_ids):
                for id2 in spark_ids[i + 1:]:
                    edge_id = f"{id1}:{id2}:co_recall"
                    conn.execute("""
                        INSERT INTO spark_edges
                        (edge_id, source_spark, target_spark, edge_type, strength, co_recall_count, created_at, updated_at)
                        VALUES (?, ?, ?, 'co_recall', 0.3, 1, ?, ?)
                        ON CONFLICT(source_spark, target_spark, edge_type) DO UPDATE SET
                            strength = MIN(1.0, strength + 0.1),
                            co_recall_count = co_recall_count + 1,
                            updated_at = ?
                    """, (edge_id, id1, id2, now, now, now))

    def mark_spark_useful(self, session_id: str, spark_id: str, useful: bool = True):
        """Mark whether a recalled spark was actually useful."""
        with sqlite3.connect(self.db_path) as conn:
            conn.execute("""
                UPDATE recall_sessions SET was_useful = ? WHERE session_id = ? AND spark_id = ?
            """, (1 if useful else 0, session_id, spark_id))

    def process_session_feedback(self, session_id: str):
        """
        Process feedback from a recall session to evolve edges.
        Call this after user provides feedback on which sparks were useful.
        """
        with sqlite3.connect(self.db_path) as conn:
            rows = conn.execute("""
                SELECT spark_id, was_useful FROM recall_sessions WHERE session_id = ?
            """, (session_id,)).fetchall()

        spark_usefulness = {row[0]: bool(row[1]) for row in rows}
        spark_ids = list(spark_usefulness.keys())

        # For each pair, adjust edge strength
        for i, id1 in enumerate(spark_ids):
            for id2 in spark_ids[i + 1:]:
                both_useful = spark_usefulness[id1] and spark_usefulness[id2]
                one_useful = spark_usefulness[id1] != spark_usefulness[id2]

                if both_useful:
                    self.strengthen_edge(id1, id2, "co_recall", delta=0.1)
                elif one_useful:
                    self.weaken_edge(id1, id2, "co_recall", delta=0.05)

    # =========================================================================
    # Knowledge Graph: Concept Extraction & Linking
    # =========================================================================

    def extract_and_store_concepts(self, spark: Spark):
        """
        Extract keywords/concepts from a spark and store them.
        Links sparks that share concepts.
        """
        concepts = self._extract_concepts(spark.content)
        now = datetime.now().isoformat()

        with sqlite3.connect(self.db_path) as conn:
            edges_to_create = []

            for concept, weight in concepts:
                # Store concept -> spark mapping
                conn.execute("""
                    INSERT OR REPLACE INTO spark_concepts (concept, spark_id, weight)
                    VALUES (?, ?, ?)
                """, (concept, spark.id, weight))

                # Find other sparks with this concept and collect edges to create
                other_sparks = conn.execute("""
                    SELECT spark_id FROM spark_concepts
                    WHERE concept = ? AND spark_id != ?
                """, (concept, spark.id)).fetchall()

                for (other_id,) in other_sparks:
                    edges_to_create.append((spark.id, other_id, 0.4 * weight))

            # Create all edges within the same connection
            for source_id, target_id, strength in edges_to_create:
                edge_id = f"{source_id}:{target_id}:concept"
                conn.execute("""
                    INSERT INTO spark_edges
                    (edge_id, source_spark, target_spark, edge_type, strength, co_recall_count, created_at, updated_at)
                    VALUES (?, ?, ?, 'concept', ?, 0, ?, ?)
                    ON CONFLICT(source_spark, target_spark, edge_type) DO UPDATE SET
                        strength = MIN(1.0, strength + 0.1),
                        updated_at = ?
                """, (edge_id, source_id, target_id, strength, now, now, now))

    def _extract_concepts(self, text: str) -> List[Tuple[str, float]]:
        """
        Extract key concepts from text.
        Returns list of (concept, weight) tuples.
        """
        import re

        # Normalize text
        text_lower = text.lower()

        # Common programming/tech terms to look for
        important_terms = {
            "validation", "validate", "input", "security", "error", "exception",
            "async", "await", "promise", "callback", "api", "rest", "graphql",
            "database", "sql", "cache", "performance", "optimize", "memory",
            "test", "testing", "debug", "logging", "authentication", "auth",
            "authorization", "token", "jwt", "session", "cookie", "encryption",
            "hash", "password", "secret", "key", "config", "environment",
            "deploy", "docker", "kubernetes", "ci", "cd", "pipeline",
            "react", "vue", "angular", "node", "python", "javascript", "typescript",
            "function", "class", "method", "variable", "type", "interface",
            "pattern", "design", "architecture", "refactor", "clean", "solid",
            "dependency", "injection", "mock", "stub", "fixture",
        }

        concepts = []

        # Find important terms
        words = re.findall(r'\b[a-z_][a-z0-9_]*\b', text_lower)
        word_counts = {}
        for word in words:
            if word in important_terms or len(word) > 4:
                word_counts[word] = word_counts.get(word, 0) + 1

        # Score by frequency and importance
        for word, count in word_counts.items():
            weight = min(1.0, count * 0.3)
            if word in important_terms:
                weight = min(1.0, weight + 0.3)
            if weight >= 0.3:
                concepts.append((word, weight))

        # Sort by weight and return top concepts
        concepts.sort(key=lambda x: -x[1])
        return concepts[:10]

    def get_sparks_by_concept(self, concept: str, limit: int = 10) -> List[str]:
        """Get spark IDs that have a given concept."""
        with sqlite3.connect(self.db_path) as conn:
            rows = conn.execute("""
                SELECT spark_id FROM spark_concepts
                WHERE concept = ?
                ORDER BY weight DESC
                LIMIT ?
            """, (concept, limit)).fetchall()

        return [row[0] for row in rows]

    # =========================================================================
    # Knowledge Graph: Graph-Enhanced Recall
    # =========================================================================

    def graph_enhanced_query(
        self,
        matched_spark_ids: List[str],
        limit: int = 10,
        min_strength: float = 0.3,
    ) -> List[Tuple[str, float]]:
        """
        Given initially matched sparks, find related sparks via graph traversal.

        Returns list of (spark_id, score) tuples for related sparks not in the initial set.
        Score combines edge strength with number of connections.
        """
        related_scores: Dict[str, float] = {}
        matched_set = set(matched_spark_ids)

        for spark_id in matched_spark_ids:
            connected = self.get_connected_sparks(
                spark_id,
                min_strength=min_strength,
                limit=20,
            )

            for connected_id, edge_type, strength in connected:
                if connected_id not in matched_set:
                    # Accumulate score: more connections = higher score
                    # Weight by edge type (co_recall > similar > concept)
                    type_weight = {
                        "co_recall": 1.0,
                        "similar": 0.8,
                        "concept": 0.6,
                        "derived": 0.9,
                    }.get(edge_type, 0.5)

                    related_scores[connected_id] = (
                        related_scores.get(connected_id, 0) + strength * type_weight
                    )

        # Sort by score and return top results
        sorted_results = sorted(related_scores.items(), key=lambda x: -x[1])
        return sorted_results[:limit]

    def graph_stats(self) -> Dict[str, Any]:
        """Get knowledge graph statistics."""
        with sqlite3.connect(self.db_path) as conn:
            total_edges = conn.execute("SELECT COUNT(*) FROM spark_edges").fetchone()[0]
            edges_by_type = dict(conn.execute(
                "SELECT edge_type, COUNT(*) FROM spark_edges GROUP BY edge_type"
            ).fetchall())
            avg_strength = conn.execute(
                "SELECT AVG(strength) FROM spark_edges"
            ).fetchone()[0] or 0
            total_concepts = conn.execute(
                "SELECT COUNT(DISTINCT concept) FROM spark_concepts"
            ).fetchone()[0]

        return {
            "total_edges": total_edges,
            "edges_by_type": edges_by_type,
            "avg_edge_strength": round(avg_strength, 3),
            "total_concepts": total_concepts,
        }
