"""
MCAL: Memory-Context Alignment Layer

Main interface providing a standalone memory architecture for AI agents.

Architecture:
    ┌─────────────────────────────────────────────┐
    │                   MCAL                       │
    │  ┌─────────────────────────────────────┐    │
    │  │  UnifiedExtractor (Single-pass)     │    │
    │  │  - Entities, Goals, Decisions       │    │
    │  │  - Relationships, Embeddings        │    │
    │  └─────────────────────────────────────┘    │
    │                    │                         │
    │                    ▼                         │
    │  ┌─────────────────────────────────────┐    │
    │  │      UnifiedGraph + VectorIndex     │    │
    │  │  (Graph storage + semantic search)  │    │
    │  └─────────────────────────────────────┘    │
    └─────────────────────────────────────────────┘

Usage:
    from mcal import MCAL
    
    # Initialize (no external dependencies!)
    mcal = MCAL(
        openai_api_key="..."  # or anthropic_api_key, or use bedrock
    )
    
    # Add conversation (extracts facts + intents + reasoning)
    result = await mcal.add(
        messages=[
            {"role": "user", "content": "Let's build a fraud detection system"},
            {"role": "assistant", "content": "Great! What's your data source?"}
        ],
        user_id="user_123"
    )
    
    # Search with goal-awareness
    context = await mcal.search(
        query="What database should we use?",
        user_id="user_123"
    )
    
    # Get assembled context for LLM
    prompt_context = mcal.get_context(
        query="What's our next step?",
        user_id="user_123",
        max_tokens=4000
    )
"""

from __future__ import annotations

import asyncio
import logging
import os
import threading
import time
from typing import Any, Optional
from dataclasses import dataclass, field

from anthropic import Anthropic
from openai import OpenAI, RateLimitError as OpenAIRateLimitError, APIError as OpenAIAPIError, APITimeoutError as OpenAITimeoutError, APIConnectionError as OpenAIConnectionError

from pathlib import Path

from .backends import MemoryEntry  # MemoryEntry still needed for compatibility
from .core.models import (
    IntentGraph,
    IntentNode,
    IntentType,
    IntentStatus,
    DecisionTrail,
    Session,
    RetrievalConfig,
)
from .core.goal_retriever import ContextAssembler
from .core.storage import MCALStorage
from .core.extraction_cache import ExtractionCache
from .core.unified_extractor import (
    UnifiedExtractor,
    UnifiedGraph,
    NodeType,
    EdgeType,
    graph_to_memories,
    memories_to_context_string,
)
from .core.retry import (
    llm_retry,
    classify_http_error,
    LLMRateLimitError,
    LLMServerError,
    LLMTimeoutError,
)
from .providers.bedrock import BedrockProvider

logger = logging.getLogger(__name__)


# =============================================================================
# LLM Client Wrapper
# =============================================================================

class AnthropicClient:
    """Wrapper around Anthropic client for MCAL components."""
    
    def __init__(self, api_key: str, model: str = "claude-sonnet-4-20250514"):
        self.client = Anthropic(api_key=api_key)
        self.model = model
    
    @llm_retry(max_attempts=3, min_wait=1.0, max_wait=10.0)
    async def complete(self, prompt: str, system: Optional[str] = None) -> str:
        """
        Generate completion with automatic retry on transient failures.
        
        Retries on rate limits and server errors with exponential backoff.
        """
        messages = [{"role": "user", "content": prompt}]
        
        try:
            response = self.client.messages.create(
                model=self.model,
                max_tokens=4096,
                system=system or "You are a helpful assistant.",
                messages=messages
            )
            return response.content[0].text
        except Exception as e:
            # Anthropic SDK raises specific exceptions we can classify
            error_str = str(e).lower()
            if 'rate' in error_str or '429' in error_str:
                raise LLMRateLimitError(f"Anthropic rate limit: {e}")
            elif 'timeout' in error_str or 'timed out' in error_str:
                raise LLMTimeoutError(f"Anthropic timeout: {e}")
            elif 'connection' in error_str or 'connect' in error_str:
                raise LLMTimeoutError(f"Anthropic connection error: {e}")
            elif '500' in error_str or '502' in error_str or '503' in error_str or '504' in error_str:
                raise LLMServerError(f"Anthropic server error: {e}")
            raise


class OpenAIClient:
    """Wrapper around OpenAI client for MCAL components."""
    
    def __init__(self, api_key: str, model: str = "gpt-4o"):
        self.client = OpenAI(api_key=api_key)
        self.model = model
        # Token tracking
        self.total_prompt_tokens = 0
        self.total_completion_tokens = 0
    
    @llm_retry(max_attempts=3, min_wait=1.0, max_wait=10.0)
    async def complete(self, prompt: str, system: Optional[str] = None) -> str:
        """
        Generate completion with automatic retry on transient failures.
        
        Retries on rate limits (429) and server errors (5xx) with exponential backoff.
        """
        messages = [{"role": "user", "content": prompt}]
        if system:
            messages.insert(0, {"role": "system", "content": system})
        
        try:
            response = self.client.chat.completions.create(
                model=self.model,
                max_tokens=4096,
                messages=messages
            )
            
            # Track token usage
            if response.usage:
                self.total_prompt_tokens += response.usage.prompt_tokens
                self.total_completion_tokens += response.usage.completion_tokens
            
            return response.choices[0].message.content
        except OpenAIRateLimitError as e:
            raise LLMRateLimitError(f"OpenAI rate limit: {e}")
        except OpenAITimeoutError as e:
            raise LLMTimeoutError(f"OpenAI timeout: {e}")
        except OpenAIConnectionError as e:
            raise LLMTimeoutError(f"OpenAI connection error: {e}")
        except OpenAIAPIError as e:
            # Check for server errors
            if hasattr(e, 'status_code') and e.status_code in (500, 502, 503, 504):
                raise LLMServerError(f"OpenAI server error ({e.status_code}): {e}")
            raise
    
    def get_token_usage(self) -> dict:
        """Get cumulative token usage."""
        return {
            "prompt_tokens": self.total_prompt_tokens,
            "completion_tokens": self.total_completion_tokens,
            "total_tokens": self.total_prompt_tokens + self.total_completion_tokens
        }
    
    def reset_token_usage(self):
        """Reset token counters."""
        self.total_prompt_tokens = 0
        self.total_completion_tokens = 0

class BedrockProviderWrapper:
    """Wrapper around Bedrock provider for MCAL components."""
    
    def __init__(self, model: str = "llama-3.3-70b", region: str = "us-east-1"):
        self.provider = BedrockProvider(model=model, region=region)
        self.model = model
    
    async def complete(self, prompt: str, system: Optional[str] = None) -> str:
        """Generate completion."""
        messages = []
        
        # Bedrock Converse API doesn't support system role - merge with user message
        if system:
            combined_prompt = f"{system}\n\n{prompt}"
            messages = [{"role": "user", "content": combined_prompt}]
        else:
            messages = [{"role": "user", "content": prompt}]
        
        return await self.provider.generate(messages)


class TieredBedrockProvider:
    """
    Tiered Bedrock provider that routes to fast/smart models based on task complexity.
    
    Fast model (8B): Simple tasks like intent detection, graph updates
    Smart model (70B): Complex tasks like decision extraction, reasoning
    """
    
    def __init__(
        self, 
        fast_model: str = "llama-3.1-8b",
        smart_model: str = "llama-3.3-70b",
        region: str = "us-east-1"
    ):
        self.fast_provider = BedrockProvider(model=fast_model, region=region)
        self.smart_provider = BedrockProvider(model=smart_model, region=region)
        self.fast_model = fast_model
        self.smart_model = smart_model
        
        # Track usage for metrics
        self.fast_calls = 0
        self.smart_calls = 0
    
    async def complete(
        self, 
        prompt: str, 
        system: Optional[str] = None,
        use_smart: bool = False
    ) -> str:
        """
        Generate completion using appropriate model tier.
        
        Args:
            prompt: The prompt to send
            system: System prompt (merged with user prompt)
            use_smart: Force use of smart model (for complex tasks)
        """
        # Build messages
        if system:
            combined_prompt = f"{system}\n\n{prompt}"
            messages = [{"role": "user", "content": combined_prompt}]
        else:
            messages = [{"role": "user", "content": prompt}]
        
        # Route to appropriate model
        if use_smart:
            self.smart_calls += 1
            logger.debug(f"Using SMART model ({self.smart_model})")
            return await self.smart_provider.generate(messages)
        else:
            self.fast_calls += 1
            logger.debug(f"Using FAST model ({self.fast_model})")
            return await self.fast_provider.generate(messages)
    
    def get_usage_stats(self) -> dict:
        """Get model usage statistics."""
        total = self.fast_calls + self.smart_calls
        return {
            "fast_calls": self.fast_calls,
            "smart_calls": self.smart_calls,
            "total_calls": total,
            "fast_ratio": self.fast_calls / total if total > 0 else 0,
        }
    
    def get_token_usage(self) -> dict:
        """Get total token usage from both providers."""
        fast_usage = self.fast_provider.get_token_usage()
        smart_usage = self.smart_provider.get_token_usage()
        return {
            "prompt_tokens": fast_usage["prompt_tokens"] + smart_usage["prompt_tokens"],
            "completion_tokens": fast_usage["completion_tokens"] + smart_usage["completion_tokens"],
            "total_tokens": fast_usage["total_tokens"] + smart_usage["total_tokens"],
            "fast_tokens": fast_usage["total_tokens"],
            "smart_tokens": smart_usage["total_tokens"],
        }
    
    def reset_token_usage(self):
        """Reset token counters on both providers."""
        self.fast_provider.reset_token_usage()
        self.smart_provider.reset_token_usage()


# =============================================================================
# MCAL Result Types
# =============================================================================

@dataclass
class TimingMetrics:
    """Timing metrics for performance monitoring."""
    total_ms: int = 0
    facts_ms: int = 0
    intents_ms: int = 0
    decisions_ms: int = 0
    parallel_savings_ms: int = 0  # Time saved by parallelization


@dataclass
class AddResult:
    """Result from adding messages to MCAL."""
    # From Mem0
    facts: list[MemoryEntry] = field(default_factory=list)
    
    # From MCAL (novel)
    intent_graph: Optional[IntentGraph] = None
    decisions: list[DecisionTrail] = field(default_factory=list)
    
    # From Unified Extractor (Issue #19)
    unified_graph: Optional[UnifiedGraph] = None
    
    # Performance metrics
    timing: Optional[TimingMetrics] = None
    
    @property
    def summary(self) -> dict:
        """Quick summary of what was extracted."""
        summary = {
            "facts_count": len(self.facts),
            "intents_count": len(self.intent_graph.nodes) if self.intent_graph else 0,
            "decisions_count": len(self.decisions),
        }
        # Add unified graph stats if present
        if self.unified_graph:
            graph_summary = self.unified_graph.summary()
            summary["unified_graph"] = {
                "nodes": graph_summary["total_nodes"],
                "edges": graph_summary["total_edges"],
                "decisions": graph_summary["decisions"],
                "goals": graph_summary["goals"],
                "actions": graph_summary["actions"],
            }
        if self.timing:
            summary["timing_ms"] = {
                "total": self.timing.total_ms,
                "facts": self.timing.facts_ms,
                "intents": self.timing.intents_ms,
                "decisions": self.timing.decisions_ms,
                "parallel_savings": self.timing.parallel_savings_ms,
            }
        return summary


@dataclass 
class SearchResult:
    """Result from searching MCAL."""
    # From Mem0 (re-ranked)
    memories: list[MemoryEntry] = field(default_factory=list)
    
    # From MCAL (enriched)
    active_goals: list[IntentNode] = field(default_factory=list)
    relevant_decisions: list[DecisionTrail] = field(default_factory=list)
    
    # Assembled context
    context: Optional[str] = None

# =============================================================================
# Main MCAL Class
# =============================================================================

class MCAL:
    """
    Memory-Context Alignment Layer.
    
    Adds reasoning preservation and goal-aware retrieval on top of Mem0.
    
    Three Pillars:
    1. Intent Graph Preservation - Track goal hierarchies
    2. Reasoning Chain Storage - Store WHY decisions were made
    3. Goal-Aware Retrieval - Retrieve based on objectives, not just similarity
    """
    
    def __init__(
        self,
        anthropic_api_key: Optional[str] = None,
        openai_api_key: Optional[str] = None,
        model: Optional[str] = None,
        llm_provider: str = "openai",
        bedrock_model: str = "llama-3.3-70b",
        bedrock_region: str = "us-east-1",
        storage_path: Optional[str] = None,
        enable_persistence: bool = True,
        enable_tiered_models: bool = False,
        bedrock_fast_model: str = "llama-3.1-8b",
        bedrock_smart_model: str = "llama-3.3-70b",
        enable_extraction_cache: bool = False,
        cache_ttl_seconds: int = 86400,
        max_graph_nodes: int = 500,
        # Deprecated parameters (Issue #53 - kept for backward compatibility)
        mem0_config: Optional[dict] = None,
        mem0_api_key: Optional[str] = None,
        use_standalone_backend: bool = False,
        use_unified_extractor: bool = True,
    ):
        """
        Initialize MCAL.
        
        Args:
            anthropic_api_key: Anthropic API key. Auto-detects from ANTHROPIC_API_KEY env var if not provided.
            openai_api_key: OpenAI API key. Auto-detects from OPENAI_API_KEY env var if not provided.
            model: Model to use for extraction (defaults based on provider)
            llm_provider: LLM provider to use ("openai", "anthropic", or "bedrock")
            bedrock_model: Bedrock model to use (default: llama-3.3-70b)
            bedrock_region: AWS region for Bedrock (default: us-east-1)
            storage_path: Path for persistent storage (default: ~/.mcal)
            enable_persistence: Enable cross-session persistence (default: True)
            enable_tiered_models: Use fast/smart model routing (bedrock only)
            bedrock_fast_model: Fast model for simple tasks (default: llama-3.1-8b)
            bedrock_smart_model: Smart model for complex tasks (default: llama-3.3-70b)
            enable_extraction_cache: Enable caching of extracted state (Issue #9)
            cache_ttl_seconds: Time-to-live for cache entries (default: 24h)
            max_graph_nodes: Maximum nodes per user graph before compaction (default: 500)
            
        Deprecated Args (Issue #53 - will be removed in v1.0):
            mem0_config: Ignored - MCAL is now standalone
            mem0_api_key: Ignored - MCAL is now standalone
            use_standalone_backend: Ignored - always standalone
            use_unified_extractor: Ignored - always uses unified extractor
        """
        self._enable_tiered_models = enable_tiered_models
        self._enable_extraction_cache = enable_extraction_cache
        self._max_graph_nodes = max_graph_nodes
        
        # Issue #53: Deprecation warnings for removed parameters
        import warnings
        if mem0_config is not None:
            warnings.warn(
                "mem0_config is deprecated and ignored. MCAL v1.0 is fully standalone.",
                DeprecationWarning,
                stacklevel=2
            )
        if mem0_api_key is not None:
            warnings.warn(
                "mem0_api_key is deprecated and ignored. MCAL v1.0 is fully standalone.",
                DeprecationWarning,
                stacklevel=2
            )
        if use_standalone_backend:
            warnings.warn(
                "use_standalone_backend is deprecated and ignored. MCAL is always standalone.",
                DeprecationWarning,
                stacklevel=2
            )
        if not use_unified_extractor:
            warnings.warn(
                "use_unified_extractor=False is deprecated. Legacy 3-pillar extraction "
                "has been removed. MCAL now always uses unified extraction.",
                DeprecationWarning,
                stacklevel=2
            )
        
        # Initialize extraction cache (Issue #9)
        if enable_extraction_cache:
            cache_path = None
            if storage_path:
                cache_path = Path(storage_path) / "extraction_cache.json"
            elif enable_persistence:
                cache_path = Path.home() / ".mcal" / "extraction_cache.json"
            self._extraction_cache = ExtractionCache(
                persist_path=cache_path,
                ttl_seconds=cache_ttl_seconds
            )
            logger.info(f"Extraction cache enabled (TTL: {cache_ttl_seconds}s)")
        else:
            self._extraction_cache = None
        
        # Issue #85: Auto-detect API keys from environment variables
        # Explicit params override env vars; env vars are fallback
        openai_api_key = openai_api_key or os.getenv("OPENAI_API_KEY")
        anthropic_api_key = anthropic_api_key or os.getenv("ANTHROPIC_API_KEY")
        
        # Initialize LLM client based on provider
        if llm_provider == "openai":
            if not openai_api_key:
                raise ValueError(
                    "No OpenAI API key found. Either pass openai_api_key= or "
                    "set the OPENAI_API_KEY environment variable."
                )
            model = model or "gpt-4o"
            self._llm = OpenAIClient(api_key=openai_api_key, model=model)
        elif llm_provider == "anthropic":
            if not anthropic_api_key:
                raise ValueError(
                    "No Anthropic API key found. Either pass anthropic_api_key= or "
                    "set the ANTHROPIC_API_KEY environment variable."
                )
            model = model or "claude-sonnet-4-20250514"
            self._llm = AnthropicClient(api_key=anthropic_api_key, model=model)
        elif llm_provider == "bedrock":
            # Bedrock uses AWS credentials from environment
            if enable_tiered_models:
                self._llm = TieredBedrockProvider(
                    fast_model=bedrock_fast_model,
                    smart_model=bedrock_smart_model,
                    region=bedrock_region
                )
                logger.info(f"Using tiered Bedrock: fast={bedrock_fast_model}, smart={bedrock_smart_model}")
            else:
                self._llm = BedrockProviderWrapper(
                    model=bedrock_model,
                    region=bedrock_region
                )
        else:
            raise ValueError(f"Unknown llm_provider: {llm_provider}. Use 'openai', 'anthropic', or 'bedrock'")
        
        # Issue #53: No memory backend needed - MCAL is standalone
        # UnifiedGraph handles all storage and search
        
        # Issue #94: Legacy IntentTracker, ReasoningStore removed - unified extractor handles everything
        self._context_assembler = ContextAssembler()
        
        # Initialize unified extractor (Issue #19 - 52x token reduction)
        self._unified_extractor = UnifiedExtractor(self._llm)
        self._user_graphs: dict[str, UnifiedGraph] = {}  # Per-user unified graphs
        # Issue #105: Thread safety for _user_graphs
        self._graph_dict_lock = threading.Lock()  # Protects dict mutations from sync methods
        self._user_locks: dict[str, asyncio.Lock] = {}  # Per-user async locks
        logger.info("Unified extractor enabled (single-pass extraction)")
        
        # Initialize persistent storage
        self._enable_persistence = enable_persistence
        if enable_persistence:
            storage_base = Path(storage_path) if storage_path else None
            self._storage = MCALStorage(base_path=storage_base)
            logger.info(f"Persistence enabled at {self._storage.base_path}")
        else:
            self._storage = None
            logger.info("Persistence disabled (in-memory only)")
        
        # Session management (in-memory cache)
        self._sessions: dict[str, Session] = {}
    
    def _get_user_lock(self, user_id: str) -> asyncio.Lock:
        """Get or create a per-user asyncio lock for safe concurrent access (Issue #105)."""
        with self._graph_dict_lock:
            if user_id not in self._user_locks:
                self._user_locks[user_id] = asyncio.Lock()
            return self._user_locks[user_id]

    # =========================================================================
    # Lifecycle (Issue #100)
    # =========================================================================
    
    def close(self) -> None:
        """
        Release resources held by this MCAL instance.
        
        Clears the in-memory graph cache and extraction cache.
        Safe to call multiple times.
        """
        with self._graph_dict_lock:
            self._user_graphs.clear()
            self._user_locks.clear()
        self._sessions.clear()
        if self._extraction_cache is not None:
            self._extraction_cache = None
        logger.info("MCAL instance closed")
    
    async def __aenter__(self) -> "MCAL":
        """Async context manager entry."""
        return self
    
    async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
        """Async context manager exit — calls close()."""
        self.close()
    
    def __enter__(self) -> "MCAL":
        """Sync context manager entry."""
        return self
    
    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        """Sync context manager exit — calls close()."""
        self.close()
    
    def _ensure_graph_loaded(self, user_id: str) -> Optional[UnifiedGraph]:
        """
        Ensure unified graph is loaded for user (lazy loading from storage).
        
        Issue #73 fix: Search and get_context were not loading persisted graphs.
        This method centralizes the lazy loading logic used across add/search/get_context.
        
        Args:
            user_id: User identifier
            
        Returns:
            UnifiedGraph if exists (in memory or on disk), None otherwise
        """
        # Check if already in memory
        if user_id in self._user_graphs:
            return self._user_graphs[user_id]
        
        # Try to load from storage if persistence is enabled
        if self._enable_persistence and self._storage:
            try:
                stored_graph = self._storage.load_unified_graph(user_id)
                if stored_graph:
                    self._user_graphs[user_id] = stored_graph
                    logger.info(f"Loaded unified graph for {user_id} from storage ({len(stored_graph.nodes)} nodes)")
                    return stored_graph
            except Exception as e:
                logger.error(f"Failed to load graph for {user_id} from storage: {e}")
                # Return None — caller will treat as new user
        
        return None
    
    async def add(
        self,
        messages: list[dict],
        user_id: str,
        session_id: Optional[str] = None,
    ) -> AddResult:
        """
        Add messages to memory with unified extraction.
        
        Single-pass extraction (Issue #19) that:
        1. Extracts entities, goals, decisions, and relationships in one LLM call
        2. Merges into per-user UnifiedGraph with deduplication
        3. Persists to disk if persistence enabled
        
        Args:
            messages: List of message dicts [{"role": "user", "content": "..."}]
            user_id: User identifier
            session_id: Optional session identifier
            
        Returns:
            AddResult with facts, unified_graph, and timing metrics
        """
        total_start = time.time()
        timing = TimingMetrics()
        result = AddResult()
        
        # Issue #2: Validate messages before processing
        valid_messages = []
        for i, msg in enumerate(messages):
            if not isinstance(msg, dict):
                logger.warning(f"Skipping message {i}: not a dict")
                continue
            if not msg.get("content"):
                logger.warning(f"Skipping message {i}: missing or empty content")
                continue
            if not msg.get("role"):
                logger.warning(f"Skipping message {i}: missing role")
                continue
            valid_messages.append(msg)
        
        if not valid_messages:
            logger.error("No valid messages to process")
            return result
        
        # =========================================================================
        # UNIFIED EXTRACTION (Issue #19 - single-pass, 52x token reduction)
        # Issue #105: Per-user async lock prevents concurrent add() lost updates
        # =========================================================================
        user_lock = self._get_user_lock(user_id)
        async with user_lock:
            logger.info("Using unified extractor (single-pass)")
            unified_start = time.time()
            
            # Issue #73: Ensure graph is loaded from storage if exists
            existing_graph = self._ensure_graph_loaded(user_id)
            
            # Issue #104: Wrap LLM extraction in error handling
            try:
                if existing_graph:
                    # Delta extraction - only process new information
                    logger.debug("Using delta extraction for existing user")
                    unified_graph = await self._unified_extractor.extract_delta(
                        valid_messages, existing_graph
                    )
                else:
                    # Full extraction for new user
                    unified_graph = await self._unified_extractor.extract(valid_messages)
            except Exception as e:
                logger.error(f"LLM extraction failed for user {user_id}: {e}")
                # Graceful degradation: return existing graph or empty graph
                unified_graph = existing_graph if existing_graph else UnifiedGraph()
            
            # Store graph for user (in memory)
            self._user_graphs[user_id] = unified_graph
            result.unified_graph = unified_graph
            
            # Issue #96: Compact graph if it exceeds max_nodes budget
            try:
                evicted = unified_graph.compact(self._max_graph_nodes)
                if evicted > 0:
                    logger.info(f"Compacted graph for {user_id}: evicted {evicted} low-importance nodes")
            except Exception as e:
                logger.warning(f"Graph compaction failed for {user_id}: {e}")
            
            # Issue #25: Persist to disk
            if self._enable_persistence and self._storage:
                try:
                    self._storage.save_unified_graph(user_id, unified_graph)
                    logger.debug(f"Saved unified graph for {user_id} to storage")
                except Exception as e:
                    logger.error(f"Failed to persist graph for {user_id}: {e}")
        
        # Map to compatible structures for backward compatibility
        # Convert graph nodes to memory entries (facts)
        result.facts = [
            MemoryEntry(
                id=node.id,
                content=f"{node.type.name}: {node.label}",
                metadata={"type": node.type.value, **node.attrs}
            )
            for node in unified_graph.nodes.values()
        ]
        
        # Extract decisions count from graph
        graph_summary = unified_graph.summary()
        
        timing.total_ms = int((time.time() - unified_start) * 1000)
        timing.facts_ms = timing.total_ms  # All in one pass
        timing.intents_ms = 0
        timing.decisions_ms = 0
        result.timing = timing
        
        logger.info(
            f"Unified extraction complete: {graph_summary['total_nodes']} nodes, "
            f"{graph_summary['total_edges']} edges, {graph_summary['decisions']} decisions "
            f"in {timing.total_ms}ms"
        )
        
        return result

    async def search(
        self,
        query: str,
        user_id: str,
        limit: int = 10,
        include_goals: bool = True,
        include_decisions: bool = True,
        retrieval_config: Optional[RetrievalConfig] = None,
    ) -> SearchResult:
        """
        Search with goal-aware retrieval.
        
        This enhances Mem0's search with:
        1. Goal-aware re-ranking (MCAL Pillar 3)
        2. Relevant decision attachment
        3. Active goal context
        
        When using unified extractor mode, searches the unified graph
        directly instead of the Mem0 backend.
        
        Args:
            query: Search query
            user_id: User identifier
            limit: Maximum results
            include_goals: Include active goals in result
            include_decisions: Include relevant decisions in result
            retrieval_config: Optional retrieval configuration
            
        Returns:
            SearchResult with memories, goals, decisions, and assembled context
        """
        result = SearchResult()
        
        # Issue #104: Input validation
        if not query or not query.strip():
            logger.warning("search() called with empty query")
            return result
        if not user_id or not user_id.strip():
            logger.warning("search() called with empty user_id")
            return result
        
        # Issue #105: Per-user async lock prevents stale reads during concurrent add()
        user_lock = self._get_user_lock(user_id)
        async with user_lock:
            # Issue #73: Ensure graph is loaded from storage if exists
            unified_graph = self._ensure_graph_loaded(user_id)
        
        if unified_graph:
            # Issue #104: Wrap graph search in error handling (embedding/FAISS failures)
            try:
                # Search unified graph directly using vector search (Issue #78 Subtask 1)
                logger.debug("Searching unified graph with semantic search...")
                graph_results = unified_graph.search(query, k=limit * 2)
                
                # Convert graph results to MemoryEntry format
                # semantic_search returns list[tuple[GraphNode, float]]
                result.memories = [
                    MemoryEntry(
                        id=node.id,
                        content=node.label,
                        metadata={"type": node.type.name.lower(), **node.attrs},
                        score=score
                    )
                    for node, score in graph_results
                ]
            except Exception as e:
                logger.error(f"Graph search failed for user {user_id}: {e}")
                result.memories = []
            
            # Get active goals from unified graph
            if include_goals:
                try:
                    active_goal_dicts = unified_graph.get_active_goals()
                    # Convert to IntentNode format for compatibility
                    result.active_goals = [
                        IntentNode(
                            id=g["id"],
                            type=self._goal_type_to_intent_type(g.get("goal_type", "goal")),
                            content=g["content"],
                            status=self._status_to_intent_status(g.get("status", "active"))
                        )
                        for g in active_goal_dicts
                    ]
                except Exception as e:
                    logger.error(f"Failed to get goals for user {user_id}: {e}")
                    result.active_goals = []
            
            # Get decisions from unified graph
            if include_decisions:
                try:
                    all_decisions = unified_graph.get_all_decisions_with_detail()
                    # Filter to relevant decisions (keyword match - any query word)
                    query_words = set(query.lower().split())
                    
                    def decision_matches(d: dict) -> bool:
                        """Check if decision matches any query word."""
                        decision_lower = d["decision"].lower()
                        # Check if any query word appears in decision
                        if any(w in decision_lower for w in query_words):
                            return True
                        # Check rationale
                        rationale = d.get("rationale", "") or ""
                        if any(w in rationale.lower() for w in query_words):
                            return True
                        # Check reasons
                        for r in d.get("reasons", []):
                            claim = (r.get("claim", "") or "").lower()
                            if any(w in claim for w in query_words):
                                return True
                        return False
                    
                    result.relevant_decisions = [
                        DecisionTrail(
                            id=d["id"],
                            decision=d["decision"],
                            rationale=d.get("rationale", ""),
                            alternatives=[],  # Could expand if needed
                            context=d.get("context", "")
                        )
                        for d in all_decisions
                        if decision_matches(d)
                    ][:5]  # Limit to top 5 relevant decisions
                except Exception as e:
                    logger.error(f"Failed to get decisions for user {user_id}: {e}")
                    result.relevant_decisions = []
        else:
            # Issue #53: No fallback needed - MCAL is always standalone
            # If no unified graph, return empty results
            logger.debug("No unified graph found for user")
            result.memories = []
            result.active_goals = []
            result.relevant_decisions = []
        
        # Assemble context
        try:
            result.context = self._context_assembler.assemble(
                retrieved=[],  # We'll pass memories differently
                active_goals=result.active_goals,
                decisions=result.relevant_decisions,
                include_goals=include_goals,
                include_decisions=include_decisions
            )
            
            # Add memories to context
            if result.memories:
                memory_lines = ["\n### RELEVANT MEMORIES ###"]
                for mem in result.memories:
                    score_str = f" (relevance: {mem.score:.2f})" if mem.score else ""
                    memory_lines.append(f"- {mem.content}{score_str}")
                result.context += "\n".join(memory_lines)
        except Exception as e:
            logger.error(f"Context assembly failed for user {user_id}: {e}")
            result.context = ""
        
        return result
    
    async def get_context(
        self,
        query: str,
        user_id: str,
        max_tokens: int = 4000,
        include_goals: bool = True,
        include_decisions: bool = True,
        include_reasoning: bool = True,
    ) -> str:
        """
        Get assembled context for LLM prompt.
        
        Convenience method that searches and formats context.
        
        Args:
            query: The query/task at hand
            user_id: User identifier
            max_tokens: Maximum tokens for context
            include_goals: Include active goals
            include_decisions: Include relevant decisions
            include_reasoning: Include decision rationale
            
        Returns:
            Formatted context string ready for LLM prompt
        """
        # Issue #104: Input validation
        if not query or not query.strip():
            logger.warning("get_context() called with empty query")
            return ""
        if not user_id or not user_id.strip():
            logger.warning("get_context() called with empty user_id")
            return ""
        
        # Search with all enhancements
        try:
            search_result = await self.search(
                query=query,
                user_id=user_id,
                include_goals=include_goals,
                include_decisions=include_decisions,
            )
        except Exception as e:
            logger.error(f"get_context() search failed for user {user_id}: {e}")
            return ""
        
        # Build context sections
        sections = []
        
        # Active goals
        if include_goals and search_result.active_goals:
            goals_text = "### ACTIVE GOALS ###\n"
            for goal in search_result.active_goals:
                status = "🔵" if goal.status.value == "active" else "⏳"
                goals_text += f"{status} {goal.content}\n"
            sections.append(goals_text)
        
        # Relevant decisions with reasoning
        if include_decisions and search_result.relevant_decisions:
            decisions_text = "### KEY DECISIONS ###\n"
            for decision in search_result.relevant_decisions[:5]:
                decisions_text += f"DECISION: {decision.decision}\n"
                if include_reasoning:
                    decisions_text += f"  Rationale: {decision.rationale}\n"
                    if decision.alternatives:
                        alts = ", ".join(a.option for a in decision.alternatives[:3])
                        decisions_text += f"  Alternatives considered: {alts}\n"
                decisions_text += "\n"
            sections.append(decisions_text)
        
        # Memories
        if search_result.memories:
            memories_text = "### RELEVANT CONTEXT ###\n"
            for mem in search_result.memories[:10]:
                memories_text += f"- {mem.content}\n"
            sections.append(memories_text)
        
        return "\n".join(sections)
    
    def get_active_goals(self, user_id: str) -> list[IntentNode]:
        """Get active goals for a user from the unified graph."""
        unified_graph = self._ensure_graph_loaded(user_id)
        if unified_graph:
            active_goal_dicts = unified_graph.get_active_goals()
            return [
                IntentNode(
                    id=g["id"],
                    type=self._goal_type_to_intent_type(g.get("goal_type", "goal")),
                    content=g["content"],
                    status=self._status_to_intent_status(g.get("status", "active"))
                )
                for g in active_goal_dicts
            ]
        return []
    
    def get_intent_graph(self, user_id: str) -> Optional[UnifiedGraph]:
        """Get unified graph for a user (replaces legacy IntentGraph)."""
        return self._ensure_graph_loaded(user_id)
    
    def get_decisions(
        self,
        user_id: str,
        goal_id: Optional[str] = None,
        include_invalidated: bool = False
    ) -> list[DecisionTrail]:
        """Get decisions from the unified graph, optionally filtered by goal."""
        unified_graph = self._ensure_graph_loaded(user_id)
        if not unified_graph:
            return []
        
        all_decisions = unified_graph.get_all_decisions_with_detail()
        
        results = []
        for d in all_decisions:
            # Filter by goal if specified
            if goal_id:
                related = d.get("related_goals", [])
                if goal_id not in related:
                    continue
            
            results.append(DecisionTrail(
                id=d["id"],
                decision=d["decision"],
                rationale=d.get("rationale", ""),
                alternatives=[],
                context=d.get("context", "")
            ))
        
        return results
    
    def clear_user_data(self, user_id: str) -> bool:
        """Clear all data for a user."""
        # Issue #105: Thread-safe dict mutation
        with self._graph_dict_lock:
            if user_id in self._user_graphs:
                del self._user_graphs[user_id]
            if user_id in self._user_locks:
                del self._user_locks[user_id]
        
        # Clear from persistent storage
        if self._enable_persistence and self._storage:
            self._storage.delete_unified_graph(user_id)
        
        if user_id in self._sessions:
            del self._sessions[user_id]
        
        # Clear extraction cache (Issue #9)
        if self._enable_extraction_cache and self._extraction_cache:
            self._extraction_cache.invalidate(user_id)
        
        return True
    
    def get_tiered_model_stats(self) -> Optional[dict]:
        """
        Get tiered model usage statistics (Issue #8).
        
        Returns:
            Dict with fast_calls, smart_calls, total, fast_ratio
            or None if tiered models not enabled
        """
        if self._enable_tiered_models and isinstance(self._llm, TieredBedrockProvider):
            return self._llm.get_usage_stats()
        return None
    
    def get_cache_stats(self) -> Optional[dict]:
        """
        Get extraction cache statistics (Issue #9).
        
        Returns:
            Dict with hits, misses, partial_hits, hit_rate, etc.
            or None if cache not enabled
        """
        if self._enable_extraction_cache and self._extraction_cache:
            return self._extraction_cache.get_stats().to_dict()
        return None
    
    def invalidate_cache(self, user_id: Optional[str] = None) -> int:
        """
        Invalidate extraction cache.
        
        Args:
            user_id: Specific user to invalidate, or None for all
            
        Returns:
            Number of entries invalidated
        """
        if not self._enable_extraction_cache or not self._extraction_cache:
            return 0
        
        if user_id:
            return 1 if self._extraction_cache.invalidate(user_id) else 0
        else:
            return self._extraction_cache.clear()
    
    def _serialize_decisions(self, decisions: list[DecisionTrail]) -> list[dict]:
        """Serialize decisions for cache storage."""
        result = []
        for d in decisions:
            result.append({
                "id": d.id,
                "decision": d.decision,
                "context": d.context,
                "rationale": d.rationale,
                "confidence": d.confidence,
                "related_goals": d.related_goals,
                "dependencies": d.dependencies,
                "invalidated_by": d.invalidated_by,
                "turn_id": d.turn_id,
                "alternatives": [
                    {"option": a.option, "pros": a.pros, "cons": a.cons, "rejection_reason": a.rejection_reason}
                    for a in (d.alternatives or [])
                ],
                "evidence": [
                    {"claim": e.claim, "source": e.source.value if e.source else "inferred", "turn_id": e.turn_id, "confidence": e.confidence}
                    for e in (d.evidence or [])
                ],
                "trade_offs": [
                    {"gained": t.gained, "sacrificed": t.sacrificed, "justification": t.justification}
                    for t in (d.trade_offs or [])
                ],
            })
        return result
    
    def _deserialize_decisions(self, data: list[dict]) -> list[DecisionTrail]:
        """Deserialize decisions from cache storage."""
        from .core.models import Alternative, Evidence, EvidenceSource, TradeOff
        
        result = []
        for d in data:
            alternatives = [
                Alternative(
                    option=a.get("option", ""),
                    pros=a.get("pros", []),
                    cons=a.get("cons", []),
                    rejection_reason=a.get("rejection_reason")
                )
                for a in d.get("alternatives", [])
            ]
            
            evidence = []
            for e in d.get("evidence", []):
                try:
                    source = EvidenceSource(e.get("source", "inferred"))
                except ValueError:
                    source = EvidenceSource.INFERRED
                evidence.append(Evidence(
                    claim=e.get("claim", ""),
                    source=source,
                    turn_id=e.get("turn_id"),
                    confidence=e.get("confidence", 0.8)
                ))
            
            trade_offs = [
                TradeOff(
                    gained=t.get("gained", ""),
                    sacrificed=t.get("sacrificed", ""),
                    justification=t.get("justification")
                )
                for t in d.get("trade_offs", [])
            ]
            
            result.append(DecisionTrail(
                id=d.get("id", ""),
                decision=d.get("decision", ""),
                context=d.get("context", ""),
                rationale=d.get("rationale", ""),
                confidence=d.get("confidence", 0.5),
                related_goals=d.get("related_goals", []),
                dependencies=d.get("dependencies", []),
                invalidated_by=d.get("invalidated_by"),
                turn_id=d.get("turn_id"),
                alternatives=alternatives,
                evidence=evidence,
                trade_offs=trade_offs,
            ))
        
        return result

    def _goal_type_to_intent_type(self, goal_type: str) -> IntentType:
        """Convert unified graph goal_type string to IntentType enum."""
        mapping = {
            "mission": IntentType.MISSION,
            "goal": IntentType.GOAL,
            "task": IntentType.TASK,
            "decision": IntentType.DECISION,
        }
        return mapping.get(goal_type.lower(), IntentType.GOAL)
    
    def _status_to_intent_status(self, status: str) -> IntentStatus:
        """Convert unified graph status string to IntentStatus enum."""
        mapping = {
            "active": IntentStatus.ACTIVE,
            "completed": IntentStatus.COMPLETED,
            "abandoned": IntentStatus.ABANDONED,
            "pending": IntentStatus.PENDING,
            "blocked": IntentStatus.BLOCKED,
        }
        return mapping.get(status.lower(), IntentStatus.ACTIVE)


# =============================================================================
# Singleton Factory (Issue #100)
# =============================================================================

# Module-level instance registry keyed by storage_path
_mcal_instances: dict[str, MCAL] = {}


def get_mcal(storage_path: str | None = None, **kwargs) -> MCAL:
    """
    Get or create a shared MCAL instance for the given storage path.
    
    This avoids the cost of re-creating MCAL (and its LLM clients,
    extractors, graphs) on every LangGraph node invocation.  The
    in-memory graph cache stays warm across calls.
    
    Usage in LangGraph nodes::
    
        from mcal import get_mcal
        
        async def retrieve_context(state):
            mcal = get_mcal(storage_path="/tmp/mcal", openai_api_key="...")
            return await mcal.search(...)
        
        async def store_memory(state):
            mcal = get_mcal(storage_path="/tmp/mcal", openai_api_key="...")
            await mcal.add(...)
    
    The first call creates the instance; subsequent calls with the same
    ``storage_path`` return the cached instance (extra kwargs are ignored
    after creation).
    
    Args:
        storage_path: Persistence directory (also used as cache key).
                      Defaults to ``~/.mcal``.
        **kwargs: Forwarded to ``MCAL(...)`` on first creation only.
        
    Returns:
        Shared MCAL instance
    """
    key = storage_path or str(Path.home() / ".mcal")
    if key not in _mcal_instances:
        _mcal_instances[key] = MCAL(storage_path=storage_path, **kwargs)
        logger.info(f"Created shared MCAL instance for {key}")
    return _mcal_instances[key]


def close_mcal(storage_path: str | None = None) -> None:
    """
    Close and remove a shared MCAL instance from the registry.
    
    Safe to call even if no instance exists for the given path.
    
    Args:
        storage_path: The key used in ``get_mcal()``.  If ``None``,
                      closes ALL registered instances.
    """
    if storage_path is None:
        keys = list(_mcal_instances.keys())
        for k in keys:
            _mcal_instances[k].close()
            del _mcal_instances[k]
        logger.info(f"Closed all {len(keys)} shared MCAL instance(s)")
    else:
        key = storage_path
        if key in _mcal_instances:
            _mcal_instances[key].close()
            del _mcal_instances[key]
            logger.info(f"Closed shared MCAL instance for {key}")
