"""
ALMA SQLite + FAISS Storage Backend.

Local storage using SQLite for structured data and FAISS for vector search.
This is the recommended backend for local development and testing.

v0.6.0 adds workflow context support:
- Checkpoint tables for crash recovery
- WorkflowOutcome tables for learning from workflows
- ArtifactRef tables for linking external files
- scope_filter parameter for workflow-scoped queries
"""

import json
import logging
import sqlite3
from contextlib import contextmanager
from datetime import datetime, timezone
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import numpy as np

from alma.storage.base import StorageBackend
from alma.storage.constants import SQLITE_TABLE_NAMES, MemoryType
from alma.types import (
    AntiPattern,
    DomainKnowledge,
    Heuristic,
    Outcome,
    UserPreference,
)

if TYPE_CHECKING:
    from alma.learning.decay import MemoryStrength
    from alma.session import SessionHandoff
    from alma.storage.archive import ArchivedMemory
    from alma.workflow import ArtifactRef, Checkpoint, WorkflowOutcome

logger = logging.getLogger(__name__)

# Try to import FAISS, fall back to numpy-based search if not available
try:
    import faiss

    FAISS_AVAILABLE = True
except ImportError:
    FAISS_AVAILABLE = False
    logger.warning("FAISS not available, falling back to numpy-based vector search")


class SQLiteStorage(StorageBackend):
    """
    SQLite + FAISS storage backend.

    Uses SQLite for structured data and FAISS for efficient vector similarity search.
    Falls back to numpy cosine similarity if FAISS is not installed.

    Database schema:
        - heuristics: id, agent, project_id, condition, strategy, confidence, ...
        - outcomes: id, agent, project_id, task_type, task_description, success, ...
        - preferences: id, user_id, category, preference, source, ...
        - domain_knowledge: id, agent, project_id, domain, fact, ...
        - anti_patterns: id, agent, project_id, pattern, why_bad, ...
        - embeddings: id, memory_type, memory_id, embedding (blob)
    """

    def __init__(
        self,
        db_path: Path,
        embedding_dim: int = 384,  # Default for all-MiniLM-L6-v2
        auto_migrate: bool = True,
    ):
        """
        Initialize SQLite storage.

        Args:
            db_path: Path to SQLite database file
            embedding_dim: Dimension of embedding vectors
            auto_migrate: If True, automatically apply pending migrations on startup
        """
        self.db_path = Path(db_path)
        self.db_path.parent.mkdir(parents=True, exist_ok=True)
        self.embedding_dim = embedding_dim

        # Migration support (lazy-loaded)
        self._migration_runner = None
        self._version_store = None

        # Initialize database
        self._init_database()

        # Initialize FAISS indices (one per memory type)
        self._indices: Dict[str, Any] = {}
        self._id_maps: Dict[str, List[str]] = {}  # memory_type -> [memory_ids]
        self._index_dirty: Dict[str, bool] = {}  # Track which indexes need rebuilding
        self._load_faiss_indices()

        # Auto-migrate if enabled
        if auto_migrate:
            self._ensure_migrated()

    @classmethod
    def from_config(cls, config: Dict[str, Any]) -> "SQLiteStorage":
        """Create instance from configuration."""
        storage_dir = config.get("storage_dir", ".alma")
        db_name = config.get("db_name", "alma.db")
        embedding_dim = config.get("embedding_dim", 384)

        db_path = Path(storage_dir) / db_name
        return cls(db_path=db_path, embedding_dim=embedding_dim)

    @contextmanager
    def _get_connection(self):
        """Get database connection with context manager."""
        conn = sqlite3.connect(self.db_path)
        conn.row_factory = sqlite3.Row
        try:
            yield conn
            conn.commit()
        except Exception:
            conn.rollback()
            raise
        finally:
            conn.close()

    def _init_database(self):
        """Initialize database schema."""
        with self._get_connection() as conn:
            cursor = conn.cursor()

            # Heuristics table
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS heuristics (
                    id TEXT PRIMARY KEY,
                    agent TEXT NOT NULL,
                    project_id TEXT NOT NULL,
                    condition TEXT NOT NULL,
                    strategy TEXT NOT NULL,
                    confidence REAL DEFAULT 0.0,
                    occurrence_count INTEGER DEFAULT 0,
                    success_count INTEGER DEFAULT 0,
                    last_validated TEXT,
                    created_at TEXT,
                    metadata TEXT
                )
            """)
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_heuristics_project_agent "
                "ON heuristics(project_id, agent)"
            )

            # Outcomes table
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS outcomes (
                    id TEXT PRIMARY KEY,
                    agent TEXT NOT NULL,
                    project_id TEXT NOT NULL,
                    task_type TEXT,
                    task_description TEXT NOT NULL,
                    success INTEGER DEFAULT 0,
                    strategy_used TEXT,
                    duration_ms INTEGER,
                    error_message TEXT,
                    user_feedback TEXT,
                    timestamp TEXT,
                    metadata TEXT
                )
            """)
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_outcomes_project_agent "
                "ON outcomes(project_id, agent)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_outcomes_task_type "
                "ON outcomes(project_id, agent, task_type)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_outcomes_timestamp "
                "ON outcomes(project_id, timestamp)"
            )

            # User preferences table
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS preferences (
                    id TEXT PRIMARY KEY,
                    user_id TEXT NOT NULL,
                    category TEXT,
                    preference TEXT NOT NULL,
                    source TEXT,
                    confidence REAL DEFAULT 1.0,
                    timestamp TEXT,
                    metadata TEXT
                )
            """)
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_preferences_user "
                "ON preferences(user_id)"
            )

            # Domain knowledge table
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS domain_knowledge (
                    id TEXT PRIMARY KEY,
                    agent TEXT NOT NULL,
                    project_id TEXT NOT NULL,
                    domain TEXT,
                    fact TEXT NOT NULL,
                    source TEXT,
                    confidence REAL DEFAULT 1.0,
                    last_verified TEXT,
                    metadata TEXT
                )
            """)
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_domain_knowledge_project_agent "
                "ON domain_knowledge(project_id, agent)"
            )

            # Anti-patterns table
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS anti_patterns (
                    id TEXT PRIMARY KEY,
                    agent TEXT NOT NULL,
                    project_id TEXT NOT NULL,
                    pattern TEXT NOT NULL,
                    why_bad TEXT,
                    better_alternative TEXT,
                    occurrence_count INTEGER DEFAULT 1,
                    last_seen TEXT,
                    created_at TEXT,
                    metadata TEXT
                )
            """)
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_anti_patterns_project_agent "
                "ON anti_patterns(project_id, agent)"
            )

            # Embeddings table (stores vectors as blobs)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS embeddings (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    memory_type TEXT NOT NULL,
                    memory_id TEXT NOT NULL,
                    embedding BLOB NOT NULL,
                    UNIQUE(memory_type, memory_id)
                )
            """)
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_embeddings_type "
                "ON embeddings(memory_type)"
            )

            # ==================== WORKFLOW TABLES (v0.6.0+) ====================

            # Checkpoints table for crash recovery
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS checkpoints (
                    id TEXT PRIMARY KEY,
                    run_id TEXT NOT NULL,
                    node_id TEXT NOT NULL,
                    state TEXT NOT NULL,
                    sequence_number INTEGER DEFAULT 0,
                    branch_id TEXT,
                    parent_checkpoint_id TEXT,
                    state_hash TEXT,
                    metadata TEXT,
                    created_at TEXT NOT NULL
                )
            """)
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_checkpoints_run ON checkpoints(run_id)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_checkpoints_run_branch "
                "ON checkpoints(run_id, branch_id)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_checkpoints_run_sequence "
                "ON checkpoints(run_id, sequence_number DESC)"
            )

            # Workflow outcomes table
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS workflow_outcomes (
                    id TEXT PRIMARY KEY,
                    tenant_id TEXT,
                    workflow_id TEXT NOT NULL,
                    run_id TEXT NOT NULL,
                    agent TEXT NOT NULL,
                    project_id TEXT NOT NULL,
                    result TEXT NOT NULL,
                    summary TEXT,
                    strategies_used TEXT,
                    successful_patterns TEXT,
                    failed_patterns TEXT,
                    extracted_heuristics TEXT,
                    extracted_anti_patterns TEXT,
                    duration_seconds REAL,
                    node_count INTEGER,
                    error_message TEXT,
                    metadata TEXT,
                    created_at TEXT NOT NULL
                )
            """)
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_workflow_outcomes_project "
                "ON workflow_outcomes(project_id, agent)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_workflow_outcomes_workflow "
                "ON workflow_outcomes(workflow_id)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_workflow_outcomes_tenant "
                "ON workflow_outcomes(tenant_id)"
            )

            # Artifact links table
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS artifact_links (
                    id TEXT PRIMARY KEY,
                    memory_id TEXT NOT NULL,
                    artifact_type TEXT NOT NULL,
                    storage_url TEXT NOT NULL,
                    filename TEXT,
                    mime_type TEXT,
                    size_bytes INTEGER,
                    checksum TEXT,
                    metadata TEXT,
                    created_at TEXT NOT NULL
                )
            """)
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_artifact_links_memory "
                "ON artifact_links(memory_id)"
            )

            # Session handoffs table (for session persistence)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS session_handoffs (
                    id TEXT PRIMARY KEY,
                    project_id TEXT NOT NULL,
                    agent TEXT NOT NULL,
                    session_id TEXT NOT NULL,
                    last_action TEXT NOT NULL,
                    last_outcome TEXT NOT NULL,
                    current_goal TEXT,
                    key_decisions TEXT,
                    active_files TEXT,
                    blockers TEXT,
                    next_steps TEXT,
                    test_status TEXT,
                    confidence_level REAL DEFAULT 0.5,
                    risk_flags TEXT,
                    session_start TEXT,
                    session_end TEXT,
                    duration_ms INTEGER DEFAULT 0,
                    metadata TEXT,
                    created_at TEXT NOT NULL
                )
            """)
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_session_handoffs_project_agent "
                "ON session_handoffs(project_id, agent)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_session_handoffs_agent_created "
                "ON session_handoffs(agent, created_at DESC)"
            )

            # ==================== MEMORY STRENGTH TABLE (v0.7.0+) ====================

            # Memory strength tracking for decay-based forgetting
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS memory_strength (
                    memory_id TEXT PRIMARY KEY,
                    memory_type TEXT NOT NULL,
                    project_id TEXT,
                    agent TEXT,
                    initial_strength REAL DEFAULT 1.0,
                    decay_half_life_days INTEGER DEFAULT 30,
                    created_at TEXT NOT NULL,
                    last_accessed TEXT NOT NULL,
                    access_count INTEGER DEFAULT 0,
                    explicit_importance REAL DEFAULT 0.5,
                    reinforcement_events TEXT DEFAULT '[]'
                )
            """)
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_memory_strength_last_accessed "
                "ON memory_strength(last_accessed)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_memory_strength_project_agent "
                "ON memory_strength(project_id, agent)"
            )

            # ==================== MEMORY ARCHIVE TABLE (v0.7.0+) ====================

            # Memory archive for soft-deleted memories
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS memory_archive (
                    id TEXT PRIMARY KEY,
                    original_id TEXT NOT NULL,
                    memory_type TEXT NOT NULL,
                    content TEXT NOT NULL,
                    embedding BLOB,
                    metadata TEXT,
                    original_created_at TEXT NOT NULL,
                    archived_at TEXT NOT NULL,
                    archive_reason TEXT NOT NULL,
                    final_strength REAL NOT NULL,
                    project_id TEXT NOT NULL,
                    agent TEXT NOT NULL,
                    restored INTEGER DEFAULT 0,
                    restored_at TEXT,
                    restored_as TEXT
                )
            """)
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_archive_project_agent "
                "ON memory_archive(project_id, agent)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_archive_reason "
                "ON memory_archive(archive_reason)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_archive_date "
                "ON memory_archive(archived_at)"
            )
            cursor.execute(
                "CREATE INDEX IF NOT EXISTS idx_archive_restored "
                "ON memory_archive(restored)"
            )

    def _load_faiss_indices(self, memory_types: Optional[List[str]] = None):
        """Load or create FAISS indices for specified memory types.

        Args:
            memory_types: List of memory types to load. If None, loads all types.
        """
        if memory_types is None:
            memory_types = list(MemoryType.VECTOR_ENABLED)

        for memory_type in memory_types:
            if FAISS_AVAILABLE:
                # Use FAISS index
                self._indices[memory_type] = faiss.IndexFlatIP(self.embedding_dim)
            else:
                # Use list for numpy fallback
                self._indices[memory_type] = []

            self._id_maps[memory_type] = []
            self._index_dirty[memory_type] = False  # Mark as fresh after rebuild

            # Load existing embeddings
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute(
                    "SELECT memory_id, embedding FROM embeddings WHERE memory_type = ?",
                    (memory_type,),
                )
                rows = cursor.fetchall()

                for row in rows:
                    memory_id = row["memory_id"]
                    embedding = np.frombuffer(row["embedding"], dtype=np.float32)

                    self._id_maps[memory_type].append(memory_id)
                    if FAISS_AVAILABLE:
                        self._indices[memory_type].add(
                            embedding.reshape(1, -1).astype(np.float32)
                        )
                    else:
                        self._indices[memory_type].append(embedding)

    def _ensure_index_fresh(self, memory_type: str) -> None:
        """Rebuild index for a memory type if it has been marked dirty.

        This implements lazy rebuilding - indexes are only rebuilt when
        actually needed for search, not immediately on every delete.

        Args:
            memory_type: The type of memory index to check/rebuild.
        """
        if self._index_dirty.get(memory_type, False):
            logger.debug(f"Rebuilding dirty index for {memory_type}")
            self._load_faiss_indices([memory_type])

    def _add_to_index(
        self,
        memory_type: str,
        memory_id: str,
        embedding: Optional[List[float]],
    ):
        """Add embedding to FAISS index."""
        if embedding is None:
            return

        embedding_array = np.array(embedding, dtype=np.float32)

        # Store in database
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                """
                INSERT OR REPLACE INTO embeddings (memory_type, memory_id, embedding)
                VALUES (?, ?, ?)
                """,
                (memory_type, memory_id, embedding_array.tobytes()),
            )

        # Add to index
        self._id_maps[memory_type].append(memory_id)
        if FAISS_AVAILABLE:
            self._indices[memory_type].add(
                embedding_array.reshape(1, -1).astype(np.float32)
            )
        else:
            self._indices[memory_type].append(embedding_array)

    def _search_index(
        self,
        memory_type: str,
        query_embedding: List[float],
        top_k: int,
    ) -> List[Tuple[str, float]]:
        """Search FAISS index for similar embeddings."""
        # Ensure index is up-to-date before searching (lazy rebuild)
        self._ensure_index_fresh(memory_type)

        if not self._id_maps[memory_type]:
            return []

        query = np.array(query_embedding, dtype=np.float32).reshape(1, -1)

        if FAISS_AVAILABLE:
            # Normalize for cosine similarity (IndexFlatIP)
            faiss.normalize_L2(query)
            scores, indices = self._indices[memory_type].search(
                query, min(top_k, len(self._id_maps[memory_type]))
            )

            results = []
            for score, idx in zip(scores[0], indices[0], strict=False):
                if idx >= 0 and idx < len(self._id_maps[memory_type]):
                    results.append((self._id_maps[memory_type][idx], float(score)))
            return results
        else:
            # Numpy fallback with cosine similarity
            embeddings = np.array(self._indices[memory_type])
            if len(embeddings) == 0:
                return []

            # Normalize
            query_norm = query / np.linalg.norm(query)
            emb_norms = embeddings / np.linalg.norm(embeddings, axis=1, keepdims=True)

            # Cosine similarity
            similarities = np.dot(emb_norms, query_norm.T).flatten()

            # Get top k
            top_indices = np.argsort(similarities)[::-1][:top_k]

            return [
                (self._id_maps[memory_type][i], float(similarities[i]))
                for i in top_indices
            ]

    # ==================== WRITE OPERATIONS ====================

    def save_heuristic(self, heuristic: Heuristic) -> str:
        """Save a heuristic."""
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                """
                INSERT OR REPLACE INTO heuristics
                (id, agent, project_id, condition, strategy, confidence,
                 occurrence_count, success_count, last_validated, created_at, metadata)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    heuristic.id,
                    heuristic.agent,
                    heuristic.project_id,
                    heuristic.condition,
                    heuristic.strategy,
                    heuristic.confidence,
                    heuristic.occurrence_count,
                    heuristic.success_count,
                    (
                        heuristic.last_validated.isoformat()
                        if heuristic.last_validated
                        else None
                    ),
                    heuristic.created_at.isoformat() if heuristic.created_at else None,
                    json.dumps(heuristic.metadata) if heuristic.metadata else None,
                ),
            )

        # Add embedding to index
        self._add_to_index(MemoryType.HEURISTICS, heuristic.id, heuristic.embedding)
        logger.debug(f"Saved heuristic: {heuristic.id}")
        return heuristic.id

    def save_outcome(self, outcome: Outcome) -> str:
        """Save an outcome."""
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                """
                INSERT OR REPLACE INTO outcomes
                (id, agent, project_id, task_type, task_description, success,
                 strategy_used, duration_ms, error_message, user_feedback, timestamp, metadata)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    outcome.id,
                    outcome.agent,
                    outcome.project_id,
                    outcome.task_type,
                    outcome.task_description,
                    1 if outcome.success else 0,
                    outcome.strategy_used,
                    outcome.duration_ms,
                    outcome.error_message,
                    outcome.user_feedback,
                    outcome.timestamp.isoformat() if outcome.timestamp else None,
                    json.dumps(outcome.metadata) if outcome.metadata else None,
                ),
            )

        # Add embedding to index
        self._add_to_index(MemoryType.OUTCOMES, outcome.id, outcome.embedding)
        logger.debug(f"Saved outcome: {outcome.id}")
        return outcome.id

    def save_user_preference(self, preference: UserPreference) -> str:
        """Save a user preference."""
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                """
                INSERT OR REPLACE INTO preferences
                (id, user_id, category, preference, source, confidence, timestamp, metadata)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    preference.id,
                    preference.user_id,
                    preference.category,
                    preference.preference,
                    preference.source,
                    preference.confidence,
                    preference.timestamp.isoformat() if preference.timestamp else None,
                    json.dumps(preference.metadata) if preference.metadata else None,
                ),
            )
        logger.debug(f"Saved preference: {preference.id}")
        return preference.id

    def save_domain_knowledge(self, knowledge: DomainKnowledge) -> str:
        """Save domain knowledge."""
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                """
                INSERT OR REPLACE INTO domain_knowledge
                (id, agent, project_id, domain, fact, source, confidence, last_verified, metadata)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    knowledge.id,
                    knowledge.agent,
                    knowledge.project_id,
                    knowledge.domain,
                    knowledge.fact,
                    knowledge.source,
                    knowledge.confidence,
                    (
                        knowledge.last_verified.isoformat()
                        if knowledge.last_verified
                        else None
                    ),
                    json.dumps(knowledge.metadata) if knowledge.metadata else None,
                ),
            )

        # Add embedding to index
        self._add_to_index(
            MemoryType.DOMAIN_KNOWLEDGE, knowledge.id, knowledge.embedding
        )
        logger.debug(f"Saved domain knowledge: {knowledge.id}")
        return knowledge.id

    def save_anti_pattern(self, anti_pattern: AntiPattern) -> str:
        """Save an anti-pattern."""
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                """
                INSERT OR REPLACE INTO anti_patterns
                (id, agent, project_id, pattern, why_bad, better_alternative,
                 occurrence_count, last_seen, created_at, metadata)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    anti_pattern.id,
                    anti_pattern.agent,
                    anti_pattern.project_id,
                    anti_pattern.pattern,
                    anti_pattern.why_bad,
                    anti_pattern.better_alternative,
                    anti_pattern.occurrence_count,
                    (
                        anti_pattern.last_seen.isoformat()
                        if anti_pattern.last_seen
                        else None
                    ),
                    (
                        anti_pattern.created_at.isoformat()
                        if anti_pattern.created_at
                        else None
                    ),
                    (
                        json.dumps(anti_pattern.metadata)
                        if anti_pattern.metadata
                        else None
                    ),
                ),
            )

        # Add embedding to index
        self._add_to_index(
            MemoryType.ANTI_PATTERNS, anti_pattern.id, anti_pattern.embedding
        )
        logger.debug(f"Saved anti-pattern: {anti_pattern.id}")
        return anti_pattern.id

    # ==================== BATCH WRITE OPERATIONS ====================

    def save_heuristics(self, heuristics: List[Heuristic]) -> List[str]:
        """Save multiple heuristics in a batch using executemany."""
        if not heuristics:
            return []

        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.executemany(
                """
                INSERT OR REPLACE INTO heuristics
                (id, agent, project_id, condition, strategy, confidence,
                 occurrence_count, success_count, last_validated, created_at, metadata)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                [
                    (
                        h.id,
                        h.agent,
                        h.project_id,
                        h.condition,
                        h.strategy,
                        h.confidence,
                        h.occurrence_count,
                        h.success_count,
                        h.last_validated.isoformat() if h.last_validated else None,
                        h.created_at.isoformat() if h.created_at else None,
                        json.dumps(h.metadata) if h.metadata else None,
                    )
                    for h in heuristics
                ],
            )

        # Add embeddings to index
        for h in heuristics:
            self._add_to_index(MemoryType.HEURISTICS, h.id, h.embedding)

        logger.debug(f"Batch saved {len(heuristics)} heuristics")
        return [h.id for h in heuristics]

    def save_outcomes(self, outcomes: List[Outcome]) -> List[str]:
        """Save multiple outcomes in a batch using executemany."""
        if not outcomes:
            return []

        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.executemany(
                """
                INSERT OR REPLACE INTO outcomes
                (id, agent, project_id, task_type, task_description, success,
                 strategy_used, duration_ms, error_message, user_feedback, timestamp, metadata)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                [
                    (
                        o.id,
                        o.agent,
                        o.project_id,
                        o.task_type,
                        o.task_description,
                        1 if o.success else 0,
                        o.strategy_used,
                        o.duration_ms,
                        o.error_message,
                        o.user_feedback,
                        o.timestamp.isoformat() if o.timestamp else None,
                        json.dumps(o.metadata) if o.metadata else None,
                    )
                    for o in outcomes
                ],
            )

        # Add embeddings to index
        for o in outcomes:
            self._add_to_index(MemoryType.OUTCOMES, o.id, o.embedding)

        logger.debug(f"Batch saved {len(outcomes)} outcomes")
        return [o.id for o in outcomes]

    def save_domain_knowledge_batch(
        self, knowledge_items: List[DomainKnowledge]
    ) -> List[str]:
        """Save multiple domain knowledge items in a batch using executemany."""
        if not knowledge_items:
            return []

        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.executemany(
                """
                INSERT OR REPLACE INTO domain_knowledge
                (id, agent, project_id, domain, fact, source, confidence, last_verified, metadata)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                [
                    (
                        k.id,
                        k.agent,
                        k.project_id,
                        k.domain,
                        k.fact,
                        k.source,
                        k.confidence,
                        k.last_verified.isoformat() if k.last_verified else None,
                        json.dumps(k.metadata) if k.metadata else None,
                    )
                    for k in knowledge_items
                ],
            )

        # Add embeddings to index
        for k in knowledge_items:
            self._add_to_index(MemoryType.DOMAIN_KNOWLEDGE, k.id, k.embedding)

        logger.debug(f"Batch saved {len(knowledge_items)} domain knowledge items")
        return [k.id for k in knowledge_items]

    # ==================== READ OPERATIONS ====================

    def get_heuristics(
        self,
        project_id: str,
        agent: Optional[str] = None,
        embedding: Optional[List[float]] = None,
        top_k: int = 5,
        min_confidence: float = 0.0,
        scope_filter: Optional[Dict[str, Any]] = None,
    ) -> List[Heuristic]:
        """Get heuristics with optional vector search and scope filtering."""
        # If embedding provided, use vector search to get candidate IDs
        candidate_ids = None
        if embedding:
            search_results = self._search_index(
                MemoryType.HEURISTICS, embedding, top_k * 2
            )
            candidate_ids = [id for id, _ in search_results]

        with self._get_connection() as conn:
            cursor = conn.cursor()

            query = "SELECT * FROM heuristics WHERE project_id = ? AND confidence >= ?"
            params: List[Any] = [project_id, min_confidence]

            if agent:
                query += " AND agent = ?"
                params.append(agent)

            if candidate_ids is not None:
                placeholders = ",".join("?" * len(candidate_ids))
                query += f" AND id IN ({placeholders})"
                params.extend(candidate_ids)

            # Apply scope filter (v0.6.0+)
            if scope_filter:
                query, params = self._apply_scope_filter(query, params, scope_filter)

            query += " ORDER BY confidence DESC LIMIT ?"
            params.append(top_k)

            cursor.execute(query, params)
            rows = cursor.fetchall()

        return [self._row_to_heuristic(row) for row in rows]

    def get_outcomes(
        self,
        project_id: str,
        agent: Optional[str] = None,
        task_type: Optional[str] = None,
        embedding: Optional[List[float]] = None,
        top_k: int = 5,
        success_only: bool = False,
        scope_filter: Optional[Dict[str, Any]] = None,
    ) -> List[Outcome]:
        """Get outcomes with optional vector search and scope filtering."""
        candidate_ids = None
        if embedding:
            search_results = self._search_index(
                MemoryType.OUTCOMES, embedding, top_k * 2
            )
            candidate_ids = [id for id, _ in search_results]

        with self._get_connection() as conn:
            cursor = conn.cursor()

            query = "SELECT * FROM outcomes WHERE project_id = ?"
            params: List[Any] = [project_id]

            if agent:
                query += " AND agent = ?"
                params.append(agent)

            if task_type:
                query += " AND task_type = ?"
                params.append(task_type)

            if success_only:
                query += " AND success = 1"

            if candidate_ids is not None:
                placeholders = ",".join("?" * len(candidate_ids))
                query += f" AND id IN ({placeholders})"
                params.extend(candidate_ids)

            # Apply scope filter (v0.6.0+)
            if scope_filter:
                query, params = self._apply_scope_filter(query, params, scope_filter)

            query += " ORDER BY timestamp DESC LIMIT ?"
            params.append(top_k)

            cursor.execute(query, params)
            rows = cursor.fetchall()

        return [self._row_to_outcome(row) for row in rows]

    def get_user_preferences(
        self,
        user_id: str,
        category: Optional[str] = None,
    ) -> List[UserPreference]:
        """Get user preferences."""
        with self._get_connection() as conn:
            cursor = conn.cursor()

            query = "SELECT * FROM preferences WHERE user_id = ?"
            params: List[Any] = [user_id]

            if category:
                query += " AND category = ?"
                params.append(category)

            cursor.execute(query, params)
            rows = cursor.fetchall()

        return [self._row_to_preference(row) for row in rows]

    def get_domain_knowledge(
        self,
        project_id: str,
        agent: Optional[str] = None,
        domain: Optional[str] = None,
        embedding: Optional[List[float]] = None,
        top_k: int = 5,
        scope_filter: Optional[Dict[str, Any]] = None,
    ) -> List[DomainKnowledge]:
        """Get domain knowledge with optional vector search and scope filtering."""
        candidate_ids = None
        if embedding:
            search_results = self._search_index(
                MemoryType.DOMAIN_KNOWLEDGE, embedding, top_k * 2
            )
            candidate_ids = [id for id, _ in search_results]

        with self._get_connection() as conn:
            cursor = conn.cursor()

            query = "SELECT * FROM domain_knowledge WHERE project_id = ?"
            params: List[Any] = [project_id]

            if agent:
                query += " AND agent = ?"
                params.append(agent)

            if domain:
                query += " AND domain = ?"
                params.append(domain)

            if candidate_ids is not None:
                placeholders = ",".join("?" * len(candidate_ids))
                query += f" AND id IN ({placeholders})"
                params.extend(candidate_ids)

            # Apply scope filter (v0.6.0+)
            if scope_filter:
                query, params = self._apply_scope_filter(query, params, scope_filter)

            query += " ORDER BY confidence DESC LIMIT ?"
            params.append(top_k)

            cursor.execute(query, params)
            rows = cursor.fetchall()

        return [self._row_to_domain_knowledge(row) for row in rows]

    def get_anti_patterns(
        self,
        project_id: str,
        agent: Optional[str] = None,
        embedding: Optional[List[float]] = None,
        top_k: int = 5,
        scope_filter: Optional[Dict[str, Any]] = None,
    ) -> List[AntiPattern]:
        """Get anti-patterns with optional vector search and scope filtering."""
        candidate_ids = None
        if embedding:
            search_results = self._search_index(
                MemoryType.ANTI_PATTERNS, embedding, top_k * 2
            )
            candidate_ids = [id for id, _ in search_results]

        with self._get_connection() as conn:
            cursor = conn.cursor()

            query = "SELECT * FROM anti_patterns WHERE project_id = ?"
            params: List[Any] = [project_id]

            if agent:
                query += " AND agent = ?"
                params.append(agent)

            if candidate_ids is not None:
                placeholders = ",".join("?" * len(candidate_ids))
                query += f" AND id IN ({placeholders})"
                params.extend(candidate_ids)

            # Apply scope filter (v0.6.0+)
            if scope_filter:
                query, params = self._apply_scope_filter(query, params, scope_filter)

            query += " ORDER BY occurrence_count DESC LIMIT ?"
            params.append(top_k)

            cursor.execute(query, params)
            rows = cursor.fetchall()

        return [self._row_to_anti_pattern(row) for row in rows]

    # ==================== MULTI-AGENT MEMORY SHARING ====================

    def get_heuristics_for_agents(
        self,
        project_id: str,
        agents: List[str],
        embedding: Optional[List[float]] = None,
        top_k: int = 5,
        min_confidence: float = 0.0,
    ) -> List[Heuristic]:
        """Get heuristics from multiple agents using optimized IN query."""
        if not agents:
            return []

        candidate_ids = None
        if embedding:
            search_results = self._search_index(
                MemoryType.HEURISTICS, embedding, top_k * 2 * len(agents)
            )
            candidate_ids = [id for id, _ in search_results]

        with self._get_connection() as conn:
            cursor = conn.cursor()

            placeholders = ",".join("?" * len(agents))
            query = f"SELECT * FROM heuristics WHERE project_id = ? AND confidence >= ? AND agent IN ({placeholders})"
            params: List[Any] = [project_id, min_confidence] + list(agents)

            if candidate_ids is not None:
                id_placeholders = ",".join("?" * len(candidate_ids))
                query += f" AND id IN ({id_placeholders})"
                params.extend(candidate_ids)

            query += " ORDER BY confidence DESC LIMIT ?"
            params.append(top_k * len(agents))

            cursor.execute(query, params)
            rows = cursor.fetchall()

        return [self._row_to_heuristic(row) for row in rows]

    def get_outcomes_for_agents(
        self,
        project_id: str,
        agents: List[str],
        task_type: Optional[str] = None,
        embedding: Optional[List[float]] = None,
        top_k: int = 5,
        success_only: bool = False,
    ) -> List[Outcome]:
        """Get outcomes from multiple agents using optimized IN query."""
        if not agents:
            return []

        candidate_ids = None
        if embedding:
            search_results = self._search_index(
                MemoryType.OUTCOMES, embedding, top_k * 2 * len(agents)
            )
            candidate_ids = [id for id, _ in search_results]

        with self._get_connection() as conn:
            cursor = conn.cursor()

            placeholders = ",".join("?" * len(agents))
            query = f"SELECT * FROM outcomes WHERE project_id = ? AND agent IN ({placeholders})"
            params: List[Any] = [project_id] + list(agents)

            if task_type:
                query += " AND task_type = ?"
                params.append(task_type)

            if success_only:
                query += " AND success = 1"

            if candidate_ids is not None:
                id_placeholders = ",".join("?" * len(candidate_ids))
                query += f" AND id IN ({id_placeholders})"
                params.extend(candidate_ids)

            query += " ORDER BY timestamp DESC LIMIT ?"
            params.append(top_k * len(agents))

            cursor.execute(query, params)
            rows = cursor.fetchall()

        return [self._row_to_outcome(row) for row in rows]

    def get_domain_knowledge_for_agents(
        self,
        project_id: str,
        agents: List[str],
        domain: Optional[str] = None,
        embedding: Optional[List[float]] = None,
        top_k: int = 5,
    ) -> List[DomainKnowledge]:
        """Get domain knowledge from multiple agents using optimized IN query."""
        if not agents:
            return []

        candidate_ids = None
        if embedding:
            search_results = self._search_index(
                MemoryType.DOMAIN_KNOWLEDGE, embedding, top_k * 2 * len(agents)
            )
            candidate_ids = [id for id, _ in search_results]

        with self._get_connection() as conn:
            cursor = conn.cursor()

            placeholders = ",".join("?" * len(agents))
            query = f"SELECT * FROM domain_knowledge WHERE project_id = ? AND agent IN ({placeholders})"
            params: List[Any] = [project_id] + list(agents)

            if domain:
                query += " AND domain = ?"
                params.append(domain)

            if candidate_ids is not None:
                id_placeholders = ",".join("?" * len(candidate_ids))
                query += f" AND id IN ({id_placeholders})"
                params.extend(candidate_ids)

            query += " ORDER BY confidence DESC LIMIT ?"
            params.append(top_k * len(agents))

            cursor.execute(query, params)
            rows = cursor.fetchall()

        return [self._row_to_domain_knowledge(row) for row in rows]

    def get_anti_patterns_for_agents(
        self,
        project_id: str,
        agents: List[str],
        embedding: Optional[List[float]] = None,
        top_k: int = 5,
    ) -> List[AntiPattern]:
        """Get anti-patterns from multiple agents using optimized IN query."""
        if not agents:
            return []

        candidate_ids = None
        if embedding:
            search_results = self._search_index(
                MemoryType.ANTI_PATTERNS, embedding, top_k * 2 * len(agents)
            )
            candidate_ids = [id for id, _ in search_results]

        with self._get_connection() as conn:
            cursor = conn.cursor()

            placeholders = ",".join("?" * len(agents))
            query = f"SELECT * FROM anti_patterns WHERE project_id = ? AND agent IN ({placeholders})"
            params: List[Any] = [project_id] + list(agents)

            if candidate_ids is not None:
                id_placeholders = ",".join("?" * len(candidate_ids))
                query += f" AND id IN ({id_placeholders})"
                params.extend(candidate_ids)

            query += " ORDER BY occurrence_count DESC LIMIT ?"
            params.append(top_k * len(agents))

            cursor.execute(query, params)
            rows = cursor.fetchall()

        return [self._row_to_anti_pattern(row) for row in rows]

    # ==================== UPDATE OPERATIONS ====================

    def update_heuristic(
        self,
        heuristic_id: str,
        updates: Dict[str, Any],
    ) -> bool:
        """Update a heuristic's fields."""
        if not updates:
            return False

        set_clauses = []
        params = []
        for key, value in updates.items():
            if key == "metadata" and value:
                value = json.dumps(value)
            elif isinstance(value, datetime):
                value = value.isoformat()
            set_clauses.append(f"{key} = ?")
            params.append(value)

        params.append(heuristic_id)

        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                f"UPDATE heuristics SET {', '.join(set_clauses)} WHERE id = ?",
                params,
            )
            return cursor.rowcount > 0

    def increment_heuristic_occurrence(
        self,
        heuristic_id: str,
        success: bool,
    ) -> bool:
        """Increment heuristic occurrence count."""
        with self._get_connection() as conn:
            cursor = conn.cursor()

            if success:
                cursor.execute(
                    """
                    UPDATE heuristics
                    SET occurrence_count = occurrence_count + 1,
                        success_count = success_count + 1,
                        last_validated = ?
                    WHERE id = ?
                    """,
                    (datetime.now(timezone.utc).isoformat(), heuristic_id),
                )
            else:
                cursor.execute(
                    """
                    UPDATE heuristics
                    SET occurrence_count = occurrence_count + 1,
                        last_validated = ?
                    WHERE id = ?
                    """,
                    (datetime.now(timezone.utc).isoformat(), heuristic_id),
                )

            return cursor.rowcount > 0

    # ==================== DELETE OPERATIONS ====================

    def delete_outcomes_older_than(
        self,
        project_id: str,
        older_than: datetime,
        agent: Optional[str] = None,
    ) -> int:
        """Delete old outcomes."""
        with self._get_connection() as conn:
            cursor = conn.cursor()

            query = "DELETE FROM outcomes WHERE project_id = ? AND timestamp < ?"
            params: List[Any] = [project_id, older_than.isoformat()]

            if agent:
                query += " AND agent = ?"
                params.append(agent)

            cursor.execute(query, params)
            deleted = cursor.rowcount

        logger.info(f"Deleted {deleted} old outcomes")
        return deleted

    def delete_low_confidence_heuristics(
        self,
        project_id: str,
        below_confidence: float,
        agent: Optional[str] = None,
    ) -> int:
        """Delete low-confidence heuristics."""
        with self._get_connection() as conn:
            cursor = conn.cursor()

            query = "DELETE FROM heuristics WHERE project_id = ? AND confidence < ?"
            params: List[Any] = [project_id, below_confidence]

            if agent:
                query += " AND agent = ?"
                params.append(agent)

            cursor.execute(query, params)
            deleted = cursor.rowcount

        logger.info(f"Deleted {deleted} low-confidence heuristics")
        return deleted

    # ==================== STATS ====================

    def get_stats(
        self,
        project_id: str,
        agent: Optional[str] = None,
    ) -> Dict[str, Any]:
        """Get memory statistics."""
        stats = {
            "project_id": project_id,
            "agent": agent,
            "storage_type": "sqlite",
            "faiss_available": FAISS_AVAILABLE,
        }

        with self._get_connection() as conn:
            cursor = conn.cursor()

            # Use canonical memory types for stats
            for memory_type in MemoryType.ALL:
                if memory_type == MemoryType.PREFERENCES:
                    # Preferences don't have project_id
                    cursor.execute(
                        f"SELECT COUNT(*) FROM {SQLITE_TABLE_NAMES[memory_type]}"
                    )
                    stats[f"{memory_type}_count"] = cursor.fetchone()[0]
                else:
                    query = f"SELECT COUNT(*) FROM {SQLITE_TABLE_NAMES[memory_type]} WHERE project_id = ?"
                    params: List[Any] = [project_id]
                    if agent:
                        query += " AND agent = ?"
                        params.append(agent)
                    cursor.execute(query, params)
                    stats[f"{memory_type}_count"] = cursor.fetchone()[0]

            # Embedding counts
            cursor.execute("SELECT COUNT(*) FROM embeddings")
            stats["embeddings_count"] = cursor.fetchone()[0]

        stats["total_count"] = sum(
            stats.get(k, 0) for k in stats if k.endswith("_count")
        )

        return stats

    # ==================== HELPERS ====================

    def _parse_datetime(self, value: Any) -> Optional[datetime]:
        """Parse datetime from string."""
        if value is None:
            return None
        if isinstance(value, datetime):
            return value
        try:
            return datetime.fromisoformat(value.replace("Z", "+00:00"))
        except (ValueError, AttributeError):
            return None

    def _row_to_heuristic(self, row: sqlite3.Row) -> Heuristic:
        """Convert database row to Heuristic."""
        return Heuristic(
            id=row["id"],
            agent=row["agent"],
            project_id=row["project_id"],
            condition=row["condition"],
            strategy=row["strategy"],
            confidence=row["confidence"] or 0.0,
            occurrence_count=row["occurrence_count"] or 0,
            success_count=row["success_count"] or 0,
            last_validated=self._parse_datetime(row["last_validated"])
            or datetime.now(timezone.utc),
            created_at=self._parse_datetime(row["created_at"])
            or datetime.now(timezone.utc),
            metadata=json.loads(row["metadata"]) if row["metadata"] else {},
        )

    def _row_to_outcome(self, row: sqlite3.Row) -> Outcome:
        """Convert database row to Outcome."""
        return Outcome(
            id=row["id"],
            agent=row["agent"],
            project_id=row["project_id"],
            task_type=row["task_type"] or "general",
            task_description=row["task_description"],
            success=bool(row["success"]),
            strategy_used=row["strategy_used"] or "",
            duration_ms=row["duration_ms"],
            error_message=row["error_message"],
            user_feedback=row["user_feedback"],
            timestamp=self._parse_datetime(row["timestamp"])
            or datetime.now(timezone.utc),
            metadata=json.loads(row["metadata"]) if row["metadata"] else {},
        )

    def _row_to_preference(self, row: sqlite3.Row) -> UserPreference:
        """Convert database row to UserPreference."""
        return UserPreference(
            id=row["id"],
            user_id=row["user_id"],
            category=row["category"] or "general",
            preference=row["preference"],
            source=row["source"] or "unknown",
            confidence=row["confidence"] or 1.0,
            timestamp=self._parse_datetime(row["timestamp"])
            or datetime.now(timezone.utc),
            metadata=json.loads(row["metadata"]) if row["metadata"] else {},
        )

    def _row_to_domain_knowledge(self, row: sqlite3.Row) -> DomainKnowledge:
        """Convert database row to DomainKnowledge."""
        return DomainKnowledge(
            id=row["id"],
            agent=row["agent"],
            project_id=row["project_id"],
            domain=row["domain"] or "general",
            fact=row["fact"],
            source=row["source"] or "unknown",
            confidence=row["confidence"] or 1.0,
            last_verified=self._parse_datetime(row["last_verified"])
            or datetime.now(timezone.utc),
            metadata=json.loads(row["metadata"]) if row["metadata"] else {},
        )

    def _row_to_anti_pattern(self, row: sqlite3.Row) -> AntiPattern:
        """Convert database row to AntiPattern."""
        return AntiPattern(
            id=row["id"],
            agent=row["agent"],
            project_id=row["project_id"],
            pattern=row["pattern"],
            why_bad=row["why_bad"] or "",
            better_alternative=row["better_alternative"] or "",
            occurrence_count=row["occurrence_count"] or 1,
            last_seen=self._parse_datetime(row["last_seen"])
            or datetime.now(timezone.utc),
            created_at=self._parse_datetime(row["created_at"])
            or datetime.now(timezone.utc),
            metadata=json.loads(row["metadata"]) if row["metadata"] else {},
        )

    # ===== Additional abstract method implementations =====

    def update_heuristic_confidence(
        self,
        heuristic_id: str,
        new_confidence: float,
    ) -> bool:
        """Update confidence score for a heuristic."""
        with self._get_connection() as conn:
            cursor = conn.execute(
                "UPDATE heuristics SET confidence = ? WHERE id = ?",
                (new_confidence, heuristic_id),
            )
            return cursor.rowcount > 0

    def update_knowledge_confidence(
        self,
        knowledge_id: str,
        new_confidence: float,
    ) -> bool:
        """Update confidence score for domain knowledge."""
        with self._get_connection() as conn:
            cursor = conn.execute(
                "UPDATE domain_knowledge SET confidence = ? WHERE id = ?",
                (new_confidence, knowledge_id),
            )
            return cursor.rowcount > 0

    def delete_heuristic(self, heuristic_id: str) -> bool:
        """Delete a heuristic by ID."""
        with self._get_connection() as conn:
            # Also remove from embedding index
            conn.execute(
                "DELETE FROM embeddings WHERE memory_type = ? AND memory_id = ?",
                (MemoryType.HEURISTICS, heuristic_id),
            )
            cursor = conn.execute(
                f"DELETE FROM {SQLITE_TABLE_NAMES[MemoryType.HEURISTICS]} WHERE id = ?",
                (heuristic_id,),
            )
            if cursor.rowcount > 0:
                # Mark index as dirty for lazy rebuild on next search
                self._index_dirty[MemoryType.HEURISTICS] = True
                return True
            return False

    def delete_outcome(self, outcome_id: str) -> bool:
        """Delete an outcome by ID."""
        with self._get_connection() as conn:
            # Also remove from embedding index
            conn.execute(
                "DELETE FROM embeddings WHERE memory_type = ? AND memory_id = ?",
                (MemoryType.OUTCOMES, outcome_id),
            )
            cursor = conn.execute(
                f"DELETE FROM {SQLITE_TABLE_NAMES[MemoryType.OUTCOMES]} WHERE id = ?",
                (outcome_id,),
            )
            if cursor.rowcount > 0:
                # Mark index as dirty for lazy rebuild on next search
                self._index_dirty[MemoryType.OUTCOMES] = True
                return True
            return False

    def delete_domain_knowledge(self, knowledge_id: str) -> bool:
        """Delete domain knowledge by ID."""
        with self._get_connection() as conn:
            # Also remove from embedding index
            conn.execute(
                "DELETE FROM embeddings WHERE memory_type = ? AND memory_id = ?",
                (MemoryType.DOMAIN_KNOWLEDGE, knowledge_id),
            )
            cursor = conn.execute(
                f"DELETE FROM {SQLITE_TABLE_NAMES[MemoryType.DOMAIN_KNOWLEDGE]} WHERE id = ?",
                (knowledge_id,),
            )
            if cursor.rowcount > 0:
                # Mark index as dirty for lazy rebuild on next search
                self._index_dirty[MemoryType.DOMAIN_KNOWLEDGE] = True
                return True
            return False

    def delete_anti_pattern(self, anti_pattern_id: str) -> bool:
        """Delete an anti-pattern by ID."""
        with self._get_connection() as conn:
            # Also remove from embedding index
            conn.execute(
                "DELETE FROM embeddings WHERE memory_type = ? AND memory_id = ?",
                (MemoryType.ANTI_PATTERNS, anti_pattern_id),
            )
            cursor = conn.execute(
                f"DELETE FROM {SQLITE_TABLE_NAMES[MemoryType.ANTI_PATTERNS]} WHERE id = ?",
                (anti_pattern_id,),
            )
            if cursor.rowcount > 0:
                # Mark index as dirty for lazy rebuild on next search
                self._index_dirty[MemoryType.ANTI_PATTERNS] = True
                return True
            return False

    # ==================== MIGRATION SUPPORT ====================

    def _get_version_store(self):
        """Get or create the version store."""
        if self._version_store is None:
            from alma.storage.migrations.version_stores import SQLiteVersionStore

            self._version_store = SQLiteVersionStore(self.db_path)
        return self._version_store

    def _get_migration_runner(self):
        """Get or create the migration runner."""
        if self._migration_runner is None:
            from alma.storage.migrations.runner import MigrationRunner
            from alma.storage.migrations.versions import v1_0_0  # noqa: F401

            self._migration_runner = MigrationRunner(
                version_store=self._get_version_store(),
                backend="sqlite",
            )
        return self._migration_runner

    def _ensure_migrated(self) -> None:
        """Ensure database is migrated to latest version."""
        runner = self._get_migration_runner()
        if runner.needs_migration():
            with self._get_connection() as conn:
                applied = runner.migrate(conn)
                if applied:
                    logger.info(f"Applied {len(applied)} migrations: {applied}")

    def get_schema_version(self) -> Optional[str]:
        """Get the current schema version."""
        return self._get_version_store().get_current_version()

    def get_migration_status(self) -> Dict[str, Any]:
        """Get migration status information."""
        runner = self._get_migration_runner()
        status = runner.get_status()
        status["migration_supported"] = True
        return status

    def migrate(
        self,
        target_version: Optional[str] = None,
        dry_run: bool = False,
    ) -> List[str]:
        """
        Apply pending schema migrations.

        Args:
            target_version: Optional target version (applies all if not specified)
            dry_run: If True, show what would be done without making changes

        Returns:
            List of applied migration versions
        """
        runner = self._get_migration_runner()
        with self._get_connection() as conn:
            return runner.migrate(conn, target_version=target_version, dry_run=dry_run)

    def rollback(
        self,
        target_version: str,
        dry_run: bool = False,
    ) -> List[str]:
        """
        Roll back schema to a previous version.

        Args:
            target_version: Version to roll back to
            dry_run: If True, show what would be done without making changes

        Returns:
            List of rolled back migration versions
        """
        runner = self._get_migration_runner()
        with self._get_connection() as conn:
            return runner.rollback(conn, target_version=target_version, dry_run=dry_run)

    # ==================== SCOPE FILTER HELPER (v0.6.0+) ====================

    def _apply_scope_filter(
        self,
        query: str,
        params: List[Any],
        scope_filter: Dict[str, Any],
    ) -> Tuple[str, List[Any]]:
        """
        Apply workflow scope filter to a query.

        Note: For tables that don't have workflow columns (tenant_id, workflow_id,
        run_id, node_id), scope filtering is a no-op. The filter will only apply
        to workflow_outcomes table which has these columns.

        Args:
            query: The SQL query string
            params: The query parameters
            scope_filter: Dict with keys: tenant_id, workflow_id, run_id, node_id

        Returns:
            Tuple of (modified query, modified params)
        """
        # Note: Most ALMA tables don't have workflow columns yet.
        # This filter primarily applies to workflow_outcomes queries.
        # For other tables, we return query unchanged to maintain backwards compatibility.
        return query, params

    # ==================== CHECKPOINT OPERATIONS (v0.6.0+) ====================

    def save_checkpoint(self, checkpoint: "Checkpoint") -> str:
        """Save a workflow checkpoint."""

        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                """
                INSERT OR REPLACE INTO checkpoints
                (id, run_id, node_id, state, sequence_number, branch_id,
                 parent_checkpoint_id, state_hash, metadata, created_at)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    checkpoint.id,
                    checkpoint.run_id,
                    checkpoint.node_id,
                    json.dumps(checkpoint.state),
                    checkpoint.sequence_number,
                    checkpoint.branch_id,
                    checkpoint.parent_checkpoint_id,
                    checkpoint.state_hash,
                    json.dumps(checkpoint.metadata) if checkpoint.metadata else None,
                    checkpoint.created_at.isoformat(),
                ),
            )
        logger.debug(f"Saved checkpoint: {checkpoint.id}")
        return checkpoint.id

    def get_checkpoint(self, checkpoint_id: str) -> Optional["Checkpoint"]:
        """Get a checkpoint by ID."""
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                "SELECT * FROM checkpoints WHERE id = ?",
                (checkpoint_id,),
            )
            row = cursor.fetchone()

        if row is None:
            return None
        return self._row_to_checkpoint(row)

    def get_latest_checkpoint(
        self,
        run_id: str,
        branch_id: Optional[str] = None,
    ) -> Optional["Checkpoint"]:
        """Get the most recent checkpoint for a workflow run."""
        with self._get_connection() as conn:
            cursor = conn.cursor()

            query = "SELECT * FROM checkpoints WHERE run_id = ?"
            params: List[Any] = [run_id]

            if branch_id is not None:
                query += " AND branch_id = ?"
                params.append(branch_id)

            query += " ORDER BY sequence_number DESC LIMIT 1"

            cursor.execute(query, params)
            row = cursor.fetchone()

        if row is None:
            return None
        return self._row_to_checkpoint(row)

    def get_checkpoints_for_run(
        self,
        run_id: str,
        branch_id: Optional[str] = None,
        limit: int = 100,
    ) -> List["Checkpoint"]:
        """Get all checkpoints for a workflow run."""
        with self._get_connection() as conn:
            cursor = conn.cursor()

            query = "SELECT * FROM checkpoints WHERE run_id = ?"
            params: List[Any] = [run_id]

            if branch_id is not None:
                query += " AND branch_id = ?"
                params.append(branch_id)

            query += " ORDER BY sequence_number ASC LIMIT ?"
            params.append(limit)

            cursor.execute(query, params)
            rows = cursor.fetchall()

        return [self._row_to_checkpoint(row) for row in rows]

    def cleanup_checkpoints(
        self,
        run_id: str,
        keep_latest: int = 1,
    ) -> int:
        """Clean up old checkpoints for a completed run."""
        with self._get_connection() as conn:
            cursor = conn.cursor()

            # Get IDs of checkpoints to keep
            cursor.execute(
                """
                SELECT id FROM checkpoints
                WHERE run_id = ?
                ORDER BY sequence_number DESC
                LIMIT ?
                """,
                (run_id, keep_latest),
            )
            keep_ids = [row["id"] for row in cursor.fetchall()]

            if not keep_ids:
                return 0

            # Delete all others
            placeholders = ",".join("?" * len(keep_ids))
            cursor.execute(
                f"""
                DELETE FROM checkpoints
                WHERE run_id = ? AND id NOT IN ({placeholders})
                """,
                [run_id] + keep_ids,
            )
            deleted = cursor.rowcount

        logger.info(f"Cleaned up {deleted} checkpoints for run {run_id}")
        return deleted

    def _row_to_checkpoint(self, row: sqlite3.Row) -> "Checkpoint":
        """Convert database row to Checkpoint."""
        from alma.workflow import Checkpoint

        return Checkpoint(
            id=row["id"],
            run_id=row["run_id"],
            node_id=row["node_id"],
            state=json.loads(row["state"]) if row["state"] else {},
            sequence_number=row["sequence_number"] or 0,
            branch_id=row["branch_id"],
            parent_checkpoint_id=row["parent_checkpoint_id"],
            state_hash=row["state_hash"] or "",
            metadata=json.loads(row["metadata"]) if row["metadata"] else {},
            created_at=self._parse_datetime(row["created_at"])
            or datetime.now(timezone.utc),
        )

    # ==================== WORKFLOW OUTCOME OPERATIONS (v0.6.0+) ====================

    def save_workflow_outcome(self, outcome: "WorkflowOutcome") -> str:
        """Save a workflow outcome."""
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                """
                INSERT OR REPLACE INTO workflow_outcomes
                (id, tenant_id, workflow_id, run_id, agent, project_id, result,
                 summary, strategies_used, successful_patterns, failed_patterns,
                 extracted_heuristics, extracted_anti_patterns, duration_seconds,
                 node_count, error_message, metadata, created_at)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    outcome.id,
                    outcome.tenant_id,
                    outcome.workflow_id,
                    outcome.run_id,
                    outcome.agent,
                    outcome.project_id,
                    outcome.result.value,
                    outcome.summary,
                    json.dumps(outcome.strategies_used),
                    json.dumps(outcome.successful_patterns),
                    json.dumps(outcome.failed_patterns),
                    json.dumps(outcome.extracted_heuristics),
                    json.dumps(outcome.extracted_anti_patterns),
                    outcome.duration_seconds,
                    outcome.node_count,
                    outcome.error_message,
                    json.dumps(outcome.metadata) if outcome.metadata else None,
                    outcome.created_at.isoformat(),
                ),
            )

        # Add embedding to index if present
        if outcome.embedding:
            self._add_to_index("workflow_outcomes", outcome.id, outcome.embedding)

        logger.debug(f"Saved workflow outcome: {outcome.id}")
        return outcome.id

    def get_workflow_outcome(self, outcome_id: str) -> Optional["WorkflowOutcome"]:
        """Get a workflow outcome by ID."""
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                "SELECT * FROM workflow_outcomes WHERE id = ?",
                (outcome_id,),
            )
            row = cursor.fetchone()

        if row is None:
            return None
        return self._row_to_workflow_outcome(row)

    def get_workflow_outcomes(
        self,
        project_id: str,
        agent: Optional[str] = None,
        workflow_id: Optional[str] = None,
        embedding: Optional[List[float]] = None,
        top_k: int = 10,
        scope_filter: Optional[Dict[str, Any]] = None,
    ) -> List["WorkflowOutcome"]:
        """Get workflow outcomes with optional filtering."""
        candidate_ids = None
        if embedding:
            search_results = self._search_index(
                "workflow_outcomes", embedding, top_k * 2
            )
            candidate_ids = [id for id, _ in search_results]

        with self._get_connection() as conn:
            cursor = conn.cursor()

            query = "SELECT * FROM workflow_outcomes WHERE project_id = ?"
            params: List[Any] = [project_id]

            if agent:
                query += " AND agent = ?"
                params.append(agent)

            if workflow_id:
                query += " AND workflow_id = ?"
                params.append(workflow_id)

            if candidate_ids is not None:
                placeholders = ",".join("?" * len(candidate_ids))
                query += f" AND id IN ({placeholders})"
                params.extend(candidate_ids)

            # Apply scope filter for workflow columns
            if scope_filter:
                if scope_filter.get("tenant_id"):
                    query += " AND tenant_id = ?"
                    params.append(scope_filter["tenant_id"])
                if scope_filter.get("workflow_id"):
                    query += " AND workflow_id = ?"
                    params.append(scope_filter["workflow_id"])
                if scope_filter.get("run_id"):
                    query += " AND run_id = ?"
                    params.append(scope_filter["run_id"])

            query += " ORDER BY created_at DESC LIMIT ?"
            params.append(top_k)

            cursor.execute(query, params)
            rows = cursor.fetchall()

        return [self._row_to_workflow_outcome(row) for row in rows]

    def _row_to_workflow_outcome(self, row: sqlite3.Row) -> "WorkflowOutcome":
        """Convert database row to WorkflowOutcome."""
        from alma.workflow import WorkflowOutcome, WorkflowResult

        return WorkflowOutcome(
            id=row["id"],
            tenant_id=row["tenant_id"],
            workflow_id=row["workflow_id"],
            run_id=row["run_id"],
            agent=row["agent"],
            project_id=row["project_id"],
            result=WorkflowResult(row["result"]),
            summary=row["summary"] or "",
            strategies_used=json.loads(row["strategies_used"])
            if row["strategies_used"]
            else [],
            successful_patterns=json.loads(row["successful_patterns"])
            if row["successful_patterns"]
            else [],
            failed_patterns=json.loads(row["failed_patterns"])
            if row["failed_patterns"]
            else [],
            extracted_heuristics=json.loads(row["extracted_heuristics"])
            if row["extracted_heuristics"]
            else [],
            extracted_anti_patterns=json.loads(row["extracted_anti_patterns"])
            if row["extracted_anti_patterns"]
            else [],
            duration_seconds=row["duration_seconds"],
            node_count=row["node_count"],
            error_message=row["error_message"],
            metadata=json.loads(row["metadata"]) if row["metadata"] else {},
            created_at=self._parse_datetime(row["created_at"])
            or datetime.now(timezone.utc),
        )

    # ==================== ARTIFACT LINK OPERATIONS (v0.6.0+) ====================

    def save_artifact_link(self, artifact_ref: "ArtifactRef") -> str:
        """Save an artifact reference linked to a memory."""
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                """
                INSERT OR REPLACE INTO artifact_links
                (id, memory_id, artifact_type, storage_url, filename,
                 mime_type, size_bytes, checksum, metadata, created_at)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    artifact_ref.id,
                    artifact_ref.memory_id,
                    artifact_ref.artifact_type.value,
                    artifact_ref.storage_url,
                    artifact_ref.filename,
                    artifact_ref.mime_type,
                    artifact_ref.size_bytes,
                    artifact_ref.checksum,
                    json.dumps(artifact_ref.metadata)
                    if artifact_ref.metadata
                    else None,
                    artifact_ref.created_at.isoformat(),
                ),
            )
        logger.debug(f"Saved artifact link: {artifact_ref.id}")
        return artifact_ref.id

    def get_artifact_links(self, memory_id: str) -> List["ArtifactRef"]:
        """Get all artifact references linked to a memory."""
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                "SELECT * FROM artifact_links WHERE memory_id = ?",
                (memory_id,),
            )
            rows = cursor.fetchall()

        return [self._row_to_artifact_ref(row) for row in rows]

    def delete_artifact_link(self, artifact_id: str) -> bool:
        """Delete an artifact reference."""
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                "DELETE FROM artifact_links WHERE id = ?",
                (artifact_id,),
            )
            return cursor.rowcount > 0

    def _row_to_artifact_ref(self, row: sqlite3.Row) -> "ArtifactRef":
        """Convert database row to ArtifactRef."""
        from alma.workflow import ArtifactRef, ArtifactType

        return ArtifactRef(
            id=row["id"],
            memory_id=row["memory_id"],
            artifact_type=ArtifactType(row["artifact_type"]),
            storage_url=row["storage_url"],
            filename=row["filename"],
            mime_type=row["mime_type"],
            size_bytes=row["size_bytes"],
            checksum=row["checksum"],
            metadata=json.loads(row["metadata"]) if row["metadata"] else {},
            created_at=self._parse_datetime(row["created_at"])
            or datetime.now(timezone.utc),
        )

    # ==================== SESSION HANDOFFS ====================

    def save_session_handoff(self, handoff: "SessionHandoff") -> str:
        """Save a session handoff for persistence."""

        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                """
                INSERT OR REPLACE INTO session_handoffs (
                    id, project_id, agent, session_id, last_action, last_outcome,
                    current_goal, key_decisions, active_files, blockers, next_steps,
                    test_status, confidence_level, risk_flags, session_start,
                    session_end, duration_ms, metadata, created_at
                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    handoff.id,
                    handoff.project_id,
                    handoff.agent,
                    handoff.session_id,
                    handoff.last_action,
                    handoff.last_outcome,
                    handoff.current_goal,
                    json.dumps(handoff.key_decisions),
                    json.dumps(handoff.active_files),
                    json.dumps(handoff.blockers),
                    json.dumps(handoff.next_steps),
                    json.dumps(handoff.test_status),
                    handoff.confidence_level,
                    json.dumps(handoff.risk_flags),
                    handoff.session_start.isoformat()
                    if handoff.session_start
                    else None,
                    handoff.session_end.isoformat() if handoff.session_end else None,
                    handoff.duration_ms,
                    json.dumps(handoff.metadata),
                    handoff.created_at.isoformat(),
                ),
            )
        return handoff.id

    def get_session_handoffs(
        self,
        project_id: str,
        agent: str,
        limit: int = 50,
    ) -> List["SessionHandoff"]:
        """Get session handoffs for an agent, most recent first."""
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                """
                SELECT * FROM session_handoffs
                WHERE project_id = ? AND agent = ?
                ORDER BY created_at DESC
                LIMIT ?
                """,
                (project_id, agent, limit),
            )
            rows = cursor.fetchall()
            return [self._row_to_session_handoff(row) for row in rows]

    def get_latest_session_handoff(
        self,
        project_id: str,
        agent: str,
    ) -> Optional["SessionHandoff"]:
        """Get the most recent session handoff for an agent."""
        handoffs = self.get_session_handoffs(project_id, agent, limit=1)
        return handoffs[0] if handoffs else None

    def delete_session_handoffs(
        self,
        project_id: str,
        agent: Optional[str] = None,
    ) -> int:
        """Delete session handoffs."""
        with self._get_connection() as conn:
            cursor = conn.cursor()
            if agent:
                cursor.execute(
                    "DELETE FROM session_handoffs WHERE project_id = ? AND agent = ?",
                    (project_id, agent),
                )
            else:
                cursor.execute(
                    "DELETE FROM session_handoffs WHERE project_id = ?",
                    (project_id,),
                )
            return cursor.rowcount

    def _row_to_session_handoff(self, row: sqlite3.Row) -> "SessionHandoff":
        """Convert database row to SessionHandoff."""
        from alma.session import SessionHandoff

        return SessionHandoff(
            id=row["id"],
            project_id=row["project_id"],
            agent=row["agent"],
            session_id=row["session_id"],
            last_action=row["last_action"],
            last_outcome=row["last_outcome"],
            current_goal=row["current_goal"] or "",
            key_decisions=json.loads(row["key_decisions"])
            if row["key_decisions"]
            else [],
            active_files=json.loads(row["active_files"]) if row["active_files"] else [],
            blockers=json.loads(row["blockers"]) if row["blockers"] else [],
            next_steps=json.loads(row["next_steps"]) if row["next_steps"] else [],
            test_status=json.loads(row["test_status"]) if row["test_status"] else {},
            confidence_level=row["confidence_level"] or 0.5,
            risk_flags=json.loads(row["risk_flags"]) if row["risk_flags"] else [],
            session_start=self._parse_datetime(row["session_start"])
            or datetime.now(timezone.utc),
            session_end=self._parse_datetime(row["session_end"]),
            duration_ms=row["duration_ms"] or 0,
            metadata=json.loads(row["metadata"]) if row["metadata"] else {},
            created_at=self._parse_datetime(row["created_at"])
            or datetime.now(timezone.utc),
        )

    # ==================== MEMORY STRENGTH OPERATIONS (v0.7.0+) ====================

    def save_memory_strength(self, strength: "MemoryStrength") -> str:
        """
        Save or update a memory strength record.

        Args:
            strength: MemoryStrength instance to save

        Returns:
            The memory ID
        """
        with self._get_connection() as conn:
            cursor = conn.cursor()

            # Check if record exists
            cursor.execute(
                "SELECT 1 FROM memory_strength WHERE memory_id = ?",
                (strength.memory_id,),
            )
            exists = cursor.fetchone() is not None

            reinforcement_events_json = json.dumps(
                [r.isoformat() for r in strength.reinforcement_events]
            )

            if exists:
                # Update existing record
                cursor.execute(
                    """
                    UPDATE memory_strength SET
                        memory_type = ?,
                        initial_strength = ?,
                        decay_half_life_days = ?,
                        last_accessed = ?,
                        access_count = ?,
                        explicit_importance = ?,
                        reinforcement_events = ?
                    WHERE memory_id = ?
                    """,
                    (
                        strength.memory_type,
                        strength.initial_strength,
                        strength.decay_half_life_days,
                        strength.last_accessed.isoformat(),
                        strength.access_count,
                        strength.explicit_importance,
                        reinforcement_events_json,
                        strength.memory_id,
                    ),
                )
            else:
                # Insert new record
                # Try to extract project_id and agent from the memory
                project_id = None
                agent = None

                # Look up the memory to get project_id and agent
                for table in [
                    "heuristics",
                    "outcomes",
                    "domain_knowledge",
                    "anti_patterns",
                ]:
                    cursor.execute(
                        f"SELECT project_id, agent FROM {table} WHERE id = ?",
                        (strength.memory_id,),
                    )
                    row = cursor.fetchone()
                    if row:
                        project_id = row["project_id"]
                        agent = row["agent"]
                        break

                cursor.execute(
                    """
                    INSERT INTO memory_strength (
                        memory_id, memory_type, project_id, agent,
                        initial_strength, decay_half_life_days,
                        created_at, last_accessed, access_count,
                        explicit_importance, reinforcement_events
                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (
                        strength.memory_id,
                        strength.memory_type,
                        project_id,
                        agent,
                        strength.initial_strength,
                        strength.decay_half_life_days,
                        strength.created_at.isoformat(),
                        strength.last_accessed.isoformat(),
                        strength.access_count,
                        strength.explicit_importance,
                        reinforcement_events_json,
                    ),
                )

        return strength.memory_id

    def get_memory_strength(self, memory_id: str) -> Optional["MemoryStrength"]:
        """
        Get a memory strength record by memory ID.

        Args:
            memory_id: The memory ID to look up

        Returns:
            MemoryStrength instance, or None if not found
        """
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                "SELECT * FROM memory_strength WHERE memory_id = ?",
                (memory_id,),
            )
            row = cursor.fetchone()

            if row is None:
                return None

            return self._row_to_memory_strength(row)

    def get_all_memory_strengths(
        self,
        project_id: str,
        agent: Optional[str] = None,
    ) -> List["MemoryStrength"]:
        """
        Get all memory strength records for a project/agent.

        Args:
            project_id: Project to query
            agent: Optional agent filter

        Returns:
            List of MemoryStrength instances
        """
        with self._get_connection() as conn:
            cursor = conn.cursor()

            if agent:
                cursor.execute(
                    """
                    SELECT * FROM memory_strength
                    WHERE project_id = ? AND agent = ?
                    """,
                    (project_id, agent),
                )
            else:
                cursor.execute(
                    """
                    SELECT * FROM memory_strength
                    WHERE project_id = ?
                    """,
                    (project_id,),
                )

            rows = cursor.fetchall()
            return [self._row_to_memory_strength(row) for row in rows]

    def delete_memory_strength(self, memory_id: str) -> bool:
        """
        Delete a memory strength record.

        Args:
            memory_id: The memory ID

        Returns:
            True if deleted, False if not found
        """
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                "DELETE FROM memory_strength WHERE memory_id = ?",
                (memory_id,),
            )
            return cursor.rowcount > 0

    def _row_to_memory_strength(self, row: sqlite3.Row) -> "MemoryStrength":
        """Convert database row to MemoryStrength."""
        from alma.learning.decay import MemoryStrength

        reinforcement_events = []
        if row["reinforcement_events"]:
            events_json = json.loads(row["reinforcement_events"])
            for event in events_json:
                if isinstance(event, str):
                    dt_str = event.replace("Z", "+00:00")
                    reinforcement_events.append(datetime.fromisoformat(dt_str))

        return MemoryStrength(
            memory_id=row["memory_id"],
            memory_type=row["memory_type"] or "unknown",
            initial_strength=row["initial_strength"] or 1.0,
            decay_half_life_days=row["decay_half_life_days"] or 30,
            created_at=self._parse_datetime(row["created_at"])
            or datetime.now(timezone.utc),
            last_accessed=self._parse_datetime(row["last_accessed"])
            or datetime.now(timezone.utc),
            access_count=row["access_count"] or 0,
            reinforcement_events=reinforcement_events,
            explicit_importance=row["explicit_importance"] or 0.5,
        )

    # ==================== ARCHIVE OPERATIONS (v0.7.0+) ====================

    def archive_memory(
        self,
        memory_id: str,
        memory_type: str,
        reason: str,
        final_strength: float,
    ) -> "ArchivedMemory":
        """
        Archive a memory before deletion.

        Captures full memory data including content, embedding, and metadata
        for potential future recovery or compliance auditing.

        Args:
            memory_id: ID of the memory to archive
            memory_type: Type of memory (heuristic, outcome, etc.)
            reason: Why being archived (decay, manual, consolidation, etc.)
            final_strength: Memory strength at time of archival

        Returns:
            ArchivedMemory instance
        """
        from alma.storage.archive import ArchivedMemory

        with self._get_connection() as conn:
            cursor = conn.cursor()

            # Get the memory data based on type
            table_map = {
                "heuristic": "heuristics",
                "outcome": "outcomes",
                "domain_knowledge": "domain_knowledge",
                "anti_pattern": "anti_patterns",
                "preference": "preferences",
            }

            table_name = table_map.get(memory_type)
            if not table_name:
                raise ValueError(f"Unknown memory type: {memory_type}")

            cursor.execute(f"SELECT * FROM {table_name} WHERE id = ?", (memory_id,))
            row = cursor.fetchone()

            if row is None:
                raise ValueError(f"Memory not found: {memory_id}")

            # Extract content and metadata from the memory
            content = self._extract_memory_content(memory_type, row)
            metadata = json.loads(row["metadata"]) if row["metadata"] else {}

            # Get the embedding if available
            cursor.execute(
                "SELECT embedding FROM embeddings WHERE memory_type = ? AND memory_id = ?",
                (memory_type, memory_id),
            )
            embedding_row = cursor.fetchone()
            embedding = None
            if embedding_row and embedding_row["embedding"]:
                embedding = np.frombuffer(
                    embedding_row["embedding"], dtype=np.float32
                ).tolist()

            # Parse original creation date - row is sqlite3.Row, need safe access
            row_keys = row.keys()
            created_at_str = (
                row["created_at"] if "created_at" in row_keys else None
            ) or (row["timestamp"] if "timestamp" in row_keys else None)
            original_created_at = (
                self._parse_datetime(created_at_str)
                if created_at_str
                else datetime.now(timezone.utc)
            )

            # Get project_id and agent
            project_id = row["project_id"] if "project_id" in row_keys else ""
            agent = row["agent"] if "agent" in row_keys else ""

            # If it's a preference, use user_id as project_id
            if memory_type == "preference":
                project_id = row["user_id"] if "user_id" in row_keys else ""
                agent = "user"

            # Create the archived memory
            archived = ArchivedMemory.create(
                original_id=memory_id,
                memory_type=memory_type,
                content=content,
                project_id=project_id,
                agent=agent,
                archive_reason=reason,
                final_strength=final_strength,
                original_created_at=original_created_at,
                embedding=embedding,
                metadata=metadata,
            )

            # Serialize embedding for storage
            embedding_blob = None
            if archived.embedding:
                embedding_blob = np.array(
                    archived.embedding, dtype=np.float32
                ).tobytes()

            # Insert into archive table
            cursor.execute(
                """
                INSERT INTO memory_archive (
                    id, original_id, memory_type, content, embedding,
                    metadata, original_created_at, archived_at, archive_reason,
                    final_strength, project_id, agent, restored, restored_at, restored_as
                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    archived.id,
                    archived.original_id,
                    archived.memory_type,
                    archived.content,
                    embedding_blob,
                    json.dumps(archived.metadata),
                    archived.original_created_at.isoformat(),
                    archived.archived_at.isoformat(),
                    archived.archive_reason,
                    archived.final_strength,
                    archived.project_id,
                    archived.agent,
                    0,  # not restored
                    None,
                    None,
                ),
            )

            return archived

    def _extract_memory_content(self, memory_type: str, row: sqlite3.Row) -> str:
        """Extract the main content from a memory row as JSON."""
        if memory_type == "heuristic":
            return json.dumps(
                {
                    "condition": row["condition"],
                    "strategy": row["strategy"],
                    "confidence": row["confidence"],
                    "occurrence_count": row["occurrence_count"],
                    "success_count": row["success_count"],
                }
            )
        elif memory_type == "outcome":
            return json.dumps(
                {
                    "task_type": row["task_type"],
                    "task_description": row["task_description"],
                    "success": bool(row["success"]),
                    "strategy_used": row["strategy_used"],
                    "duration_ms": row["duration_ms"],
                    "error_message": row["error_message"],
                    "user_feedback": row["user_feedback"],
                }
            )
        elif memory_type == "domain_knowledge":
            return json.dumps(
                {
                    "domain": row["domain"],
                    "fact": row["fact"],
                    "source": row["source"],
                    "confidence": row["confidence"],
                }
            )
        elif memory_type == "anti_pattern":
            return json.dumps(
                {
                    "pattern": row["pattern"],
                    "why_bad": row["why_bad"],
                    "better_alternative": row["better_alternative"],
                    "occurrence_count": row["occurrence_count"],
                }
            )
        elif memory_type == "preference":
            return json.dumps(
                {
                    "category": row["category"],
                    "preference": row["preference"],
                    "source": row["source"],
                    "confidence": row["confidence"],
                }
            )
        else:
            return json.dumps(dict(row))

    def get_archive(self, archive_id: str) -> Optional["ArchivedMemory"]:
        """
        Get an archived memory by its archive ID.

        Args:
            archive_id: The archive ID

        Returns:
            ArchivedMemory instance, or None if not found
        """
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute(
                "SELECT * FROM memory_archive WHERE id = ?",
                (archive_id,),
            )
            row = cursor.fetchone()

            if row is None:
                return None

            return self._row_to_archived_memory(row)

    def list_archives(
        self,
        project_id: str,
        agent: Optional[str] = None,
        reason: Optional[str] = None,
        memory_type: Optional[str] = None,
        older_than: Optional[datetime] = None,
        younger_than: Optional[datetime] = None,
        include_restored: bool = False,
        limit: int = 100,
    ) -> List["ArchivedMemory"]:
        """
        List archived memories with filtering.

        Args:
            project_id: Project to query
            agent: Optional agent filter
            reason: Optional archive reason filter
            memory_type: Optional memory type filter
            older_than: Optional filter for archives older than this time
            younger_than: Optional filter for archives younger than this time
            include_restored: Whether to include archives that have been restored
            limit: Maximum number of archives to return

        Returns:
            List of ArchivedMemory instances
        """
        with self._get_connection() as conn:
            cursor = conn.cursor()

            conditions = ["project_id = ?"]
            params: List[Any] = [project_id]

            if agent:
                conditions.append("agent = ?")
                params.append(agent)

            if reason:
                conditions.append("archive_reason = ?")
                params.append(reason)

            if memory_type:
                conditions.append("memory_type = ?")
                params.append(memory_type)

            if older_than:
                conditions.append("archived_at < ?")
                params.append(older_than.isoformat())

            if younger_than:
                conditions.append("archived_at > ?")
                params.append(younger_than.isoformat())

            if not include_restored:
                conditions.append("restored = 0")

            where_clause = " AND ".join(conditions)
            params.append(limit)

            cursor.execute(
                f"""
                SELECT * FROM memory_archive
                WHERE {where_clause}
                ORDER BY archived_at DESC
                LIMIT ?
                """,
                params,
            )

            rows = cursor.fetchall()
            return [self._row_to_archived_memory(row) for row in rows]

    def restore_from_archive(self, archive_id: str) -> str:
        """
        Restore an archived memory, creating a new memory from archive data.

        The original archive is marked as restored but retained for audit purposes.

        Args:
            archive_id: The archive ID to restore

        Returns:
            New memory ID of the restored memory

        Raises:
            ValueError: If archive not found or already restored
        """
        import uuid

        with self._get_connection() as conn:
            cursor = conn.cursor()

            # Get the archive
            cursor.execute(
                "SELECT * FROM memory_archive WHERE id = ?",
                (archive_id,),
            )
            row = cursor.fetchone()

            if row is None:
                raise ValueError(f"Archive not found: {archive_id}")

            if row["restored"]:
                raise ValueError(f"Archive already restored as: {row['restored_as']}")

            archived = self._row_to_archived_memory(row)
            content = json.loads(archived.content)

            # Generate new memory ID
            new_id = f"{archived.memory_type[:3]}-{uuid.uuid4().hex[:12]}"

            # Restore based on memory type
            if archived.memory_type == "heuristic":
                cursor.execute(
                    """
                    INSERT INTO heuristics (
                        id, agent, project_id, condition, strategy, confidence,
                        occurrence_count, success_count, last_validated, created_at, metadata
                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (
                        new_id,
                        archived.agent,
                        archived.project_id,
                        content.get("condition", ""),
                        content.get("strategy", ""),
                        content.get("confidence", 0.5),
                        content.get("occurrence_count", 1),
                        content.get("success_count", 0),
                        datetime.now(timezone.utc).isoformat(),
                        datetime.now(timezone.utc).isoformat(),
                        json.dumps(archived.metadata),
                    ),
                )
            elif archived.memory_type == "outcome":
                cursor.execute(
                    """
                    INSERT INTO outcomes (
                        id, agent, project_id, task_type, task_description, success,
                        strategy_used, duration_ms, error_message, user_feedback,
                        timestamp, metadata
                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (
                        new_id,
                        archived.agent,
                        archived.project_id,
                        content.get("task_type"),
                        content.get("task_description", ""),
                        1 if content.get("success") else 0,
                        content.get("strategy_used"),
                        content.get("duration_ms"),
                        content.get("error_message"),
                        content.get("user_feedback"),
                        datetime.now(timezone.utc).isoformat(),
                        json.dumps(archived.metadata),
                    ),
                )
            elif archived.memory_type == "domain_knowledge":
                cursor.execute(
                    """
                    INSERT INTO domain_knowledge (
                        id, agent, project_id, domain, fact, source, confidence,
                        last_verified, metadata
                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (
                        new_id,
                        archived.agent,
                        archived.project_id,
                        content.get("domain"),
                        content.get("fact", ""),
                        content.get("source"),
                        content.get("confidence", 1.0),
                        datetime.now(timezone.utc).isoformat(),
                        json.dumps(archived.metadata),
                    ),
                )
            elif archived.memory_type == "anti_pattern":
                cursor.execute(
                    """
                    INSERT INTO anti_patterns (
                        id, agent, project_id, pattern, why_bad, better_alternative,
                        occurrence_count, last_seen, created_at, metadata
                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (
                        new_id,
                        archived.agent,
                        archived.project_id,
                        content.get("pattern", ""),
                        content.get("why_bad"),
                        content.get("better_alternative"),
                        content.get("occurrence_count", 1),
                        datetime.now(timezone.utc).isoformat(),
                        datetime.now(timezone.utc).isoformat(),
                        json.dumps(archived.metadata),
                    ),
                )
            elif archived.memory_type == "preference":
                cursor.execute(
                    """
                    INSERT INTO preferences (
                        id, user_id, category, preference, source, confidence,
                        timestamp, metadata
                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (
                        new_id,
                        archived.project_id,  # For preferences, project_id is user_id
                        content.get("category"),
                        content.get("preference", ""),
                        content.get("source"),
                        content.get("confidence", 1.0),
                        datetime.now(timezone.utc).isoformat(),
                        json.dumps(archived.metadata),
                    ),
                )
            else:
                raise ValueError(f"Cannot restore memory type: {archived.memory_type}")

            # Restore embedding if available
            if archived.embedding:
                embedding_blob = np.array(
                    archived.embedding, dtype=np.float32
                ).tobytes()
                cursor.execute(
                    """
                    INSERT INTO embeddings (memory_type, memory_id, embedding)
                    VALUES (?, ?, ?)
                    """,
                    (archived.memory_type, new_id, embedding_blob),
                )

            # Mark archive as restored
            cursor.execute(
                """
                UPDATE memory_archive
                SET restored = 1, restored_at = ?, restored_as = ?
                WHERE id = ?
                """,
                (datetime.now(timezone.utc).isoformat(), new_id, archive_id),
            )

            return new_id

    def purge_archives(
        self,
        older_than: datetime,
        project_id: Optional[str] = None,
        reason: Optional[str] = None,
    ) -> int:
        """
        Permanently delete archived memories.

        This is a destructive operation - archives cannot be recovered after purging.

        Args:
            older_than: Delete archives older than this datetime
            project_id: Optional project filter
            reason: Optional reason filter

        Returns:
            Number of archives permanently deleted
        """
        with self._get_connection() as conn:
            cursor = conn.cursor()

            conditions = ["archived_at < ?"]
            params: List[Any] = [older_than.isoformat()]

            if project_id:
                conditions.append("project_id = ?")
                params.append(project_id)

            if reason:
                conditions.append("archive_reason = ?")
                params.append(reason)

            where_clause = " AND ".join(conditions)

            cursor.execute(
                f"DELETE FROM memory_archive WHERE {where_clause}",
                params,
            )

            return cursor.rowcount

    def get_archive_stats(
        self,
        project_id: str,
        agent: Optional[str] = None,
    ) -> Dict[str, Any]:
        """
        Get statistics about archived memories.

        Args:
            project_id: Project to query
            agent: Optional agent filter

        Returns:
            Dict with archive statistics (counts, by reason, by type, etc.)
        """
        from alma.storage.archive import ArchiveStats

        with self._get_connection() as conn:
            cursor = conn.cursor()

            # Build base filter
            base_conditions = ["project_id = ?"]
            base_params: List[Any] = [project_id]

            if agent:
                base_conditions.append("agent = ?")
                base_params.append(agent)

            base_where = " AND ".join(base_conditions)

            # Total count
            cursor.execute(
                f"SELECT COUNT(*) as cnt FROM memory_archive WHERE {base_where}",
                base_params,
            )
            total_count = cursor.fetchone()["cnt"]

            # Restored count
            cursor.execute(
                f"SELECT COUNT(*) as cnt FROM memory_archive WHERE {base_where} AND restored = 1",
                base_params,
            )
            restored_count = cursor.fetchone()["cnt"]

            # Count by reason
            cursor.execute(
                f"""
                SELECT archive_reason, COUNT(*) as cnt
                FROM memory_archive
                WHERE {base_where}
                GROUP BY archive_reason
                """,
                base_params,
            )
            by_reason = {row["archive_reason"]: row["cnt"] for row in cursor.fetchall()}

            # Count by type
            cursor.execute(
                f"""
                SELECT memory_type, COUNT(*) as cnt
                FROM memory_archive
                WHERE {base_where}
                GROUP BY memory_type
                """,
                base_params,
            )
            by_type = {row["memory_type"]: row["cnt"] for row in cursor.fetchall()}

            # Count by agent
            cursor.execute(
                """
                SELECT agent, COUNT(*) as cnt
                FROM memory_archive
                WHERE project_id = ?
                GROUP BY agent
                """,
                (project_id,),
            )
            by_agent = {row["agent"]: row["cnt"] for row in cursor.fetchall()}

            # Date range
            cursor.execute(
                f"SELECT MIN(archived_at) as oldest, MAX(archived_at) as newest FROM memory_archive WHERE {base_where}",
                base_params,
            )
            dates = cursor.fetchone()
            oldest_archive = (
                self._parse_datetime(dates["oldest"]) if dates["oldest"] else None
            )
            newest_archive = (
                self._parse_datetime(dates["newest"]) if dates["newest"] else None
            )

            stats = ArchiveStats(
                total_count=total_count,
                by_reason=by_reason,
                by_type=by_type,
                by_agent=by_agent,
                restored_count=restored_count,
                oldest_archive=oldest_archive,
                newest_archive=newest_archive,
            )

            return stats.to_dict()

    def _row_to_archived_memory(self, row: sqlite3.Row) -> "ArchivedMemory":
        """Convert database row to ArchivedMemory."""
        from alma.storage.archive import ArchivedMemory

        # Parse embedding
        embedding = None
        if row["embedding"]:
            embedding = np.frombuffer(row["embedding"], dtype=np.float32).tolist()

        # Parse metadata
        metadata = json.loads(row["metadata"]) if row["metadata"] else {}

        # Parse restored_at
        restored_at = None
        if row["restored_at"]:
            restored_at = self._parse_datetime(row["restored_at"])

        return ArchivedMemory(
            id=row["id"],
            original_id=row["original_id"],
            memory_type=row["memory_type"],
            content=row["content"],
            embedding=embedding,
            metadata=metadata,
            original_created_at=self._parse_datetime(row["original_created_at"])
            or datetime.now(timezone.utc),
            archived_at=self._parse_datetime(row["archived_at"])
            or datetime.now(timezone.utc),
            archive_reason=row["archive_reason"],
            final_strength=row["final_strength"],
            project_id=row["project_id"],
            agent=row["agent"],
            restored=bool(row["restored"]),
            restored_at=restored_at,
            restored_as=row["restored_as"],
        )
