# handlers/memory_handlers.py - Memory-related endpoint handlers

import hashlib
import time
import uuid
import re
import logging
from datetime import datetime, timedelta, timezone
from typing import List, Dict, Any, Set, Optional
# Removed FastAPI dependencies for stdio-only MCP server
import httpx

logger = logging.getLogger(__name__)

# Simple exception class to replace FastAPI ValidationError
class ValidationError(Exception):
    def __init__(self, detail: str, status_code: int = 400):
        self.detail = detail
        self.status_code = status_code
        super().__init__(detail)
from qdrant_client.http import models
from collections import defaultdict

from ..config import (
    QDRANT_COLLECTION_NAME, ENABLE_GEMMA_SUMMARIZATION, SEARCH_RESULT_MULTIPLIER,
    MAX_SEARCH_RESULTS, MAX_SUMMARIZATION_CHUNKS
)
from ..models import (
    MemoryItemIn, MemorySearchRequest, RetrievedChunk,
    SearchResponse, AddMemoryResponse, GetProjectMemoriesRequest,
    UpdateMemoryRequest, DeleteRunMemoriesRequest, GetRecentMemoriesRequest,
    PathGroupedResults, HierarchyOverviewRequest, HierarchyOverviewResponse,
    ProjectSummary, TicketSummary, RunSummary,
    ExportMemoriesRequest, ExportMemoriesResponse, ExportedMemory
)
from ..services import get_embedding, synthesize_search_results, AppConfig
from ..chunking import create_semantic_chunker
from ..utils.dependencies import get_http_client
from ..utils.search_utils import (
    expand_query_with_keywords, calculate_keyword_enhanced_score,
    generate_chunk_keywords
)
from ..utils.validation import validate_hierarchy, HierarchyValidationError

# Initialize semantic chunker
try:
    semantic_chunker = create_semantic_chunker(chunk_size=90)
except Exception as e:
    logger.error(f"Failed to initialize semantic chunker: {e}")
    raise ValidationError(status_code=500, detail="Failed to initialize semantic chunker")

def limit_by_memory_count(chunks: List[RetrievedChunk], limit: int, sort_by_timestamp: bool = True) -> List[RetrievedChunk]:
    """
    Groups chunks by original_content_hash and limits by number of memories (not chunks).

    Args:
        chunks: List of chunks to process
        limit: Number of distinct memories to return
        sort_by_timestamp: Whether to sort memory groups by timestamp

    Returns:
        List of chunks representing the limited number of memories
    """
    if not chunks:
        return []

    # Group chunks by original_content_hash to reconstruct complete memories
    memory_groups_by_hash = defaultdict(list)
    for chunk in chunks:
        content_hash = chunk.metadata.get('original_content_hash', 'unknown')
        memory_groups_by_hash[content_hash].append(chunk)

    # Sort memory groups by timestamp (using the most recent chunk's timestamp in each group)
    if sort_by_timestamp:
        sorted_memory_hashes = sorted(
            memory_groups_by_hash.keys(),
            key=lambda h: max(
                chunk.metadata.get('timestamp_iso', '')
                for chunk in memory_groups_by_hash[h]
            ),
            reverse=True  # Most recent first
        )
    else:
        sorted_memory_hashes = list(memory_groups_by_hash.keys())

    # Apply limit to number of MEMORIES (not chunks)
    limited_memory_hashes = sorted_memory_hashes[:limit]

    logger.info(f"Limiting to {limit} memories (from {len(sorted_memory_hashes)} total memories)")

    # Flatten back to chunks, maintaining chunk order within each memory
    result_chunks = []
    for content_hash in limited_memory_hashes:
        chunks_for_memory = memory_groups_by_hash[content_hash]
        # Sort chunks by chunk_index to maintain proper order
        chunks_for_memory.sort(key=lambda x: x.metadata.get('chunk_index', 0))
        result_chunks.extend(chunks_for_memory)

    logger.info(f"Returning {len(result_chunks)} chunks representing {len(limited_memory_hashes)} memories")

    return result_chunks

def safe_int_conversion(value, default=1):
    """
    Safely convert version field to integer, handling string floats like "1.0".
    
    Args:
        value: The value to convert (could be int, float, string)
        default: Default value if conversion fails
        
    Returns:
        int: The converted integer value
    """
    if value is None:
        return default
    
    try:
        # Handle string values that might be "1.0"
        if isinstance(value, str):
            # Convert string to float first, then to int
            return int(float(value))
        # Handle numeric values
        elif isinstance(value, (int, float)):
            return int(value)
        else:
            logger.warning(f"Unexpected version type {type(value)}: {value}, using default {default}")
            return default
    except (ValueError, TypeError) as e:
        logger.warning(f"Failed to convert version '{value}' to int: {e}, using default {default}")
        return default


def parse_iso8601_to_utc(timestamp_str: str) -> datetime:
    """
    Parses an ISO8601 timestamp string into a timezone-naive UTC datetime.
    Raises ValidationError on parse failure.
    """
    try:
        cleaned = timestamp_str.strip()
        if cleaned.endswith("Z"):
            cleaned = cleaned[:-1] + "+00:00"
        dt = datetime.fromisoformat(cleaned)
        # Normalize to UTC and drop tzinfo for comparisons
        if dt.tzinfo:
            dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
        return dt
    except Exception as e:
        raise ValidationError(
            status_code=400,
            detail=f"Invalid ISO8601 timestamp '{timestamp_str}': {e}"
        )


def determine_time_bounds(
    hours: Optional[int] = None,
    start_time_iso: Optional[str] = None,
    end_time_iso: Optional[str] = None
) -> (datetime, datetime):
    """
    Determines start and end time bounds based on explicit ISO inputs or hours window.
    """
    if start_time_iso:
        start_time = parse_iso8601_to_utc(start_time_iso)
        end_time = parse_iso8601_to_utc(end_time_iso) if end_time_iso else datetime.utcnow()
    elif hours is not None:
        end_time = datetime.utcnow()
        start_time = end_time - timedelta(hours=hours)
    else:
        start_time = datetime.min
        end_time = datetime.utcnow()

    if start_time > end_time:
        raise ValidationError(status_code=400, detail="start_time must be before end_time")

    return start_time, end_time


def normalize_tags_value(raw_tags: Any) -> List[str]:
    """
    Normalizes tags input (string or iterable) into a sorted list of unique strings.
    """
    if raw_tags is None:
        return []

    tags: Set[str] = set()
    if isinstance(raw_tags, str):
        # Allow comma or whitespace separated inputs
        candidates = re.split(r"[,\s]+", raw_tags)
        tags.update(tag.strip() for tag in candidates if tag.strip())
    elif isinstance(raw_tags, (list, tuple, set)):
        for tag in raw_tags:
            tag_str = str(tag).strip()
            if tag_str:
                tags.add(tag_str)
    else:
        logger.warning(f"Unsupported tags type {type(raw_tags)}; skipping tag normalization")

    return sorted(tags)


def chunk_matches_filters(metadata: Dict[str, Any], memory_types: List[str], tags: List[str]) -> bool:
    """
    Returns True if metadata matches provided memory_types and tags filters.
    """
    if memory_types:
        if metadata.get("type") not in memory_types:
            return False

    if tags:
        metadata_tags_raw = metadata.get("tags", [])
        metadata_tags = normalize_tags_value(metadata_tags_raw)
        if not set(metadata_tags).intersection(tags):
            return False

    return True


def filter_superseded_chunks(chunks: List[RetrievedChunk]) -> List[RetrievedChunk]:
    """
    Removes any chunk whose original_content_hash is referenced by another chunk's
    'supersedes' metadata field within the same result set.
    """
    superseded_hashes: Set[str] = set()
    for chunk in chunks:
        supersedes_value = chunk.metadata.get("supersedes")
        if supersedes_value:
            supersedes_list = supersedes_value if isinstance(supersedes_value, (list, tuple, set)) else [supersedes_value]
            superseded_hashes.update(str(val) for val in supersedes_list if str(val))

    if not superseded_hashes:
        return chunks

    filtered = [
        chunk for chunk in chunks
        if str(chunk.metadata.get("original_content_hash", "")) not in superseded_hashes
    ]
    logger.info(f" Supersession filter removed {len(chunks) - len(filtered)} chunks superseded by newer entries")
    return filtered

async def add_memory(memory_item: MemoryItemIn, config: AppConfig):
    """
    Adds memory content. Chunks content, gets embeddings, and stores in Qdrant.
    Supports flexible 4-level hierarchy: app_id (required), project_id (optional), ticket_id (optional), run_id (optional).
    run_id requires ticket_id - used for AutoStack multi-run scenarios.
    """
    try:
        # Validate that app_id is provided
        if not memory_item.metadata.get("app_id"):
            raise ValidationError(status_code=400, detail="app_id is required in metadata")

        # Validate hierarchical structure
        try:
            validate_hierarchy(
                app_id=memory_item.metadata.get("app_id"),
                project_id=memory_item.metadata.get("project_id"),
                ticket_id=memory_item.metadata.get("ticket_id"),
                run_id=memory_item.metadata.get("run_id")
            )
        except HierarchyValidationError as e:
            raise ValidationError(status_code=e.status_code, detail=e.detail)

        # Normalize metadata types to prevent future conversion issues
        normalized_metadata = dict(memory_item.metadata)

        # Ensure version is stored as integer
        if 'version' in normalized_metadata:
            normalized_metadata['version'] = safe_int_conversion(normalized_metadata['version'])
        else:
            # Default version if not provided
            normalized_metadata['version'] = 1

        # Ensure timestamp_iso is provided if not present
        if "timestamp_iso" not in normalized_metadata:
            from datetime import datetime, timedelta
            normalized_metadata["timestamp_iso"] = datetime.utcnow().isoformat() + "Z"

        # Normalize tags to a predictable list format for filtering/searching
        if "tags" in normalized_metadata:
            normalized_metadata["tags"] = normalize_tags_value(normalized_metadata.get("tags"))

        # Normalize supersession metadata to a list for easier downstream filtering
        if "supersedes" in normalized_metadata:
            supersedes_raw = normalized_metadata.get("supersedes")
            if isinstance(supersedes_raw, (list, tuple, set)):
                normalized_metadata["supersedes"] = [str(val) for val in supersedes_raw if str(val)]
            elif supersedes_raw:
                normalized_metadata["supersedes"] = [str(supersedes_raw)]
            else:
                normalized_metadata["supersedes"] = []

        app_id = normalized_metadata.get('app_id', 'N/A')
        project_id = normalized_metadata.get('project_id', None)
        ticket_id = normalized_metadata.get('ticket_id', None)
        run_id = normalized_metadata.get('run_id', None)

        # Determine hierarchy level for logging
        if run_id:
            level = "run-level"
            context = f"app: {app_id}, project: {project_id}, ticket: {ticket_id}, run: {run_id}"
        elif ticket_id:
            level = "ticket-level"
            context = f"app: {app_id}, project: {project_id}, ticket: {ticket_id}"
        elif project_id:
            level = "project-level"
            context = f"app: {app_id}, project: {project_id}"
        else:
            level = "app-level"
            context = f"app: {app_id}"
        
        logger.info(f"Received /add_memory for {level} context ({context}), chunking={memory_item.chunking}")

        # Conditional chunking based on chunking parameter
        if memory_item.chunking:
            # Use semantic chunking for searchable content
            try:
                chunks = semantic_chunker(memory_item.content)  # Using actual semchunk
            except Exception as e:
                logger.error(f"Failed to chunk content for {level} context ({context}): {e}")
                raise ValidationError(status_code=500, detail=f"Content chunking failed: {str(e)}")

            if not chunks:
                logger.warning(f"No chunks generated for {level} context ({context}). Content: '{memory_item.content[:100]}'")
                # This might happen if content is very short or only whitespace
                if not memory_item.content.strip():
                    raise ValidationError(status_code=400, detail="Content is empty or only whitespace.")
                # If content is not empty but semchunk didn't chunk, store the original content as a single chunk
                chunks = [memory_item.content.strip()]
        else:
            # Store as single chunk without semantic splitting
            logger.info(" Chunking disabled - storing as single unit for {level} context ({context})")
            if not memory_item.content.strip():
                raise ValidationError(status_code=400, detail="Content is empty or only whitespace.")
            chunks = [memory_item.content.strip()]

        points_to_upsert = []
        original_content_hash = hashlib.sha256(memory_item.content.encode()).hexdigest()

        for i, chunk_text in enumerate(chunks):
            if not chunk_text: # Skip empty chunks
                continue
            try:
                embedding = await get_embedding(chunk_text, config.http_client, config)
                
                # Generate chunk-specific keywords using Gemma
                chunk_keywords = await generate_chunk_keywords(chunk_text, config)
                
                chunk_metadata = normalized_metadata.copy() # Start with normalized metadata
                chunk_metadata["original_content_hash"] = original_content_hash
                chunk_metadata["chunk_index"] = i
                chunk_metadata["total_chunks"] = len(chunks)
                chunk_metadata["keywords"] = chunk_keywords  # Use chunk keywords as primary keywords
                # Add the actual chunk text to the payload for Qdrant, so we can retrieve it.
                # Qdrant payload can be any JSON-serializable dict.
                payload = {"text_chunk": chunk_text, **chunk_metadata} 

                points_to_upsert.append(models.PointStruct(
                    id=str(uuid.uuid4()), # Generate unique ID for each chunk point
                    vector=embedding,
                    payload=payload
                ))
            except Exception as e:
                logger.error(f" Failed to process chunk {i} for {level} context ({context}): {e}")
                # Decide on error strategy: skip chunk, fail all? For now, skip faulty chunks.

        if not points_to_upsert:
            logger.error(f" No valid points generated for upsertion for {level} context ({context})")
            raise ValidationError(status_code=500, detail="No data could be prepared for storage after chunking/embedding.")

        try:
            config.qdrant_client.upsert(
                collection_name=QDRANT_COLLECTION_NAME,
                points=points_to_upsert,
                wait=True # Wait for operation to complete
            )
            logger.info(f" Successfully upserted {len(points_to_upsert)} points for {level} context ({context})")
            return AddMemoryResponse(
                message=f"Memory added. {len(points_to_upsert)} chunks stored.",
                chunks_stored=len(points_to_upsert),
                original_content_hash=original_content_hash
            )
        except Exception as e:
            logger.error(f" Failed to upsert points to Qdrant for {level} context ({context}): {e}")
            raise ValidationError(status_code=500, detail=f"Storage in Qdrant failed: {str(e)}")

    except Exception as e:
        logger.error(f" Failed to add memory: {e}")
        raise ValidationError(status_code=500, detail=f"Failed to add memory: {str(e)}")

async def search_memories(search_request: MemorySearchRequest, config: AppConfig):
    """
    Searches memories in Qdrant with keyword-enhanced querying, then uses LM Studio to synthesize results.
    """
    try:
        # Validate input
        if not search_request.query_text or not search_request.query_text.strip():
            raise ValidationError(
                status_code=400,
                detail="query_text is required and cannot be empty"
            )

        # Validate hierarchical structure in metadata_filters
        other_filters = {}
        if search_request.metadata_filters:
            try:
                validate_hierarchy(
                    app_id=search_request.metadata_filters.get("app_id"),
                    project_id=search_request.metadata_filters.get("project_id"),
                    ticket_id=search_request.metadata_filters.get("ticket_id")
                )
            except HierarchyValidationError as e:
                raise ValidationError(status_code=e.status_code, detail=e.detail)

        filters_str = f", filters: {search_request.metadata_filters}" if search_request.metadata_filters else ""
        keyword_str = f", keyword_filters: {search_request.keyword_filters}" if search_request.keyword_filters else ""
        type_str = f", memory_types: {search_request.memory_types}" if search_request.memory_types else ""
        tags_str = f", tags: {search_request.tag_filters}" if search_request.tag_filters else ""
        logger.info(f" Received /search_memories for query: '{search_request.query_text[:50]}...'{filters_str}{keyword_str}{type_str}{tags_str}")
        
        # Step 1: Expand query with relevant keywords for better semantic matching
        try:
            expanded_query = await expand_query_with_keywords(search_request.query_text, search_request.metadata_filters, config)
            logger.info(f" Expanded query: '{expanded_query[:100]}...'")
        except Exception as e:
            logger.warning(f" Query expansion failed: {e}")
            expanded_query = search_request.query_text
        
        try:
            query_embedding = await get_embedding(expanded_query, config.http_client, config)
        except Exception as e:
            logger.error(f" Failed to get embedding for search query: {e}")
            raise ValidationError(status_code=500, detail=f"Query embedding failed: {str(e)}")

        # Qdrant metadata filtering:
        # Build the filter condition based on provided metadata_filters and cascade setting
        qdrant_filter = None
        if search_request.metadata_filters:
            app_id = search_request.metadata_filters.get("app_id")
            project_id = search_request.metadata_filters.get("project_id")
            ticket_id = search_request.metadata_filters.get("ticket_id")
            run_id = search_request.metadata_filters.get("run_id")

            # Extract non-hierarchy filters
            other_filters = {
                k: v for k, v in search_request.metadata_filters.items()
                if k not in ["app_id", "project_id", "ticket_id", "run_id"]
            }

            def append_additional_filters(condition_list: List[models.FieldCondition]):
                for key, value in other_filters.items():
                    if key == "tags":
                        tag_values = list(value) if isinstance(value, (list, tuple, set)) else [value]
                        condition_list.append(models.FieldCondition(key="tags", match=models.MatchAny(any=tag_values)))
                    else:
                        condition_list.append(models.FieldCondition(key=key, match=models.MatchValue(value=value)))
                if search_request.memory_types:
                    condition_list.append(models.FieldCondition(
                        key="type",
                        match=models.MatchAny(any=search_request.memory_types)
                    ))
                if search_request.tag_filters:
                    condition_list.append(models.FieldCondition(
                        key="tags",
                        match=models.MatchAny(any=search_request.tag_filters)
                    ))

            # Build filter conditions based on cascade setting
            should_conditions = []

            if not search_request.cascade and (app_id or project_id or ticket_id or run_id):
                # EXACT MATCH MODE - Only search at the specified level
                must_conditions = []

                if run_id:
                    # Exact run-level match (all four levels)
                    must_conditions.extend([
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=ticket_id)),
                        models.FieldCondition(key="run_id", match=models.MatchValue(value=run_id))
                    ])
                elif ticket_id:
                    # Ticket-level match (returns ticket-level + all runs)
                    must_conditions.extend([
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=ticket_id))
                    ])
                elif project_id:
                    # Exact project-level match
                    must_conditions.extend([
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=project_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                    ])
                elif app_id:
                    # Exact app-level match
                    must_conditions.extend([
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ])

                append_additional_filters(must_conditions)
                qdrant_filter = models.Filter(must=must_conditions)

            elif search_request.cascade and (app_id or project_id or ticket_id or run_id):
                # CASCADE MODE - Search across hierarchy levels
                if run_id:
                    # Search app + project + ticket + run levels
                    should_conditions.extend([
                        # 1. App-level memories
                        models.Filter(must=[
                            models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                            models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                        ]),
                        # 2. Project-level memories
                        models.Filter(must=[
                            models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                            models.FieldCondition(key="project_id", match=models.MatchValue(value=project_id)),
                            models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                        ]),
                        # 3. Ticket-level memories (no run_id)
                        models.Filter(must=[
                            models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                            models.FieldCondition(key="project_id", match=models.MatchValue(value=project_id)),
                            models.FieldCondition(key="ticket_id", match=models.MatchValue(value=ticket_id)),
                            models.IsEmptyCondition(is_empty=models.PayloadField(key="run_id"))
                        ]),
                        # 4. Run-level memories (all four match)
                        models.Filter(must=[
                            models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                            models.FieldCondition(key="project_id", match=models.MatchValue(value=project_id)),
                            models.FieldCondition(key="ticket_id", match=models.MatchValue(value=ticket_id)),
                            models.FieldCondition(key="run_id", match=models.MatchValue(value=run_id))
                        ])
                    ])
                elif ticket_id:
                    # Search app + project + ticket levels (includes all runs)
                    should_conditions.extend([
                        models.Filter(must=[
                            models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                            models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                        ]),
                        models.Filter(must=[
                            models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                            models.FieldCondition(key="project_id", match=models.MatchValue(value=project_id)),
                            models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                        ]),
                        models.Filter(must=[
                            models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                            models.FieldCondition(key="project_id", match=models.MatchValue(value=project_id)),
                            models.FieldCondition(key="ticket_id", match=models.MatchValue(value=ticket_id))
                        ])
                    ])
                elif project_id:
                    # Search app + project levels
                    should_conditions.extend([
                        models.Filter(must=[
                            models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                            models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                        ]),
                        models.Filter(must=[
                            models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                            models.FieldCondition(key="project_id", match=models.MatchValue(value=project_id)),
                            models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                        ])
                    ])
                elif app_id:
                    # Search app level only
                    should_conditions.append(
                        models.Filter(must=[
                            models.FieldCondition(key="app_id", match=models.MatchValue(value=app_id)),
                            models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                        ])
                    )

                if other_filters or search_request.memory_types or search_request.tag_filters:
                    for condition in should_conditions:
                        append_additional_filters(condition.must)

                qdrant_filter = models.Filter(should=should_conditions)
            else:
                # No hierarchy filters, just apply other filters
                must_conditions = []
                for key, value in search_request.metadata_filters.items():
                    if key == "tags":
                        tag_values = list(value) if isinstance(value, (list, tuple, set)) else [value]
                        must_conditions.append(models.FieldCondition(key=key, match=models.MatchAny(any=tag_values)))
                    else:
                        must_conditions.append(models.FieldCondition(key=key, match=models.MatchValue(value=value)))
                append_additional_filters(must_conditions)
                if must_conditions:
                    qdrant_filter = models.Filter(must=must_conditions)
        elif search_request.memory_types or search_request.tag_filters:
            must_conditions = []
            if search_request.memory_types:
                must_conditions.append(models.FieldCondition(
                    key="type",
                    match=models.MatchAny(any=search_request.memory_types)
                ))
            if search_request.tag_filters:
                must_conditions.append(models.FieldCondition(
                    key="tags",
                    match=models.MatchAny(any=search_request.tag_filters)
                ))
            qdrant_filter = models.Filter(must=must_conditions)
        
        try:
            # Step 2: Get more results initially for keyword-based re-ranking
            search_limit = min(search_request.limit * SEARCH_RESULT_MULTIPLIER, MAX_SEARCH_RESULTS)
            search_results = config.qdrant_client.search(
                collection_name=QDRANT_COLLECTION_NAME,
                query_vector=query_embedding,
                query_filter=qdrant_filter,
                limit=search_limit,
                with_payload=True, # Crucial: get the metadata and text_chunk back
                with_vectors=False # Usually don't need vectors in response
            )
        except Exception as e:
            logger.error(f" Qdrant search failed: {e}")
            raise ValidationError(status_code=500, detail=f"Qdrant search failed: {str(e)}")

        retrieved_chunks_for_response: List[RetrievedChunk] = []
        for hit in search_results:
            # The actual text chunk is in the payload, along with all original metadata
            chunk_content = hit.payload.get("text_chunk", "")
            metadata_from_payload = {k: v for k, v in hit.payload.items() if k != "text_chunk"}
            
            # Step 3: Calculate keyword-enhanced score
            enhanced_score = calculate_keyword_enhanced_score(
                hit.score, 
                search_request.query_text, 
                metadata_from_payload.get("keywords", [])
            )
            
            retrieved_chunks_for_response.append(RetrievedChunk(
                chunk_id=str(hit.id), # Qdrant point ID
                text_chunk=chunk_content,
                metadata=metadata_from_payload,
                score=enhanced_score  # Use enhanced score instead of raw vector score
            ))

        # Step 3.5: Apply explicit type/tag filters before further processing
        if search_request.memory_types or search_request.tag_filters:
            filtered_chunks = []
            for chunk in retrieved_chunks_for_response:
                if chunk_matches_filters(chunk.metadata, search_request.memory_types, search_request.tag_filters):
                    filtered_chunks.append(chunk)
            logger.info(f" Type/tag post-filter reduced results from {len(retrieved_chunks_for_response)} to {len(filtered_chunks)}")
            retrieved_chunks_for_response = filtered_chunks
        
        # Step 4: Apply keyword filtering if specified
        if search_request.keyword_filters:
            filtered_chunks = []
            for chunk in retrieved_chunks_for_response:
                keywords = chunk.metadata.get("keywords", [])
                all_chunk_keywords = [kw.lower() for kw in keywords]
                
                # Check if chunk contains at least one of the required keywords
                required_keywords = [kw.lower() for kw in search_request.keyword_filters]
                if any(req_kw in all_chunk_keywords for req_kw in required_keywords):
                    filtered_chunks.append(chunk)
            
            retrieved_chunks_for_response = filtered_chunks
            logger.info(f" Keyword filtering reduced results from {len(search_results)} to {len(retrieved_chunks_for_response)}")
        
        # Step 4.5: Version-aware deduplication - prefer highest version within each logical memory group
        memory_groups = defaultdict(list)
        for chunk in retrieved_chunks_for_response:
            # Create a key that uniquely identifies a logical memory
            app_id = chunk.metadata.get('app_id', '')
            project_id = chunk.metadata.get('project_id', '') or 'none'  # Handle None values
            ticket_id = chunk.metadata.get('ticket_id', '') or 'none'
            run_id = chunk.metadata.get('run_id', '') or 'none'
            memory_type = chunk.metadata.get('type', '') or 'none'

            memory_key = f"{app_id}|{project_id}|{ticket_id}|{run_id}|{memory_type}"
            memory_groups[memory_key].append(chunk)
        
        # Within each group, prefer chunks from the highest version
        version_filtered_chunks = []
        for memory_key, chunks_in_group in memory_groups.items():
            if len(chunks_in_group) == 1:
                # No versioning conflict, keep the chunk
                version_filtered_chunks.extend(chunks_in_group)
            else:
                # Find the highest version in this group
                max_version = max(safe_int_conversion(chunk.metadata.get('version', 1)) for chunk in chunks_in_group)
                highest_version_chunks = [chunk for chunk in chunks_in_group 
                                        if safe_int_conversion(chunk.metadata.get('version', 1)) == max_version]
                version_filtered_chunks.extend(highest_version_chunks)
                
                logger.info(f" Version deduplication for {memory_key}: {len(chunks_in_group)} chunks reduced to {len(highest_version_chunks)} (version {max_version})")
        
        if search_request.hide_superseded:
            version_filtered_chunks = filter_superseded_chunks(version_filtered_chunks)

        retrieved_chunks_for_response = version_filtered_chunks
        
        # Step 5: Re-rank by enhanced scores and limit to requested amount
        retrieved_chunks_for_response.sort(key=lambda x: x.score, reverse=True)
        retrieved_chunks_for_response = retrieved_chunks_for_response[:search_request.limit]

        if not retrieved_chunks_for_response:
            logger.info(" No chunks found matching the search criteria.")
            return SearchResponse(retrieved_chunks=[], total_results=0)

        grouped_results = None
        if search_request.group_by_path:
            path_groups = {}
            for chunk in retrieved_chunks_for_response:
                path_parts = [
                    chunk.metadata.get("app_id"),
                    chunk.metadata.get("project_id"),
                    chunk.metadata.get("ticket_id"),
                    chunk.metadata.get("run_id")
                ]
                path_label_parts = [str(part) for part in path_parts if part]
                path_label = "/".join(path_label_parts) if path_label_parts else "unscoped"

                if path_label not in path_groups:
                    path_groups[path_label] = {
                        "app_id": chunk.metadata.get("app_id"),
                        "project_id": chunk.metadata.get("project_id"),
                        "ticket_id": chunk.metadata.get("ticket_id"),
                        "run_id": chunk.metadata.get("run_id"),
                        "chunks": [],
                        "memory_hashes": set(),
                        "top_score": chunk.score
                    }

                group = path_groups[path_label]
                group["chunks"].append(chunk)
                group["memory_hashes"].add(str(chunk.metadata.get("original_content_hash", chunk.chunk_id)))
                group["top_score"] = max(group["top_score"], chunk.score)

            grouped_results = [
                PathGroupedResults(
                    path=path_label,
                    app_id=data["app_id"],
                    project_id=data["project_id"],
                    ticket_id=data["ticket_id"],
                    run_id=data["run_id"],
                    memory_count=len(data["memory_hashes"]),
                    top_score=data["top_score"],
                    chunks=data["chunks"]
                )
                for path_label, data in path_groups.items()
            ]

        # --- New Enhancement: Synthesize search results (configurable) ---
        synthesized_summary = None
        if ENABLE_GEMMA_SUMMARIZATION and search_request.query_text:
            try:
                # Use the http_client from the config object
                summary = await synthesize_search_results(search_request.query_text, retrieved_chunks_for_response, config.http_client, config)
                if summary:
                    logger.info(" LM Studio summary generated successfully.")
                    synthesized_summary = summary
                else:
                    logger.warning(" LM Studio summarization returned no content.")
            except Exception as e:
                logger.warning(f" Search result summarization failed: {e}. Returning raw chunks.")
                # Proceed to return raw chunks if summarization fails
        else:
            logger.info(f" Gemma summarization disabled via ENABLE_GEMMA_SUMMARIZATION=false")

        if synthesized_summary:
            return SearchResponse(
                synthesized_summary=synthesized_summary,
                retrieved_chunks=retrieved_chunks_for_response, # Still return chunks for reference or if summary is brief
                total_results=len(retrieved_chunks_for_response),
                grouped_results=grouped_results
            )
        else:
            return SearchResponse(
                retrieved_chunks=retrieved_chunks_for_response,
                total_results=len(retrieved_chunks_for_response),
                grouped_results=grouped_results
            )

    except ValidationError:
        # Re-raise ValidationErrors as-is
        raise
    except Exception as e:
        logger.error(f" Unexpected error in search_memories: {e}")
        raise ValidationError(
            status_code=500, 
            detail=f"Internal server error during search: {str(e)}"
        )

async def get_project_memories(request: GetProjectMemoriesRequest, config: AppConfig):
    """
    Retrieves all memories for a specific app_id/project_id/ticket_id/run_id combination with cascading.

    Cascading behavior (default, cascade=true):
    - app_id only: Returns app-level memories (no project_id)
    - app_id + project_id: Returns app-level + project-level memories
    - app_id + project_id + ticket_id: Returns app-level + project-level + ticket-level memories (includes all runs)
    - app_id + project_id + ticket_id + run_id: Returns app-level + project-level + ticket-level + run-level memories

    With cascade=false:
    - Returns ONLY the exact level specified
    - If run_id provided: Returns only that specific run
    - If run_id omitted but ticket_id provided: Returns ticket-level + all runs for that ticket

    This allows agents to get full context including parent-level information.
    """
    try:
        # Validate input
        if not request.app_id:
            raise ValidationError(
                status_code=400,
                detail="app_id is required"
            )

        # Validate hierarchical structure
        try:
            validate_hierarchy(
                app_id=request.app_id,
                project_id=request.project_id,
                ticket_id=request.ticket_id,
                run_id=request.run_id
            )
        except HierarchyValidationError as e:
            raise ValidationError(status_code=e.status_code, detail=e.detail)

        # Build filter based on cascade setting and hierarchy level
        filter_desc = f"app_id: {request.app_id}"
        should_conditions = []

        if not request.cascade:
            # EXACT MATCH MODE (cascade=false) - Only return memories at the specified level
            if request.run_id:
                # Return ONLY run-level memories (exact match on all four)
                filter_desc += f", project_id: {request.project_id}, ticket_id: {request.ticket_id}, run_id: {request.run_id} (exact match, no cascade)"
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id)),
                        models.FieldCondition(key="run_id", match=models.MatchValue(value=request.run_id))
                    ])
                )
            elif request.ticket_id:
                # Return ticket-level memories (no run_id) + ALL runs for this ticket
                # Don't filter on run_id at all - returns all memories with this ticket_id
                filter_desc += f", project_id: {request.project_id}, ticket_id: {request.ticket_id} (ticket + all runs, no cascade)"
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id))
                    ])
                )
            elif request.project_id:
                # Return ONLY project-level memories (project_id matches, ticket_id is empty)
                filter_desc += f", project_id: {request.project_id} (exact match, no cascade)"
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                    ])
                )
            else:
                # Return ONLY app-level memories (project_id is empty)
                filter_desc += " (app-level only, no cascade)"
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ])
                )
        else:
            # CASCADE MODE (cascade=true) - Return parent levels + current level
            if request.run_id:
                # Run-level query: Get app-level + project-level + ticket-level + run-level
                filter_desc += f", project_id: {request.project_id}, ticket_id: {request.ticket_id}, run_id: {request.run_id} (with cascading)"

                # 1. App-level memories (app_id matches, no project_id)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ])
                )

                # 2. Project-level memories (app_id + project_id match, no ticket_id)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                    ])
                )

                # 3. Ticket-level memories (app_id + project_id + ticket_id match, no run_id)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="run_id"))
                    ])
                )

                # 4. Run-level memories (all four match)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id)),
                        models.FieldCondition(key="run_id", match=models.MatchValue(value=request.run_id))
                    ])
                )

            elif request.ticket_id:
                # Ticket-level query: Get app-level + project-level + ticket-level (includes all runs)
                filter_desc += f", project_id: {request.project_id}, ticket_id: {request.ticket_id} (with cascading)"

                # 1. App-level memories (app_id matches, no project_id)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ])
                )

                # 2. Project-level memories (app_id + project_id match, no ticket_id)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                    ])
                )

                # 3. Ticket-level memories (all three match, includes all runs)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id))
                    ])
                )

            elif request.project_id:
                # Project-level query: Get app-level + project-level
                filter_desc += f", project_id: {request.project_id} (with cascading)"

                # 1. App-level memories (app_id matches, no project_id)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ])
                )

                # 2. Project-level memories (app_id + project_id match, no ticket_id)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                    ])
                )

            else:
                # App-level query: Get only app-level memories
                filter_desc += " (app-level only)"

                # App-level memories (app_id matches, no project_id)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ])
                )

        import logging
        logger = logging.getLogger(__name__)
        logger.info(f"Retrieving memories for {filter_desc}")

        # For cascade mode, use "should" (OR logic) to get multiple levels
        # For exact match mode, if there's only one condition, unwrap it to avoid unnecessary nesting
        if not request.cascade and len(should_conditions) == 1:
            # Exact match mode with single condition - unwrap the nested filter
            qdrant_filter = should_conditions[0]
            logger.debug(f"Using unwrapped filter for exact match (cascade=false, single condition)")
        else:
            # Cascade mode or multiple conditions - use "should" (OR logic)
            qdrant_filter = models.Filter(should=should_conditions)
            logger.debug(f"Using wrapped filter with {len(should_conditions)} should conditions (cascade={request.cascade})")

        logger.debug(f"Qdrant filter structure: {qdrant_filter}")

        try:
            # Scroll through all matching points
            all_points = []
            offset = None
            batch_size = 100
            
            while True:
                result = config.qdrant_client.scroll(
                    collection_name=QDRANT_COLLECTION_NAME,
                    scroll_filter=qdrant_filter,
                    limit=batch_size,
                    offset=offset,
                    with_payload=True,
                    with_vectors=False
                )
                
                points, next_offset = result
                all_points.extend(points)
                
                if next_offset is None or len(points) < batch_size:
                    break
                    
                offset = next_offset

            logger.info(f"Found {len(all_points)} total memory chunks for {filter_desc}")
            
        except Exception as e:
            logger.error(f" Qdrant scroll failed: {e}")
            raise ValidationError(status_code=500, detail=f"Qdrant retrieval failed: {str(e)}")
        
        # Convert to RetrievedChunk format
        retrieved_chunks: List[RetrievedChunk] = []
        for point in all_points:
            chunk_content = point.payload.get("text_chunk", "")
            metadata_from_payload = {k: v for k, v in point.payload.items() if k != "text_chunk"}

            # Log the run_id from Qdrant to verify filtering
            logger.debug(f"Point from Qdrant: run_id={metadata_from_payload.get('run_id', 'NONE')}, type={metadata_from_payload.get('type', 'NONE')}")

            retrieved_chunks.append(RetrievedChunk(
                chunk_id=str(point.id),
                text_chunk=chunk_content,
                metadata=metadata_from_payload,
                score=1.0  # No similarity score needed for direct retrieval
            ))

        # Apply optional type/tag filters before deduplication
        if request.memory_types or request.tags:
            filtered_chunks = [
                chunk for chunk in retrieved_chunks
                if chunk_matches_filters(chunk.metadata, request.memory_types, request.tags)
            ]
            logger.info(f" Type/tag filters reduced project memories from {len(retrieved_chunks)} to {len(filtered_chunks)}")
            retrieved_chunks = filtered_chunks
        
        # Version-aware deduplication - prefer highest version within each logical memory group
        memory_groups = defaultdict(list)
        for chunk in retrieved_chunks:
            # Create a key that uniquely identifies a logical memory
            app_id = chunk.metadata.get('app_id', '')
            project_id = chunk.metadata.get('project_id', '') or 'none'
            ticket_id = chunk.metadata.get('ticket_id', '') or 'none'
            run_id = chunk.metadata.get('run_id', '') or 'none'
            memory_type = chunk.metadata.get('type', '') or 'none'

            memory_key = f"{app_id}|{project_id}|{ticket_id}|{run_id}|{memory_type}"
            memory_groups[memory_key].append(chunk)
        
        # Within each group, prefer chunks from the highest version
        version_filtered_chunks = []
        for memory_key, chunks_in_group in memory_groups.items():
            if len(chunks_in_group) == 1:
                version_filtered_chunks.extend(chunks_in_group)
            else:
                # Find the highest version in this group
                max_version = max(safe_int_conversion(chunk.metadata.get('version', 1)) for chunk in chunks_in_group)
                highest_version_chunks = [chunk for chunk in chunks_in_group 
                                        if safe_int_conversion(chunk.metadata.get('version', 1)) == max_version]
                version_filtered_chunks.extend(highest_version_chunks)
                
                logger.info(f" Version deduplication for {memory_key}: {len(chunks_in_group)} chunks reduced to {len(highest_version_chunks)} (version {max_version})")
        
        if request.hide_superseded:
            version_filtered_chunks = filter_superseded_chunks(version_filtered_chunks)

        # Apply limit by memory count (not chunk count)
        retrieved_chunks = limit_by_memory_count(
            version_filtered_chunks,
            request.limit,
            sort_by_timestamp=(request.sort_by == "timestamp")
        )

        # Generate summary based on return_format
        synthesized_summary = None
        if request.return_format in ["summary_only", "both"] and ENABLE_GEMMA_SUMMARIZATION and retrieved_chunks:
            try:
                # Create a context-aware prompt for summarization
                context_prompt = f"Summarize all memories for {filter_desc}"
                summary = await synthesize_search_results(
                    context_prompt,
                    retrieved_chunks[:MAX_SUMMARIZATION_CHUNKS],
                    config.http_client,
                    config
                )
                if summary:
                    logger.info(f" LM Studio summary generated successfully for project memories (return_format={request.return_format}).")
                    synthesized_summary = summary
            except Exception as e:
                logger.warning(f" Project memories summarization failed: {e}. Returning raw chunks.")

        # Return based on requested format
        if request.return_format == "summary_only":
            # Return only summary, empty chunks for token efficiency
            return SearchResponse(
                synthesized_summary=synthesized_summary,
                retrieved_chunks=[],
                total_results=len(retrieved_chunks)
            )
        elif request.return_format == "chunks_only":
            # Return only chunks, no summary
            return SearchResponse(
                synthesized_summary=None,
                retrieved_chunks=retrieved_chunks,
                total_results=len(retrieved_chunks)
            )
        else:  # "both" (default)
            # Return both summary and chunks
            return SearchResponse(
                synthesized_summary=synthesized_summary,
                retrieved_chunks=retrieved_chunks,
                total_results=len(retrieved_chunks)
            )
        
    except ValidationError:
        raise
    except Exception as e:
        logger.error(f" Unexpected error in get_project_memories: {e}")
        raise ValidationError(
            status_code=500, 
            detail=f"Internal server error during retrieval: {str(e)}"
        )

async def update_memory(request: UpdateMemoryRequest, config: AppConfig):
    """
    Updates an existing memory by finding it based on app_id/project_id/ticket_id/run_id/type combination,
    then replacing its content and incrementing the version.
    """
    try:
        # Validate input
        if not request.app_id:
            raise ValidationError(
                status_code=400,
                detail="app_id is required"
            )

        # Validate hierarchical structure
        try:
            validate_hierarchy(
                app_id=request.app_id,
                project_id=request.project_id,
                ticket_id=request.ticket_id,
                run_id=request.run_id
            )
        except HierarchyValidationError as e:
            raise ValidationError(status_code=e.status_code, detail=e.detail)

        # Build filter to find the memory to update
        filter_conditions = [
            models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id))
        ]

        if request.project_id:
            filter_conditions.append(
                models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id))
            )

        if request.ticket_id:
            filter_conditions.append(
                models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id))
            )

        if request.run_id:
            filter_conditions.append(
                models.FieldCondition(key="run_id", match=models.MatchValue(value=request.run_id))
            )

        if request.memory_type:
            filter_conditions.append(
                models.FieldCondition(key="type", match=models.MatchValue(value=request.memory_type))
            )
        
        qdrant_filter = models.Filter(must=filter_conditions)
        
        # Find existing memory chunks
        existing_points = []
        offset = None
        
        while True:
            result = config.qdrant_client.scroll(
                collection_name=QDRANT_COLLECTION_NAME,
                scroll_filter=qdrant_filter,
                limit=100,
                offset=offset,
                with_payload=True,
                with_vectors=False
            )
            
            points, next_offset = result
            existing_points.extend(points)
            
            if next_offset is None or len(points) < 100:
                break
            offset = next_offset
        
        if not existing_points:
            raise ValidationError(
                status_code=404,
                detail="No memory found matching the specified criteria"
            )
        
        # Group by original_content_hash to find unique memories
        memory_groups = defaultdict(list)
        for point in existing_points:
            content_hash = point.payload.get("original_content_hash", "unknown")
            memory_groups[content_hash].append(point)
        
        # Find the latest version
        max_version = 0
        for points in memory_groups.values():
            for point in points:
                version = safe_int_conversion(point.payload.get("version", 1))
                max_version = max(max_version, version)
        
        # Prepare new version
        new_version = max_version + 1
        
        # Delete old chunks
        point_ids_to_delete = [point.id for point in existing_points]
        config.qdrant_client.delete(
            collection_name=QDRANT_COLLECTION_NAME,
            points_selector=models.PointIdsList(points=point_ids_to_delete)
        )
        
        # Create new memory with updated content
        new_metadata = {
            "app_id": request.app_id,
            "version": new_version,
            "timestamp_iso": datetime.utcnow().isoformat() + "Z"
        }

        # Add optional fields if provided
        if request.project_id:
            new_metadata["project_id"] = request.project_id
        if request.ticket_id:
            new_metadata["ticket_id"] = request.ticket_id
        if request.run_id:
            new_metadata["run_id"] = request.run_id
        if request.memory_type:
            new_metadata["type"] = request.memory_type
            
        # Apply any additional metadata updates
        new_metadata.update(request.metadata_updates)
        
        # Create new memory item and add it
        memory_item = MemoryItemIn(
            content=request.new_content,
            metadata=new_metadata
        )
        
        add_result = await add_memory(memory_item, config)
        
        return {
            "message": f"Memory updated successfully to version {new_version}",
            "previous_version": max_version,
            "new_version": new_version,
            "chunks_replaced": len(existing_points),
            "chunks_stored": add_result.chunks_stored,
            "original_content_hash": add_result.original_content_hash
        }
        
    except ValidationError:
        raise
    except Exception as e:
        logger.error(f" Unexpected error in update_memory: {e}")
        raise ValidationError(
            status_code=500,
            detail=f"Internal server error during update: {str(e)}"
        )

async def delete_run_memories(request: DeleteRunMemoriesRequest, config: AppConfig):
    """
    Deletes all memories for a specific run_id.
    Requires exact 4-level match: app_id + project_id + ticket_id + run_id.
    This is a destructive operation with no undo capability.

    Args:
        request: DeleteRunMemoriesRequest with app_id, project_id, ticket_id, run_id, and optional dry_run
        config: AppConfig with Qdrant client

    Returns:
        dict with message, deleted_count, deleted_memories, run_id

    Raises:
        ValidationError: If hierarchy is invalid or run_id not found
    """
    try:
        # Validate hierarchical structure - all 4 levels required
        try:
            validate_hierarchy(
                app_id=request.app_id,
                project_id=request.project_id,
                ticket_id=request.ticket_id,
                run_id=request.run_id
            )
        except HierarchyValidationError as e:
            raise ValidationError(status_code=e.status_code, detail=e.detail)

        # Build exact match filter for all 4 levels
        qdrant_filter = models.Filter(must=[
            models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
            models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
            models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id)),
            models.FieldCondition(key="run_id", match=models.MatchValue(value=request.run_id))
        ])

        # Scroll to find all points to delete
        all_points = []
        offset = None

        while True:
            result = config.qdrant_client.scroll(
                collection_name=QDRANT_COLLECTION_NAME,
                scroll_filter=qdrant_filter,
                limit=100,
                offset=offset,
                with_payload=True,
                with_vectors=False
            )

            points, next_offset = result
            all_points.extend(points)

            if next_offset is None or len(points) < 100:
                break
            offset = next_offset

        if not all_points:
            raise ValidationError(
                status_code=404,
                detail=f"No memories found for run_id: {request.run_id}"
            )

        # Count distinct memories (by original_content_hash)
        unique_hashes = set(point.payload.get("original_content_hash") for point in all_points)

        # If dry_run, return preview without deleting
        if request.dry_run:
            logger.info(f"DRY RUN: Would delete {len(all_points)} chunks ({len(unique_hashes)} memories) for run_id: {request.run_id}")
            return {
                "message": f"DRY RUN: Would delete all memories for run_id: {request.run_id}",
                "deleted_count": len(all_points),
                "deleted_memories": len(unique_hashes),
                "run_id": request.run_id,
                "dry_run": True
            }

        # Delete all points
        point_ids = [point.id for point in all_points]
        config.qdrant_client.delete(
            collection_name=QDRANT_COLLECTION_NAME,
            points_selector=models.PointIdsList(points=point_ids)
        )

        logger.info(f"Deleted {len(point_ids)} chunks ({len(unique_hashes)} memories) for run_id: {request.run_id}")

        return {
            "message": f"Successfully deleted all memories for run_id: {request.run_id}",
            "deleted_count": len(point_ids),
            "deleted_memories": len(unique_hashes),
            "run_id": request.run_id,
            "dry_run": False
        }

    except ValidationError:
        raise
    except Exception as e:
        logger.error(f"Unexpected error in delete_run_memories: {e}")
        raise ValidationError(
            status_code=500,
            detail=f"Deletion failed: {str(e)}"
        )

async def get_recent_memories(request: GetRecentMemoriesRequest, config: AppConfig):
    """
    Retrieves memories from the last N hours, optionally filtered by app_id/project_id/ticket_id/run_id.
    Perfect for agents resuming work after a break.

    Supports 4-level hierarchy filtering with cascade behavior:
    - With run_id: Returns specific run (cascade=false) or run + parents (cascade=true)
    - With ticket_id but no run_id: Returns ticket-level + all runs (cascade=false) or + parents (cascade=true)
    """
    try:
        # Validate hierarchical structure if filters provided
        if request.app_id or request.project_id or request.ticket_id or request.run_id:
            try:
                validate_hierarchy(
                    app_id=request.app_id,
                    project_id=request.project_id,
                    ticket_id=request.ticket_id,
                    run_id=request.run_id
                )
            except HierarchyValidationError as e:
                raise ValidationError(status_code=e.status_code, detail=e.detail)

        # Calculate timestamp window
        start_time, end_time = determine_time_bounds(
            hours=request.hours,
            start_time_iso=request.start_time_iso,
            end_time_iso=request.end_time_iso
        )
        logger.info(f"Retrieving recent memories between {start_time.isoformat()} and {end_time.isoformat()}")
        
        # Build filter conditions based on cascade setting
        should_conditions = []

        if not request.cascade:
            # EXACT MATCH MODE (cascade=false) - Only return memories at the specified level
            if request.run_id:
                # Return ONLY run-level memories (all four match)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id)),
                        models.FieldCondition(key="run_id", match=models.MatchValue(value=request.run_id))
                    ])
                )
            elif request.ticket_id:
                # Return ticket-level memories + ALL runs for this ticket
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id))
                    ])
                )
            elif request.project_id:
                # Return ONLY project-level memories (project_id matches, ticket_id is empty)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                    ])
                )
            elif request.app_id:
                # Return ONLY app-level memories (project_id is empty)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ])
                )
        else:
            # CASCADE MODE (cascade=true) - Return parent levels + current level
            if request.run_id:
                # Return app-level + project-level + ticket-level + run-level memories
                # 1. App-level memories
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ])
                )
                # 2. Project-level memories
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                    ])
                )
                # 3. Ticket-level memories (no run_id)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="run_id"))
                    ])
                )
                # 4. Run-level memories (all four match)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id)),
                        models.FieldCondition(key="run_id", match=models.MatchValue(value=request.run_id))
                    ])
                )
            elif request.ticket_id:
                # Return app-level + project-level + ticket-level memories (includes all runs)
                # 1. App-level memories
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ])
                )
                # 2. Project-level memories
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                    ])
                )
                # 3. Ticket-level memories (includes all runs)
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id))
                    ])
                )
            elif request.project_id:
                # Return app-level + project-level memories
                # 1. App-level memories
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ])
                )
                # 2. Project-level memories
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                    ])
                )
            elif request.app_id:
                # Return only app-level memories
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ])
                )

        # Note: Qdrant doesn't support direct date range filtering in the same way as other DBs
        # We'll need to fetch all matching records and filter by timestamp in memory
        qdrant_filter = models.Filter(should=should_conditions) if should_conditions else None
        
        # Scroll through all matching points
        all_points = []
        offset = None
        batch_size = 100
        
        while True:
            result = config.qdrant_client.scroll(
                collection_name=QDRANT_COLLECTION_NAME,
                scroll_filter=qdrant_filter,
                limit=batch_size,
                offset=offset,
                with_payload=True,
                with_vectors=False
            )
            
            points, next_offset = result
            
            # Filter by timestamp
            for point in points:
                timestamp_iso = point.payload.get("timestamp_iso", "")
                if not timestamp_iso:
                    continue
                try:
                    timestamp_dt = parse_iso8601_to_utc(timestamp_iso)
                except ValidationError:
                    continue
                if start_time <= timestamp_dt <= end_time:
                    all_points.append(point)
            
            if next_offset is None or len(points) < batch_size:
                break
                
            offset = next_offset
        
        logger.info(f" Found {len(all_points)} recent memory chunks from the last {request.hours} hours")
        
        # Convert to RetrievedChunk format
        retrieved_chunks: List[RetrievedChunk] = []
        for point in all_points:
            chunk_content = point.payload.get("text_chunk", "")
            metadata_from_payload = {k: v for k, v in point.payload.items() if k != "text_chunk"}
            
            retrieved_chunks.append(RetrievedChunk(
                chunk_id=str(point.id),
                text_chunk=chunk_content,
                metadata=metadata_from_payload,
                score=1.0  # No similarity score for time-based retrieval
            ))

        # Apply optional type/tag filters before deduplication
        if request.memory_types or request.tags:
            filtered_chunks = [
                chunk for chunk in retrieved_chunks
                if chunk_matches_filters(chunk.metadata, request.memory_types, request.tags)
            ]
            logger.info(f" Type/tag filters reduced recent memories from {len(retrieved_chunks)} to {len(filtered_chunks)}")
            retrieved_chunks = filtered_chunks
        
        # Apply version deduplication
        memory_groups = defaultdict(list)
        for chunk in retrieved_chunks:
            app_id = chunk.metadata.get('app_id', '')
            project_id = chunk.metadata.get('project_id', '') or 'none'
            ticket_id = chunk.metadata.get('ticket_id', '') or 'none'
            run_id = chunk.metadata.get('run_id', '') or 'none'
            memory_type = chunk.metadata.get('type', '') or 'none'

            memory_key = f"{app_id}|{project_id}|{ticket_id}|{run_id}|{memory_type}"
            memory_groups[memory_key].append(chunk)
        
        # Keep only highest version per group
        version_filtered_chunks = []
        for memory_key, chunks_in_group in memory_groups.items():
            if len(chunks_in_group) == 1:
                version_filtered_chunks.extend(chunks_in_group)
            else:
                max_version = max(safe_int_conversion(chunk.metadata.get('version', 1)) for chunk in chunks_in_group)
                highest_version_chunks = [chunk for chunk in chunks_in_group 
                                        if safe_int_conversion(chunk.metadata.get('version', 1)) == max_version]
                version_filtered_chunks.extend(highest_version_chunks)
        
        if request.hide_superseded:
            version_filtered_chunks = filter_superseded_chunks(version_filtered_chunks)

        # Apply limit by memory count (not chunk count)
        retrieved_chunks = limit_by_memory_count(
            version_filtered_chunks,
            request.limit,
            sort_by_timestamp=True
        )

        # Generate summary based on return_format
        synthesized_summary = None
        if request.return_format in ["summary_only", "both"] and ENABLE_GEMMA_SUMMARIZATION and retrieved_chunks:
            try:
                context_prompt = f"Summarize the recent activities and updates from the last {request.hours} hours"
                summary = await synthesize_search_results(
                    context_prompt,
                    retrieved_chunks[:MAX_SUMMARIZATION_CHUNKS],
                    config.http_client,
                    config
                )
                if summary:
                    logger.info(f" LM Studio summary generated successfully for recent memories (return_format={request.return_format}).")
                    synthesized_summary = summary
            except Exception as e:
                logger.warning(f" Recent memories summarization failed: {e}")

        # Return based on requested format
        if request.return_format == "summary_only":
            # Return only summary, empty chunks for token efficiency
            return SearchResponse(
                synthesized_summary=synthesized_summary,
                retrieved_chunks=[],
                total_results=len(retrieved_chunks)
            )
        elif request.return_format == "chunks_only":
            # Return only chunks, no summary
            return SearchResponse(
                synthesized_summary=None,
                retrieved_chunks=retrieved_chunks,
                total_results=len(retrieved_chunks)
            )
        else:  # "both" (default)
            # Return both summary and chunks
            return SearchResponse(
                synthesized_summary=synthesized_summary,
                retrieved_chunks=retrieved_chunks,
                total_results=len(retrieved_chunks)
            )
        
    except Exception as e:
        logger.error(f" Unexpected error in get_recent_memories: {e}")
        raise ValidationError(
            status_code=500,
            detail=f"Internal server error during retrieval: {str(e)}"
        ) 

async def get_hierarchy_overview(request: HierarchyOverviewRequest, config: AppConfig):
    """
    Returns a hierarchical overview for a given scope, including child IDs and optional counts.
    - app_id only: returns projects with ticket/run counts
    - app_id + project_id: returns tickets under the project (with run counts)
    - app_id + project_id + ticket_id: returns runs under the ticket
    """
    try:
        if not request.app_id:
            raise ValidationError(status_code=400, detail="app_id is required")

        try:
            validate_hierarchy(
                app_id=request.app_id,
                project_id=request.project_id,
                ticket_id=request.ticket_id
            )
        except HierarchyValidationError as e:
            raise ValidationError(status_code=e.status_code, detail=e.detail)

        # Time window (unbounded unless explicitly provided)
        if request.start_time_iso or request.end_time_iso:
            start_time, end_time = determine_time_bounds(
                hours=None,
                start_time_iso=request.start_time_iso,
                end_time_iso=request.end_time_iso
            )
        else:
            start_time, end_time = datetime.min, datetime.utcnow()

        # Build basic filter
        filter_conditions = [
            models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id))
        ]
        if request.project_id:
            filter_conditions.append(
                models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id))
            )
        if request.ticket_id:
            filter_conditions.append(
                models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id))
            )

        qdrant_filter = models.Filter(must=filter_conditions)

        # Scroll through all matching points
        all_points = []
        offset = None
        batch_size = 100

        while True:
            result = config.qdrant_client.scroll(
                collection_name=QDRANT_COLLECTION_NAME,
                scroll_filter=qdrant_filter,
                limit=batch_size,
                offset=offset,
                with_payload=True,
                with_vectors=False
            )

            points, next_offset = result

            for point in points:
                ts = point.payload.get("timestamp_iso")
                if not ts:
                    continue
                try:
                    ts_dt = parse_iso8601_to_utc(ts)
                except ValidationError:
                    continue
                if start_time <= ts_dt <= end_time:
                    all_points.append(point)

            if next_offset is None or len(points) < batch_size:
                break

            offset = next_offset

        # Convert to RetrievedChunk for consistent filtering
        retrieved_chunks: List[RetrievedChunk] = []
        for point in all_points:
            metadata = {k: v for k, v in point.payload.items() if k != "text_chunk"}
            retrieved_chunks.append(RetrievedChunk(
                chunk_id=str(point.id),
                text_chunk=point.payload.get("text_chunk", ""),
                metadata=metadata,
                score=1.0
            ))

        # Apply filters
        if request.memory_types or request.tags:
            retrieved_chunks = [
                chunk for chunk in retrieved_chunks
                if chunk_matches_filters(chunk.metadata, request.memory_types, request.tags)
            ]

        if request.hide_superseded:
            retrieved_chunks = filter_superseded_chunks(retrieved_chunks)

        track_counts = request.include_counts

        projects: Dict[str, Dict[str, Any]] = {}
        all_memory_hashes: Set[str] = set()

        for chunk in retrieved_chunks:
            metadata = chunk.metadata
            memory_hash = str(metadata.get("original_content_hash", chunk.chunk_id))
            project_id = metadata.get("project_id")
            ticket_id = metadata.get("ticket_id")
            run_id = metadata.get("run_id")

            if track_counts:
                all_memory_hashes.add(memory_hash)

            # For overview we only care about entries that live at project level or below
            if not project_id:
                continue

            project_entry = projects.setdefault(project_id, {
                "tickets": {},
                "memory_hashes": set()
            })

            if track_counts:
                project_entry["memory_hashes"].add(memory_hash)

            if ticket_id:
                ticket_entry = project_entry["tickets"].setdefault(ticket_id, {
                    "runs": {},
                    "memory_hashes": set(),
                    "run_ids": set()
                })
                if track_counts:
                    ticket_entry["memory_hashes"].add(memory_hash)
                if run_id:
                    run_entry = ticket_entry["runs"].setdefault(run_id, {"memory_hashes": set()})
                    ticket_entry["run_ids"].add(run_id)
                    if track_counts:
                        run_entry["memory_hashes"].add(memory_hash)

        project_summaries: List[ProjectSummary] = []
        for project_id, project_data in sorted(projects.items()):
            ticket_summaries: List[TicketSummary] = []
            for ticket_id, ticket_data in sorted(project_data["tickets"].items()):
                run_summaries: List[RunSummary] = []
                for run_id, run_data in sorted(ticket_data["runs"].items()):
                    run_summaries.append(RunSummary(
                        run_id=run_id,
                        memory_count=len(run_data["memory_hashes"]) if track_counts else 0
                    ))

                ticket_summaries.append(TicketSummary(
                    ticket_id=ticket_id,
                    run_count=len(ticket_data["run_ids"]),
                    memory_count=len(ticket_data["memory_hashes"]) if track_counts else 0,
                    runs=run_summaries
                ))

            project_summaries.append(ProjectSummary(
                project_id=project_id,
                ticket_count=len(ticket_summaries),
                run_count=sum(t.run_count for t in ticket_summaries),
                memory_count=len(project_data["memory_hashes"]) if track_counts else 0,
                tickets=ticket_summaries
            ))

        tickets_scope: List[TicketSummary] = []
        runs_scope: List[RunSummary] = []

        if request.project_id:
            target_project = next((p for p in project_summaries if p.project_id == request.project_id), None)
            if target_project:
                tickets_scope = target_project.tickets
                if request.ticket_id:
                    target_ticket = next((t for t in tickets_scope if t.ticket_id == request.ticket_id), None)
                    if target_ticket:
                        runs_scope = target_ticket.runs

        total_projects = len(project_summaries)
        total_tickets = sum(p.ticket_count for p in project_summaries)
        total_runs = sum(p.run_count for p in project_summaries)
        total_memories = len(all_memory_hashes) if track_counts else 0

        filters_applied = {}
        if request.memory_types:
            filters_applied["memory_types"] = request.memory_types
        if request.tags:
            filters_applied["tags"] = request.tags
        if request.hide_superseded:
            filters_applied["hide_superseded"] = True
        if request.start_time_iso or request.end_time_iso:
            filters_applied["start_time_iso"] = request.start_time_iso
            filters_applied["end_time_iso"] = request.end_time_iso or "now"

        return HierarchyOverviewResponse(
            app_id=request.app_id,
            project_id=request.project_id,
            ticket_id=request.ticket_id,
            total_projects=total_projects,
            total_tickets=total_tickets,
            total_runs=total_runs,
            total_memories=total_memories,
            projects=project_summaries,
            tickets=tickets_scope,
            runs=runs_scope,
            filters_applied=filters_applied
        )

    except ValidationError:
        raise
    except Exception as e:
        logger.error(f" Unexpected error in get_hierarchy_overview: {e}")
        raise ValidationError(
            status_code=500,
            detail=f"Internal server error during hierarchy overview: {str(e)}"
        )


async def export_memories(request: ExportMemoriesRequest, config: AppConfig):
    """
    Exports memories as reconstructed JSON blobs for the requested scope.
    """
    try:
        if not request.app_id:
            raise ValidationError(status_code=400, detail="app_id is required")

        try:
            validate_hierarchy(
                app_id=request.app_id,
                project_id=request.project_id,
                ticket_id=request.ticket_id,
                run_id=request.run_id
            )
        except HierarchyValidationError as e:
            raise ValidationError(status_code=e.status_code, detail=e.detail)

        start_time, end_time = determine_time_bounds(
            hours=None,
            start_time_iso=request.start_time_iso,
            end_time_iso=request.end_time_iso
        )

        filter_desc = f"app_id: {request.app_id}"
        should_conditions = []

        if not request.cascade:
            if request.run_id:
                filter_desc += f", project_id: {request.project_id}, ticket_id: {request.ticket_id}, run_id: {request.run_id} (exact match)"
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id)),
                        models.FieldCondition(key="run_id", match=models.MatchValue(value=request.run_id))
                    ])
                )
            elif request.ticket_id:
                filter_desc += f", project_id: {request.project_id}, ticket_id: {request.ticket_id} (ticket + runs)"
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id))
                    ])
                )
            elif request.project_id:
                filter_desc += f", project_id: {request.project_id} (project only)"
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                    ])
                )
            else:
                filter_desc += " (app-level only)"
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ])
                )
        else:
            if request.run_id:
                filter_desc += f", project_id: {request.project_id}, ticket_id: {request.ticket_id}, run_id: {request.run_id} (cascade)"
                should_conditions.extend([
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ]),
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                    ]),
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="run_id"))
                    ]),
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id)),
                        models.FieldCondition(key="run_id", match=models.MatchValue(value=request.run_id))
                    ])
                ])
            elif request.ticket_id:
                filter_desc += f", project_id: {request.project_id}, ticket_id: {request.ticket_id} (cascade)"
                should_conditions.extend([
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ]),
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                    ]),
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.FieldCondition(key="ticket_id", match=models.MatchValue(value=request.ticket_id))
                    ])
                ])
            elif request.project_id:
                filter_desc += f", project_id: {request.project_id} (cascade)"
                should_conditions.extend([
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ]),
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.FieldCondition(key="project_id", match=models.MatchValue(value=request.project_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="ticket_id"))
                    ])
                ])
            else:
                filter_desc += " (app-level)"
                should_conditions.append(
                    models.Filter(must=[
                        models.FieldCondition(key="app_id", match=models.MatchValue(value=request.app_id)),
                        models.IsEmptyCondition(is_empty=models.PayloadField(key="project_id"))
                    ])
                )

        qdrant_filter = should_conditions[0] if (not request.cascade and len(should_conditions) == 1) else models.Filter(should=should_conditions)

        try:
            all_points = []
            offset = None
            batch_size = 200

            while True:
                result = config.qdrant_client.scroll(
                    collection_name=QDRANT_COLLECTION_NAME,
                    scroll_filter=qdrant_filter,
                    limit=batch_size,
                    offset=offset,
                    with_payload=True,
                    with_vectors=False
                )

                points, next_offset = result
                for point in points:
                    ts = point.payload.get("timestamp_iso")
                    if not ts:
                        continue
                    try:
                        ts_dt = parse_iso8601_to_utc(ts)
                    except ValidationError:
                        continue
                    if start_time <= ts_dt <= end_time:
                        all_points.append(point)

                if next_offset is None or len(points) < batch_size:
                    break

                offset = next_offset

            logger.info(f"Exporting {len(all_points)} chunks for {filter_desc}")
        except Exception as e:
            logger.error(f" Qdrant scroll failed during export: {e}")
            raise ValidationError(status_code=500, detail=f"Qdrant retrieval failed: {str(e)}")

        retrieved_chunks: List[RetrievedChunk] = []
        for point in all_points:
            chunk_content = point.payload.get("text_chunk", "")
            metadata_from_payload = {k: v for k, v in point.payload.items() if k != "text_chunk"}
            retrieved_chunks.append(RetrievedChunk(
                chunk_id=str(point.id),
                text_chunk=chunk_content,
                metadata=metadata_from_payload,
                score=1.0
            ))

        if request.memory_types or request.tags:
            retrieved_chunks = [
                chunk for chunk in retrieved_chunks
                if chunk_matches_filters(chunk.metadata, request.memory_types, request.tags)
            ]

        # Version deduplication to keep highest version of each logical memory
        memory_groups = defaultdict(list)
        for chunk in retrieved_chunks:
            app_id = chunk.metadata.get('app_id', '')
            project_id = chunk.metadata.get('project_id', '') or 'none'
            ticket_id = chunk.metadata.get('ticket_id', '') or 'none'
            run_id = chunk.metadata.get('run_id', '') or 'none'
            memory_type = chunk.metadata.get('type', '') or 'none'

            memory_key = f"{app_id}|{project_id}|{ticket_id}|{run_id}|{memory_type}"
            memory_groups[memory_key].append(chunk)

        version_filtered_chunks = []
        for memory_key, chunks_in_group in memory_groups.items():
            if len(chunks_in_group) == 1:
                version_filtered_chunks.extend(chunks_in_group)
            else:
                max_version = max(safe_int_conversion(chunk.metadata.get('version', 1)) for chunk in chunks_in_group)
                highest_version_chunks = [chunk for chunk in chunks_in_group 
                                        if safe_int_conversion(chunk.metadata.get('version', 1)) == max_version]
                version_filtered_chunks.extend(highest_version_chunks)

        if request.hide_superseded:
            version_filtered_chunks = filter_superseded_chunks(version_filtered_chunks)

        # Reconstruct full memories grouped by original_content_hash
        memory_chunks = defaultdict(list)
        for chunk in version_filtered_chunks:
            content_hash = chunk.metadata.get("original_content_hash", chunk.chunk_id)
            memory_chunks[content_hash].append(chunk)

        exported_memories: List[ExportedMemory] = []
        total_chunks = 0

        for content_hash, chunks_in_memory in memory_chunks.items():
            chunks_in_memory.sort(key=lambda c: c.metadata.get("chunk_index", 0))
            total_chunks += len(chunks_in_memory)
            content = "".join(c.text_chunk for c in chunks_in_memory)
            base_metadata = {
                k: v for k, v in chunks_in_memory[0].metadata.items()
                if k not in ["text_chunk", "chunk_index", "total_chunks", "keywords"]
            }
            exported_memories.append(ExportedMemory(
                original_content_hash=str(content_hash),
                metadata=base_metadata,
                content=content,
                chunk_count=len(chunks_in_memory)
            ))

        return ExportMemoriesResponse(
            app_id=request.app_id,
            project_id=request.project_id,
            ticket_id=request.ticket_id,
            run_id=request.run_id,
            total_memories=len(exported_memories),
            total_chunks=total_chunks,
            memories=exported_memories
        )

    except ValidationError:
        raise
    except Exception as e:
        logger.error(f" Unexpected error in export_memories: {e}")
        raise ValidationError(
            status_code=500,
            detail=f"Internal server error during export: {str(e)}"
        )
