"""
SPARQL Agent Core - Main orchestrator for AGS SPARQL Agent

This module contains the main SPARQLAgent class that orchestrates all the 
functionality including ontology discovery, query generation, execution,
and learning.
"""

import os
import sys
import json
import asyncio
from typing import Dict, List, Optional, Any
from datetime import datetime

from .models import (
    GraphmartConfig,
    OntologyInfo,
    AgentMemory,
    QuestionResponse,
    KnowledgeExploration,
    ExamplesResponse,
    AgentInsights,
    QueryResult,
    SPARQLQuery
)
from .interaction_logger import get_logger

# Debug function for MCP compliance - logs to stderr only
def debug_log(message: str):
    """Log debug messages to stderr to avoid MCP protocol interference."""
    # Respect ENABLE_AGENT_DEBUG environment variable for MCP compatibility
    if os.getenv('ENABLE_AGENT_DEBUG', 'false').lower() == 'true':
        print(f"[DEBUG] {message}", file=sys.stderr)


class SPARQLAgent:
    """Main orchestrator for the AGS SPARQL Agent."""
    
    def __init__(self, config: GraphmartConfig):
        """Initialize the SPARQL agent with configuration."""
        self.config = config
        self.ontology_info: Optional[OntologyInfo] = None
        self.agent_memory: Optional[AgentMemory] = None
        self.anzo_client = None
        self.openai_client = None
        
        # Load agent configuration
        self.agent_config = self._load_agent_config()
        
        # Initialize components (will be created in subsequent files)
        self.ontology_manager = None
        self.query_engine = None
        self.discovery_engine = None
        self.memory_system = None
    
    def _load_agent_config(self) -> Dict[str, Any]:
        """Load agent configuration from JSON file."""
        config_path = os.path.join(
            os.path.dirname(__file__), 
            '..', 
            'config', 
            'agent_config.json'
        )
        
        try:
            with open(config_path, 'r') as f:
                return json.load(f)
        except FileNotFoundError:
            # Return default configuration
            return {
                "max_iterations": 3,
                "memory_file": "data/agent_memory.json",
                "query_timeout": 30,
                "confidence_threshold": 0.6,
                "auto_discovery": True,
                "cache_ontologies": True,
                "simple_mode": True
            }
    
    async def initialize(self):
        """Initialize all components of the agent."""
        debug_log(f"Initializing AGS SPARQL Agent for: {self.config.graphmart_uri}")
        
        try:
            # Initialize clients (reusing patterns from GraphRAG pipeline)
            await self._setup_clients()
            
            # Load agent memory
            await self._load_memory()
            
            # Initialize core components (will be implemented next)
            await self._initialize_components()
            
            # Discover ontologies if auto-discovery is enabled
            if self.agent_config.get("auto_discovery", True):
                await self._discover_ontologies()
            
            debug_log("AGS SPARQL Agent initialization complete")
            
        except Exception as e:
            debug_log(f"Failed to initialize AGS SPARQL Agent: {e}")
            raise
    
    async def _setup_clients(self):
        """Set up AGS and OpenAI clients using GraphRAG patterns."""
        # Setup Anzo client (same pattern as GraphRAG)
        from pyanzo import AnzoClient
        
        self.anzo_client = AnzoClient(
            domain=self.config.ags_server,
            port=self.config.ags_port,
            username=self.config.username,
            password=self.config.password
        )
        
        # Setup OpenAI client (same pattern as GraphRAG)
        import instructor
        from openai import AzureOpenAI
        
        api_key = os.getenv('API_KEY')
        if not api_key:
            raise ValueError("API_KEY environment variable is required")
        
        api_version = "2023-07-01-preview"
        # endpoint = "https://promptdesignerv1.openai.azure.com/"
        endpoint = "badendpoint"

        azure_client = AzureOpenAI(
            api_version=api_version,
            api_key=api_key,
            azure_endpoint=endpoint,
        )
        
        # Patch with instructor for structured outputs
        self.openai_client = instructor.from_openai(azure_client)
        
        debug_log(f"Connected to AGS: {self.config.ags_server}:{self.config.ags_port}")
        debug_log(f"Connected to OpenAI: {endpoint}")
    
    async def _load_memory(self):
        """Load persistent agent memory."""
        memory_file = os.path.join(
            os.path.dirname(__file__),
            '..',
            self.agent_config.get("memory_file", "data/agent_memory.json")
        )
        
        # Ensure directory exists
        os.makedirs(os.path.dirname(memory_file), exist_ok=True)
        
        self.agent_memory = AgentMemory.load_from_file(memory_file)
        debug_log(f"Loaded agent memory: {len(self.agent_memory.successful_queries)} successful patterns")
    
    async def _initialize_components(self):
        """Initialize core agent components."""
        # Initialize ontology discovery engine
        from ontology_discovery import OntologyDiscoveryEngine
        self.ontology_manager = OntologyDiscoveryEngine(self.anzo_client, self.config)
        
        # Initialize SPARQL query engine
        from sparql_query_engine import SPARQLQueryEngine
        self.query_engine = SPARQLQueryEngine(self.anzo_client, self.openai_client, self.config)
        
        # TODO: Initialize other components as we create them
        # self.discovery_engine = DiscoveryEngine(self.ontology_manager, self.query_engine)
        # self.memory_system = MemorySystem(self.agent_memory, self.agent_config)
        
        debug_log("Core agent components initialized")
    
    async def _discover_ontologies(self):
        """Discover and cache ontology information."""
        try:
            # Try to load from cache first
            from ontology_cache import OntologyCacheManager, serialize_ontology_info, deserialize_ontology_info
            cache_manager = OntologyCacheManager()
            
            debug_log("Checking ontology cache...")
            if cache_manager.is_cache_valid(self.config.graphmart_uri):
                debug_log("Loading ontology from cache...")
                cached_data = cache_manager.load_cache(self.config.graphmart_uri)
                if cached_data:
                    self.ontology_info = deserialize_ontology_info(cached_data)
                    debug_log(f"Ontology loaded from cache: {len(self.ontology_info.classes)} classes, {len(self.ontology_info.properties)} properties")
                    return
            
            # Cache miss or invalid - discover from graphmart
            debug_log("Cache miss - starting fresh ontology discovery...")
            self.ontology_info = await self.ontology_manager.discover_graphmart_ontologies()
            debug_log(f"Ontology discovery complete: {len(self.ontology_info.classes)} classes, {len(self.ontology_info.properties)} properties")
            
            # Save to cache for next time
            try:
                serialized_data = serialize_ontology_info(self.ontology_info)
                cache_manager.save_cache(self.config.graphmart_uri, serialized_data)
                debug_log("Ontology cached successfully")
            except Exception as cache_error:
                debug_log(f"Warning: Failed to cache ontology data: {cache_error}")
                # Don't fail the entire operation if caching fails
                
        except Exception as e:
            debug_log(f"Error during ontology discovery: {e}")
            # Create fallback ontology info
            self.ontology_info = OntologyInfo(
                classes={},
                properties={},
                namespaces={},
                domain_summary=f"Ontology discovery failed: {e}",
                total_instances=0
            )
    
    async def refresh_ontology_cache(self):
        """Force refresh of ontology cache by bypassing cache and re-discovering."""
        try:
            from ontology_cache import OntologyCacheManager, serialize_ontology_info
            
            debug_log("Forcing ontology cache refresh...")
            
            # Clear existing cache for this graphmart
            cache_manager = OntologyCacheManager()
            cache_manager.clear_cache(self.config.graphmart_uri)
            
            # Force fresh discovery
            self.ontology_info = await self.ontology_manager.discover_graphmart_ontologies()
            debug_log(f"Fresh ontology discovery complete: {len(self.ontology_info.classes)} classes, {len(self.ontology_info.properties)} properties")
            
            # Save new cache
            serialized_data = serialize_ontology_info(self.ontology_info)
            cache_manager.save_cache(self.config.graphmart_uri, serialized_data)
            debug_log("New ontology cache saved")
            
            return True
            
        except Exception as e:
            debug_log(f"Error during ontology cache refresh: {e}")
            return False
    
    # --- Core MCP Tool Implementations ---
    
    async def ask_question(self, question: str, include_reasoning: bool = False, include_execution_details: bool = False) -> QuestionResponse:
        """
        Primary interface: ask any question about the knowledge graph.
        
        This method auto-discovers ontology info as needed and handles
        iterations internally to provide a simple interface.
        """
        debug_log(f"Processing question: {question}")
        
        # Initialize logging for this interaction
        logger = get_logger()
        interaction_id = logger.start_interaction(question, include_reasoning, include_execution_details)
        
        try:
            # Ensure ontology info is available
            ontology_start_time = datetime.now()
            if not self.ontology_info:
                await self._discover_ontologies()
            ontology_discovery_time = (datetime.now() - ontology_start_time).total_seconds()
            
            # Log ontology context that will be provided to LLM
            if self.ontology_info and hasattr(self.ontology_info, 'llm_context'):
                logger.log_ontology_context(self.ontology_info.llm_context, ontology_discovery_time)
            
            # Step 1: Generate initial SPARQL query
            query_start_time = datetime.now()
            sparql_query = await self.query_engine.generate_sparql_query(
                question, self.ontology_info, self.agent_memory
            )
            query_generation_time = (datetime.now() - query_start_time).total_seconds()
            
            # Log the prompts used (we'll need to extract these from the query engine)
            # For now, we'll create placeholder prompts based on available info
            system_prompt = f"You are an expert SPARQL query generator for a knowledge graph with {len(self.ontology_info.classes)} classes and {len(self.ontology_info.properties)} properties."
            user_prompt = f"Generate a SPARQL query to answer: {question}\n\nOntology Context:\n{self.ontology_info.llm_context[:500]}..."
            logger.log_prompts(system_prompt, user_prompt)
            
            # No need to store original separately - query object tracks its own history
            
            if include_reasoning:
                reasoning_steps = ["Generated initial SPARQL query"]
            else:
                reasoning_steps = None
            
            # Step 2: Execute query with iterative improvement
            final_result = None
            attempts = 0
            max_attempts = self.agent_config.get("max_iterations", 3)
            start_time = datetime.now()
            
            while attempts < max_attempts:
                attempts += 1
                debug_log(f"🔄 Query attempt {attempts}/{max_attempts}")
                
                # Execute the current query
                exec_start_time = datetime.now()
                query_result = await self.query_engine.execute_sparql_query(sparql_query)
                exec_time = (datetime.now() - exec_start_time).total_seconds()
                
                # Log this SPARQL attempt
                logger.log_sparql_attempt(
                    sparql_query, 
                    attempts, 
                    generation_time=query_generation_time if attempts == 1 else None,
                    execution_time=exec_time,
                    result=query_result
                )
                
                if query_result.success and len(query_result.data) > 0:
                    # Success! Process results into natural language
                    final_result = query_result
                    if reasoning_steps:
                        reasoning_steps.append(f"Query executed successfully on attempt {attempts}")
                    break
                
                elif attempts < max_attempts:
                    # Try to improve the query
                    if reasoning_steps:
                        reasoning_steps.append(f"Attempt {attempts} failed: {query_result.error_message or 'No results'}. Improving query...")
                    
                    improved_query = await self.query_engine.improve_query_based_on_results(
                        sparql_query, query_result, self.ontology_info
                    )
                    
                    if improved_query:
                        sparql_query = improved_query
                        debug_log(f"🔧 Query improved for attempt {attempts + 1}")
                        # Reset generation time for improved query
                        query_generation_time = 0.1  # Estimated time for improvement
                    else:
                        debug_log(f"❌ Could not improve query")
                        final_result = query_result  # Use the failed result
                        break
                else:
                    debug_log(f"🔍 DEBUG - Final attempt failed")
                    # Final attempt failed
                    final_result = query_result
                    if reasoning_steps:
                        reasoning_steps.append(f"All {max_attempts} attempts failed")
                    break
            
            # Step 3: Generate natural language answer from results
            natural_answer = self._generate_natural_language_answer(
                question, final_result, sparql_query
            )
            
            # Step 4: Update agent memory if successful
            if final_result.success and len(final_result.data) > 0:
                await self._update_agent_memory(question, sparql_query, final_result)
            
            # Calculate confidence based on results
            confidence = self._calculate_confidence(final_result, sparql_query)
            
            # Prepare execution details if requested
            execution_details = None
            if include_execution_details:
                total_time = (datetime.now() - start_time).total_seconds()
                from models import ExecutionDetails
                execution_details = ExecutionDetails(
                    final_query_result=final_result,
                    final_sparql_query=sparql_query,  # Contains complete history
                    retry_attempts=attempts,
                    total_execution_time=total_time
                )
            
            # Create response
            response = QuestionResponse(
                answer=natural_answer,
                confidence=confidence,
                sources_used=len(final_result.data) if final_result else 0,
                reasoning_steps=reasoning_steps,
                sparql_generated=sparql_query.latest_expanded_query if include_reasoning else None,
                execution_details=execution_details
            )
            
            # Complete logging for this interaction
            execution_times = {
                "ontology_discovery": ontology_discovery_time,
                "query_generation": query_generation_time,
                "query_execution": (datetime.now() - start_time).total_seconds()
            }
            logger.complete_interaction(response, execution_times)
            
            return response
            
        except Exception as e:
            debug_log(f"❌ Question processing failed: {e}")
            
            # Create error response
            error_response = QuestionResponse(
                answer=f"I encountered an error while processing your question: {e}",
                confidence=0.0,
                sources_used=0,
                reasoning_steps=[f"Error: {e}"] if include_reasoning else None,
                sparql_generated=None
            )
            
            # Try to complete logging even for errors
            try:
                logger.complete_interaction(error_response)
            except:
                pass  # Don't let logging errors break the response
            
            return error_response
    
    async def explore_knowledge(self, focus: str = "overview") -> KnowledgeExploration:
        """Get overview of what's available in the knowledge graph."""
        debug_log(f"Exploring knowledge with focus: {focus}")
        
        try:
            if not self.ontology_info:
                await self._discover_ontologies()
            
            # Generate key classes info
            key_classes = []
            if self.ontology_info and self.ontology_info.classes:
                # Sort classes by instance count and take top 10
                sorted_classes = sorted(
                    self.ontology_info.classes.values(),
                    key=lambda c: c.instance_count or 0,
                    reverse=True
                )[:10]
                
                for cls in sorted_classes:
                    key_classes.append({
                        "name": cls.name,
                        "uri": cls.uri,
                        "description": cls.description or f"Class with {cls.instance_count or 0} instances",
                        "instance_count": cls.instance_count or 0,
                        "sample_instances": cls.sample_instances[:3]  # First 3 samples
                    })
            
            # Generate key properties info:
            #   1. ALL object properties (cross-domain links — the interesting joins)
            #   2. Top 3-4 data properties per key class (the queryable attributes)
            key_properties = []
            if self.ontology_info and self.ontology_info.properties:
                from collections import defaultdict

                # Build a set of the top-class URIs for fast lookup
                top_class_uris = {c["uri"] for c in key_classes}

                seen_uris = set()

                # --- Pass 1: ALL object properties (regardless of domain class) ---
                all_props = list(self.ontology_info.properties.values())
                object_props = sorted(
                    [p for p in all_props if p.is_object_property is True],
                    key=lambda p: p.usage_frequency or 0,
                    reverse=True
                )
                for prop in object_props:
                    if prop.uri not in seen_uris:
                        seen_uris.add(prop.uri)
                        key_properties.append({
                            "name": prop.name,
                            "uri": prop.uri,
                            "description": prop.description or f"Object property used {prop.usage_frequency or 0} times",
                            "usage_frequency": prop.usage_frequency or 0,
                            "domain_classes": prop.domain_classes[:3],
                            "range_classes": prop.range_classes[:3],
                            "property_type": "object"
                        })

                # --- Pass 2: Top 3-4 data properties per key class ---
                data_props_by_class = defaultdict(list)
                unmatched_data = []
                for prop in all_props:
                    if prop.is_object_property is True:
                        continue  # already handled above
                    matched = False
                    for domain_uri in (prop.domain_classes or []):
                        if domain_uri in top_class_uris:
                            data_props_by_class[domain_uri].append(prop)
                            matched = True
                            break
                    if not matched:
                        unmatched_data.append(prop)

                for cls_info in key_classes:
                    cls_uri = cls_info["uri"]
                    cls_data_props = sorted(
                        data_props_by_class.get(cls_uri, []),
                        key=lambda p: p.usage_frequency or 0,
                        reverse=True
                    )[:4]
                    for prop in cls_data_props:
                        if prop.uri not in seen_uris:
                            seen_uris.add(prop.uri)
                            key_properties.append({
                                "name": prop.name,
                                "uri": prop.uri,
                                "description": prop.description or f"Property used {prop.usage_frequency or 0} times",
                                "usage_frequency": prop.usage_frequency or 0,
                                "domain_classes": prop.domain_classes[:3],
                                "property_type": "data"
                            })

                # Fill any remaining budget (up to 50 total) with high-frequency
                # unmatched data properties that span multiple classes
                remaining = 50 - len(key_properties)
                if remaining > 0:
                    for prop in sorted(unmatched_data, key=lambda p: p.usage_frequency or 0, reverse=True)[:remaining]:
                        if prop.uri not in seen_uris:
                            seen_uris.add(prop.uri)
                            key_properties.append({
                                "name": prop.name,
                                "uri": prop.uri,
                                "description": prop.description or f"Property used {prop.usage_frequency or 0} times",
                                "usage_frequency": prop.usage_frequency or 0,
                                "domain_classes": prop.domain_classes[:3],
                                "property_type": "data"
                            })
            
            # Generate sample questions based on discovered content
            sample_questions = [
                "What types of entities are in this knowledge graph?",
                "What relationships exist between entities?",
                "Can you show me some example data?"
            ]
            
            # Add domain-specific questions if we have classes
            if key_classes:
                top_class = key_classes[0]
                sample_questions.extend([
                    f"How many {top_class['name']} instances are there?",
                    f"Show me examples of {top_class['name']} entities",
                    f"What properties does {top_class['name']} have?"
                ])
            
            return KnowledgeExploration(
                domain_summary=self.ontology_info.domain_summary if self.ontology_info else "Knowledge graph exploration in progress",
                key_classes=key_classes,
                key_properties=key_properties,
                sample_questions=sample_questions,
                total_classes=len(self.ontology_info.classes) if self.ontology_info else 0,
                total_properties=len(self.ontology_info.properties) if self.ontology_info else 0
            )
            
        except Exception as e:
            debug_log(f"Error exploring knowledge: {e}")
            return KnowledgeExploration(
                domain_summary=f"Error exploring knowledge: {e}",
                key_classes=[],
                key_properties=[],
                sample_questions=[],
                total_classes=0,
                total_properties=0
            )
    
    async def get_examples(self, category: str = "questions") -> ExamplesResponse:
        """Get sample questions and data patterns for learning."""
        debug_log(f"Getting examples for category: {category}")
        
        try:
            # TODO: Implement example generation
            return ExamplesResponse(
                sample_questions=[
                    {"complexity": "simple", "question": "What types of things are in this knowledge graph?"},
                    {"complexity": "intermediate", "question": "Show me relationships between different entities"},
                    {"complexity": "advanced", "question": "Find patterns in the data that indicate specific behaviors"}
                ],
                data_examples=[],
                query_patterns=[]
            )
            
        except Exception as e:
            return ExamplesResponse(
                sample_questions=[],
                data_examples=[],
                query_patterns=[]
            )
    
    async def query_sparql_direct(
        self, 
        sparql: Optional[str] = None, 
        question: Optional[str] = None, 
        explain: bool = False
    ) -> QueryResult:
        """Execute SPARQL directly or generate from question."""
        debug_log(f"Direct SPARQL query - sparql provided: {sparql is not None}, question: {question}")
        
        try:
            if sparql:
                # Execute the provided SPARQL directly
                # Create a minimal SPARQLQuery object for the engine
                sparql_query = SPARQLQuery(
                    latest_compact_query=sparql,
                    latest_expanded_query=sparql,
                    original_question=question or "Direct SPARQL execution"
                )
                
                # Execute using the query engine
                result = await self.query_engine.execute_sparql_query(sparql_query)
                return result
                
            elif question:
                # Generate SPARQL from question and execute
                # Ensure ontology info is available
                if not self.ontology_info:
                    await self._discover_ontologies()
                
                # Generate SPARQL query
                sparql_query = await self.query_engine.generate_sparql_query(
                    question, self.ontology_info, self.agent_memory
                )
                
                # Execute the generated query
                result = await self.query_engine.execute_sparql_query(sparql_query)
                return result
                
            else:
                return QueryResult(
                    success=False,
                    data=[],
                    error_message="Either 'sparql' query or 'question' parameter must be provided",
                    sparql_query="",
                    execution_time=0.0
                )
            
        except Exception as e:
            return QueryResult(
                success=False,
                data=[],
                error_message=str(e),
                sparql_query=sparql or "",
                execution_time=0.0
            )
    
    async def get_class_info(self, class_name: str) -> Dict[str, Any]:
        """Get detailed information about a specific class."""
        debug_log(f"Getting class info for: {class_name}")
        
        try:
            if not self.ontology_info:
                await self._discover_ontologies()
            
            # Find the class by name or URI
            target_class = None
            for class_uri, class_info in self.ontology_info.classes.items():
                if (class_info.name.lower() == class_name.lower() or 
                    class_uri == class_name or 
                    class_uri.endswith(f"#{class_name}") or
                    class_uri.endswith(f"/{class_name}")):
                    target_class = class_info
                    break
            
            if not target_class:
                return {
                    "error": f"Class '{class_name}' not found",
                    "class_name": class_name,
                    "available_classes": [cls.name for cls in list(self.ontology_info.classes.values())[:10]]
                }
            
            # Find properties that have this class as domain
            related_properties = []
            for prop_uri, prop_info in self.ontology_info.properties.items():
                if target_class.uri in prop_info.domain_classes:
                    related_properties.append({
                        "name": prop_info.name,
                        "uri": prop_uri,
                        "description": prop_info.description,
                        "usage_frequency": prop_info.usage_frequency
                    })
            
            return {
                "class_name": target_class.name,
                "uri": target_class.uri,
                "description": target_class.description or f"Class with {target_class.instance_count or 0} instances",
                "instance_count": target_class.instance_count or 0,
                "sample_instances": target_class.sample_instances,
                "properties": related_properties,
                "total_properties": len(related_properties)
            }
            
        except Exception as e:
            return {
                "error": str(e),
                "class_name": class_name
            }
    
    async def get_property_info(self, property_name: str) -> Dict[str, Any]:
        """Get detailed information about a specific property."""
        print(f"Getting property info for: {property_name}")
        
        try:
            if not self.ontology_info:
                await self._discover_ontologies()
            
            # Find the property by name or URI
            target_property = None
            for prop_uri, prop_info in self.ontology_info.properties.items():
                if (prop_info.name.lower() == property_name.lower() or 
                    prop_uri == property_name or 
                    prop_uri.endswith(f"#{property_name}") or
                    prop_uri.endswith(f"/{property_name}")):
                    target_property = prop_info
                    break
            
            if not target_property:
                return {
                    "error": f"Property '{property_name}' not found",
                    "property_name": property_name,
                    "available_properties": [prop.name for prop in list(self.ontology_info.properties.values())[:10]]
                }
            
            # Get domain and range class details
            domain_class_details = []
            for class_uri in target_property.domain_classes:
                if class_uri in self.ontology_info.classes:
                    cls = self.ontology_info.classes[class_uri]
                    domain_class_details.append({
                        "name": cls.name,
                        "uri": class_uri,
                        "instance_count": cls.instance_count
                    })
            
            range_class_details = []
            for class_uri in target_property.range_classes:
                if class_uri in self.ontology_info.classes:
                    cls = self.ontology_info.classes[class_uri]
                    range_class_details.append({
                        "name": cls.name,
                        "uri": class_uri,
                        "instance_count": cls.instance_count
                    })
            
            return {
                "property_name": target_property.name,
                "uri": target_property.uri,
                "description": target_property.description or f"Property used {target_property.usage_frequency or 0} times",
                "usage_frequency": target_property.usage_frequency or 0,
                "domain_classes": domain_class_details,
                "range_classes": range_class_details,
                "total_domain_classes": len(domain_class_details),
                "total_range_classes": len(range_class_details)
            }
            
        except Exception as e:
            return {
                "error": str(e),
                "property_name": property_name
            }
    
    async def get_agent_insights(self) -> AgentInsights:
        """View agent learning patterns and performance statistics."""
        print("Getting agent insights and performance statistics")
        
        try:
            # TODO: Implement agent insights generation
            return AgentInsights(
                query_success_rate=0.0,
                avg_iterations_per_query=0.0,
                most_popular_classes=[],
                most_popular_properties=[],
                learning_trends={"status": "Implementation in progress"},
                memory_stats={
                    "successful_queries": len(self.agent_memory.successful_queries),
                    "failed_patterns": len(self.agent_memory.failed_patterns),
                    "domain_templates": len(self.agent_memory.domain_templates)
                }
            )
            
        except Exception as e:
            return AgentInsights(
                query_success_rate=0.0,
                avg_iterations_per_query=0.0,
                most_popular_classes=[],
                most_popular_properties=[],
                learning_trends={"error": str(e)},
                memory_stats={}
            )
    
    def _generate_natural_language_answer(
        self, 
        question: str, 
        query_result: QueryResult, 
        sparql_query: SPARQLQuery
    ) -> str:
        """Generate a natural language answer from SPARQL query results."""
        
        if not query_result.success:
            if query_result.error_message:
                return f"I encountered an error while searching the knowledge graph: {query_result.error_message}"
            else:
                return "I couldn't find any results for your question in the knowledge graph."
        
        if len(query_result.data) == 0:
            return "I didn't find any matching results for your question in the knowledge graph."
        
        # Format results based on query type and content
        try:
            results = query_result.data
            
            # Handle count queries (either by query type or column names)
            is_count_query = (sparql_query.query_type == "count" or 
                            (query_result.column_names and 'count' in query_result.column_names))
            
            if is_count_query and len(results) == 1:
                # For count queries, result is a simple list like ['5']
                count_value = results[0] if isinstance(results[0], (str, int)) else results[0][0]
                return f"I found {count_value} results for your question."
            
            # Handle regular SELECT queries
            if query_result.column_names and len(query_result.column_names) > 0:
                # Format as a structured answer
                answer_parts = [f"I found {len(query_result.data)} result{'s' if len(query_result.data) != 1 else ''}:"]
                
                # Show first few results
                max_results_to_show = min(5, len(results))
                for i, row in enumerate(results[:max_results_to_show]):
                    if len(row) >= 1:
                        # Use first column as main result, second as label if available
                        main_value = str(row[0]) if row[0] else "Unknown"
                        
                        # If there's a label column, use it
                        if len(row) >= 2 and row[1]:
                            label = str(row[1])
                            answer_parts.append(f"  {i+1}. {label} ({main_value})")
                        else:
                            # Extract readable name from URI if possible
                            if main_value.startswith('http'):
                                readable_name = main_value.split('/')[-1].split('#')[-1]
                                answer_parts.append(f"  {i+1}. {readable_name}")
                            else:
                                answer_parts.append(f"  {i+1}. {main_value}")
                
                if len(results) > max_results_to_show:
                    answer_parts.append(f"  ... and {len(results) - max_results_to_show} more")
                
                return "\n".join(answer_parts)
            
            else:
                # Fallback for unstructured results
                return f"I found {len(query_result.data)} results in the knowledge graph."
            
        except Exception as e:
            return f"I found {len(query_result.data)} results, but encountered an error formatting the answer: {e}"
    
    def _calculate_confidence(self, query_result: QueryResult, sparql_query: SPARQLQuery) -> float:
        """Calculate confidence score for the answer based on query success and results."""
        
        if not query_result.success:
            return 0.0
        
        if len(query_result.data) == 0:
            return 0.1  # Low confidence for no results
        
        # Base confidence from query generation
        base_confidence = sparql_query.confidence
        
        # Adjust based on result characteristics
        if len(query_result.data) >= 1:
            result_confidence = min(0.9, 0.6 + (len(query_result.data) * 0.05))
        else:
            result_confidence = 0.3
        
        # Combine confidences
        final_confidence = (base_confidence + result_confidence) / 2
        
        return min(0.95, max(0.1, final_confidence))
    
    async def _update_agent_memory(
        self, 
        question: str, 
        sparql_query: SPARQLQuery, 
        query_result: QueryResult
    ):
        """Update agent memory with successful query patterns."""
        
        if not self.agent_memory:
            return
        
        try:
            # Add to successful queries
            memory_entry = {
                "question": question,
                "sparql": sparql_query.latest_expanded_query,
                "classes_used": sparql_query.ontology_classes_used,
                "properties_used": sparql_query.ontology_properties_used,
                "query_type": sparql_query.query_type,
                "result_count": len(query_result.data),
                "execution_time": query_result.execution_time,
                "timestamp": datetime.now().isoformat(),
                "confidence": sparql_query.confidence
            }
            
            self.agent_memory.successful_queries.append(memory_entry)
            
            # Keep only recent successful queries (last 100)
            if len(self.agent_memory.successful_queries) > 100:
                self.agent_memory.successful_queries = self.agent_memory.successful_queries[-100:]
            
            # Update learning patterns
            await self._update_learning_patterns(memory_entry)
            
        except Exception as e:
            print(f"❌ Failed to update agent memory: {e}")
    
    async def _update_learning_patterns(self, memory_entry: Dict):
        """Update learning patterns based on successful queries."""
        
        # Track popular classes and properties
        for class_uri in memory_entry.get("classes_used", []):
            # Update popular classes count
            if class_uri not in self.agent_memory.popular_classes:
                self.agent_memory.popular_classes[class_uri] = 0
            self.agent_memory.popular_classes[class_uri] += 1
        
        for prop_uri in memory_entry.get("properties_used", []):
            # Update popular properties count
            if prop_uri not in self.agent_memory.popular_properties:
                self.agent_memory.popular_properties[prop_uri] = 0
            self.agent_memory.popular_properties[prop_uri] += 1
    
    async def shutdown(self):
        """Clean shutdown of the agent."""
        print("Shutting down AGS SPARQL Agent...")
        
        # Save agent memory
        if self.agent_memory:
            memory_file = os.path.join(
                os.path.dirname(__file__),
                '..',
                self.agent_config.get("memory_file", "data/agent_memory.json")
            )
            self.agent_memory.save_to_file(memory_file)
            print("Agent memory saved")
        
        # Close connections
        # TODO: Properly close anzo_client if needed
        
        print("AGS SPARQL Agent shutdown complete")
