#!/usr/bin/env python3
# Copyright (c) 2025 Infinidatum
# Author: Duraimurugan Rajamanickam
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#     http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
LLM Training and RAG Building Observability
===========================================

Comprehensive observability platform for Large Language Model training,
RAG (Retrieval-Augmented Generation) building, and Agentic AI processes
in enterprise environments.

This module addresses critical gaps in:
- LLM training pipeline monitoring
- RAG system performance tracking
- Agent workflow observability
- Token usage and cost optimization
- Model performance and drift detection
- Data pipeline health monitoring
- Enterprise governance and compliance
"""

import time
import json
import os
import hashlib
import psutil
from datetime import datetime, timedelta
from typing import Dict, List, Any, Optional, Union, Tuple, Callable
from dataclasses import dataclass, field
from enum import Enum
from collections import deque, defaultdict
import numpy as np
import logging
import threading
from concurrent.futures import ThreadPoolExecutor
import asyncio

# Import OpenFinOps components for professional visualization
try:
    from finopsmetrics.vizlychart import LineChart, ScatterChart, BarChart, VizlyFigure
    from finopsmetrics.vizlychart.theme import get_theme
    VIZLYCHART_AVAILABLE = True
except ImportError:
    VIZLYCHART_AVAILABLE = False


class LLMTrainingStage(Enum):
    """LLM training pipeline stages."""
    DATA_PREPROCESSING = "data_preprocessing"
    TOKENIZATION = "tokenization"
    PRE_TRAINING = "pre_training"
    FINE_TUNING = "fine_tuning"
    INSTRUCTION_TUNING = "instruction_tuning"
    RLHF = "rlhf"  # Reinforcement Learning from Human Feedback
    EVALUATION = "evaluation"
    DEPLOYMENT = "deployment"


class RAGStage(Enum):
    """RAG pipeline stages."""
    DOCUMENT_INGESTION = "document_ingestion"
    CHUNKING = "chunking"
    EMBEDDING_GENERATION = "embedding_generation"
    INDEX_BUILDING = "index_building"
    RETRIEVAL = "retrieval"
    GENERATION = "generation"
    EVALUATION = "evaluation"


class AgentWorkflowStage(Enum):
    """Agent workflow stages."""
    PLANNING = "planning"
    TOOL_SELECTION = "tool_selection"
    EXECUTION = "execution"
    REFLECTION = "reflection"
    MEMORY_UPDATE = "memory_update"
    RESPONSE_GENERATION = "response_generation"


class ModelType(Enum):
    """Model types for monitoring."""
    FOUNDATION_MODEL = "foundation"
    FINE_TUNED = "fine_tuned"
    ADAPTER = "adapter"
    EMBEDDING_MODEL = "embedding"
    REWARD_MODEL = "reward"


@dataclass
class LLMTrainingMetrics:
    """Comprehensive LLM training metrics."""
    # Identifiers
    run_id: str
    model_name: str
    model_type: ModelType
    stage: LLMTrainingStage
    timestamp: float

    # Training Progress
    epoch: int
    step: int
    global_step: int
    total_steps: int
    progress_percent: float

    # Loss Metrics
    train_loss: float
    validation_loss: float
    perplexity: float
    gradient_norm: float
    learning_rate: float

    # Performance Metrics
    tokens_per_second: float
    sequences_per_second: float
    gpu_memory_allocated: float  # GB
    gpu_memory_cached: float    # GB
    gpu_utilization: float      # Percentage

    # Data Metrics
    batch_size: int
    sequence_length: int
    vocab_size: int
    total_parameters: int
    trainable_parameters: int

    # Quality Metrics
    bleu_score: Optional[float] = None
    rouge_score: Optional[float] = None
    bert_score: Optional[float] = None
    human_eval_score: Optional[float] = None

    # Resource Usage
    cpu_usage: float = 0.0
    memory_usage: float = 0.0
    disk_io: float = 0.0
    network_io: float = 0.0

    # Cost Metrics
    compute_cost_per_hour: float = 0.0
    estimated_total_cost: float = 0.0

    # Metadata
    hyperparameters: Dict[str, Any] = field(default_factory=dict)
    dataset_info: Dict[str, Any] = field(default_factory=dict)
    infrastructure: Dict[str, Any] = field(default_factory=dict)


@dataclass
class RAGPipelineMetrics:
    """RAG pipeline performance metrics."""
    # Identifiers
    pipeline_id: str
    stage: RAGStage
    timestamp: float

    # Document Processing
    documents_processed: int
    chunks_generated: int
    embeddings_created: int
    index_size: int

    # Performance Metrics
    processing_time: float      # seconds
    throughput: float          # items/second
    latency_p50: float         # milliseconds
    latency_p95: float         # milliseconds
    latency_p99: float         # milliseconds

    # Quality Metrics
    retrieval_accuracy: float
    relevance_score: float
    diversity_score: float
    coverage_score: float

    # Generation Metrics
    generation_time: float
    response_length: int
    coherence_score: float
    factuality_score: float

    # Resource Usage
    cpu_usage: float
    memory_usage: float
    gpu_usage: float
    storage_usage: float

    # Cost and Token Usage
    input_tokens: int
    output_tokens: int
    total_tokens: int
    cost_per_query: float

    # Error Tracking
    error_rate: float
    timeout_rate: float
    retry_count: int

    # Metadata
    model_info: Dict[str, Any] = field(default_factory=dict)
    dataset_info: Dict[str, Any] = field(default_factory=dict)
    configuration: Dict[str, Any] = field(default_factory=dict)


@dataclass
class AgentWorkflowMetrics:
    """Agent workflow and orchestration metrics."""
    # Identifiers
    agent_id: str
    workflow_id: str
    stage: AgentWorkflowStage
    timestamp: float

    # Workflow Progress
    task_id: str
    subtask_count: int
    completed_subtasks: int
    failed_subtasks: int

    # Performance Metrics
    execution_time: float
    planning_time: float
    tool_execution_time: float
    reflection_time: float

    # Decision Making
    tools_selected: List[str]
    confidence_score: float
    decision_rationale: str
    alternative_plans: int

    # Resource Utilization
    memory_operations: int
    retrieval_calls: int
    generation_calls: int
    tool_invocations: int

    # Quality Metrics
    task_success_rate: float
    goal_achievement_score: float
    efficiency_score: float
    creativity_score: float

    # Cost Tracking
    total_tokens_used: int
    api_calls_made: int
    compute_cost: float
    time_to_completion: float

    # Error Handling
    error_count: int
    recovery_attempts: int
    fallback_used: bool
    human_intervention: bool

    # Metadata
    agent_config: Dict[str, Any] = field(default_factory=dict)
    environment_state: Dict[str, Any] = field(default_factory=dict)
    user_context: Dict[str, Any] = field(default_factory=dict)


@dataclass
class EnterpriseGovernanceMetrics:
    """Enterprise AI governance and compliance metrics."""
    # Identifiers
    organization_id: str
    project_id: str
    timestamp: float

    # Usage Metrics
    active_models: int
    total_inference_calls: int
    total_training_hours: float
    data_processed_gb: float

    # Cost Management
    monthly_spend: float
    budget_utilization: float
    cost_per_model: Dict[str, float]
    cost_optimization_savings: float

    # Compliance
    data_privacy_score: float
    security_compliance_score: float
    bias_detection_results: Dict[str, float]
    fairness_metrics: Dict[str, float]

    # Performance
    model_performance_scores: Dict[str, float]
    latency_sla_compliance: float
    availability_percentage: float
    error_rates: Dict[str, float]

    # Risk Management
    model_drift_detected: bool
    data_drift_detected: bool
    anomaly_count: int
    security_incidents: int

    # Resource Efficiency
    gpu_utilization_avg: float
    compute_efficiency_score: float
    carbon_footprint_kg: float
    sustainability_score: float


@dataclass
class LLMFinOpsMetrics:
    """Comprehensive FinOps metrics for LLM operations cost tracking."""
    # Identifiers
    operation_id: str
    operation_type: str  # training, inference, fine_tuning, evaluation
    model_name: str
    organization_id: str
    project_id: str
    team_id: str
    timestamp: float

    # Compute Costs
    gpu_cost_per_hour: float
    cpu_cost_per_hour: float
    memory_cost_per_gb_hour: float
    storage_cost_per_gb_hour: float
    network_cost_per_gb: float

    # Resource Usage (for cost calculation)
    gpu_hours_used: float
    gpu_type: str  # A100, H100, V100, etc.
    gpu_count: int
    cpu_hours_used: float
    memory_gb_hours: float
    storage_gb_hours: float
    network_gb_transferred: float

    # LLM-Specific Costs
    token_processing_cost: float  # Cost per token processed
    model_loading_cost: float    # Cost to load model into memory
    checkpoint_storage_cost: float
    data_ingestion_cost: float

    # Training-Specific Costs
    training_data_cost: float
    preprocessing_cost: float
    evaluation_cost: float
    experiment_tracking_cost: float

    # Inference-Specific Costs
    api_call_cost: float
    latency_sla_penalty: float
    scaling_cost: float  # Auto-scaling overhead
    load_balancing_cost: float

    # Total Costs
    total_compute_cost: float
    total_infrastructure_cost: float
    total_operational_cost: float
    total_cost: float

    # Cost Optimization Metrics
    efficiency_score: float  # Cost per unit of work
    waste_percentage: float  # Unused resource percentage
    optimization_potential: float  # Estimated savings opportunity
    cost_per_token: float
    cost_per_request: float
    cost_per_model_parameter: float

    # Budget and Forecasting
    budget_allocated: float
    budget_consumed: float
    budget_remaining: float
    projected_monthly_cost: float
    cost_trend: str  # increasing, decreasing, stable

    # Chargeback and Allocation
    department_allocation: Dict[str, float]
    project_allocation: Dict[str, float]
    user_allocation: Dict[str, float]

    # Carbon and Sustainability Costs
    carbon_cost_usd: float
    sustainability_penalty: float
    green_compute_premium: float

    # Alert Thresholds
    cost_threshold_exceeded: bool
    budget_alert_triggered: bool
    efficiency_below_target: bool

    # Metadata
    pricing_model: str  # on_demand, reserved, spot, hybrid
    region: str
    availability_zone: str
    cost_center: str
    tags: Dict[str, str] = field(default_factory=dict)


@dataclass
class ComputeResourceCosts:
    """Real-time compute resource cost tracking."""
    # Identifiers
    resource_id: str
    resource_type: str  # gpu, cpu, memory, storage, network
    timestamp: float

    # Resource Specifications
    resource_specs: Dict[str, Any]
    provider: str  # aws, azure, gcp, on_premise
    instance_type: str
    region: str

    # Usage Metrics
    utilization_percentage: float
    active_time_hours: float
    idle_time_hours: float
    peak_usage: float
    average_usage: float

    # Cost Breakdown
    base_cost_per_hour: float
    usage_based_cost: float
    reserved_instance_discount: float
    spot_instance_savings: float
    volume_discount: float

    # Real-time Costs
    current_hourly_cost: float
    total_session_cost: float
    estimated_daily_cost: float
    estimated_monthly_cost: float

    # Efficiency Metrics
    cost_efficiency_score: float
    waste_cost: float  # Cost of unused resources
    optimization_recommendation: str


@dataclass
class CostOptimizationInsights:
    """AI-driven cost optimization insights and recommendations."""
    # Identifiers
    analysis_id: str
    organization_id: str
    timestamp: float
    analysis_period: str  # daily, weekly, monthly

    # Cost Analysis
    total_analyzed_cost: float
    potential_savings: float
    savings_percentage: float

    # Resource Optimization
    underutilized_resources: List[Dict[str, Any]]
    oversized_instances: List[Dict[str, Any]]
    rightsizing_recommendations: List[Dict[str, Any]]

    # Scheduling Optimization
    non_peak_hour_savings: float
    spot_instance_opportunities: List[Dict[str, Any]]
    reserved_instance_recommendations: List[Dict[str, Any]]

    # Model Optimization
    model_compression_savings: float
    quantization_opportunities: List[Dict[str, Any]]
    distillation_recommendations: List[Dict[str, Any]]

    # Infrastructure Optimization
    multi_cloud_optimization: Dict[str, float]
    region_cost_comparison: Dict[str, float]
    auto_scaling_recommendations: List[Dict[str, Any]]

    # Operational Optimization
    automated_shutdown_savings: float
    workload_scheduling_optimization: float
    data_transfer_optimization: float

    # Implementation Priority
    quick_wins: List[Dict[str, Any]]  # <1 week implementation
    medium_term: List[Dict[str, Any]]  # 1-4 weeks
    long_term: List[Dict[str, Any]]   # >1 month

    # Risk Assessment
    implementation_risk: str  # low, medium, high
    business_impact: str
    confidence_score: float


@dataclass
class VectorDBMetrics:
    """Real-time vector database monitoring metrics."""
    # Identifiers
    db_id: str
    db_type: str  # pinecone, weaviate, chroma, faiss
    index_name: str
    timestamp: float

    # Storage Metrics
    total_vectors: int
    vector_dimensions: int
    index_size_bytes: int
    index_size_gb: float
    memory_usage_gb: float

    # Performance Metrics
    query_latency_ms: float
    index_build_time_ms: float
    queries_per_second: float
    insert_throughput: int

    # Quality Metrics
    index_efficiency: float
    recall_at_k: Dict[int, float]  # recall@1, recall@5, recall@10
    precision_at_k: Dict[int, float]

    # Health Metrics
    connection_status: str  # connected, degraded, disconnected
    error_rate: float
    availability_percentage: float

    # Cost Metrics
    storage_cost_per_gb: float
    query_cost_per_1k: float
    total_cost_usd: float


@dataclass
class RetrievalMetrics:
    """Detailed retrieval quality and performance metrics."""
    # Identifiers
    query_id: str
    pipeline_id: str
    timestamp: float

    # Query Information
    query_text: str
    query_embedding: Optional[List[float]]
    query_type: str  # semantic, keyword, hybrid

    # Retrieval Results
    retrieved_docs: List[Dict[str, Any]]
    retrieval_time_ms: float
    total_candidates: int
    final_results_count: int

    # Quality Scores
    relevance_scores: List[float]
    semantic_similarity_scores: List[float]
    diversity_score: float
    coverage_score: float

    # Ranking Metrics
    ndcg_at_k: Dict[int, float]  # NDCG@1, @5, @10
    mrr: float  # Mean Reciprocal Rank
    map_score: float  # Mean Average Precision

    # Ground Truth Comparison (if available)
    ground_truth_docs: Optional[List[str]]
    precision_at_k: Optional[Dict[int, float]]
    recall_at_k: Optional[Dict[int, float]]

    # Performance Breakdown
    embedding_time_ms: float
    vector_search_time_ms: float
    reranking_time_ms: float
    post_processing_time_ms: float

    # Context
    user_feedback: Optional[str]  # positive, negative, neutral
    business_context: Dict[str, Any]


class VectorDBConnector:
    """Real vector database monitoring and integration."""

    def __init__(self):
        self.connections = {}
        self.metrics_history = deque(maxlen=10000)
        self.monitoring_active = False
        self.monitor_thread = None

    def connect_pinecone(self, api_key: str, environment: str, index_name: str):
        """Connect to Pinecone vector database."""
        try:
            # Simulated Pinecone connection (replace with actual pinecone-client)
            connection_config = {
                'type': 'pinecone',
                'api_key': api_key,
                'environment': environment,
                'index_name': index_name,
                'connected_at': time.time()
            }
            self.connections[f"pinecone_{index_name}"] = connection_config
            return True
        except Exception as e:
            logging.error(f"Failed to connect to Pinecone: {e}")
            return False

    def connect_weaviate(self, url: str, api_key: Optional[str] = None):
        """Connect to Weaviate vector database."""
        try:
            # Simulated Weaviate connection (replace with actual weaviate-client)
            connection_config = {
                'type': 'weaviate',
                'url': url,
                'api_key': api_key,
                'connected_at': time.time()
            }
            self.connections[f"weaviate_{hashlib.md5(url.encode()).hexdigest()[:8]}"] = connection_config
            return True
        except Exception as e:
            logging.error(f"Failed to connect to Weaviate: {e}")
            return False

    def connect_chroma(self, persist_directory: str):
        """Connect to ChromaDB."""
        try:
            # Simulated ChromaDB connection (replace with actual chromadb)
            connection_config = {
                'type': 'chroma',
                'persist_directory': persist_directory,
                'connected_at': time.time()
            }
            self.connections[f"chroma_{hashlib.md5(persist_directory.encode()).hexdigest()[:8]}"] = connection_config
            return True
        except Exception as e:
            logging.error(f"Failed to connect to ChromaDB: {e}")
            return False

    def connect_faiss(self, index_path: str):
        """Connect to FAISS index."""
        try:
            if os.path.exists(index_path):
                connection_config = {
                    'type': 'faiss',
                    'index_path': index_path,
                    'connected_at': time.time(),
                    'index_size': os.path.getsize(index_path)
                }
                self.connections[f"faiss_{hashlib.md5(index_path.encode()).hexdigest()[:8]}"] = connection_config
                return True
            else:
                logging.error(f"FAISS index not found: {index_path}")
                return False
        except Exception as e:
            logging.error(f"Failed to connect to FAISS: {e}")
            return False

    def get_real_vector_metrics(self, db_id: str) -> Optional[VectorDBMetrics]:
        """Get actual vector database metrics."""
        if db_id not in self.connections:
            return None

        connection = self.connections[db_id]
        db_type = connection['type']

        try:
            if db_type == 'pinecone':
                return self._get_pinecone_metrics(db_id, connection)
            elif db_type == 'weaviate':
                return self._get_weaviate_metrics(db_id, connection)
            elif db_type == 'chroma':
                return self._get_chroma_metrics(db_id, connection)
            elif db_type == 'faiss':
                return self._get_faiss_metrics(db_id, connection)
        except Exception as e:
            logging.error(f"Failed to get metrics for {db_id}: {e}")

        return None

    def _get_pinecone_metrics(self, db_id: str, connection: Dict) -> VectorDBMetrics:
        """Get Pinecone-specific metrics."""
        # Simulated metrics (replace with actual Pinecone API calls)
        index_stats = {
            'totalVectorCount': 2340000 + int(time.time() % 1000),
            'dimension': 1536,
            'indexFullness': 0.847
        }

        return VectorDBMetrics(
            db_id=db_id,
            db_type='pinecone',
            index_name=connection['index_name'],
            timestamp=time.time(),
            total_vectors=index_stats['totalVectorCount'],
            vector_dimensions=index_stats['dimension'],
            index_size_bytes=index_stats['totalVectorCount'] * index_stats['dimension'] * 4,  # 4 bytes per float
            index_size_gb=(index_stats['totalVectorCount'] * index_stats['dimension'] * 4) / (1024**3),
            memory_usage_gb=847.3 + (time.time() % 100),
            query_latency_ms=45.2 + (np.random.random() * 20),
            index_build_time_ms=0,  # Not applicable for Pinecone
            queries_per_second=1247.5 + (np.random.random() * 200),
            insert_throughput=850 + int(np.random.random() * 100),
            index_efficiency=0.987 + (np.random.random() * 0.01),
            recall_at_k={1: 0.947, 5: 0.989, 10: 0.995},
            precision_at_k={1: 0.947, 5: 0.923, 10: 0.889},
            connection_status='connected',
            error_rate=0.001 + (np.random.random() * 0.005),
            availability_percentage=99.97 + (np.random.random() * 0.03),
            storage_cost_per_gb=0.70,
            query_cost_per_1k=0.0004,
            total_cost_usd=592.80 + (time.time() % 50)
        )

    def _get_weaviate_metrics(self, db_id: str, connection: Dict) -> VectorDBMetrics:
        """Get Weaviate-specific metrics."""
        # Simulated metrics (replace with actual Weaviate API calls)
        return VectorDBMetrics(
            db_id=db_id,
            db_type='weaviate',
            index_name='default',
            timestamp=time.time(),
            total_vectors=1890000 + int(time.time() % 1000),
            vector_dimensions=768,
            index_size_bytes=1890000 * 768 * 4,
            index_size_gb=(1890000 * 768 * 4) / (1024**3),
            memory_usage_gb=534.2 + (time.time() % 50),
            query_latency_ms=32.1 + (np.random.random() * 15),
            index_build_time_ms=125000 + int(np.random.random() * 50000),
            queries_per_second=2840.3 + (np.random.random() * 300),
            insert_throughput=1200 + int(np.random.random() * 200),
            index_efficiency=0.934 + (np.random.random() * 0.02),
            recall_at_k={1: 0.912, 5: 0.967, 10: 0.987},
            precision_at_k={1: 0.912, 5: 0.898, 10: 0.856},
            connection_status='connected',
            error_rate=0.002 + (np.random.random() * 0.008),
            availability_percentage=99.89 + (np.random.random() * 0.11),
            storage_cost_per_gb=0.25,
            query_cost_per_1k=0.0002,
            total_cost_usd=118.50 + (time.time() % 30)
        )

    def _get_chroma_metrics(self, db_id: str, connection: Dict) -> VectorDBMetrics:
        """Get ChromaDB-specific metrics."""
        persist_dir = connection['persist_directory']
        index_size = 0

        try:
            if os.path.exists(persist_dir):
                for root, dirs, files in os.walk(persist_dir):
                    for file in files:
                        index_size += os.path.getsize(os.path.join(root, file))
        except:
            index_size = 0

        return VectorDBMetrics(
            db_id=db_id,
            db_type='chroma',
            index_name='default_collection',
            timestamp=time.time(),
            total_vectors=560000 + int(time.time() % 500),
            vector_dimensions=384,
            index_size_bytes=index_size,
            index_size_gb=index_size / (1024**3),
            memory_usage_gb=89.7 + (time.time() % 20),
            query_latency_ms=18.3 + (np.random.random() * 10),
            index_build_time_ms=45000 + int(np.random.random() * 20000),
            queries_per_second=4200.1 + (np.random.random() * 500),
            insert_throughput=2100 + int(np.random.random() * 300),
            index_efficiency=0.891 + (np.random.random() * 0.03),
            recall_at_k={1: 0.876, 5: 0.945, 10: 0.978},
            precision_at_k={1: 0.876, 5: 0.843, 10: 0.812},
            connection_status='connected',
            error_rate=0.001 + (np.random.random() * 0.004),
            availability_percentage=99.95 + (np.random.random() * 0.05),
            storage_cost_per_gb=0.10,
            query_cost_per_1k=0.0001,
            total_cost_usd=23.40 + (time.time() % 10)
        )

    def _get_faiss_metrics(self, db_id: str, connection: Dict) -> VectorDBMetrics:
        """Get FAISS-specific metrics."""
        index_path = connection['index_path']
        index_size = connection.get('index_size', 0)

        # Get memory usage for the process
        process = psutil.Process()
        memory_info = process.memory_info()

        return VectorDBMetrics(
            db_id=db_id,
            db_type='faiss',
            index_name=os.path.basename(index_path),
            timestamp=time.time(),
            total_vectors=1200000 + int(time.time() % 800),
            vector_dimensions=512,
            index_size_bytes=index_size,
            index_size_gb=index_size / (1024**3),
            memory_usage_gb=memory_info.rss / (1024**3),
            query_latency_ms=8.7 + (np.random.random() * 5),
            index_build_time_ms=180000 + int(np.random.random() * 60000),
            queries_per_second=8500.2 + (np.random.random() * 1000),
            insert_throughput=0,  # FAISS requires rebuild for new vectors
            index_efficiency=0.967 + (np.random.random() * 0.02),
            recall_at_k={1: 0.945, 5: 0.987, 10: 0.995},
            precision_at_k={1: 0.945, 5: 0.934, 10: 0.901},
            connection_status='connected',
            error_rate=0.0005 + (np.random.random() * 0.002),
            availability_percentage=99.99 + (np.random.random() * 0.01),
            storage_cost_per_gb=0.05,  # Local storage cost
            query_cost_per_1k=0.00005,
            total_cost_usd=12.80 + (time.time() % 5)
        )

    def start_monitoring(self, interval_seconds: float = 30.0):
        """Start continuous monitoring of all connected databases."""
        self.monitoring_active = True

        def monitor_loop():
            while self.monitoring_active:
                for db_id in self.connections:
                    metrics = self.get_real_vector_metrics(db_id)
                    if metrics:
                        self.metrics_history.append(metrics)
                time.sleep(interval_seconds)

        self.monitor_thread = threading.Thread(target=monitor_loop, daemon=True)
        self.monitor_thread.start()
        logging.info("Vector database monitoring started")

    def stop_monitoring(self):
        """Stop continuous monitoring."""
        self.monitoring_active = False
        if self.monitor_thread:
            self.monitor_thread.join(timeout=5)
        logging.info("Vector database monitoring stopped")

    def get_latest_metrics(self) -> Dict[str, VectorDBMetrics]:
        """Get latest metrics for all connected databases."""
        latest_metrics = {}
        for db_id in self.connections:
            metrics = self.get_real_vector_metrics(db_id)
            if metrics:
                latest_metrics[db_id] = metrics
        return latest_metrics


class RetrievalEvaluator:
    """Advanced retrieval quality measurement and evaluation."""

    def __init__(self):
        self.evaluation_cache = {}
        self.ground_truth_store = {}
        self.relevance_judgments = defaultdict(dict)

    def calculate_ndcg_at_k(self, retrieved_docs: List[Dict], ground_truth: List[str], k: int = 10) -> float:
        """Calculate Normalized Discounted Cumulative Gain at k."""
        if not ground_truth or not retrieved_docs:
            return 0.0

        # Create relevance scores (1 if doc in ground truth, 0 otherwise)
        relevance_scores = []
        for i, doc in enumerate(retrieved_docs[:k]):
            doc_id = doc.get('id', str(i))
            relevance_scores.append(1 if doc_id in ground_truth else 0)

        # Calculate DCG
        dcg = 0.0
        for i, rel in enumerate(relevance_scores):
            dcg += rel / np.log2(i + 2)  # i+2 because log2(1) = 0

        # Calculate IDCG (Ideal DCG)
        ideal_relevance = sorted([1] * min(len(ground_truth), k), reverse=True)
        idcg = 0.0
        for i, rel in enumerate(ideal_relevance):
            idcg += rel / np.log2(i + 2)

        return dcg / idcg if idcg > 0 else 0.0

    def calculate_mrr(self, retrieved_docs: List[Dict], ground_truth: List[str]) -> float:
        """Calculate Mean Reciprocal Rank."""
        for i, doc in enumerate(retrieved_docs):
            doc_id = doc.get('id', str(i))
            if doc_id in ground_truth:
                return 1.0 / (i + 1)
        return 0.0

    def calculate_map(self, retrieved_docs: List[Dict], ground_truth: List[str], k: int = 10) -> float:
        """Calculate Mean Average Precision at k."""
        if not ground_truth:
            return 0.0

        relevant_found = 0
        precision_sum = 0.0

        for i, doc in enumerate(retrieved_docs[:k]):
            doc_id = doc.get('id', str(i))
            if doc_id in ground_truth:
                relevant_found += 1
                precision_at_i = relevant_found / (i + 1)
                precision_sum += precision_at_i

        return precision_sum / len(ground_truth) if len(ground_truth) > 0 else 0.0

    def measure_semantic_similarity(self, query: str, retrieved_docs: List[Dict]) -> List[float]:
        """Measure semantic similarity between query and retrieved documents."""
        # Simulated semantic similarity calculation
        # In real implementation, use sentence transformers or similar
        similarities = []
        for doc in retrieved_docs:
            # Simulate similarity based on text overlap and randomization
            doc_text = doc.get('text', doc.get('content', ''))
            base_similarity = len(set(query.lower().split()) & set(doc_text.lower().split())) / max(len(query.split()), 1)
            noise = np.random.normal(0, 0.1)
            similarity = max(0, min(1, base_similarity + noise))
            similarities.append(similarity)
        return similarities

    def calculate_diversity_score(self, retrieved_docs: List[Dict]) -> float:
        """Calculate diversity score of retrieved documents."""
        if len(retrieved_docs) < 2:
            return 1.0

        # Simulate diversity calculation based on document content
        unique_topics = set()
        for doc in retrieved_docs:
            # Simple topic extraction simulation
            content = doc.get('text', doc.get('content', ''))
            words = content.lower().split()
            # Use first few words as topic indicators
            topic = '_'.join(sorted(words[:3]))
            unique_topics.add(topic)

        return len(unique_topics) / len(retrieved_docs)

    def evaluate_retrieval(self, query: str, retrieved_docs: List[Dict],
                         ground_truth: Optional[List[str]] = None) -> RetrievalMetrics:
        """Comprehensive retrieval evaluation."""
        query_id = hashlib.md5(f"{query}_{time.time()}".encode()).hexdigest()

        # Calculate semantic similarities
        semantic_similarities = self.measure_semantic_similarity(query, retrieved_docs)

        # Calculate relevance scores (simulated)
        relevance_scores = [max(0.3, sim + np.random.normal(0, 0.1)) for sim in semantic_similarities]

        # Calculate diversity and coverage
        diversity_score = self.calculate_diversity_score(retrieved_docs)
        coverage_score = min(1.0, len(retrieved_docs) / 10)  # Assuming 10 is ideal number

        # Calculate ranking metrics
        ndcg_scores = {}
        precision_scores = {}
        recall_scores = {}
        mrr = 0.0
        map_score = 0.0

        if ground_truth:
            for k in [1, 5, 10]:
                ndcg_scores[k] = self.calculate_ndcg_at_k(retrieved_docs, ground_truth, k)
                # Calculate precision@k and recall@k
                relevant_at_k = sum(1 for doc in retrieved_docs[:k]
                                  if doc.get('id', '') in ground_truth)
                precision_scores[k] = relevant_at_k / k if k > 0 else 0
                recall_scores[k] = relevant_at_k / len(ground_truth) if len(ground_truth) > 0 else 0

            mrr = self.calculate_mrr(retrieved_docs, ground_truth)
            map_score = self.calculate_map(retrieved_docs, ground_truth)

        return RetrievalMetrics(
            query_id=query_id,
            pipeline_id=f"pipeline_{int(time.time())}",
            timestamp=time.time(),
            query_text=query,
            query_embedding=None,  # Would be populated with actual embedding
            query_type='semantic',
            retrieved_docs=retrieved_docs,
            retrieval_time_ms=45.2 + (np.random.random() * 30),
            total_candidates=len(retrieved_docs) * 10,  # Simulate larger candidate pool
            final_results_count=len(retrieved_docs),
            relevance_scores=relevance_scores,
            semantic_similarity_scores=semantic_similarities,
            diversity_score=diversity_score,
            coverage_score=coverage_score,
            ndcg_at_k=ndcg_scores,
            mrr=mrr,
            map_score=map_score,
            ground_truth_docs=ground_truth,
            precision_at_k=precision_scores if ground_truth else None,
            recall_at_k=recall_scores if ground_truth else None,
            embedding_time_ms=12.3 + (np.random.random() * 8),
            vector_search_time_ms=23.1 + (np.random.random() * 15),
            reranking_time_ms=8.7 + (np.random.random() * 5),
            post_processing_time_ms=1.1 + (np.random.random() * 2),
            user_feedback=None,
            business_context={}
        )

    def track_relevance_judgments(self, query_id: str, doc_id: str, relevance_score: float):
        """Track human relevance judgments for continuous improvement."""
        self.relevance_judgments[query_id][doc_id] = {
            'score': relevance_score,
            'timestamp': time.time()
        }

    def add_ground_truth(self, query: str, relevant_doc_ids: List[str]):
        """Add ground truth data for evaluation."""
        query_hash = hashlib.md5(query.encode()).hexdigest()
        self.ground_truth_store[query_hash] = relevant_doc_ids


class RAGPipelineMonitor:
    """Real-time RAG pipeline monitoring and tracking."""

    def __init__(self):
        self.active_pipelines = {}
        self.document_processor = None
        self.embedding_model = None
        self.vector_db_connector = VectorDBConnector()
        self.retrieval_evaluator = RetrievalEvaluator()
        self.metrics_history = deque(maxlen=50000)
        self.real_time_stats = defaultdict(list)

    def track_document_ingestion(self, doc_processor: Any):
        """Track document processing pipeline."""
        self.document_processor = doc_processor

    def monitor_embedding_generation(self, embedding_model: Any):
        """Monitor embedding model performance."""
        self.embedding_model = embedding_model

    def measure_vector_store_operations(self, vector_db: Any):
        """Measure vector database operations."""
        # This would integrate with actual vector database
        pass

    def simulate_document_processing(self) -> Dict[str, Any]:
        """Simulate real document processing metrics."""
        return {
            'documents_processed': 2847 + int(np.random.random() * 100),
            'chunks_generated': 89300 + int(np.random.random() * 500),
            'processing_rate_docs_per_hour': 2847 + int(np.random.random() * 200),
            'avg_chunk_size': 512 + int(np.random.random() * 100),
            'processing_time_ms': 234.5 + (np.random.random() * 50),
            'errors_count': int(np.random.random() * 5),
            'success_rate': 0.991 + (np.random.random() * 0.008)
        }

    def simulate_embedding_generation(self) -> Dict[str, Any]:
        """Simulate embedding generation metrics."""
        return {
            'embeddings_generated': 87100 + int(np.random.random() * 300),
            'generation_rate_per_min': 2847.3 + (np.random.random() * 100),
            'avg_generation_time_ms': 23.4 + (np.random.random() * 10),
            'model_type': 'text-embedding-ada-002',
            'embedding_dimensions': 1536,
            'batch_size': 100,
            'gpu_utilization': 78.4 + (np.random.random() * 20),
            'memory_usage_gb': 12.7 + (np.random.random() * 3)
        }

    def get_real_time_pipeline_metrics(self, pipeline_id: str) -> Dict[str, Any]:
        """Get comprehensive real-time pipeline metrics."""
        doc_metrics = self.simulate_document_processing()
        embedding_metrics = self.simulate_embedding_generation()

        # Get vector DB metrics if connected
        vector_metrics = {}
        if self.vector_db_connector.connections:
            latest_db_metrics = self.vector_db_connector.get_latest_metrics()
            if latest_db_metrics:
                db_id = list(latest_db_metrics.keys())[0]
                db_metrics = latest_db_metrics[db_id]
                vector_metrics = {
                    'total_vectors': db_metrics.total_vectors,
                    'index_size_gb': db_metrics.index_size_gb,
                    'query_latency_ms': db_metrics.query_latency_ms,
                    'queries_per_second': db_metrics.queries_per_second,
                    'index_efficiency': db_metrics.index_efficiency
                }

        return {
            'pipeline_id': pipeline_id,
            'timestamp': time.time(),
            'document_processing': doc_metrics,
            'embedding_generation': embedding_metrics,
            'vector_storage': vector_metrics,
            'overall_health': 'healthy',
            'throughput_docs_per_hour': doc_metrics['processing_rate_docs_per_hour'],
            'end_to_end_latency_ms': 187.3 + (np.random.random() * 40)
        }


class LLMObservabilityHub:
    """
    Central hub for LLM, RAG, and Agent observability.
    """

    def __init__(self):
        # Metrics storage
        self.llm_training_metrics = deque(maxlen=100000)
        self.rag_pipeline_metrics = deque(maxlen=50000)
        self.agent_workflow_metrics = deque(maxlen=50000)
        self.governance_metrics = deque(maxlen=10000)

        # FinOps metrics storage
        self.finops_metrics = deque(maxlen=50000)
        self.compute_cost_metrics = deque(maxlen=100000)
        self.cost_optimization_insights = deque(maxlen=1000)

        # Real-time tracking
        self.active_training_runs = {}
        self.active_rag_pipelines = {}
        self.active_agent_workflows = {}
        self.active_compute_resources = {}

        # Alerts and thresholds
        self.alert_thresholds = self._initialize_alert_thresholds()
        self.active_alerts = deque(maxlen=1000)

        # Performance baselines
        self.performance_baselines = {}

        # Enhanced cost tracking
        self.cost_budgets = {}
        self.spend_tracking = defaultdict(float)
        self.cost_allocation_matrix = {}
        self.optimization_recommendations = []

        # FinOps configuration
        self.pricing_models = self._initialize_pricing_models()

        # Real RAG measurement components
        self.vector_db_connector = VectorDBConnector()
        self.retrieval_evaluator = RetrievalEvaluator()
        self.rag_pipeline_monitor = RAGPipelineMonitor()

        # Real-time data streaming
        self.real_time_enabled = False
        self.data_stream_thread = None

    def _initialize_alert_thresholds(self) -> Dict[str, Any]:
        """Initialize alert thresholds for different metrics."""
        return {
            'llm_training': {
                'loss_increase_threshold': 0.1,
                'gpu_memory_threshold': 90.0,
                'learning_rate_min': 1e-7,
                'gradient_norm_max': 10.0
            },
            'rag_pipeline': {
                'latency_p95_threshold': 2000,  # milliseconds
                'error_rate_threshold': 0.05,
                'retrieval_accuracy_min': 0.8,
                'cost_per_query_max': 0.01
            },
            'agent_workflow': {
                'execution_time_max': 300,  # seconds
                'success_rate_min': 0.9,
                'token_usage_max': 100000,
                'error_rate_max': 0.1
            },
            'governance': {
                'monthly_spend_threshold': 50000,
                'model_drift_threshold': 0.1,
                'security_score_min': 0.9,
                'bias_score_max': 0.2
            }
        }

    def collect_llm_training_metrics(self, metrics: LLMTrainingMetrics):
        """Collect LLM training metrics."""
        self.llm_training_metrics.append(metrics)
        self.active_training_runs[metrics.run_id] = metrics
        self._check_llm_training_alerts(metrics)

    def collect_rag_pipeline_metrics(self, metrics: RAGPipelineMetrics):
        """Collect RAG pipeline metrics."""
        self.rag_pipeline_metrics.append(metrics)
        self.active_rag_pipelines[metrics.pipeline_id] = metrics
        self._check_rag_pipeline_alerts(metrics)

    def collect_agent_workflow_metrics(self, metrics: AgentWorkflowMetrics):
        """Collect agent workflow metrics."""
        self.agent_workflow_metrics.append(metrics)
        self.active_agent_workflows[f"{metrics.agent_id}_{metrics.workflow_id}"] = metrics
        self._check_agent_workflow_alerts(metrics)

    def collect_governance_metrics(self, metrics: EnterpriseGovernanceMetrics):
        """Collect enterprise governance metrics."""
        self.governance_metrics.append(metrics)
        self._check_governance_alerts(metrics)

    def _check_llm_training_alerts(self, metrics: LLMTrainingMetrics):
        """Check for LLM training alerts."""
        alerts = []
        thresholds = self.alert_thresholds['llm_training']

        if metrics.gpu_memory_allocated > thresholds['gpu_memory_threshold']:
            alerts.append({
                'type': 'GPU_MEMORY_HIGH',
                'severity': 'WARNING',
                'message': f"GPU memory usage {metrics.gpu_memory_allocated:.1f}% exceeds threshold",
                'run_id': metrics.run_id,
                'timestamp': metrics.timestamp
            })

        if metrics.gradient_norm > thresholds['gradient_norm_max']:
            alerts.append({
                'type': 'GRADIENT_EXPLOSION',
                'severity': 'CRITICAL',
                'message': f"Gradient norm {metrics.gradient_norm:.2f} indicates potential gradient explosion",
                'run_id': metrics.run_id,
                'timestamp': metrics.timestamp
            })

        for alert in alerts:
            self.active_alerts.append(alert)

    def _check_rag_pipeline_alerts(self, metrics: RAGPipelineMetrics):
        """Check for RAG pipeline alerts."""
        alerts = []
        thresholds = self.alert_thresholds['rag_pipeline']

        if metrics.latency_p95 > thresholds['latency_p95_threshold']:
            alerts.append({
                'type': 'HIGH_LATENCY',
                'severity': 'WARNING',
                'message': f"RAG pipeline latency {metrics.latency_p95:.0f}ms exceeds threshold",
                'pipeline_id': metrics.pipeline_id,
                'timestamp': metrics.timestamp
            })

        if metrics.error_rate > thresholds['error_rate_threshold']:
            alerts.append({
                'type': 'HIGH_ERROR_RATE',
                'severity': 'CRITICAL',
                'message': f"RAG pipeline error rate {metrics.error_rate:.2%} exceeds threshold",
                'pipeline_id': metrics.pipeline_id,
                'timestamp': metrics.timestamp
            })

        for alert in alerts:
            self.active_alerts.append(alert)

    def _check_agent_workflow_alerts(self, metrics: AgentWorkflowMetrics):
        """Check for agent workflow alerts."""
        alerts = []
        thresholds = self.alert_thresholds['agent_workflow']

        if metrics.execution_time > thresholds['execution_time_max']:
            alerts.append({
                'type': 'SLOW_EXECUTION',
                'severity': 'WARNING',
                'message': f"Agent workflow execution time {metrics.execution_time:.1f}s exceeds threshold",
                'agent_id': metrics.agent_id,
                'workflow_id': metrics.workflow_id,
                'timestamp': metrics.timestamp
            })

        if metrics.task_success_rate < thresholds['success_rate_min']:
            alerts.append({
                'type': 'LOW_SUCCESS_RATE',
                'severity': 'CRITICAL',
                'message': f"Agent success rate {metrics.task_success_rate:.2%} below threshold",
                'agent_id': metrics.agent_id,
                'timestamp': metrics.timestamp
            })

        for alert in alerts:
            self.active_alerts.append(alert)

    def _check_governance_alerts(self, metrics: EnterpriseGovernanceMetrics):
        """Check for governance alerts."""
        alerts = []
        thresholds = self.alert_thresholds['governance']

        if metrics.monthly_spend > thresholds['monthly_spend_threshold']:
            alerts.append({
                'type': 'BUDGET_EXCEEDED',
                'severity': 'CRITICAL',
                'message': f"Monthly spend ${metrics.monthly_spend:,.0f} exceeds budget threshold",
                'organization_id': metrics.organization_id,
                'timestamp': metrics.timestamp
            })

        if metrics.model_drift_detected:
            alerts.append({
                'type': 'MODEL_DRIFT',
                'severity': 'WARNING',
                'message': "Model drift detected - performance degradation possible",
                'organization_id': metrics.organization_id,
                'timestamp': metrics.timestamp
            })

        for alert in alerts:
            self.active_alerts.append(alert)

    def collect_finops_metrics(self, metrics: LLMFinOpsMetrics):
        """Collect FinOps metrics for cost tracking."""
        self.finops_metrics.append(metrics)

        # Update spend tracking
        self.spend_tracking[metrics.organization_id] += metrics.total_cost
        self.spend_tracking[f"{metrics.organization_id}_{metrics.project_id}"] += metrics.total_cost

        # Check cost-related alerts
        self._check_finops_alerts(metrics)

        # Trigger cost optimization analysis if needed
        if metrics.cost_threshold_exceeded:
            self._trigger_cost_optimization_analysis(metrics)

    def collect_compute_cost_metrics(self, metrics: ComputeResourceCosts):
        """Collect real-time compute cost metrics."""
        self.compute_cost_metrics.append(metrics)
        self.active_compute_resources[metrics.resource_id] = metrics

        # Check for cost efficiency alerts
        if metrics.cost_efficiency_score < 0.7:  # Below 70% efficiency
            self.active_alerts.append({
                'type': 'LOW_COST_EFFICIENCY',
                'severity': 'WARNING',
                'message': f"Resource {metrics.resource_id} efficiency: {metrics.cost_efficiency_score:.1%}",
                'resource_id': metrics.resource_id,
                'timestamp': metrics.timestamp
            })

    def collect_cost_optimization_insights(self, insights: CostOptimizationInsights):
        """Collect cost optimization insights and recommendations."""
        self.cost_optimization_insights.append(insights)
        self.optimization_recommendations.extend(insights.quick_wins)

    def _check_finops_alerts(self, metrics: LLMFinOpsMetrics):
        """Check for FinOps-related alerts."""
        alerts = []

        if metrics.cost_threshold_exceeded:
            alerts.append({
                'type': 'COST_THRESHOLD_EXCEEDED',
                'severity': 'WARNING',
                'message': f"Operation {metrics.operation_id} cost ${metrics.total_cost:.2f} exceeds threshold",
                'operation_id': metrics.operation_id,
                'timestamp': metrics.timestamp
            })

        if metrics.budget_alert_triggered:
            alerts.append({
                'type': 'BUDGET_ALERT',
                'severity': 'CRITICAL',
                'message': f"Budget consumption {metrics.budget_consumed/metrics.budget_allocated:.1%} for {metrics.project_id}",
                'project_id': metrics.project_id,
                'timestamp': metrics.timestamp
            })

        if metrics.efficiency_below_target:
            alerts.append({
                'type': 'LOW_EFFICIENCY',
                'severity': 'WARNING',
                'message': f"Operation efficiency {metrics.efficiency_score:.1%} below target",
                'operation_id': metrics.operation_id,
                'timestamp': metrics.timestamp
            })

        for alert in alerts:
            self.active_alerts.append(alert)

    def _trigger_cost_optimization_analysis(self, metrics: LLMFinOpsMetrics):
        """Trigger automated cost optimization analysis."""
        # This would integrate with the cost optimization engine
        analysis_request = {
            'organization_id': metrics.organization_id,
            'operation_type': metrics.operation_type,
            'cost_threshold_breach': metrics.total_cost,
            'timestamp': metrics.timestamp
        }
        # Schedule analysis (would be implemented with actual optimization engine)
        pass

    def _initialize_pricing_models(self) -> Dict[str, Dict]:
        """Initialize pricing models for different cloud providers and services."""
        return {
            'aws': {
                'p4d.24xlarge': {'gpu_cost_per_hour': 32.77, 'gpu_type': 'A100', 'gpu_count': 8},
                'p3.2xlarge': {'gpu_cost_per_hour': 3.06, 'gpu_type': 'V100', 'gpu_count': 1},
                'g5.xlarge': {'gpu_cost_per_hour': 1.006, 'gpu_type': 'A10G', 'gpu_count': 1},
            },
            'azure': {
                'Standard_ND96asr_v4': {'gpu_cost_per_hour': 27.20, 'gpu_type': 'A100', 'gpu_count': 8},
                'Standard_NC6s_v3': {'gpu_cost_per_hour': 3.06, 'gpu_type': 'V100', 'gpu_count': 1},
            },
            'gcp': {
                'a2-highgpu-8g': {'gpu_cost_per_hour': 29.39, 'gpu_type': 'A100', 'gpu_count': 8},
                'n1-standard-4-k80': {'gpu_cost_per_hour': 0.45, 'gpu_type': 'K80', 'gpu_count': 1},
            }
        }

    def calculate_operation_cost(self, operation_type: str, resources: Dict, duration_hours: float) -> float:
        """Calculate total cost for an operation."""
        total_cost = 0.0

        # GPU costs
        if 'gpu_type' in resources and 'gpu_count' in resources:
            gpu_cost = self._get_gpu_cost_per_hour(resources['gpu_type']) * resources['gpu_count']
            total_cost += gpu_cost * duration_hours

        # Add other resource costs (CPU, memory, storage, network)
        if 'cpu_hours' in resources:
            total_cost += resources['cpu_hours'] * 0.05  # $0.05 per CPU hour

        if 'memory_gb_hours' in resources:
            total_cost += resources['memory_gb_hours'] * 0.01  # $0.01 per GB hour

        if 'storage_gb_hours' in resources:
            total_cost += resources['storage_gb_hours'] * 0.001  # $0.001 per GB hour

        return total_cost

    def _get_gpu_cost_per_hour(self, gpu_type: str) -> float:
        """Get GPU cost per hour based on type."""
        gpu_costs = {
            'H100': 8.00,
            'A100': 4.00,
            'V100': 3.06,
            'A10G': 1.006,
            'T4': 0.35,
            'K80': 0.45
        }
        return gpu_costs.get(gpu_type, 2.00)  # Default cost

    def get_cost_summary(self, organization_id: str, time_period: str = 'monthly') -> Dict[str, Any]:
        """Get comprehensive cost summary for an organization."""
        relevant_metrics = [m for m in self.finops_metrics
                          if m.organization_id == organization_id]

        if not relevant_metrics:
            return {'error': 'No cost data found for organization'}

        total_cost = sum(m.total_cost for m in relevant_metrics)
        total_compute_cost = sum(m.total_compute_cost for m in relevant_metrics)
        total_waste = sum(m.total_cost * m.waste_percentage / 100 for m in relevant_metrics)

        return {
            'organization_id': organization_id,
            'period': time_period,
            'total_cost': total_cost,
            'compute_cost': total_compute_cost,
            'waste_cost': total_waste,
            'efficiency_score': (total_cost - total_waste) / total_cost if total_cost > 0 else 0,
            'cost_breakdown': {
                'training': sum(m.total_cost for m in relevant_metrics if m.operation_type == 'training'),
                'inference': sum(m.total_cost for m in relevant_metrics if m.operation_type == 'inference'),
                'fine_tuning': sum(m.total_cost for m in relevant_metrics if m.operation_type == 'fine_tuning'),
            },
            'optimization_potential': sum(m.optimization_potential for m in relevant_metrics),
            'carbon_cost': sum(m.carbon_cost_usd for m in relevant_metrics)
        }

    def get_training_summary(self, run_id: str) -> Dict[str, Any]:
        """Get comprehensive training run summary."""
        run_metrics = [m for m in self.llm_training_metrics if m.run_id == run_id]
        if not run_metrics:
            return {}

        latest = run_metrics[-1]
        return {
            'run_id': run_id,
            'model_name': latest.model_name,
            'stage': latest.stage.value,
            'progress': latest.progress_percent,
            'current_loss': latest.train_loss,
            'best_loss': min(m.train_loss for m in run_metrics),
            'total_steps': latest.total_steps,
            'current_step': latest.step,
            'tokens_per_second_avg': np.mean([m.tokens_per_second for m in run_metrics[-10:]]),
            'gpu_utilization_avg': np.mean([m.gpu_utilization for m in run_metrics[-10:]]),
            'estimated_cost': latest.estimated_total_cost,
            'duration_hours': (time.time() - run_metrics[0].timestamp) / 3600
        }

    def get_rag_pipeline_summary(self, pipeline_id: str) -> Dict[str, Any]:
        """Get RAG pipeline performance summary."""
        pipeline_metrics = [m for m in self.rag_pipeline_metrics if m.pipeline_id == pipeline_id]
        if not pipeline_metrics:
            return {}

        latest = pipeline_metrics[-1]
        return {
            'pipeline_id': pipeline_id,
            'current_stage': latest.stage.value,
            'documents_processed': latest.documents_processed,
            'avg_latency_p95': np.mean([m.latency_p95 for m in pipeline_metrics[-50:]]),
            'avg_retrieval_accuracy': np.mean([m.retrieval_accuracy for m in pipeline_metrics[-50:]]),
            'total_tokens_used': sum(m.total_tokens for m in pipeline_metrics),
            'total_cost': sum(m.cost_per_query for m in pipeline_metrics),
            'error_rate': np.mean([m.error_rate for m in pipeline_metrics[-50:]]),
            'throughput': latest.throughput
        }

    def get_agent_performance_summary(self, agent_id: str) -> Dict[str, Any]:
        """Get agent performance summary."""
        agent_metrics = [m for m in self.agent_workflow_metrics if m.agent_id == agent_id]
        if not agent_metrics:
            return {}

        return {
            'agent_id': agent_id,
            'total_workflows': len(set(m.workflow_id for m in agent_metrics)),
            'avg_success_rate': np.mean([m.task_success_rate for m in agent_metrics]),
            'avg_execution_time': np.mean([m.execution_time for m in agent_metrics]),
            'total_tasks_completed': sum(m.completed_subtasks for m in agent_metrics),
            'total_tokens_used': sum(m.total_tokens_used for m in agent_metrics),
            'total_cost': sum(m.compute_cost for m in agent_metrics),
            'error_rate': np.mean([m.error_count / max(m.subtask_count, 1) for m in agent_metrics]),
            'efficiency_score': np.mean([m.efficiency_score for m in agent_metrics])
        }

    def get_enterprise_dashboard_data(self) -> Dict[str, Any]:
        """Get comprehensive enterprise dashboard data."""
        if not self.governance_metrics:
            return {}

        latest_governance = list(self.governance_metrics)[-1]

        # Active training runs summary
        active_training = len(self.active_training_runs)

        # RAG pipelines summary
        active_rag = len(self.active_rag_pipelines)

        # Agent workflows summary
        active_agents = len(set(m.agent_id for m in self.agent_workflow_metrics if time.time() - m.timestamp < 3600))

        return {
            'governance_metrics': {
                'monthly_spend': latest_governance.monthly_spend,
                'budget_utilization': latest_governance.budget_utilization,
                'active_models': latest_governance.active_models,
                'security_score': latest_governance.security_compliance_score,
                'bias_score': np.mean(list(latest_governance.bias_detection_results.values())) if latest_governance.bias_detection_results else 0,
                'sustainability_score': latest_governance.sustainability_score
            },
            'activity_summary': {
                'active_training_runs': active_training,
                'active_rag_pipelines': active_rag,
                'active_agents': active_agents,
                'total_alerts': len(self.active_alerts)
            },
            'performance_overview': {
                'avg_model_performance': np.mean(list(latest_governance.model_performance_scores.values())) if latest_governance.model_performance_scores else 0,
                'sla_compliance': latest_governance.latency_sla_compliance,
                'availability': latest_governance.availability_percentage,
                'compute_efficiency': latest_governance.compute_efficiency_score
            }
        }

    # Real RAG Measurement Methods
    def connect_vector_database(self, db_type: str, **kwargs) -> bool:
        """Connect to a real vector database for monitoring."""
        if db_type == 'pinecone':
            return self.vector_db_connector.connect_pinecone(
                kwargs.get('api_key', ''),
                kwargs.get('environment', ''),
                kwargs.get('index_name', '')
            )
        elif db_type == 'weaviate':
            return self.vector_db_connector.connect_weaviate(
                kwargs.get('url', ''),
                kwargs.get('api_key')
            )
        elif db_type == 'chroma':
            return self.vector_db_connector.connect_chroma(
                kwargs.get('persist_directory', './chroma_db')
            )
        elif db_type == 'faiss':
            return self.vector_db_connector.connect_faiss(
                kwargs.get('index_path', '')
            )
        return False

    def start_real_time_monitoring(self, interval_seconds: float = 30.0):
        """Start real-time monitoring of all connected systems."""
        if self.real_time_enabled:
            return

        self.real_time_enabled = True

        # Start vector DB monitoring
        if self.vector_db_connector.connections:
            self.vector_db_connector.start_monitoring(interval_seconds)

        def data_stream_loop():
            """Continuous data streaming for real-time updates."""
            while self.real_time_enabled:
                try:
                    # Update RAG pipeline metrics
                    if hasattr(self, 'active_rag_pipelines'):
                        for pipeline_id in self.active_rag_pipelines:
                            metrics = self.rag_pipeline_monitor.get_real_time_pipeline_metrics(pipeline_id)
                            # Store for demo consumption
                            self.rag_pipeline_monitor.real_time_stats[pipeline_id].append(metrics)

                    # Limit history size
                    for pipeline_id in self.rag_pipeline_monitor.real_time_stats:
                        if len(self.rag_pipeline_monitor.real_time_stats[pipeline_id]) > 100:
                            self.rag_pipeline_monitor.real_time_stats[pipeline_id] = \
                                self.rag_pipeline_monitor.real_time_stats[pipeline_id][-100:]

                    time.sleep(interval_seconds)

                except Exception as e:
                    logging.error(f"Error in data stream loop: {e}")
                    time.sleep(interval_seconds)

        self.data_stream_thread = threading.Thread(target=data_stream_loop, daemon=True)
        self.data_stream_thread.start()
        logging.info("Real-time monitoring started")

    def stop_real_time_monitoring(self):
        """Stop real-time monitoring."""
        self.real_time_enabled = False

        # Stop vector DB monitoring
        self.vector_db_connector.stop_monitoring()

        if self.data_stream_thread:
            self.data_stream_thread.join(timeout=5)

        logging.info("Real-time monitoring stopped")

    def get_real_rag_metrics(self) -> Dict[str, Any]:
        """Get actual RAG metrics from connected systems."""
        metrics = {
            'timestamp': time.time(),
            'vector_databases': {},
            'pipeline_performance': {},
            'retrieval_quality': {}
        }

        # Get vector database metrics
        if self.vector_db_connector.connections:
            latest_db_metrics = self.vector_db_connector.get_latest_metrics()
            for db_id, db_metrics in latest_db_metrics.items():
                metrics['vector_databases'][db_id] = {
                    'total_vectors': db_metrics.total_vectors,
                    'index_size_gb': round(db_metrics.index_size_gb, 2),
                    'query_latency_ms': round(db_metrics.query_latency_ms, 1),
                    'queries_per_second': round(db_metrics.queries_per_second, 1),
                    'index_efficiency': round(db_metrics.index_efficiency, 3),
                    'error_rate': round(db_metrics.error_rate, 4),
                    'availability': round(db_metrics.availability_percentage, 2),
                    'total_cost_usd': round(db_metrics.total_cost_usd, 2)
                }

        # Get pipeline metrics
        for pipeline_id in ['main_rag_pipeline']:  # Default pipeline
            pipeline_metrics = self.rag_pipeline_monitor.get_real_time_pipeline_metrics(pipeline_id)
            metrics['pipeline_performance'][pipeline_id] = pipeline_metrics

        return metrics

    def evaluate_query_quality(self, query: str, retrieved_docs: List[Dict],
                             ground_truth: Optional[List[str]] = None) -> Dict[str, Any]:
        """Evaluate the quality of a RAG query result."""
        retrieval_metrics = self.retrieval_evaluator.evaluate_retrieval(
            query, retrieved_docs, ground_truth
        )

        return {
            'query_id': retrieval_metrics.query_id,
            'retrieval_time_ms': round(retrieval_metrics.retrieval_time_ms, 1),
            'relevance_scores': [round(score, 3) for score in retrieval_metrics.relevance_scores],
            'diversity_score': round(retrieval_metrics.diversity_score, 3),
            'semantic_similarities': [round(sim, 3) for sim in retrieval_metrics.semantic_similarity_scores],
            'ndcg_scores': {k: round(v, 3) for k, v in retrieval_metrics.ndcg_at_k.items()},
            'mrr': round(retrieval_metrics.mrr, 3),
            'map_score': round(retrieval_metrics.map_score, 3),
            'precision_at_k': {k: round(v, 3) for k, v in retrieval_metrics.precision_at_k.items()} if retrieval_metrics.precision_at_k else {},
            'recall_at_k': {k: round(v, 3) for k, v in retrieval_metrics.recall_at_k.items()} if retrieval_metrics.recall_at_k else {}
        }

    def generate_demo_data_feed(self) -> Dict[str, Any]:
        """Generate realistic data feed for demo purposes."""
        # This combines real metrics where available with realistic simulations
        real_metrics = self.get_real_rag_metrics()

        demo_data = {
            'timestamp': time.time(),
            'pipeline_status': {
                'documents_processed': 2847 + int(time.time() % 100),
                'chunks_generated': 89300 + int(time.time() % 500),
                'embeddings_created': 87100 + int(time.time() % 300),
                'vectors_stored': real_metrics['vector_databases'].get(
                    list(real_metrics['vector_databases'].keys())[0], {}
                ).get('total_vectors', 2340000) if real_metrics['vector_databases'] else 2340000,
                'queries_served': 12400 + int(time.time() % 200)
            },
            'performance_metrics': {
                'retrieval_accuracy': 94.7 + (np.random.random() - 0.5) * 2,
                'avg_latency': real_metrics['vector_databases'].get(
                    list(real_metrics['vector_databases'].keys())[0], {}
                ).get('query_latency_ms', 187) if real_metrics['vector_databases'] else 187 + (np.random.random() * 20),
                'relevance_score': 0.847 + (np.random.random() - 0.5) * 0.1,
                'cache_hit_rate': 78.2 + (np.random.random() * 10)
            },
            'vector_storage': real_metrics['vector_databases'],
            'cost_metrics': {
                'total_cost': sum([db.get('total_cost_usd', 0) for db in real_metrics['vector_databases'].values()]),
                'query_cost_avg': 0.0004 + (np.random.random() * 0.0002)
            }
        }

        return demo_data

    def get_ai_recommendations(self) -> Dict[str, Any]:
        """Get AI-powered recommendations for observability insights and FinOps optimization."""
        try:
            from .ai_recommendations import generate_ai_recommendations, get_recommendation_dashboard

            # Get current metrics
            rag_metrics = self.get_current_rag_metrics()
            finops_metrics = self.get_current_finops_metrics()
            vector_db_metrics = self.get_current_vector_db_metrics()

            # Generate recommendations
            recommendations = generate_ai_recommendations(
                rag_metrics=rag_metrics,
                finops_metrics=finops_metrics,
                vector_db_metrics=vector_db_metrics
            )

            # Get dashboard summary
            dashboard = get_recommendation_dashboard()

            return {
                'recommendations': recommendations,
                'dashboard': dashboard,
                'timestamp': time.time(),
                'status': 'success'
            }

        except ImportError:
            logging.warning("AI recommendations module not available")
            return {
                'recommendations': [],
                'dashboard': {'summary': {'total': 0}, 'top_recommendations': []},
                'timestamp': time.time(),
                'status': 'ai_module_unavailable'
            }
        except Exception as e:
            logging.error(f"Error generating AI recommendations: {e}")
            return {
                'recommendations': [],
                'dashboard': {'summary': {'total': 0}, 'top_recommendations': []},
                'timestamp': time.time(),
                'status': 'error',
                'error': str(e)
            }

    def get_current_rag_metrics(self) -> 'RAGPipelineMetrics':
        """Get current RAG pipeline metrics for AI analysis."""
        # Generate realistic current metrics based on live data
        real_metrics = self.get_real_rag_metrics()

        avg_latency = float(str(real_metrics.get('performance_metrics', {}).get('avg-latency', '65ms')).replace('ms', ''))

        return RAGPipelineMetrics(
            pipeline_id="current_pipeline",
            stage=RAGStage.RETRIEVAL,
            timestamp=time.time(),

            # Document Processing
            documents_processed=real_metrics.get('pipeline_stages', {}).get('docs-processed', 2500),
            chunks_generated=real_metrics.get('pipeline_stages', {}).get('chunks-created', 85000),
            embeddings_created=real_metrics.get('pipeline_stages', {}).get('embeddings-generated', 83000),
            index_size=2500000,

            # Performance Metrics
            processing_time=45.3,
            throughput=150.5,
            latency_p50=avg_latency,
            latency_p95=avg_latency * 1.8,  # Estimated p95
            latency_p99=avg_latency * 2.5,  # Estimated p99

            # Quality Metrics
            retrieval_accuracy=float(str(real_metrics.get('performance_metrics', {}).get('retrieval-accuracy', '93.5%')).replace('%', '')) / 100,
            relevance_score=0.847,
            diversity_score=0.78,
            coverage_score=0.92,

            # Generation Metrics
            generation_time=2.5,
            response_length=512,
            coherence_score=0.89,
            factuality_score=0.91,

            # Resource Usage
            cpu_usage=0.45,
            memory_usage=8.5,
            gpu_usage=0.72,
            storage_usage=15.2,

            # Cost and Token Usage
            input_tokens=450,
            output_tokens=180,
            total_tokens=630,
            cost_per_query=real_metrics.get('cost_summary', {}).get('query_cost_avg', 0.0005),

            # Error Tracking
            error_rate=0.008,
            timeout_rate=0.002,
            retry_count=0
        )

    def get_current_finops_metrics(self) -> 'LLMFinOpsMetrics':
        """Get current FinOps metrics for AI analysis."""
        real_metrics = self.get_real_rag_metrics()
        total_cost = real_metrics.get('cost_summary', {}).get('total_cost', 750.0)

        return LLMFinOpsMetrics(
            operation_id="ai_ops_001",
            operation_type="inference",
            model_name="enterprise_llm",
            organization_id="infinidatum",
            project_id="rag_system",
            team_id="ai_team",
            timestamp=time.time(),

            # Cost components
            gpu_cost_per_hour=2.50,
            cpu_cost_per_hour=0.15,
            memory_cost_per_gb_hour=0.05,
            storage_cost_per_gb_hour=0.02,
            network_cost_per_gb=0.01,

            # Resource usage
            gpu_hours_used=120.5,
            gpu_type="A100",
            gpu_count=2,
            cpu_hours_used=450.2,
            memory_gb_hours=2400,
            storage_gb_hours=25600,
            network_gb_transferred=150.5,

            # LLM-specific costs
            token_processing_cost=0.0005,
            model_loading_cost=15.50,
            checkpoint_storage_cost=8.20,
            data_ingestion_cost=12.30,

            # Training-specific costs
            training_data_cost=0.0,
            preprocessing_cost=0.0,
            evaluation_cost=25.80,
            experiment_tracking_cost=5.40,

            # Inference-specific costs
            api_call_cost=total_cost * 0.40,
            latency_sla_penalty=0.0,
            scaling_cost=total_cost * 0.05,
            load_balancing_cost=total_cost * 0.02,

            # Total costs
            total_compute_cost=total_cost * 0.65,
            total_infrastructure_cost=total_cost * 0.20,
            total_operational_cost=total_cost * 0.15,
            total_cost=total_cost,

            # Cost optimization metrics
            efficiency_score=0.85,
            waste_percentage=12.5,
            optimization_potential=total_cost * 0.18,
            cost_per_token=0.0005,
            cost_per_request=real_metrics.get('cost_summary', {}).get('query_cost_avg', 0.0005),
            cost_per_model_parameter=0.0000001,

            # Budget and forecasting
            budget_allocated=1000.0,
            budget_consumed=total_cost,
            budget_remaining=1000.0 - total_cost,
            projected_monthly_cost=total_cost * 1.15,
            cost_trend="increasing",

            # Chargeback and allocation
            department_allocation={"engineering": 0.6, "research": 0.4},
            project_allocation={"rag_system": 0.8, "ml_ops": 0.2},
            user_allocation={"team_lead": 0.3, "engineers": 0.7},

            # Carbon and sustainability
            carbon_cost_usd=8.50,
            sustainability_penalty=0.0,
            green_compute_premium=12.30,

            # Alert thresholds
            cost_threshold_exceeded=total_cost > 800.0,
            budget_alert_triggered=total_cost > 1000.0 * 0.8,
            efficiency_below_target=0.85 < 0.9,

            # Metadata
            pricing_model="hybrid",
            region="us-west-2",
            availability_zone="us-west-2a",
            cost_center="ai_operations"
        )

    def get_current_vector_db_metrics(self) -> Dict[str, 'VectorDBMetrics']:
        """Get current vector database metrics for AI analysis."""
        real_metrics = self.get_real_rag_metrics()
        vector_dbs = real_metrics.get('vector_databases', {})

        metrics_dict = {}
        for db_id, db_data in vector_dbs.items():
            metrics_dict[db_id] = VectorDBMetrics(
                db_id=db_id,
                db_type="vector_db",
                index_name=f"index_{db_id}",
                timestamp=time.time(),
                total_vectors=db_data.get('total_vectors', 1000000),
                vector_dimensions=1536,
                index_size_bytes=int(db_data.get('index_size_gb', 5.0) * 1024**3),
                index_size_gb=db_data.get('index_size_gb', 5.0),
                memory_usage_gb=16.0,
                query_latency_ms=db_data.get('query_latency_ms', 50.0),
                index_build_time_ms=3600000,  # 1 hour in ms
                queries_per_second=db_data.get('queries_per_second', 100.0),
                insert_throughput=1000,
                index_efficiency=db_data.get('index_efficiency', 0.85),
                recall_at_k={1: 0.95, 5: 0.98, 10: 0.99},
                precision_at_k={1: 0.90, 5: 0.85, 10: 0.80},
                error_rate=db_data.get('error_rate', 0.001),
                uptime_percentage=db_data.get('availability', 99.9),
                cache_hit_ratio=0.82,
                concurrent_connections=12,
                cost_per_hour=db_data.get('total_cost_usd', 50.0) / (24 * 30)  # Monthly to hourly
            )

        # Add default metrics if no real data
        if not metrics_dict:
            metrics_dict['default_vectordb'] = VectorDBMetrics(
                db_id="default_vectordb",
                db_type="pinecone",
                index_name="default_index",
                timestamp=time.time(),
                total_vectors=2500000,
                vector_dimensions=1536,
                index_size_bytes=int(12.5 * 1024**3),
                index_size_gb=12.5,
                memory_usage_gb=24.0,
                query_latency_ms=45.0,
                index_build_time_ms=2400000,  # 40 minutes in ms
                queries_per_second=425.0,
                insert_throughput=1500,
                index_efficiency=0.91,
                recall_at_k={1: 0.96, 5: 0.99, 10: 0.995},
                precision_at_k={1: 0.92, 5: 0.88, 10: 0.82},
                error_rate=0.002,
                uptime_percentage=99.95,
                cache_hit_ratio=0.88,
                concurrent_connections=8,
                cost_per_hour=2.15
            )

        return metrics_dict