"""
Main Orchestrator Module

The Semantica orchestrator coordinates all framework components and manages
the overall execution flow.

This module provides the main entry point for the Semantica framework, handling:
- Framework initialization and lifecycle management
- Knowledge base construction from various data sources
- Pipeline execution and resource management
- Plugin system coordination
- System health monitoring

Example Usage:
    >>> from semantica import Semantica
    >>> framework = Semantica()
    >>> result = framework.build_knowledge_base(
    ...     sources=["doc1.pdf", "doc2.docx"],
    ...     embeddings=True,
    ...     graph=True
    ... )

Author: Semantica Contributors
License: MIT
"""

from typing import Any, Dict, List, Optional, Union
from pathlib import Path

from .config_manager import Config, ConfigManager
from .lifecycle import LifecycleManager, SystemState
from .plugin_registry import PluginRegistry
from ..utils.exceptions import ConfigurationError, ProcessingError
from ..utils.logging import get_logger, log_execution_time
from ..utils.progress_tracker import get_progress_tracker


class Semantica:
    """
    Main Semantica framework class.
    
    This is the primary entry point for using the framework. It coordinates
    all modules and provides a unified API for semantic processing.
    
    Attributes:
        config: Configuration object
        config_manager: Configuration management system
        plugin_registry: Plugin management system
        lifecycle_manager: Lifecycle management
        
    Methods:
        initialize(): Initialize all framework components
        build_knowledge_base(): Build knowledge base from sources
        run_pipeline(): Execute processing pipeline
        get_status(): Get system health and status
    """
    
    def __init__(self, config: Optional[Union[Config, Dict[str, Any]]] = None, **kwargs):
        """
        Initialize Semantica framework.
        
        Args:
            config: Configuration object or dict
            **kwargs: Additional configuration parameters
        """
        self.logger = get_logger("semantica")
        self.config_manager = ConfigManager()
        
        # Load configuration
        if isinstance(config, Config):
            self.config = config
        elif isinstance(config, dict):
            self.config = self.config_manager.load_from_dict(config)
        else:
            # Load from kwargs or defaults
            self.config = self.config_manager.load_from_dict(kwargs or {})
        
        # Initialize core components
        self.lifecycle_manager = LifecycleManager()
        self.plugin_registry = PluginRegistry()
        
        # Register lifecycle manager with itself
        self.lifecycle_manager.register_component("lifecycle_manager", self.lifecycle_manager)
        self.lifecycle_manager.register_component("plugin_registry", self.plugin_registry)
        self.lifecycle_manager.register_component("config_manager", self.config_manager)
        
        # Module placeholders (to be initialized)
        self._modules: Dict[str, Any] = {}
        self._initialized: bool = False
        
        # Initialize progress tracker (automatic, zero configuration)
        self.progress_tracker = get_progress_tracker()
        
        self.logger.info("Semantica framework initialized")
    
    @log_execution_time
    def initialize(self) -> None:
        """
        Initialize all framework components.
        
        This method sets up all modules, loads plugins, and prepares
        the system for processing.
        
        Raises:
            ConfigurationError: If configuration is invalid
            SemanticaError: If initialization fails
        """
        try:
            self.logger.info("Initializing Semantica framework")
            
            # Validate configuration
            self.config.validate()
            
            # Register startup hooks
            self._register_startup_hooks()
            
            # Execute startup sequence
            self.lifecycle_manager.startup()
            
            # Initialize modules
            self._initialize_modules()
            
            # Load plugins if configured
            if self.config.get("plugins", {}):
                self._load_plugins()
            
            # Run health checks
            health_summary = self.lifecycle_manager.get_health_summary()
            if not health_summary["is_healthy"]:
                self.logger.warning(
                    "Some components are unhealthy",
                    extra={"health_summary": health_summary}
                )
            
            self._initialized = True
            self.logger.info("Semantica framework initialization completed")
            
        except Exception as e:
            self.logger.error(f"Initialization failed: {e}")
            raise
    
    def _ensure_initialized(self) -> None:
        """Ensure framework is initialized (lazy initialization)."""
        if not self._initialized:
            self.initialize()
    
    @log_execution_time
    def build_knowledge_base(self, sources: List[Union[str, Path]], **kwargs) -> Dict[str, Any]:
        """
        Build knowledge base from data sources.
        
        This is a high-level method that orchestrates the complete
        knowledge base construction process.
        
        Args:
            sources: List of data sources (files, URLs, streams)
            **kwargs: Additional processing options:
                - pipeline: Custom pipeline configuration
                - embeddings: Whether to generate embeddings
                - graph: Whether to build knowledge graph
                - normalize: Whether to normalize data
                
        Returns:
            Dictionary containing:
                - knowledge_graph: Knowledge graph data
                - embeddings: Embedding vectors
                - metadata: Processing metadata
                - statistics: Processing statistics
                
        Raises:
            ProcessingError: If processing fails
        """
        # Auto-initialize if not already initialized
        self._ensure_initialized()
        
        # Start overall progress tracking
        overall_tracking_id = self.progress_tracker.start_tracking(
            module="core",
            submodule="Semantica",
            message=f"Building knowledge base from {len(sources)} sources"
        )
        
        try:
            self.logger.info(f"Building knowledge base from {len(sources)} sources")
            
            # Validate sources
            validated_sources = self._validate_sources(sources)
            
            # Create processing pipeline
            pipeline_config = kwargs.get("pipeline", {})
            pipeline = self._create_pipeline(pipeline_config)
            
            # Process sources
            results = []
            for source in validated_sources:
                try:
                    # Track file processing
                    file_str = str(source)
                    file_tracking_id = self.progress_tracker.start_tracking(
                        file=file_str,
                        module="core",
                        submodule="build_knowledge_base",
                        message=f"Processing {Path(file_str).name if file_str else 'source'}"
                    )
                    try:
                        result = self.run_pipeline(pipeline, source)
                        results.append(result)
                        self.progress_tracker.stop_tracking(file_tracking_id, status="completed")
                    except Exception as e:
                        self.progress_tracker.stop_tracking(file_tracking_id, status="failed", message=str(e))
                        self.logger.error(f"Failed to process source {source}: {e}")
                        if kwargs.get("fail_fast", False):
                            raise ProcessingError(f"Failed to process source {source}: {e}")
                except Exception as e:
                    self.logger.error(f"Failed to process source {source}: {e}")
                    if kwargs.get("fail_fast", False):
                        raise ProcessingError(f"Failed to process source {source}: {e}")
            
            # Build knowledge graph if requested
            knowledge_graph = None
            if kwargs.get("graph", True):
                knowledge_graph = self._build_knowledge_graph(results)
            
            # Generate embeddings if requested
            embeddings = None
            if kwargs.get("embeddings", True):
                embeddings = self._generate_embeddings(results)
            
            # Compile statistics
            statistics = {
                "sources_processed": len(results),
                "sources_total": len(sources),
                "success_rate": len([r for r in results if r.get("success")]) / len(results) if results else 0.0,
            }
            
            # Stop overall tracking
            self.progress_tracker.stop_tracking(overall_tracking_id, status="completed", 
                                               message=f"Processed {len(results)} sources")
            
            # Show summary
            self.progress_tracker.show_summary()
            
            return {
                "knowledge_graph": knowledge_graph,
                "embeddings": embeddings,
                "results": results,
                "statistics": statistics,
                "metadata": {
                    "sources": validated_sources,
                    "pipeline": pipeline_config,
                }
            }
            
        except Exception as e:
            self.progress_tracker.stop_tracking(overall_tracking_id, status="failed", message=str(e))
            self.logger.error(f"Failed to build knowledge base: {e}")
            raise ProcessingError(f"Failed to build knowledge base: {e}")
    
    @log_execution_time
    def run_pipeline(self, pipeline: Union[Dict[str, Any], Any], data: Any) -> Dict[str, Any]:
        """
        Execute a processing pipeline.
        
        Args:
            pipeline: Pipeline object or configuration dictionary
            data: Input data for pipeline
            
        Returns:
            Dictionary containing:
                - output: Pipeline output data
                - metadata: Processing metadata
                - metrics: Performance metrics
                
        Raises:
            ProcessingError: If pipeline execution fails
        """
        pipeline_tracking_id = None
        try:
            self.logger.info("Executing processing pipeline")
            
            # Track pipeline execution
            pipeline_tracking_id = self.progress_tracker.start_tracking(
                file=str(data) if isinstance(data, (str, Path)) else None,
                module="pipeline",
                submodule="ExecutionEngine",
                message="Executing pipeline"
            )
            
            # Validate pipeline
            if isinstance(pipeline, dict):
                pipeline = self._create_pipeline_from_dict(pipeline)
            
            # Validate pipeline object
            if not hasattr(pipeline, "execute"):
                raise ProcessingError("Pipeline must have execute() method")
            
            # Allocate resources
            resources = self._allocate_resources(pipeline)
            
            try:
                # Execute pipeline
                result = pipeline.execute(data)
                
                # Collect metrics
                metrics = self._collect_metrics(pipeline)
                
                if pipeline_tracking_id:
                    self.progress_tracker.stop_tracking(pipeline_tracking_id, status="completed")
                
                return {
                    "success": True,
                    "output": result,
                    "metrics": metrics,
                    "metadata": {
                        "pipeline": str(pipeline),
                        "resources": resources,
                    }
                }
                
            finally:
                # Release resources
                self._release_resources(resources)
                
        except Exception as e:
            if pipeline_tracking_id:
                self.progress_tracker.stop_tracking(pipeline_tracking_id, status="failed", message=str(e))
            self.logger.error(f"Pipeline execution failed: {e}")
            raise ProcessingError(f"Pipeline execution failed: {e}")
    
    def get_status(self) -> Dict[str, Any]:
        """
        Get system health and status.
        
        Returns:
            Dictionary containing:
                - state: System state
                - health: Health summary
                - modules: Module status
                - plugins: Plugin status
                - metrics: System metrics
        """
        health_summary = self.lifecycle_manager.get_health_summary()
        
        # Get module status
        module_status = {}
        for name, module in self._modules.items():
            module_status[name] = {
                "initialized": module is not None,
                "status": "ready" if module else "not_initialized"
            }
        
        # Get plugin status
        plugin_status = {}
        try:
            plugins = self.plugin_registry.list_plugins()
            for plugin_info in plugins:
                plugin_status[plugin_info.get("name", "unknown")] = {
                    "loaded": plugin_info.get("loaded", False),
                    "version": plugin_info.get("version", "unknown")
                }
        except Exception as e:
            self.logger.warning(f"Failed to get plugin status: {e}")
        
        return {
            "state": self.lifecycle_manager.get_state().value,
            "health": health_summary,
            "modules": module_status,
            "plugins": plugin_status,
            "config": {
                "loaded": self.config is not None,
                "validated": True  # Assuming validated if initialized
            }
        }
    
    def shutdown(self, graceful: bool = True) -> None:
        """
        Shutdown the framework.
        
        Args:
            graceful: Whether to shutdown gracefully (default: True)
        """
        self.logger.info("Shutting down Semantica framework")
        self.lifecycle_manager.shutdown(graceful=graceful)
        self.logger.info("Semantica framework shutdown completed")
    
    def _register_startup_hooks(self) -> None:
        """
        Register framework startup hooks.
        
        This method registers all framework-level startup hooks that execute
        during initialization. Hooks are registered with specific priorities
        to ensure correct execution order.
        
        Registered Hooks:
            - Config validation (priority 10): Validates configuration before other operations
            - Module initialization (priority 30): Initializes framework modules
        """
        # Register config validation hook (high priority - runs first)
        def validate_config_hook():
            """Validate configuration during startup."""
            self.config.validate()
        
        self.lifecycle_manager.register_startup_hook(validate_config_hook, priority=10)
        
        # Register module initialization hook (lower priority - runs after validation)
        def initialize_modules_hook():
            """Initialize framework modules during startup."""
            self._initialize_modules()
        
        self.lifecycle_manager.register_startup_hook(initialize_modules_hook, priority=30)
    
    def _initialize_modules(self) -> None:
        """
        Initialize framework modules.
        
        This method attempts to import and verify availability of key framework
        modules. It's called during startup to ensure modules are ready for use.
        Import failures are logged but don't stop initialization (modules may be optional).
        
        Modules Checked:
            - GraphBuilder: Knowledge graph construction
            - PipelineBuilder: Processing pipeline creation
            - FileIngestor: File ingestion capabilities
            - DocumentParser: Document parsing capabilities
        """
        try:
            # Import key modules to verify they're available
            # These imports don't create instances, just verify module availability
            from ..kg import GraphBuilder
            from ..pipeline import PipelineBuilder
            from ..ingest import FileIngestor
            from ..parse import DocumentParser
            
            self.logger.debug("Framework modules verified and available")
        except ImportError as e:
            # Log but don't fail - modules may be optional or not installed
            self.logger.warning(
                f"Some framework modules could not be imported: {e}. "
                "They may be optional or not installed."
            )
    
    def _load_plugins(self) -> None:
        """
        Load configured plugins from configuration.
        
        This method reads the plugin configuration from the framework config
        and loads each plugin using the plugin registry. Plugin loading failures
        are logged but don't stop the initialization process.
        
        Configuration Format:
            plugins:
                plugin_name:
                    config_key: config_value
                    ...
        """
        plugins_config = self.config.get("plugins", {})
        
        if not plugins_config:
            self.logger.debug("No plugins configured")
            return
        
        self.logger.info(f"Loading {len(plugins_config)} configured plugin(s)")
        
        for plugin_name, plugin_config in plugins_config.items():
            try:
                # Load plugin with its configuration
                self.plugin_registry.load_plugin(plugin_name, **plugin_config)
                self.logger.info(f"Successfully loaded plugin: {plugin_name}")
            except Exception as e:
                # Log error but continue loading other plugins
                self.logger.error(
                    f"Failed to load plugin '{plugin_name}': {e}. "
                    "Continuing with other plugins."
                )
    
    def _validate_sources(self, sources: List[Union[str, Path]]) -> List[Union[str, Path]]:
        """
        Validate and filter data sources.
        
        This method checks if sources exist (for file paths) or are valid URLs.
        Invalid sources are logged as warnings but don't stop processing.
        
        Args:
            sources: List of source paths or URLs to validate
            
        Returns:
            List of validated sources
            
        Raises:
            ProcessingError: If no valid sources are found
        """
        validated_sources = []
        
        for source in sources:
            # Convert string to Path for easier handling
            source_path = Path(source) if isinstance(source, str) else source
            
            # Check if source is valid:
            # 1. File path exists on filesystem
            # 2. URL starts with http:// or https://
            is_valid_file = source_path.exists()
            is_valid_url = isinstance(source, str) and source.startswith(("http://", "https://"))
            
            if is_valid_file or is_valid_url:
                validated_sources.append(source)
            else:
                self.logger.warning(
                    f"Source not found or invalid: {source}. "
                    "Skipping this source."
                )
        
        # Ensure we have at least one valid source
        if not validated_sources:
            error_msg = (
                f"No valid sources provided. "
                f"Checked {len(sources)} source(s), all were invalid."
            )
            raise ProcessingError(error_msg)
        
        return validated_sources
    
    def _create_pipeline(self, pipeline_config: Dict[str, Any]) -> Any:
        """
        Create processing pipeline from configuration.
        
        This method creates a pipeline instance from the provided configuration.
        The actual pipeline creation is delegated to the pipeline module.
        
        Args:
            pipeline_config: Pipeline configuration dictionary
            
        Returns:
            Pipeline object or configuration dict (if pipeline module not available)
        """
        try:
            # Try to use PipelineBuilder if available
            from ..pipeline import PipelineBuilder
            pipeline_builder = PipelineBuilder()
            return pipeline_builder.build_from_config(pipeline_config)
        except ImportError:
            # Fallback: return config as-is if pipeline module not available
            self.logger.debug("Pipeline module not available, using config directly")
            return pipeline_config
    
    def _create_pipeline_from_dict(self, pipeline_dict: Dict[str, Any]) -> Any:
        """
        Create pipeline object from dictionary configuration.
        
        Args:
            pipeline_dict: Dictionary containing pipeline configuration
            
        Returns:
            Pipeline object or dict if pipeline module not available
        """
        return self._create_pipeline(pipeline_dict)
    
    def _build_knowledge_graph(self, results: List[Dict[str, Any]]) -> Dict[str, Any]:
        """
        Build knowledge graph from processing results.
        
        This method extracts entities and relationships from processing results
        and constructs a knowledge graph structure.
        
        Args:
            results: List of processing results containing entities and relationships
            
        Returns:
            Dictionary containing knowledge graph structure
        """
        try:
            from ..kg import GraphBuilder
            
            # Extract entities and relationships from results
            graph_sources = []
            for result in results:
                if isinstance(result, dict):
                    source_data = {}
                    if "entities" in result:
                        source_data["entities"] = result["entities"]
                    if "relationships" in result:
                        source_data["relationships"] = result["relationships"]
                    if source_data:
                        graph_sources.append(source_data)
            
            if not graph_sources:
                self.logger.warning("No entities or relationships found in results")
                return {"entities": [], "relationships": [], "metadata": {}}
            
            # Track knowledge graph building
            kg_tracking_id = self.progress_tracker.start_tracking(
                module="kg",
                submodule="GraphBuilder",
                message="Building knowledge graph"
            )
            
            try:
                # Build knowledge graph
                graph_builder = GraphBuilder(merge_entities=True, resolve_conflicts=True)
                knowledge_graph = graph_builder.build(graph_sources)
                self.progress_tracker.stop_tracking(kg_tracking_id, status="completed")
                return knowledge_graph
            except Exception as e:
                self.progress_tracker.stop_tracking(kg_tracking_id, status="failed", message=str(e))
                raise
            
        except ImportError:
            self.logger.warning("KG module not available, returning placeholder")
            return {"status": "placeholder", "results": results}
    
    def _generate_embeddings(self, results: List[Dict[str, Any]]) -> Dict[str, Any]:
        """
        Generate embeddings from processing results.
        
        This method extracts text content from processing results and generates
        embeddings using the embeddings module.
        
        Args:
            results: List of processing results containing text content
            
        Returns:
            Dictionary containing generated embeddings
        """
        try:
            from ..embeddings import EmbeddingGenerator
            
            # Extract text content from results
            texts_to_embed = []
            for result in results:
                if isinstance(result, dict):
                    # Try to find text content in various possible keys
                    text = (
                        result.get("text") or
                        result.get("content") or
                        result.get("output", {}).get("text") if isinstance(result.get("output"), dict) else None
                    )
                    if text:
                        texts_to_embed.append(text)
            
            if not texts_to_embed:
                self.logger.warning("No text content found in results for embedding")
                return {"embeddings": [], "metadata": {}}
            
            # Track embedding generation
            embed_tracking_id = self.progress_tracker.start_tracking(
                module="embeddings",
                submodule="EmbeddingGenerator",
                message=f"Generating embeddings for {len(texts_to_embed)} texts"
            )
            
            try:
                # Generate embeddings
                embedding_generator = EmbeddingGenerator()
                embeddings_result = embedding_generator.process_batch(texts_to_embed)
                self.progress_tracker.stop_tracking(embed_tracking_id, status="completed",
                                                   message=f"Generated {embeddings_result.get('success_count', 0)} embeddings")
                
                return {
                    "embeddings": embeddings_result["embeddings"],
                    "metadata": {
                        "total_texts": len(texts_to_embed),
                        "successful": embeddings_result["success_count"],
                        "failed": embeddings_result["failure_count"]
                    }
                }
            except Exception as e:
                self.progress_tracker.stop_tracking(embed_tracking_id, status="failed", message=str(e))
                raise
            
        except ImportError:
            self.logger.warning("Embeddings module not available, returning placeholder")
            return {"status": "placeholder", "results": results}
    
    def _allocate_resources(self, pipeline: Any) -> Dict[str, Any]:
        """
        Allocate resources for pipeline execution.
        
        This method prepares and allocates necessary resources (connections,
        memory, etc.) for pipeline execution. Currently returns a placeholder
        but can be extended for actual resource management.
        
        Args:
            pipeline: Pipeline object that will be executed
            
        Returns:
            Dictionary containing resource allocation information:
                - allocated: Whether resources were successfully allocated
                - resources: List of allocated resource references (optional)
        """
        # Placeholder for resource allocation logic
        # Can be extended to allocate database connections, file handles, etc.
        return {
            "allocated": True,
            "resources": []
        }
    
    def _release_resources(self, resources: Dict[str, Any]) -> None:
        """
        Release allocated resources safely.
        
        This method ensures all resources (connections, files, etc.) are properly
        closed and cleaned up, even if errors occur during cleanup.
        
        Args:
            resources: Dictionary containing resource references to release
        """
        if not resources:
            return
        
        # Release database/network connections
        connections = resources.get("connections", [])
        for connection in connections:
            try:
                if hasattr(connection, "close"):
                    connection.close()
                    self.logger.debug("Closed connection resource")
            except Exception as e:
                self.logger.warning(f"Error closing connection: {e}")
        
        # Release file handles
        file_handles = resources.get("files", [])
        for file_handle in file_handles:
            try:
                if hasattr(file_handle, "close"):
                    file_handle.close()
                    self.logger.debug("Closed file resource")
            except Exception as e:
                self.logger.warning(f"Error closing file: {e}")
        
        # Clear resource dictionary to prevent reuse
        resources.clear()
        self.logger.debug("All resources released")
    
    def _collect_metrics(self, pipeline: Any) -> Dict[str, Any]:
        """
        Collect performance metrics from pipeline execution.
        
        This method gathers execution metrics such as execution time, memory usage,
        and other performance indicators from the pipeline.
        
        Args:
            pipeline: Pipeline object that was executed
            
        Returns:
            Dictionary containing performance metrics
        """
        metrics = {
            "execution_time": 0.0,
            "memory_usage": 0,
            "cpu_usage": 0.0
        }
        
        # Try to get metrics from pipeline if it has them
        try:
            if hasattr(pipeline, "get_metrics"):
                pipeline_metrics = pipeline.get_metrics()
                metrics.update(pipeline_metrics)
        except Exception as e:
            self.logger.debug(f"Pipeline metrics not available: {e}")
        
        # Try to get system metrics using psutil (optional dependency)
        try:
            import psutil
            import os
            
            process = psutil.Process(os.getpid())
            memory_info = process.memory_info()
            metrics["memory_usage"] = memory_info.rss  # Resident Set Size in bytes
            
            # Get CPU usage (non-blocking)
            try:
                cpu_percent = process.cpu_percent(interval=0.1)
                metrics["cpu_usage"] = cpu_percent
            except Exception:
                # CPU percent may not be available immediately
                pass
                
        except ImportError:
            # psutil not available, use basic metrics
            self.logger.debug("psutil not available, using basic metrics")
        except Exception as e:
            self.logger.debug(f"Error collecting system metrics: {e}")
        
        return metrics