"""
Chatbot Manager.

Tool for instanciate, managing and interacting with Chatbot through APIs.
"""
from typing import Any, Dict, Type, Optional, Tuple, List
from importlib import import_module
import contextlib
import time
import asyncio
import copy
from aiohttp import web
from datamodel.exceptions import ValidationError  # pylint: disable=E0611 # noqa
# Navigator:
from navconfig.logging import logging
from asyncdb.exceptions import NoDataFound
from ..bots.abstract import AbstractBot
from ..bots.basic import BasicBot
from ..bots.chatbot import Chatbot
from ..bots.agent import BasicAgent
from ..handlers.chat import ChatHandler, BotHandler
from ..handlers.agent import AgentTalk
from ..handlers.datasets import DatasetManagerHandler
from ..handlers.chat_interaction import ChatInteractionHandler
from ..storage import ChatStorage
from ..handlers import ChatbotHandler
from ..handlers.config_handler import BotConfigHandler
from ..handlers.test_handler import BotConfigTestHandler
from ..handlers.dashboard_handler import (
    DashboardHandler,
    DashboardTabHandler,
    _ensure_dashboard_indexes,
)
from ..handlers.models import BotModel
from ..handlers.stream import StreamHandler
from ..registry import agent_registry, AgentRegistry, BotConfigStorage
# Crew:
from ..bots.orchestration.crew import AgentCrew
from ..handlers.crew.models import CrewDefinition, ExecutionMode
from ..handlers.crew.handler import CrewHandler
from ..handlers.crew.execution_handler import CrewExecutionHandler
from ..handlers.crew.redis_persistence import CrewRedis
from ..openapi.config import setup_swagger
from ..conf import (
    ENABLE_CREWS,
    ENABLE_DATABASE_BOTS,
    ENABLE_DASHBOARDS,
    ENABLE_REGISTRY_BOTS,
    ENABLE_SWAGGER,
)
# Credentials handler
from ..handlers.credentials import setup_credentials_routes
# Telegram integration
# Integrations (Telegram, MS Teams)
from ..integrations import IntegrationBotManager


class BotManager:
    """BotManager.

    Manage Bots/Agents and interact with them through via aiohttp App.
    Deploy and manage chatbots and agents using a RESTful API.

    """
    app: web.Application = None

    def __init__(
        self,
        enable_database_bots: bool = ENABLE_DATABASE_BOTS,
        enable_crews: bool = ENABLE_CREWS,
        enable_registry_bots: bool = ENABLE_REGISTRY_BOTS,
        enable_swagger_api: bool = ENABLE_SWAGGER,
    ) -> None:
        """Initialize BotManager.

        Args:
            enable_database_bots: When True, load bots from the database via
                ``_load_database_bots()``. Defaults to ``ENABLE_DATABASE_BOTS``
                (False unless overridden in config/env).
            enable_crews: When True, initialize ``CrewRedis`` and call
                ``load_crews()`` during startup. Defaults to ``ENABLE_CREWS``.
            enable_registry_bots: When True, run the full ``AgentRegistry``
                pipeline (load_modules, discover_config_agents,
                load_agent_definitions, instantiate_startup_agents). Defaults
                to ``ENABLE_REGISTRY_BOTS`` (True).
            enable_swagger_api: When True, register the OpenAPI/Swagger routes
                via ``setup_swagger()``. Defaults to ``ENABLE_SWAGGER``.
        """
        self.app = None
        self._bots: Dict[str, AbstractBot] = {}
        self._botdef: Dict[str, Type] = {}  # Store class definitions for each bot
        self._bot_expiration: Dict[str, float] = {}  # Track expiration timestamps for temporary bots
        self._cleanup_task: Optional[asyncio.Task] = None  # Background cleanup task
        self.logger = logging.getLogger(
            name='Parrot.Manager'
        )
        self.registry: AgentRegistry = agent_registry
        self._crews: Dict[str, Tuple[AgentCrew, CrewDefinition]] = {}
        # Store flags as instance attributes
        self.enable_database_bots: bool = enable_database_bots
        self.enable_crews: bool = enable_crews
        self.enable_registry_bots: bool = enable_registry_bots
        self.enable_swagger_api: bool = enable_swagger_api
        # Initialize Redis persistence for crews — keyed off instance attr
        self.crew_redis = CrewRedis() if self.enable_crews else None
        # Integration manager
        self._integration_manager: Optional[IntegrationBotManager] = None

    @staticmethod
    def _normalize_tenant(tenant: Optional[str]) -> str:
        return tenant or "global"

    def _get_crew_key(self, tenant: str, name: str) -> str:
        return f"{tenant}:{name}"

    def _split_crew_key(self, key: str) -> Tuple[str, str]:
        """
        Split a crew cache key into (tenant, name).

        Legacy or malformed keys may not contain a tenant prefix
        separated by ":", in which case we assume the global tenant
        and treat the whole key as the name.
        """
        if ":" not in key:
            # Handle legacy or malformed keys gracefully instead of raising ValueError
            self.logger.warning(
                "Malformed or legacy crew key without tenant prefix: %r. "
                "Assuming global tenant.",
                key,
            )
            return self._normalize_tenant(None), key

        tenant, name = key.split(":", 1)
        # Normalize empty or falsy tenant values to the default
        tenant = tenant or self._normalize_tenant(None)
        return tenant, name

    def get_bot_class(self, bot_name: str) -> Optional[Type]:
        """
        Get bot class by name, searching in:
        1. parrot.bots (core bots)
        2. parrot.agents (plugin agents)

        Args:
            bot_name: Name of the bot/agent class

        Returns:
            Bot class if found, None otherwise
        """
        if not bot_name:
            self.logger.warning(
                "Empty bot_name provided to get_bot_class, defaulting to 'BasicAgent'"
            )
            bot_name = "BasicAgent"

        # First, try to import from core bots
        with contextlib.suppress(ImportError, AttributeError):
            module = import_module("parrot.bots")
            if hasattr(module, bot_name):
                return getattr(module, bot_name)

        # Second, try to import from plugin agents
        with contextlib.suppress(ImportError, AttributeError):
            agent_module_name = f"parrot.agents.{bot_name.lower()}"
            module = import_module(agent_module_name)
            if hasattr(module, bot_name):
                return getattr(module, bot_name)

        # Third, try direct import from parrot.agents package
        # (in case the agent is defined in plugins/agents/__init__.py)
        with contextlib.suppress(ImportError, AttributeError):
            module = import_module("parrot.agents")
            if hasattr(module, bot_name):
                return getattr(module, bot_name)

        self.logger.warning(
            f"Warning: Bot class '{bot_name}' not found in parrot.bots or parrot.agents"
        )
        return None

    def get_or_create_bot(self, bot_name: str, **kwargs):
        """
        Get existing bot or create new one from class name.

        Args:
            bot_name: Name of the bot/agent class
            **kwargs: Arguments to pass to bot constructor

        Returns:
            Bot instance
        """
        # Check if already instantiated
        if bot_name in self._bots:
            return self._bots[bot_name]

        # Get the class and instantiate
        bot_class = self.get_bot_class(bot_name)
        if bot_class is None:
            raise ValueError(f"Bot class '{bot_name}' not found")

        return self.create_bot(class_name=bot_class, name=bot_name, **kwargs)

    def _log_final_state(self) -> None:
        """Log the final state of bot loading."""
        registry_info = self.registry.get_registration_info()
        self.logger.notice("=== Bot Loading Complete ===")
        self.logger.notice(f"Registered agents: {registry_info['total_registered']}")
        # self.logger.info(f"Startup agents: {startup_info['total_startup_agents']}")
        self.logger.notice(f"Active bots: {len(self._bots)}")

    async def _process_startup_results(self, startup_results: Dict[str, Any]) -> None:
        """Process startup instantiation results."""
        for agent_name, result in startup_results.items():
            print('===========================================')
            print('Agent startup result:', agent_name, result)
            print('===========================================')
            if result["status"] == "success":
                if instance := result.get("instance"):
                    self._bots[agent_name] = instance
                    self.logger.info(
                        f"Added startup agent to active bots: {agent_name}"
                    )
            else:
                self.logger.error(
                    f"Startup agent {agent_name} failed: {result['error']}"
                )

    async def load_bots(self, app: web.Application) -> None:
        """Load and register all bots using the registry and optional database.

        Args:
            app: The aiohttp Application instance passed during startup.
        """
        self.logger.info("Starting bot loading with global registry")

        if self.enable_registry_bots:
            # Step 1: Import modules to trigger decorator registration
            await self.registry.load_modules()

            # Step 2: Register config-based agents
            config_count = self.registry.discover_config_agents()
            self.logger.info(
                f"Registered {config_count} agents from config"
            )

            # Step 2b: Load YAML agent definitions from agents/agents/
            definitions_dir = self.registry.agents_dir / "agents"
            if definitions_dir.is_dir():
                def_count = self.registry.load_agent_definitions(definitions_dir)
                self.logger.info(
                    f"Loaded {def_count} agents from YAML definitions"
                )

            # Step 3: Instantiate startup agents
            startup_results = await self.registry.instantiate_startup_agents(app)
            await self._process_startup_results(startup_results)
        else:
            self.logger.info(
                "AgentRegistry loading skipped (enable_registry_bots=False)"
            )

        # Step 4: Load database bots
        if self.enable_database_bots:
            await self._load_database_bots(app)
        else:
            self.logger.debug(
                "Database bot loading skipped (enable_database_bots=False)"
            )

        # Step 5: Report final state
        self._log_final_state()

    async def _load_database_bots(self, app: web.Application) -> None:
        """Load bots from database."""
        try:
            # Import here to avoid circular imports
            from ..handlers.models import BotModel  # pylint: disable=import-outside-toplevel # noqa
            db = app['database']
            async with await db.acquire() as conn:
                BotModel.Meta.connection = conn
                try:
                    all_bots = await BotModel.filter(enabled=True)
                except Exception as e:
                    self.logger.error(
                        f"Failed to load bots from DB: {e}"
                    )
                    return

            for bot_model in all_bots:
                self.logger.notice(
                    f"Loading bot '{bot_model.name}' (mode: {bot_model.operation_mode})..."
                )
                if bot_model.name in self._bots:
                    self.logger.debug(
                        f"Bot {bot_model.name} already active, skipping"
                    )
                    continue
                try:
                    # Use the factory function from models.py or create bot directly
                    if hasattr(self, 'get_bot_class') and hasattr(bot_model, 'bot_class'):
                        # If you have a bot_class field and get_bot_class method
                        class_name = self.get_bot_class(getattr(bot_model, 'bot_class', None))
                    else:
                        # Default to BasicBot or your default bot class
                        class_name = BasicBot
                    bot_instance = class_name(
                        chatbot_id=bot_model.chatbot_id,
                        name=bot_model.name,
                        description=bot_model.description,
                        # LLM configuration
                        use_llm=bot_model.llm,
                        model_name=bot_model.model_name,
                        model_config=bot_model.model_config,
                        temperature=bot_model.temperature,
                        max_tokens=bot_model.max_tokens,
                        top_k=bot_model.top_k,
                        top_p=bot_model.top_p,
                        # Bot personality
                        role=bot_model.role,
                        goal=bot_model.goal,
                        backstory=bot_model.backstory,
                        rationale=bot_model.rationale,
                        capabilities=bot_model.capabilities,
                        # Prompt configuration
                        system_prompt=bot_model.system_prompt_template,
                        human_prompt=bot_model.human_prompt_template,
                        pre_instructions=bot_model.pre_instructions,
                        # Vector store configuration
                        embedding_model=bot_model.embedding_model,
                        use_vectorstore=bot_model.use_vector,
                        vector_store_config=bot_model.vector_store_config,
                        context_search_limit=bot_model.context_search_limit,
                        context_score_threshold=bot_model.context_score_threshold,
                        # Tool and agent configuration
                        tools_enabled=bot_model.tools_enabled,
                        auto_tool_detection=bot_model.auto_tool_detection,
                        tool_threshold=bot_model.tool_threshold,
                        available_tools=bot_model.tools,
                        operation_mode=bot_model.operation_mode,
                        # Memory configuration
                        memory_type=bot_model.memory_type,
                        memory_config=bot_model.memory_config,
                        max_context_turns=bot_model.max_context_turns,
                        use_conversation_history=bot_model.use_conversation_history,
                        # Security and permissions
                        permissions=bot_model.permissions,
                        # Metadata
                        language=bot_model.language,
                        disclaimer=bot_model.disclaimer,
                    )
                    # Set the model ID reference
                    bot_instance.model_id = bot_model.chatbot_id

                    await bot_instance.configure(app)
                    self.add_bot(bot_instance)
                    self.logger.info(
                        f"Successfully loaded bot '{bot_model.name}' "
                        f"with {len(bot_model.tools) if bot_model.tools else 0} tools"
                    )
                except ValidationError as e:
                    self.logger.error(
                        f"Invalid configuration for bot '{bot_model.name}': {e}"
                    )
                except Exception as e:
                    self.logger.error(
                        f"Failed to load database bot {bot_instance.name}: {str(e)}"
                    )
            self.logger.info(
                f":: Bots loaded successfully. Total active bots: {len(self._bots)}"
            )
        except Exception as e:
            self.logger.error(
                f"Database bot loading failed: {str(e)}"
            )

    # Alternative approach using the factory function from models.py
    async def load_bots_with_factory(self, app: web.Application) -> None:
        """Load all bots from DB using the factory function."""
        self.logger.info("Loading bots from DB...")
        db = app['database']
        async with await db.acquire() as conn:
            BotModel.Meta.connection = conn
            try:
                bot_models = await BotModel.filter(enabled=True)
            except Exception as e:
                self.logger.error(
                    f"Failed to load bots from DB: {e}"
                )
                return

            for bot_model in bot_models:
                self.logger.notice(
                    f"Loading bot '{bot_model.name}' (mode: {bot_model.operation_mode})..."
                )

                try:
                    # Use the factory function from models.py
                    # Determine bot class if you have custom classes
                    bot_class = None
                    if hasattr(self, 'get_bot_class') and hasattr(bot_model, 'bot_class'):
                        bot_class = self.get_bot_class(getattr(bot_model, 'bot_class', None))
                    else:
                        # Default to BasicBot or your default bot class
                        bot_class = BasicBot

                    # Create bot using factory function
                    chatbot = bot_class(bot_model, bot_class)

                    # Configure the bot
                    try:
                        await chatbot.configure(app=app)
                        self.add_bot(chatbot)
                        self.logger.info(
                            f"Successfully loaded bot '{bot_model.name}' "
                            f"with {len(bot_model.tools) if bot_model.tools else 0} tools"
                        )
                    except ValidationError as e:
                        self.logger.error(
                            f"Invalid configuration for bot '{bot_model.name}': {e}"
                        )
                    except Exception as e:
                        self.logger.error(
                            f"Failed to configure bot '{bot_model.name}': {e}"
                        )

                except Exception as e:
                    self.logger.error(
                        f"Failed to create bot instance for '{bot_model.name}': {e}"
                    )
                    continue

        self.logger.info(
            f":: Bots loaded successfully. Total active bots: {len(self._bots)}"
        )

    @staticmethod
    def _build_prompt_builder(prompt_config: dict) -> 'PromptBuilder':
        """Build a PromptBuilder from a declarative prompt config dict.

        Args:
            prompt_config: Dict with keys: preset, remove, add, customize.

        Returns:
            Configured PromptBuilder instance.
        """
        from ..bots.prompts.presets import get_preset
        from ..bots.prompts.layers import PromptLayer, LayerPriority, RenderPhase
        from ..bots.prompts.domain_layers import get_domain_layer

        preset_name = prompt_config.get("preset", "default")
        builder = get_preset(preset_name)

        # Remove layers
        for layer_name in prompt_config.get("remove", []):
            builder.remove(layer_name)

        # Add layers
        for item in prompt_config.get("add", []):
            if isinstance(item, str):
                builder.add(get_domain_layer(item))
            elif isinstance(item, dict):
                phase_str = item.get("phase", "configure")
                phase = (
                    RenderPhase.CONFIGURE
                    if phase_str == "configure"
                    else RenderPhase.REQUEST
                )
                layer = PromptLayer(
                    name=item["name"],
                    priority=item.get("priority", LayerPriority.CUSTOM),
                    phase=phase,
                    template=item.get("template", ""),
                )
                builder.add(layer)

        # Customize existing layers
        for layer_name, overrides in prompt_config.get("customize", {}).items():
            existing = builder.get(layer_name)
            if existing:
                replacement = PromptLayer(
                    name=existing.name,
                    priority=existing.priority,
                    phase=existing.phase,
                    template=overrides.get("template", existing.template),
                    condition=existing.condition,
                    required_vars=existing.required_vars,
                )
                builder.replace(layer_name, replacement)

        return builder

    def create_bot(self, class_name: Any = None, name: str = None, **kwargs) -> AbstractBot:
        """Create a Bot and add it to the manager."""
        if class_name is None:
            class_name = Chatbot
        chatbot = class_name(**kwargs)
        chatbot.name = name
        return chatbot

    def add_bot(self, bot: AbstractBot) -> None:
        """Add a Bot to the manager."""
        self._bots[bot.name] = bot
        # Store the class definition for future instance creation
        self._botdef[bot.name] = bot.__class__

    async def get_bot(
        self,
        name: str,
        new: bool = False,
        session_id: str = "",
        **kwargs
    ) -> AbstractBot:
        """Get a Bot by name.

        Args:
            name: Name of the bot to get
            new: If True, create a new instance instead of returning existing one
            session_id: Session identifier for creating unique temporary instances
            **kwargs: Additional arguments to pass to bot constructor when new=True

        Returns:
            Bot instance (existing or newly created)
        """
        # Handle new instance creation
        if new:
            # Get the class definition for this bot
            cls = self._botdef.get(name, BasicAgent)

            # Create unique name to avoid duplicates
            new_name = f"{name}_{session_id}" if session_id else f"{name}_{int(time.time())}"

            # Prepare configuration to inherit from base bot
            base_bot = self._bots.get(name)
            bot_kwargs = kwargs.copy()

            if base_bot:
                # 1. Inherit LLM Configuration if not explicitly provided
                if 'use_llm' not in bot_kwargs and hasattr(base_bot, '_llm_raw'):
                    bot_kwargs['use_llm'] = base_bot._llm_raw

                if 'model' not in bot_kwargs and hasattr(base_bot, '_llm_model'):
                    bot_kwargs['model'] = base_bot._llm_model

                # 2. Clone Tools
                if 'tools' not in bot_kwargs and hasattr(base_bot, 'tool_manager'):
                    try:
                        # Deep copy tools to ensure isolation
                        base_tools = base_bot.tool_manager.get_all_tools()
                        new_tools = []
                        for tool in base_tools:
                            try:
                                # Attempt deep copy
                                new_tool = copy.deepcopy(tool)
                                new_tools.append(new_tool)
                            except Exception as e:
                                self.logger.warning(
                                    f"Failed to copy tool {tool.name}, sharing instance. Error: {e}"
                                )
                                # Fallback to shared instance
                                new_tools.append(tool)
                        bot_kwargs['tools'] = new_tools
                    except Exception as e:
                        self.logger.error(f"Error cloning tools from {name}: {e}")

                # 3. Clone Vector Store Configuration
                if 'vector_store_config' not in bot_kwargs and hasattr(base_bot, '_vector_store'):
                    try:
                        if base_bot._vector_store:
                            bot_kwargs['vector_store_config'] = copy.deepcopy(base_bot._vector_store)
                    except Exception as e:
                        self.logger.warning(f"Failed to copy vector store config: {e}")
                        bot_kwargs['vector_store_config'] = base_bot._vector_store

                if 'use_vectorstore' not in bot_kwargs and hasattr(base_bot, '_use_vector'):
                    bot_kwargs['use_vectorstore'] = getattr(base_bot, '_use_vector', False)

            # Create new instance with merged configuration
            bot = cls(name=new_name, **bot_kwargs)

            # Configure the bot
            await bot.configure(self.app)

            # Add to bots dictionary
            self._bots[new_name] = bot

            # Set expiration time (1 hour from now)
            self._bot_expiration[new_name] = time.time() + 3600

            self.logger.info(
                f"Created new temporary bot instance '{new_name}' from '{name}' "
                f"(expires in 1 hour)"
            )

            return bot

        # Existing behavior for getting/creating bots
        if name not in self._bots:
            self.logger.warning(
                f"Bot '{name}' not in _bots. Available: {list(self._bots.keys())}"
            )
        if name in self._bots:
            _bot = self._bots[name]
            if not getattr(_bot, 'is_configured', False):
                self.logger.warning(f"Bot '{name}' found in _bots and is not configured.")
                await _bot.configure(self.app)
            return self._bots[name]
        if self.registry.has(name):
            try:
                # Get instance (returns singleton if at_startup=True)
                bot_instance = await self.registry.get_instance(name)
                if bot_instance:
                    # Only configure if NOT already configured
                    if not getattr(bot_instance, 'is_configured', False):
                        self.logger.info(f"Configuring bot {name} on demand.")
                        await bot_instance.configure(self.app)
                    self.add_bot(bot_instance)
                    return bot_instance
            except Exception as e:
                self.logger.error(
                    f"Failed to get bot instance from registry: {e}"
                )
        return None

    def remove_bot(self, name: str) -> None:
        """Remove a Bot by name."""
        del self._bots[name]
        # Clean up expiration tracking if it exists (but keep class definition)
        self._bot_expiration.pop(name, None)

    def get_bots(self) -> Dict[str, AbstractBot]:
        """Get all Bots declared on Manager."""
        return self._bots

    async def create_agent(self, class_name: Any = None, name: str = None, **kwargs) -> AbstractBot:
        if class_name is None:
            class_name = BasicAgent
        return class_name(name=name, **kwargs)

    def add_agent(self, agent: AbstractBot) -> None:
        """Add a Agent to the manager."""
        self._bots[str(agent.chatbot_id)] = agent

    def remove_agent(self, agent: AbstractBot) -> None:
        """Remove a Bot by name."""
        del self._bots[str(agent.chatbot_id)]

    async def save_agent(self, name: str, **kwargs) -> None:
        """Save a Agent to the DB."""
        self.logger.info(f"Saving Agent {name} into DB ...")
        db = self.app['database']
        async with await db.acquire() as conn:
            BotModel.Meta.connection = conn
            try:
                try:
                    bot = await BotModel.get(name=name)
                except NoDataFound:
                    bot = None
                if bot:
                    self.logger.info(f"Bot {name} already exists.")
                    for key, val in kwargs.items():
                        bot.set(key, val)
                    await bot.update()
                    self.logger.info(f"Bot {name} updated.")
                else:
                    self.logger.info(f"Bot {name} not found. Creating new one.")
                    # Create a new Bot
                    new_bot = BotModel(
                        name=name,
                        **kwargs
                    )
                    await new_bot.insert()
                self.logger.info(f"Bot {name} saved into DB.")
                return True
            except Exception as e:
                self.logger.error(
                    f"Failed to Create new Bot {name} from DB: {e}"
                )
                return None

    def get_app(self) -> web.Application:
        """Get the app."""
        if self.app is None:
            raise RuntimeError("App is not set.")
        return self.app

    def setup(self, app: web.Application) -> web.Application:
        self.app = None
        if app:
            self.app = app if isinstance(app, web.Application) else app.get_app()
        # register signals for startup and shutdown
        self.app.on_startup.append(self.on_startup)
        self.app.on_shutdown.append(self.on_shutdown)
        # Add Manager to main Application:
        self.app['bot_manager'] = self
        ## Configure Routes
        router = self.app.router
        # Chat Information Router
        router.add_view(
            '/api/v1/chats',
            ChatHandler
        )
        router.add_view(
            '/api/v1/chat/{chatbot_name}',
            ChatHandler
        )
        router.add_view(
            '/api/v1/chat/{chatbot_name}/{method_name}',
            ChatHandler
        )
        # Talk with agents:
        router.add_view(
            '/api/v1/agents/chat/{agent_id}',
            AgentTalk
        )
        router.add_view(
            '/api/v1/agents/chat/{agent_id}/{method_name}',
            AgentTalk
        )
        # Dataset Manager for agents:
        router.add_view(
            '/api/v1/agents/datasets/{agent_id}',
            DatasetManagerHandler
        )
        router.add_view(
            '/api/v1/agents/datasets/{agent_id}/{dataset_id}',
            DatasetManagerHandler
        )
        # ChatBot Manager
        ChatbotHandler.configure(self.app, '/api/v1/bots')
        # Bot Handler
        router.add_view(
            '/api/v1/chatbots',
            BotHandler
        )
        router.add_view(
            '/api/v1/chatbots/{name}',
            BotHandler
        )
        # Streaming Handler:
        st = StreamHandler()
        st.configure_routes(self.app)
        # Crew Configuration
        if ENABLE_CREWS:
            CrewHandler.configure(self.app, '/api/v1/crew')
            CrewExecutionHandler.configure(self.app, '/api/v1/crews')
        # Agent Config CRUD
        router.add_view(
            '/api/v1/agents/config',
            BotConfigHandler
        )
        router.add_view(
            '/api/v1/agents/config/{agent_name}',
            BotConfigHandler
        )
        # Agent Testing (session-based)
        router.add_view(
            '/api/v1/agents/test/{agent_name}',
            BotConfigTestHandler
        )
        # Chat Interaction Persistence
        router.add_view(
            '/api/v1/chat/interactions',
            ChatInteractionHandler
        )
        router.add_view(
            '/api/v1/chat/interactions/{session_id}',
            ChatInteractionHandler
        )
        # Dashboard Persistence
        if ENABLE_DASHBOARDS:
            router.add_view(
                '/api/v1/dashboards',
                DashboardHandler
            )
            router.add_view(
                '/api/v1/dashboards/{dashboard_id}',
                DashboardHandler
            )
            router.add_view(
                '/api/v1/dashboards/{dashboard_id}/tabs',
                DashboardTabHandler
            )
            router.add_view(
                '/api/v1/dashboards/{dashboard_id}/tabs/{tab_id}',
                DashboardTabHandler
            )
        # User credential management routes
        setup_credentials_routes(self.app)
        if self.enable_swagger_api:
            self.logger.info("Setting up OpenAPI documentation...")
            setup_swagger(self.app)
        self.logger.info("""
✅ OpenAPI Documentation configured successfully!

Available documentation UIs:
- Swagger UI:  http://localhost:5000/api/docs
- ReDoc:       http://localhost:5000/api/docs/redoc
- RapiDoc:     http://localhost:5000/api/docs/rapidoc
- OpenAPI Spec: http://localhost:5000/api/docs/swagger.json
        """)
        return self.app

    async def _cleanup_expired_bots(self) -> None:
        """Background task to cleanup expired temporary bot instances.

        Runs every 5 minutes to check for and remove bot instances that have
        exceeded their expiration time (typically 1 hour after creation).
        """
        while True:
            try:
                await asyncio.sleep(300)  # Check every 5 minutes
                current_time = time.time()

                # Find all expired bots
                expired = [
                    name for name, expiry in self._bot_expiration.items()
                    if current_time > expiry
                ]

                # Remove expired bots
                for name in expired:
                    try:
                        self.logger.info(f"Removing expired bot instance: {name}")
                        self.remove_bot(name)
                        del self._bot_expiration[name]
                    except Exception as e:
                        self.logger.error(
                            f"Error removing expired bot '{name}': {e}"
                        )
                        # Remove from expiration tracking even if removal failed
                        self._bot_expiration.pop(name, None)

                if expired:
                    self.logger.info(
                        f"Cleaned up {len(expired)} expired bot instance(s). "
                        f"Active bots: {len(self._bots)}, "
                        f"Tracked expirations: {len(self._bot_expiration)}"
                    )
            except asyncio.CancelledError:
                self.logger.info("Cleanup task cancelled")
                raise
            except Exception as e:
                self.logger.error(
                    f"Error in cleanup task: {e}",
                    exc_info=True
                )
                # Continue running even if there's an error

    async def on_startup(self, app: web.Application) -> None:
        """On startup."""
        # configure all pre-configured chatbots:
        await self.load_bots(app)
        # Initialize BotConfigStorage and attach to app
        if self.enable_registry_bots:
            app['bot_config_storage'] = BotConfigStorage()
        # Load crews from Redis
        if self.enable_crews:
            await self.load_crews()
        # Start background cleanup task for expired bots
        self._cleanup_task = asyncio.create_task(self._cleanup_expired_bots())
        self.logger.info("Started background cleanup task for temporary bot instances")
        self.logger.info("Started background cleanup task for temporary bot instances")
        # Initialize ChatStorage (Redis + DocumentDB)
        chat_storage = ChatStorage()
        try:
            await chat_storage.initialize()
            self.logger.info("ChatStorage initialized (Redis + DocumentDB)")
        except Exception as exc:
            self.logger.warning(f"ChatStorage initialization failed: {exc}")
        # Initialize Dashboard indexes
        if ENABLE_DASHBOARDS:
            await _ensure_dashboard_indexes(app)
        app['chat_storage'] = chat_storage
        # Start Integration bots
        self._integration_manager = IntegrationBotManager(self)
        await self._integration_manager.startup()

    async def on_shutdown(self, app: web.Application) -> None:
        """On shutdown."""
        # Cancel background cleanup task
        if self._cleanup_task:
            self._cleanup_task.cancel()
            try:
                await self._cleanup_task
            except asyncio.CancelledError:
                pass
            self.logger.info("Stopped background cleanup task")
        # Stop Integration bots
        if self._integration_manager:
            await self._integration_manager.shutdown()
        # Close ChatStorage
        chat_storage = app.get('chat_storage')
        if chat_storage:
            await chat_storage.close()
            self.logger.info("ChatStorage closed")

    async def add_crew(
        self,
        name: str,
        crew: AgentCrew,
        crew_def: CrewDefinition
    ) -> None:
        """
        Register a crew in the manager and persist to Redis.

        Args:
            name: Unique name for the crew
            crew: AgentCrew instance
            crew_def: Crew definition containing metadata

        Raises:
            ValueError: If crew with same name already exists
        """
        tenant = self._normalize_tenant(crew_def.tenant)
        crew_key = self._get_crew_key(tenant, name)
        if crew_key in self._crews:
            raise ValueError(f"Crew '{name}' already exists")

        # Add to memory
        self._crews[crew_key] = (crew, crew_def)

        # Persist to Redis
        try:
            await self.crew_redis.save_crew(crew_def)
            self.logger.info(
                f"Registered crew '{name}' with {len(crew.agents)} agents "
                f"in {crew_def.execution_mode.value} mode and saved to Redis"
            )
        except Exception as e:
            self.logger.error(f"Failed to save crew '{name}' to Redis: {e}")
            # Don't fail the operation if Redis fails, crew is still in memory
            self.logger.info(
                f"Crew '{name}' registered in memory only (Redis persistence failed)"
            )

    async def get_crew(
        self,
        identifier: str,
        as_new: bool = False,
        tenant: Optional[str] = None
    ) -> Optional[Tuple[AgentCrew, CrewDefinition]]:
        """
        Get a crew by name or ID. Loads from Redis if not in memory.

        Args:
            identifier: Crew name or crew_id
            as_new: If True, creates a new instance (default False)
            tenant: Tenant identifier

        Returns:
            Tuple of (AgentCrew, CrewDefinition) if found, None otherwise
        """
        crew_def = None
        cached_crew = None
        tenant = self._normalize_tenant(tenant)
        crew_key = self._get_crew_key(tenant, identifier)

        # 1. Resolve Crew Definition from Memory
        if crew_key in self._crews:
            cached_crew, crew_def = self._crews[crew_key]
        else:
            # Check by crew_id in memory
            for _, (c, cd) in self._crews.items():
                if cd.crew_id == identifier and cd.tenant == tenant:
                    cached_crew, crew_def = c, cd
                    break

        # 2. If valid definition found in memory
        if crew_def:
            if as_new:
                # Create fresh instance from definition
                try:
                    new_crew = await self._create_crew_from_definition(crew_def)
                    return (new_crew, crew_def)
                except Exception as e:
                    self.logger.error(
                        f"Failed to create new crew instance: {e}"
                    )
                    return (None, None)
            else:
                return (cached_crew, crew_def)

        # 3. If not in memory, try Redis
        try:
            # Try to load by name first
            crew_def = await self.crew_redis.load_crew(identifier, tenant)
            # If not found by name, try by ID
            if not crew_def:
                crew_def = await self.crew_redis.load_crew_by_id(identifier, tenant)

            if crew_def:
                # We found it in Redis!
                # We need to instantiate it to cache it (so we have definition for next time)
                base_crew = await self._create_crew_from_definition(crew_def)

                # Update Cache
                cache_key = self._get_crew_key(crew_def.tenant, crew_def.name)
                self._crews[cache_key] = (base_crew, crew_def)

                self.logger.info(
                    f"Loaded crew '{crew_def.name}' from Redis "
                    f"(ID: {crew_def.crew_id})"
                )

                if as_new:
                    return (await self._create_crew_from_definition(crew_def), crew_def)
                else:
                    return (base_crew, crew_def)

        except Exception as e:
            self.logger.error(
                f"Error loading crew '{identifier}' from Redis: {e}"
            )
            return (None, None)

        return (None, None)

    def list_crews(
        self,
        tenant: Optional[str] = None
    ) -> Dict[str, Tuple[AgentCrew, CrewDefinition]]:
        """
        List all registered crews.

        Returns:
            Dictionary mapping crew names to (AgentCrew, CrewDefinition) tuples
        """
        if tenant is None:
            return self._crews.copy()
        tenant = self._normalize_tenant(tenant)
        return {
            crew_def.name: (crew, crew_def)
            for _, (crew, crew_def) in self._crews.items()
            if crew_def.tenant == tenant
        }

    async def remove_crew(
        self,
        identifier: str,
        tenant: Optional[str] = None
    ) -> bool:
        """
        Remove a crew from the manager and Redis.

        Args:
            identifier: Crew name or crew_id
            tenant: Tenant identifier

        Returns:
            True if removed, False if not found
        """
        crew_name = None
        crew_def = None
        tenant = self._normalize_tenant(tenant)

        # Try by name first
        crew_key = self._get_crew_key(tenant, identifier)
        if crew_key in self._crews:
            crew_name = identifier
            _, crew_def = self._crews[crew_key]
            del self._crews[crew_key]
        else:
            # Try by crew_id
            for key, (_, def_) in list(self._crews.items()):
                if def_.crew_id == identifier and def_.tenant == tenant:
                    crew_name = def_.name
                    crew_def = def_
                    del self._crews[key]
                    break

        if crew_name and crew_def:
            # Remove from Redis
            try:
                await self.crew_redis.delete_crew(crew_def.name, crew_def.tenant)
                self.logger.info(
                    f"Removed crew '{crew_name}' (ID: {crew_def.crew_id}) "
                    f"from memory and Redis"
                )
            except Exception as e:
                self.logger.error(
                    f"Failed to delete crew '{crew_name}' from Redis: {e}"
                )
                self.logger.info(
                    f"Crew '{crew_name}' removed from memory only"
                )
            return True

        return False

    def update_crew(
        self,
        identifier: str,
        crew: AgentCrew,
        crew_def: CrewDefinition
    ) -> bool:
        """
        Update an existing crew.

        Args:
            identifier: Crew name or crew_id
            crew: Updated AgentCrew instance
            crew_def: Updated crew definition

        Returns:
            True if updated, False if not found
        """
        crew_key = self._get_crew_key(crew_def.tenant, identifier)
        if crew_key in self._crews:
            self._crews[crew_key] = (crew, crew_def)
            self.logger.info(f"Updated crew '{identifier}'")
            return True

        for key, (_, def_) in self._crews.items():
            if def_.crew_id == identifier and def_.tenant == crew_def.tenant:
                self._crews[key] = (crew, crew_def)
                self.logger.info(f"Updated crew '{def_.name}'")
                return True

        return False

    async def load_crews(self) -> None:
        """
        Load all crews from Redis on startup.

        This method is called during application startup to restore
        all previously saved crews from Redis into memory.
        """
        try:
            # Check Redis connection
            if not await self.crew_redis.ping():
                self.logger.warning("Redis connection failed, skipping crew loading")
                return

            # Get all crew definitions from Redis
            crew_defs = await self.crew_redis.get_all_crews()

            if not crew_defs:
                self.logger.info("No crews found in Redis")
                return

            self.logger.info(f"Loading {len(crew_defs)} crews from Redis...")

            loaded_count = 0
            for crew_def in crew_defs:
                try:
                    # Reconstruct the crew from definition
                    crew = await self._create_crew_from_definition(crew_def)

                    # Add to memory (without saving back to Redis)
                    crew_key = self._get_crew_key(crew_def.tenant, crew_def.name)
                    self._crews[crew_key] = (crew, crew_def)

                    loaded_count += 1
                    self.logger.info(
                        f"Loaded crew '{crew_def.name}' with {len(crew_def.agents)} agents "
                        f"in {crew_def.execution_mode.value} mode"
                    )
                except Exception as e:
                    self.logger.error(
                        f"Failed to load crew '{crew_def.name}': {e}",
                        exc_info=True
                    )

            self.logger.info(
                f":: Crews loaded successfully. Total active crews: {loaded_count}"
            )
        except Exception as e:
            self.logger.error(
                f"Failed to load crews from Redis: {e}",
                exc_info=True
            )

    async def sync_crews(self) -> None:
        """
        Synchronize in-memory crews with Redis.

        This handles:
        1. Loading new crews added by other workers
        2. Removing crews deleted by other workers
        """
        try:
            # Get all crew names from Redis
            remote_entries = await self.crew_redis.list_all_crews()
            remote_names = {
                self._get_crew_key(entry["tenant"], entry["name"])
                for entry in remote_entries
            }
            local_names = set(self._crews.keys())

            # Identify additions and removals
            added = remote_names - local_names
            removed = local_names - remote_names

            if not added and not removed:
                return

            self.logger.debug(
                f"Syncing crews: {len(added)} to add, {len(removed)} to remove"
            )

            # Handle additions
            for key in added:
                try:
                    tenant, name = self._split_crew_key(key)
                    crew_def = await self.crew_redis.load_crew(name, tenant)
                    if crew_def:
                        crew = await self._create_crew_from_definition(crew_def)
                        self._crews[key] = (crew, crew_def)
                        self.logger.info(f"Synced new crew '{name}' from Redis")
                except Exception as e:
                    # Provide more specific diagnostics, especially for malformed keys.
                    if isinstance(e, ValueError):
                        self.logger.error(
                            f"Failed to sync crew: invalid key format {key!r}: {e}"
                        )
                    else:
                        self.logger.error(
                            f"Failed to sync crew for key {key!r}: {e}",
                            exc_info=True,
                        )

            # Handle removals
            for key in removed:
                self._crews.pop(key, None)
                self.logger.info(f"Synced removal of crew '{key}'")

        except Exception as e:
            self.logger.error(f"Error syncing crews: {e}", exc_info=True)

    async def _create_crew_from_definition(
        self,
        crew_def: CrewDefinition
    ) -> AgentCrew:
        """
        Create an AgentCrew instance from a CrewDefinition.

        This method reconstructs a crew from its JSON definition,
        creating all agents and setting up flow relations.

        Args:
            crew_def: Crew definition

        Returns:
            AgentCrew instance
        """

        # Create agents
        agents = []
        for agent_def in crew_def.agents:
            # Get agent class
            agent_class = self.get_bot_class(agent_def.agent_class)
            if not agent_class:
                self.logger.warning(
                    f"Agent class '{agent_def.agent_class}' not found, "
                    f"using BasicAgent as fallback"
                )
                agent_class = BasicAgent

            # Collect tools
            tools = []
            if agent_def.tools:
                tools.extend(iter(agent_def.tools))

            # Create agent instance
            agent = agent_class(
                name=agent_def.name or agent_def.agent_id,
                tools=tools,
                **agent_def.config
            )

            # Set system prompt if provided
            if agent_def.system_prompt:
                agent.system_prompt = agent_def.system_prompt

            agents.append(agent)

        # Create crew
        crew = AgentCrew(
            name=crew_def.name,
            agents=agents,
            max_parallel_tasks=crew_def.max_parallel_tasks
        )

        # Add shared tools
        for tool_name in crew_def.shared_tools:
            # Try to get tool from registry or bot manager
            # This is a placeholder - implement tool retrieval as needed
            try:
                # You may need to implement get_tool method
                # For now, we'll skip tools that aren't available
                self.logger.debug(
                    f"Shared tool '{tool_name}' for crew '{crew_def.name}' "
                    f"(implement tool retrieval as needed)"
                )
            except Exception as e:
                self.logger.warning(
                    f"Could not add shared tool '{tool_name}': {e}"
                )

        # Setup flow relations if in flow mode
        if crew_def.execution_mode == ExecutionMode.FLOW and crew_def.flow_relations:
            for relation in crew_def.flow_relations:
                try:
                    # Convert agent IDs to agent objects
                    source_agents = self._get_agents_by_ids(
                        crew,
                        relation.source if isinstance(relation.source, list) else [relation.source]
                    )
                    target_agents = self._get_agents_by_ids(
                        crew,
                        relation.target if isinstance(relation.target, list) else [relation.target]
                    )

                    # Setup flow
                    crew.task_flow(
                        source_agents if len(source_agents) > 1 else source_agents[0],
                        target_agents if len(target_agents) > 1 else target_agents[0]
                    )
                except Exception as e:
                    self.logger.error(
                        f"Failed to setup flow relation for crew '{crew_def.name}': {e}"
                    )

        return crew

    def _get_agents_by_ids(
        self,
        crew: AgentCrew,
        agent_ids: List[str]
    ) -> List[Any]:
        """
        Get agent objects from crew by their IDs.

        Args:
            crew: AgentCrew instance
            agent_ids: List of agent IDs

        Returns:
            List of agent objects
        """
        agents = []
        for agent_id in agent_ids:
            if agent := crew.agents.get(agent_id):
                agents.append(agent)
            else:
                self.logger.warning(f"Agent '{agent_id}' not found in crew")
        return agents

    def get_crew_stats(self) -> Dict[str, Any]:
        """
        Get statistics about registered crews.

        Returns:
            Dictionary with crew statistics
        """
        stats = {
            'total_crews': len(self._crews),
            'crews_by_mode': {
                'sequential': 0,
                'parallel': 0,
                'flow': 0
            },
            'total_agents': 0,
            'crews': []
        }

        for name, (crew, crew_def) in self._crews.items():
            mode = crew_def.execution_mode.value
            stats['crews_by_mode'][mode] = stats['crews_by_mode'].get(mode, 0) + 1
            stats['total_agents'] += len(crew.agents)

            stats['crews'].append({
                'name': crew_def.name,
                'tenant': crew_def.tenant,
                'crew_id': crew_def.crew_id,
                'mode': mode,
                'agent_count': len(crew.agents)
            })

        return stats
