"""Provide the ImJoy core API interface."""

import asyncio
import inspect
import io
import json
import logging
import sys
import os
import uuid
import time
from enum import Enum
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from pydantic import BaseModel, Field, field_validator
import msgpack
from pydantic import (  # pylint: disable=no-name-in-module
    BaseModel,
    EmailStr,
    PrivateAttr,
    constr,
    SerializeAsAny,
    ConfigDict,
    AnyHttpUrl,
)

from hypha.utils import EventBus
import jsonschema
from hypha.core.activity import ActivityTracker

from prometheus_client import Counter, Gauge

LOGLEVEL = os.environ.get("HYPHA_LOGLEVEL", "WARNING").upper()
logging.basicConfig(level=LOGLEVEL, stream=sys.stdout)
logger = logging.getLogger("core")
logger.setLevel(LOGLEVEL)


class VisibilityEnum(str, Enum):
    """Represent the visibility of the workspace."""

    public = "public"
    protected = "protected"


class StatusEnum(str, Enum):
    """Represent the status of a component."""

    ready = "ready"
    initializing = "initializing"
    not_initialized = "not_initialized"


class ServiceConfig(BaseModel):
    """Represent service config."""

    model_config = ConfigDict(extra="allow", use_enum_values=False)

    visibility: VisibilityEnum = VisibilityEnum.protected
    require_context: Union[Tuple[str], List[str], bool] = False
    workspace: Optional[str] = None
    flags: List[str] = []
    singleton: Optional[bool] = False
    created_by: Optional[Dict] = None
    service_embedding: Optional[Any] = None


class ServiceInfo(BaseModel):
    """Represent service."""

    model_config = ConfigDict(extra="allow")

    config: Optional[SerializeAsAny[ServiceConfig]] = None
    id: str
    name: Optional[str] = None
    type: Optional[str] = "generic"
    description: Optional[constr(max_length=1024)] = None  # type: ignore
    docs: Optional[str] = None
    app_id: Optional[str] = None
    service_schema: Optional[Dict[str, Any]] = None
    score: Optional[float] = None  # score generated by vector search

    def is_singleton(self):
        """Check if the service is singleton."""
        return "single-instance" in self.config.flags

    def to_redis_dict(self):
        """
        Serialize the model to a Redis-compatible dictionary based on field types.
        """
        data = self.model_dump()
        redis_data = {}
        # Note: Here we only store the fields that are in the model
        # and ignore any extra fields that might be present in the data
        # Iterate over fields and encode based on their type
        for field_name, field_info in self.model_fields.items():
            value = data.get(field_name)
            if value is None or field_name == "score":
                continue
            elif field_name == "config":
                redis_data[field_name] = value
            elif field_info.annotation in {str, Optional[str]}:
                redis_data[field_name] = value
            elif field_info.annotation in {list, List[str], Optional[List[str]]}:
                redis_data[field_name] = ",".join(value)
            else:
                redis_data[field_name] = json.dumps(value)

        # Expand config fields to store as separate keys
        if "config" in redis_data and redis_data["config"]:
            # Iterate through the keys for ServiceConfig and store them to self
            for field_name, field_info in ServiceConfig.model_fields.items():
                value = redis_data["config"].get(field_name)
                if value is None:
                    continue
                elif field_name == "service_embedding":
                    redis_data[field_name] = value
                elif field_info.annotation in {str, Optional[str]}:
                    redis_data[field_name] = value
                elif field_info.annotation in {list, List[str], Optional[List[str]]}:
                    redis_data[field_name] = ",".join(value)
                else:
                    redis_data[field_name] = json.dumps(value)
            del redis_data["config"]
        return redis_data

    @classmethod
    def from_redis_dict(cls, service_data: Dict[str, Any], in_bytes=True):
        """
        Deserialize a Redis-compatible dictionary back to a model instance.
        """
        converted_data = {}
        # Note: Here we only convert the fields that are in the model
        # and ignore any extra fields that might be present in the data
        # Iterate over fields and decode based on their type
        # Extract the fields form ServiceConfig first
        config_data = {}
        for field_name, field_info in ServiceConfig.model_fields.items():
            if not in_bytes:
                if field_name not in service_data:
                    continue
                value = service_data.get(field_name)
                del service_data[field_name]
            else:
                if field_name.encode("utf-8") not in service_data:
                    continue
                value = service_data.get(field_name.encode("utf-8"))
                del service_data[field_name.encode("utf-8")]
            if value is None:
                config_data[field_name] = None
            elif field_name == "service_embedding":
                config_data[field_name] = value
            elif field_info.annotation in {str, Optional[str]}:
                config_data[field_name] = (
                    value if isinstance(value, str) else value.decode("utf-8")
                )
            elif field_info.annotation in {list, List[str], Optional[List[str]]}:
                config_data[field_name] = (
                    value.split(",")
                    if isinstance(value, str)
                    else value.decode("utf-8").split(",")
                )
            else:
                value_str = value if isinstance(value, str) else value.decode("utf-8")
                config_data[field_name] = json.loads(value_str)

        if config_data:
            if in_bytes:
                service_data[b"config"] = ServiceConfig.model_validate(config_data)
            else:
                service_data["config"] = ServiceConfig.model_validate(config_data)

        for field_name, field_info in cls.model_fields.items():
            if not in_bytes:
                value = service_data.get(field_name)
            else:
                value = service_data.get(field_name.encode("utf-8"))
            if value is None:
                converted_data[field_name] = None
            elif field_name == "config":
                converted_data[field_name] = value
            elif field_info.annotation in {str, Optional[str]}:
                converted_data[field_name] = (
                    value if isinstance(value, str) else value.decode("utf-8")
                )
            elif field_info.annotation in {list, List[str], Optional[List[str]]}:
                config_data[field_name] = (
                    value.split(",")
                    if isinstance(value, str)
                    else value.decode("utf-8").split(",")
                )
            else:
                value_str = value if isinstance(value, str) else value.decode("utf-8")
                converted_data[field_name] = json.loads(value_str)
        return cls.model_validate(converted_data)

    @classmethod
    def model_validate(cls, data):
        data = data.copy()
        if "config" in data and data["config"] is not None:
            data["config"] = ServiceConfig.model_validate(data["config"])
        return super().model_validate(data)


class RemoteService(ServiceInfo):
    pass


class UserTokenInfo(BaseModel):
    """Represent user profile."""

    token: constr(max_length=1024)  # type: ignore
    workspace: Optional[str] = None
    expires_in: Optional[int] = None
    email: Optional[EmailStr] = None
    email_verified: Optional[bool] = None
    name: Optional[constr(max_length=64)] = None  # type: ignore
    nickname: Optional[constr(max_length=64)] = None  # type: ignore
    user_id: Optional[constr(max_length=64)] = None  # type: ignore
    picture: Optional[AnyHttpUrl] = None


class UserPermission(str, Enum):
    """Represent the permission of the workspace."""

    read = "r"
    read_write = "rw"
    admin = "a"


class ScopeInfo(BaseModel):
    """Represent scope info."""

    model_config = ConfigDict(extra="allow")

    current_workspace: Optional[str] = None
    workspaces: Optional[Dict[str, UserPermission]] = {}
    client_id: Optional[str] = None
    extra_scopes: Optional[List[str]] = []  # extra scopes


class UserInfo(BaseModel):
    """Represent user info."""

    id: str
    roles: List[str]
    is_anonymous: bool
    scope: Optional[ScopeInfo] = None
    email: Optional[EmailStr] = None
    parent: Optional[str] = None
    expires_at: Optional[float] = None
    _metadata: Dict[str, Any] = PrivateAttr(
        default_factory=lambda: {}
    )  # e.g. s3 credential

    def get_workspace(self):
        return f"ws-user-{self.id}"

    def get_metadata(self, key=None) -> Dict[str, Any]:
        """Return the metadata."""
        if key:
            return self._metadata.get(key)
        return self._metadata

    def set_metadata(self, key, value):
        """Set the metadata."""
        self._metadata[key] = value

    def get_permission(self, workspace: str):
        """Get the workspace permission."""
        if not self.scope:
            return None
        assert isinstance(workspace, str)
        if self.scope.workspaces.get("*"):
            return self.scope.workspaces["*"]
        return self.scope.workspaces.get(workspace, None)

    def check_permission(self, workspace: str, minimal_permission: UserPermission):
        permission = self.get_permission(workspace)
        if not permission:
            return False
        if minimal_permission == UserPermission.read:
            if permission in [
                UserPermission.read,
                UserPermission.read_write,
                UserPermission.admin,
            ]:
                return True
        elif minimal_permission == UserPermission.read_write:
            if permission in [UserPermission.read_write, UserPermission.admin]:
                return True
        elif minimal_permission == UserPermission.admin:
            if permission == UserPermission.admin:
                return True

        return False


class ClientInfo(BaseModel):
    """Represent service."""

    id: str
    parent: Optional[str] = None
    name: Optional[str] = None
    workspace: str
    services: List[SerializeAsAny[ServiceInfo]] = []
    user_info: UserInfo

    @classmethod
    def model_validate(cls, data):
        data = data.copy()
        data["user_info"] = UserInfo.model_validate(data["user_info"])
        data["services"] = [
            ServiceInfo.model_validate(service) for service in data["services"]
        ]
        return super().model_validate(data)


class Artifact(BaseModel):
    """Represent resource description file object."""

    model_config = ConfigDict(extra="allow")

    type: Optional[str] = "generic"
    format_version: str = "0.2.1"
    name: Optional[str] = None
    id: Optional[str] = None
    description: Optional[str] = None
    tags: Optional[List[str]] = None
    documentation: Optional[str] = None
    covers: Optional[List[str]] = None
    badges: Optional[List[str]] = None
    authors: Optional[List[Dict[str, str]]] = None
    attachments: Optional[Dict[str, List[Any]]] = None
    files: Optional[List[Dict[str, Any]]] = None
    config: Optional[Dict[str, Any]] = None
    version: Optional[str] = "0.1.0"
    links: Optional[List[str]] = None
    maintainers: Optional[List[Dict[str, str]]] = None
    license: Optional[str] = None
    git_repo: Optional[str] = None
    source: Optional[str] = None
    config: Optional[Dict[str, Any]] = None

    @classmethod
    def model_validate(cls, data):
        data = data.copy()
        if "services" in data and data["services"] is not None:
            data["services"] = [
                ServiceInfo.model_validate(service) for service in data["services"]
            ]
        return super().model_validate(data)


class CollectionArtifact(Artifact):
    """Represent collection artifact."""

    type: Optional[str] = "collection"
    collection: Optional[List[str]] = []
    collection_schema: Optional[Dict[str, Any]] = None

    @classmethod
    def model_validate(cls, data):
        data = data.copy()
        if "collection" in data and data["collection"] is not None:
            data["collection"] = [
                CollectionArtifact.model_validate(artifact)
                for artifact in data["collection"]
            ]
        if "collection_schema" in data and data["collection_schema"] is not None:
            # make sure the schema is a valid json schema
            jsonschema.Draft7Validator.check_schema(data["collection_schema"])
        return super().model_validate(data)


class ApplicationManifest(Artifact):
    """Represent application artifact."""

    passive: Optional[bool] = False
    dependencies: Optional[List[str]] = []
    requirements: Optional[List[str]] = []
    api_version: Optional[str] = "0.1.0"
    icon: Optional[str] = None
    env: Optional[Union[str, Dict[str, Any]]] = None
    type: Optional[str] = "application"
    entry_point: Optional[str] = None  # entry point for the application
    daemon: Optional[bool] = False  # whether the application is a daemon
    singleton: Optional[bool] = False  # whether the application is a singleton
    services: Optional[List[SerializeAsAny[ServiceInfo]]] = None  # for application
    stop_after_inactive: Optional[float] = None  # stop the application after inactivity


class ServiceTypeInfo(BaseModel):
    """Represent service type info."""

    id: str
    name: str
    definition: Dict[str, Any]
    description: Optional[str] = None
    config: Optional[Dict[str, Any]] = {}
    docs: Optional[str] = None


class WorkspaceInfo(BaseModel):
    """Represent a workspace."""

    type: str = "workspace"
    id: Optional[str] = None  # we will use name as id if not provided
    name: str
    description: Optional[str] = None
    persistent: Optional[bool] = False
    owners: Optional[List[str]] = []
    read_only: Optional[bool] = False
    icon: Optional[str] = None
    covers: Optional[List[str]] = None
    docs: Optional[str] = None
    service_types: Optional[Dict[str, ServiceTypeInfo]] = {}
    config: Optional[Dict[str, Any]] = {}
    status: Optional[Dict[str, Any]] = None

    def __init__(self, **data):
        super().__init__(**data)
        if self.id is None:
            self.id = self.name

    @classmethod
    def model_validate(cls, data):
        data = data.copy()
        return super().model_validate(data)


class TokenConfig(BaseModel):
    expires_in: Optional[int] = Field(None, description="Expiration time in seconds")
    workspace: Optional[str] = Field(None, description="Workspace name")
    permission: Optional[str] = Field(
        "read_write",
        description="Permission level",
        pattern="^(read|read_write|admin)$",
    )
    extra_scopes: Optional[List[str]] = Field(None, description="List of extra scopes")

    @field_validator("extra_scopes")
    def validate_scopes(cls, v):
        if ":" in v and v.count(":") == 1:
            prefix = v.split(":")[0]
            if prefix in ["ws", "cid"]:
                raise ValueError("Invalid scope, cannot start with ws or cid")
        return v


background_tasks = set()


class RedisRPCConnection:
    """Represent a Redis connection for handling RPC-like messaging."""

    _counter = Counter("rpc_call", "Counts the RPC calls", ["workspace"])
    _tracker = None

    @classmethod
    def set_activity_tracker(cls, tracker: ActivityTracker):
        cls._tracker = tracker

    def __init__(
        self,
        event_bus: EventBus,
        workspace: str,
        client_id: str,
        user_info: UserInfo,
        manager_id: str,
    ):
        """Initialize Redis RPC Connection."""
        assert workspace and "/" not in client_id, "Invalid workspace or client ID"
        self._workspace = workspace
        self._client_id = client_id
        self._user_info = user_info.model_dump()
        self._stop = False
        self._event_bus = event_bus
        self._handle_connected = None
        self._handle_disconnected = None
        self._handle_message = None
        self.manager_id = manager_id

    def on_disconnected(self, handler):
        """Register a disconnection event handler."""
        self._handle_disconnected = handler

    def on_connected(self, handler):
        """Register a connection open event handler."""
        self._handle_connected = handler
        assert inspect.iscoroutinefunction(
            handler
        ), "Connect handler must be a coroutine"

    def on_message(self, handler: Callable):
        """Set message handler."""
        self._handle_message = handler
        self._event_bus.on(f"{self._workspace}/{self._client_id}:msg", handler)
        # for broadcast messages
        self._event_bus.on(f"{self._workspace}/*:msg", handler)
        if self._handle_connected:
            task = asyncio.create_task(self._handle_connected(self))
            background_tasks.add(task)
            task.add_done_callback(background_tasks.discard)

    async def emit_message(self, data: Union[dict, bytes]):
        """Send message after packing additional info."""
        assert isinstance(data, bytes), "Data must be bytes"
        if self._stop:
            raise ValueError(
                f"Connection has already been closed (client: {self._workspace}/{self._client_id})"
            )
        unpacker = msgpack.Unpacker(io.BytesIO(data))
        message = unpacker.unpack()
        pos = unpacker.tell()
        target_id = message.get("to")
        if "/" not in target_id:
            if "/ws-" in target_id:
                raise ValueError(
                    f"Invalid target ID: {target_id}, it appears that the target is a workspace manager (target_id should starts with */)"
                )
            target_id = f"{self._workspace}/{target_id}"

        source_id = f"{self._workspace}/{self._client_id}"

        message.update(
            {
                "ws": (
                    target_id.split("/")[0]
                    if self._workspace == "*"
                    else self._workspace
                ),
                "to": target_id,
                "from": source_id,
                "user": self._user_info,
            }
        )

        packed_message = msgpack.packb(message) + data[pos:]
        # logger.info(f"Sending message to channel {target_id}:msg")
        await self._event_bus.emit(f"{target_id}:msg", packed_message)
        RedisRPCConnection._counter.labels(workspace=self._workspace).inc()
        await RedisRPCConnection._tracker.reset_timer(
            self._workspace + "/" + self._client_id, "client"
        )

    async def disconnect(self, reason=None):
        """Handle disconnection."""
        self._stop = True
        if self._handle_message:
            self._event_bus.off(
                f"{self._workspace}/{self._client_id}:msg", self._handle_message
            )
            self._event_bus.off(f"{self._workspace}/*:msg", self._handle_message)

        self._handle_message = None
        logger.debug(
            f"Redis Connection Disconnected: {self._workspace}/{self._client_id}"
        )
        if self._handle_disconnected:
            await self._handle_disconnected(reason)

        await RedisRPCConnection._tracker.remove_entity(
            self._workspace + "/" + self._client_id, "client"
        )


class RedisEventBus:
    """Represent a redis event bus."""

    _counter = Counter(
        "event_bus", "Counts the events on the redis event bus", ["event", "status"]
    )
    _pubsub_latency = Gauge(
        "redis_pubsub_latency_seconds", "Redis pubsub latency in seconds"
    )

    def __init__(self, redis) -> None:
        """Initialize the event bus."""
        self._redis = redis
        self._handle_connected = None
        self._stop = False
        self._local_event_bus = EventBus(logger)
        self._redis_event_bus = EventBus(logger)
        self._reconnect_delay = 1  # Start with 1 second delay
        self._max_reconnect_delay = 30  # Maximum delay between reconnection attempts
        self._health_check_interval = 5  # Health check every 5 seconds
        self._consecutive_failures = 0
        self._max_failures = 3  # Circuit breaker threshold
        self._circuit_breaker_open = False
        self._last_successful_connection = None
        self._health_check_channel = "health_check:" + str(uuid.uuid4())
        self._health_check_pubsub = None
        self._pubsub_health_future = None

    async def init(self):
        """Setup the event bus."""
        loop = asyncio.get_running_loop()
        self._ready = loop.create_future()
        self._loop = loop

        # Start the Redis subscription task
        self._subscribe_task = loop.create_task(self._subscribe_redis())
        self._health_check_task = loop.create_task(self._health_check())

        # Wait for readiness signal
        await self._ready

    def on(self, event_name, func):
        """Register a callback for an event from Redis."""
        self._redis_event_bus.on(event_name, func)

    def off(self, event_name, func=None):
        """Unregister a callback for an event from Redis."""
        self._redis_event_bus.off(event_name, func)

    def once(self, event_name, func):
        """Register a callback for an event and remove it once triggered."""

        def once_wrapper(*args, **kwargs):
            result = func(*args, **kwargs)
            self._redis_event_bus.off(event_name, once_wrapper)
            return result

        self._redis_event_bus.on(event_name, once_wrapper)

    async def wait_for(self, event_name, match=None, timeout=None):
        """Wait for an event from either local or Redis event bus."""
        local_future = asyncio.create_task(
            self._local_event_bus.wait_for(event_name, match, timeout)
        )
        redis_future = asyncio.create_task(
            self._redis_event_bus.wait_for(event_name, match, timeout)
        )
        done, pending = await asyncio.wait(
            [local_future, redis_future], return_when=asyncio.FIRST_COMPLETED
        )
        for task in pending:
            task.cancel()
        return done.pop().result()

    def on_local(self, event_name, func):
        """Register a callback for a local event."""
        return self._local_event_bus.on(event_name, func)

    def off_local(self, event_name, func=None):
        """Unregister a callback for a local event."""
        return self._local_event_bus.off(event_name, func)

    def once_local(self, event_name, func):
        """Register a callback for a local event and remove it once triggered."""

        def once_wrapper(*args, **kwargs):
            result = func(*args, **kwargs)
            self._local_event_bus.off(event_name, once_wrapper)
            return result

        return self._local_event_bus.on(event_name, once_wrapper)

    async def wait_for_local(self, event_name, match=None, timeout=None):
        """Wait for local event."""
        return await self._local_event_bus.wait_for(event_name, match, timeout)

    async def emit_local(self, event_name, data=None):
        """Emit a local event."""
        if not self._ready.done():
            self._ready.set_result(True)
        local_task = self._local_event_bus.emit(event_name, data)
        if asyncio.iscoroutine(local_task):
            await local_task

    def emit(self, event_name, data):
        """Emit an event."""
        if not self._ready.done():
            self._ready.set_result(True)
        tasks = []

        # Emit locally
        local_task = self._local_event_bus.emit(event_name, data)
        if asyncio.iscoroutine(local_task):
            tasks.append(local_task)

        # Emit globally via Redis
        data_type = "b:"
        if isinstance(data, dict):
            data = json.dumps(data)
            data_type = "d:"
        elif isinstance(data, str):
            data_type = "s:"
            data = data.encode("utf-8")
        else:
            assert data and isinstance(
                data, (str, bytes)
            ), "Data must be a string or bytes"
        global_task = self._loop.create_task(
            self._redis.publish("event:" + data_type + event_name, data)
        )
        tasks.append(global_task)

        if tasks:
            return asyncio.gather(*tasks)
        else:
            fut = asyncio.get_event_loop().create_future()
            fut.set_result(None)
            return fut

    async def stop(self):
        """Stop the event bus."""
        self._stop = True
        if self._subscribe_task:
            self._subscribe_task.cancel()
        if self._health_check_task:
            self._health_check_task.cancel()
        if self._health_check_pubsub:
            await self._health_check_pubsub.unsubscribe()
            await self._health_check_pubsub.close()
        try:
            await asyncio.gather(
                self._subscribe_task, self._health_check_task, return_exceptions=True
            )
        except asyncio.CancelledError:
            pass

    async def _check_pubsub_health(self):
        """Check if pubsub is working by sending and receiving a test message."""
        try:
            if not self._health_check_pubsub:
                self._health_check_pubsub = self._redis.pubsub()
                await self._health_check_pubsub.subscribe(self._health_check_channel)

            # Create a future to wait for the message
            self._pubsub_health_future = asyncio.Future()

            # Send test message with timestamp
            test_message = str(time.time())
            await self._redis.publish(self._health_check_channel, test_message)

            # Wait for message with timeout
            try:
                start_time = time.time()
                received = await asyncio.wait_for(
                    self._pubsub_health_future, timeout=2.0
                )
                end_time = time.time()

                # Verify it's the message we sent
                if received == test_message:
                    latency = end_time - start_time
                    self._pubsub_latency.set(latency)
                    self._counter.labels(event="pubsub_health", status="success").inc()
                    return True
                else:
                    logger.warning("Received unexpected message in pubsub health check")
                    self._counter.labels(event="pubsub_health", status="mismatch").inc()
                    return False

            except asyncio.TimeoutError:
                logger.warning("Pubsub health check timed out")
                self._counter.labels(event="pubsub_health", status="timeout").inc()
                return False

        except Exception as e:
            logger.error(f"Pubsub health check failed: {str(e)}")
            self._counter.labels(event="pubsub_health", status="error").inc()
            return False

    async def _process_health_check_message(self, message):
        """Process incoming health check messages."""
        try:
            if self._pubsub_health_future and not self._pubsub_health_future.done():
                self._pubsub_health_future.set_result(message.decode("utf-8"))
        except Exception as e:
            logger.error(f"Error processing health check message: {str(e)}")

    async def _health_check(self):
        """Periodically check Redis health including pubsub and performance."""
        while not self._stop:
            try:
                await self._redis.ping()
                await asyncio.sleep(self._health_check_interval)
                if self._circuit_breaker_open:
                    continue

                # Check pubsub functionality
                pubsub_healthy = await self._check_pubsub_health()
                if not pubsub_healthy:
                    self._consecutive_failures += 1
                    if self._consecutive_failures >= self._max_failures:
                        logger.error("Circuit breaker opened due to pubsub failures")
                        self._circuit_breaker_open = True
                        self._counter.labels(
                            event="circuit_breaker", status="open"
                        ).inc()
                        await self._attempt_reconnection()
                    continue

                # Reset failure counter on successful health check
                self._consecutive_failures = 0
                self._last_successful_connection = asyncio.get_event_loop().time()
                self._counter.labels(event="health_check", status="success").inc()

            except Exception as e:
                logger.warning(f"Redis health check failed: {str(e)}")
                self._counter.labels(event="health_check", status="failure").inc()
                self._consecutive_failures += 1

                if self._consecutive_failures >= self._max_failures:
                    logger.error("Circuit breaker opened due to multiple failures")
                    self._circuit_breaker_open = True
                    await self._attempt_reconnection()

    async def _attempt_reconnection(self):
        """Attempt to reconnect with exponential backoff."""
        while self._circuit_breaker_open and not self._stop:
            try:
                # Cancel existing subscription task
                if self._subscribe_task:
                    self._subscribe_task.cancel()

                await asyncio.sleep(self._reconnect_delay)

                # Try to ping Redis
                await self._redis.ping()

                # If successful, reset circuit breaker and restart subscription
                self._circuit_breaker_open = False
                self._consecutive_failures = 0
                self._reconnect_delay = 1
                self._counter.labels(event="reconnection", status="success").inc()

                self._subscribe_task = self._loop.create_task(self._subscribe_redis())
                logger.info("Successfully reconnected to Redis")
                return

            except Exception as e:
                logger.error(f"Reconnection attempt failed: {str(e)}")
                self._counter.labels(event="reconnection", status="failure").inc()

                # Exponential backoff with max delay
                self._reconnect_delay = min(
                    self._reconnect_delay * 2, self._max_reconnect_delay
                )

    async def _subscribe_redis(self):
        """Handle Redis subscription with automatic reconnection."""
        cpu_count = os.cpu_count() or 1
        concurrent_tasks = cpu_count * 10
        logger.info(
            f"Starting Redis event bus with {concurrent_tasks} concurrent task processing"
        )

        while not self._stop:
            try:
                pubsub = self._redis.pubsub()
                self._stop = False
                semaphore = asyncio.Semaphore(concurrent_tasks)

                # Subscribe to both events and health check channel
                await pubsub.psubscribe("event:*")
                await pubsub.subscribe(self._health_check_channel)

                self._ready.set_result(True) if not self._ready.done() else None
                self._counter.labels(event="subscription", status="success").inc()

                while not self._stop:
                    if self._circuit_breaker_open:
                        await asyncio.sleep(0.1)
                        continue

                    try:
                        msg = await pubsub.get_message(
                            ignore_subscribe_messages=True, timeout=0.05
                        )
                        if msg:
                            channel = msg["channel"].decode("utf-8")
                            if channel == self._health_check_channel:
                                await self._process_health_check_message(msg["data"])
                            else:
                                task = asyncio.create_task(
                                    self._process_message(msg, semaphore)
                                )
                                background_tasks.add(task)
                                task.add_done_callback(background_tasks.discard)
                    except Exception as e:
                        logger.warning(f"Error getting message: {str(e)}")
                        self._counter.labels(
                            event="message_processing", status="failure"
                        ).inc()
                        await asyncio.sleep(0.1)  # Prevent tight loop on errors

            except Exception as exp:
                logger.error(f"Subscription error: {str(exp)}")
                self._counter.labels(event="subscription", status="failure").inc()
                if not self._ready.done():
                    self._ready.set_exception(exp)
                await asyncio.sleep(1)  # Prevent tight loop on connection errors

    async def _process_message(self, msg, semaphore):
        """Process a single message while respecting the semaphore."""
        async with semaphore:
            try:
                channel = msg["channel"].decode("utf-8")
                RedisEventBus._counter.labels(event="*", status="processed").inc()

                if channel.startswith("event:b:"):
                    event_type = channel[8:]
                    data = msg["data"]
                    await self._redis_event_bus.emit(event_type, data)
                    if ":" not in event_type:
                        RedisEventBus._counter.labels(
                            event=event_type, status="processed"
                        ).inc()
                elif channel.startswith("event:d:"):
                    event_type = channel[8:]
                    data = json.loads(msg["data"])
                    await self._redis_event_bus.emit(event_type, data)
                    if ":" not in event_type:
                        RedisEventBus._counter.labels(
                            event=event_type, status="processed"
                        ).inc()
                elif channel.startswith("event:s:"):
                    event_type = channel[8:]
                    data = msg["data"].decode("utf-8")
                    await self._redis_event_bus.emit(event_type, data)
                    if ":" not in event_type:
                        RedisEventBus._counter.labels(
                            event=event_type, status="processed"
                        ).inc()
                else:
                    logger.info("Unknown channel: %s", channel)
            except Exception as exp:
                logger.exception(f"Error processing message: {exp}")
                RedisEventBus._counter.labels(
                    event="message_processing", status="error"
                ).inc()
