"""LLMHosts backend dispatcher -- routes unified requests to inference backends.

Routing priority:

1. **LLMHosts Core** (local GPU inference via custom Rust engine) -- always preferred
   when a model is loaded.
2. **Remote devices** (Phase 5): tunnel URLs registered via :meth:`register_remote_backend`
   or :meth:`register_remote_backends`; requests are forwarded to the device's
   OpenAI-compatible proxy at ``{tunnel_url}/v1/chat/completions``.
3. **Cloud APIs** via BYOK keys forwarded through LiteLLM (when available).

The dispatcher converts :class:`UnifiedRequest` into the appropriate
backend format, calls the backend, and returns a :class:`UnifiedResponse`.
Streaming is handled via :meth:`dispatch_stream`, which yields raw text tokens.
"""

from __future__ import annotations

import asyncio
import json
import logging
import time
from collections import deque
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any

import httpx

from llmhosts.config import ConnectionPoolConfig, LLMHostsConfig, RemitConfig
from llmhosts.proxy.errors import BackendTimeoutError, BackendUnavailableError, KarmaTooLowError
from llmhosts.proxy.models import UnifiedRequest, UnifiedResponse

try:
    from llmhosts.metrics.savings import record_savings as _record_savings

    _savings_available = True
except Exception:  # pragma: no cover
    _savings_available = False
    _record_savings = None  # type: ignore[assignment]

from llmhosts.inference.core import LLMHOSTS_CORE_AVAILABLE, LLMHostsInferenceEngine

if TYPE_CHECKING:
    from collections.abc import AsyncIterator, Callable

    from llmhosts.audit.logger import AuditLogger
    from llmhosts.backends.manager import BackendManager
    from llmhosts.billing.store import BillingStore
    from llmhosts.discovery.gpu_monitor import GPUMonitor
    from llmhosts.hive.karma import KarmaEngine
    from llmhosts.hive.manager import HiveManager
    from llmhosts.marketplace.probes import ProbeStore
    from llmhosts.remit import RemitWatcher
    from llmhosts.router.engine import Router

try:
    from llmhosts.telemetry.events import TelemetryTier as _TelemetryTier
    from llmhosts.telemetry.manager import emit_routing_event as _emit_telemetry

    _telemetry_available = True
except ImportError:
    _telemetry_available = False

# Karma: block requests when below this; deprioritize when below 1.0
KARMA_THRESHOLD_BLOCK = 0.5
COMPUTE_HOURS_PER_REQUEST = 0.001

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Backend registry entries
# ---------------------------------------------------------------------------

BACKEND_TIMEOUT = 120.0  # generous default for large-model generation
REMOTE_TIMEOUT = 30.0  # default timeout for remote/marketplace device forwarding


@dataclass
class BackendEntry:
    """Metadata for a single inference backend."""

    name: str
    base_url: str
    backend_type: str  # "llmhosts-core" | "openai" | "anthropic" | "litellm" | "remote" | "generic" | ...
    models: list[str] = field(default_factory=list)
    priority: int = 0  # lower = preferred
    healthy: bool = True
    member_id: str | None = None  # Hive member ID when backend is a Hive peer (for karma routing)
    device_id: str | None = None  # Phase 5: device ID when backend is a remote marketplace node


# ---------------------------------------------------------------------------
# Cloud provider detection -- heuristics for model → provider mapping
# ---------------------------------------------------------------------------

_CLOUD_MODEL_PREFIXES: dict[str, list[str]] = {
    "openai": ["gpt-", "o1-", "o3-", "chatgpt-", "ft:gpt-", "dall-e", "tts-", "whisper"],
    "anthropic": ["claude-"],
    "google": ["gemini-", "gemini/"],
    "mistral": ["mistral-", "mixtral-", "codestral-", "open-mistral-", "open-mixtral-"],
    "groq": ["groq/"],
    "together": ["together_ai/", "together/"],
    "deepseek": ["deepseek-", "deepseek/"],
    "openrouter": ["openrouter/"],
}


def _detect_cloud_model(model: str) -> bool:
    """Return *True* if *model* looks like a cloud model rather than a local model.

    Uses prefix heuristics -- any model matching a known cloud provider
    prefix is treated as a cloud request.  Models containing ``/`` that
    match a known LiteLLM provider prefix are also considered cloud.
    """
    model_lower = model.lower()
    for prefixes in _CLOUD_MODEL_PREFIXES.values():
        for prefix in prefixes:
            if model_lower.startswith(prefix):
                return True
    return False


def _litellm_dict_to_unified(data: dict[str, Any], model: str) -> UnifiedResponse:
    """Convert a normalized LiteLLM response dict to :class:`UnifiedResponse`."""
    return UnifiedResponse(
        content=data.get("content", ""),
        finish_reason=data.get("finish_reason", "stop") or "stop",
        model=data.get("model", model),
        prompt_tokens=data.get("prompt_tokens", 0),
        completion_tokens=data.get("completion_tokens", 0),
        total_tokens=data.get("total_tokens", 0),
        tool_calls=data.get("tool_calls"),
    )


# ---------------------------------------------------------------------------
# BackendDispatcher
# ---------------------------------------------------------------------------


class BackendDispatcher:
    """Discover backends, dispatch requests, and manage health state.

    Routing priority:

    1. **LLMHosts Core** -- when a model is loaded, all local inference goes
       through the custom Rust engine.  This is the normal state.
    2. **Remote devices** -- marketplace backends via tunnel URLs.
    3. **Cloud (LiteLLM)** -- BYOK cloud fallback when :meth:`attach_backend_manager`
       is called with an initialised :class:`BackendManager`.

    Typical lifecycle::

        dispatcher = BackendDispatcher()
        await dispatcher.startup()

        # Optionally attach cloud backends
        dispatcher.attach_backend_manager(backend_manager)

        response = await dispatcher.dispatch(unified_request)
        await dispatcher.shutdown()
    """

    def __init__(
        self,
        pool_config: ConnectionPoolConfig | None = None,
        *,
        remit_allowlist: list[str] | None = None,
        config: LLMHostsConfig | None = None,
        remote_timeout: float = REMOTE_TIMEOUT,
    ) -> None:
        self._backends: list[BackendEntry] = []
        self._pool_config = pool_config or ConnectionPoolConfig()
        self._pool_registry: dict[str, httpx.AsyncClient] = {}
        self._backend_manager: BackendManager | None = None
        self._router: Router | None = None
        # At-capacity fallback tracking
        self._config = config or LLMHostsConfig()
        self._active_requests: int = 0
        self._latency_history: deque[float] = deque(
            maxlen=self._config.fallback.latency_window_size,
        )
        self._hive_manager: HiveManager | None = None
        self._karma_engine: KarmaEngine | None = None
        self._audit_logger: AuditLogger | None = None
        self._billing_callback: Callable[[dict[str, Any]], None] | None = None
        self._billing_store: BillingStore | None = None
        self._dashboard_usage_url: str | None = None
        self._dashboard_usage_secret: str | None = None
        self._dashboard_usage_tasks: set[asyncio.Task[None]] = set()
        # Remit allowlist: when non-empty, only backends whose name, base_url, or device_id
        # is in this list are registered (per config.remit.remit_allowlist).
        self._remit_allowlist: list[str] = list(remit_allowlist) if remit_allowlist else []
        self._remit_watcher: RemitWatcher | None = None
        # Remote/marketplace settings
        self._remote_timeout: float = remote_timeout
        self._probe_store: ProbeStore | None = None
        # GPU Respect Architecture: passive monitor for yield-first routing
        self._gpu_monitor: GPUMonitor | None = None

        # LLMHosts Core inference engine — the primary local backend
        self._core_engine: LLMHostsInferenceEngine | None = None
        if LLMHOSTS_CORE_AVAILABLE:
            self._core_engine = LLMHostsInferenceEngine()
            logger.info("LLMHosts Core engine initialized (no model loaded yet)")

    # -- lifecycle -----------------------------------------------------------

    async def startup(self) -> None:
        """Discover local backends. Connection pools are created lazily on first use."""
        # Register Core engine as the primary local backend when a model is loaded
        if self._core_engine is not None and self._core_engine.is_loaded:
            self._register_core_model()
        await self._discover_lan_nodes()
        logger.info(
            "BackendDispatcher ready - %d backend(s), %d model(s), core=%s",
            len(self._backends),
            sum(len(b.models) for b in self._backends),
            "loaded" if (self._core_engine is not None and self._core_engine.is_loaded) else "not loaded",
        )

    async def shutdown(self) -> None:
        """Close all per-backend connection pools."""
        for url, client in self._pool_registry.items():
            await client.aclose()
            logger.debug("Closed connection pool for %s", url)
        self._pool_registry.clear()
        logger.info("BackendDispatcher shut down")

    # -- connection pooling --------------------------------------------------

    def _get_or_create_pool(self, backend: BackendEntry) -> httpx.AsyncClient:
        """Return the connection pool for *backend*, creating one if needed.

        Each unique ``base_url`` gets its own :class:`httpx.AsyncClient` with
        HTTP/2 enabled (when the ``h2`` package is available) and configurable
        pool limits from :class:`ConnectionPoolConfig`.
        """
        key = backend.base_url
        if key in self._pool_registry:
            return self._pool_registry[key]

        cfg = self._pool_config
        use_http2 = cfg.http2
        try:
            client = httpx.AsyncClient(
                http2=use_http2,
                limits=httpx.Limits(
                    max_connections=cfg.max_connections,
                    max_keepalive_connections=cfg.max_keepalive,
                    keepalive_expiry=cfg.idle_timeout,
                ),
                timeout=httpx.Timeout(BACKEND_TIMEOUT, connect=10.0),
            )
        except ImportError:
            # h2 package not installed -- fall back to HTTP/1.1
            use_http2 = False
            logger.info("h2 package not installed; falling back to HTTP/1.1 for %s", key)
            client = httpx.AsyncClient(
                http2=False,
                limits=httpx.Limits(
                    max_connections=cfg.max_connections,
                    max_keepalive_connections=cfg.max_keepalive,
                    keepalive_expiry=cfg.idle_timeout,
                ),
                timeout=httpx.Timeout(BACKEND_TIMEOUT, connect=10.0),
            )
        self._pool_registry[key] = client
        logger.info(
            "Created connection pool for %s (max_conn=%d, keepalive=%d, http2=%s)",
            key,
            cfg.max_connections,
            cfg.max_keepalive,
            use_http2,
        )
        return client

    # -- BackendManager integration ------------------------------------------

    def attach_backend_manager(self, manager: BackendManager) -> None:
        """Attach a :class:`BackendManager` for cloud backend routing.

        Once attached, cloud model requests are routed through LiteLLM
        via the manager.  Local inference continues through LLMHosts Core.

        Parameters
        ----------
        manager:
            An initialised :class:`BackendManager`.
        """
        self._backend_manager = manager
        # Register a synthetic BackendEntry so cloud models show up
        # in model listing and backend selection
        if manager.has_cloud:
            cloud_entry = BackendEntry(
                name="litellm-cloud",
                base_url="",  # no direct URL -- routed via LiteLLM
                backend_type="litellm",
                models=[],  # cloud models are resolved dynamically
                priority=10,  # prefer local over cloud
                healthy=True,
            )
            self._backends.append(cloud_entry)
            logger.info("Attached BackendManager with cloud backend support")
        else:
            logger.info("Attached BackendManager (no cloud providers configured)")

    @property
    def backend_manager(self) -> BackendManager | None:
        """The attached :class:`BackendManager`, or *None*."""
        return self._backend_manager

    # -- Router integration --------------------------------------------------

    def attach_router(self, router: Router) -> None:
        """Attach a :class:`Router` for intelligent routing and privacy enforcement.

        When attached, :meth:`dispatch` and :meth:`dispatch_stream` call
        ``router.route()`` before backend selection to apply PII detection and
        privacy constraints.  SENSITIVE-tier PII is enforced to local backends only.
        """
        self._router = router
        logger.info("Router attached to BackendDispatcher (privacy enforcement active)")

    # -- Hive karma integration --------------------------------------------------

    def attach_hive_karma(self, hive_manager: HiveManager, karma_engine: KarmaEngine) -> None:
        """Attach Hive karma for contribution/consumption tracking and karma-weighted routing.

        When attached, the local Core backend is tagged with current member ID,
        and after each successful dispatch karma is updated (contribution if local,
        consumption if forwarded to a Hive peer). Backends are sorted by karma when
        multiple candidates exist.
        """
        self._hive_manager = hive_manager
        self._karma_engine = karma_engine
        current_member_id = hive_manager.get_current_member_id()
        if current_member_id:
            for b in self._backends:
                if b.name == "llmhosts-core" or b.backend_type == "llmhosts-core":
                    b.member_id = current_member_id
                    break
        logger.info("Hive karma attached to BackendDispatcher")

    # -- Audit logger integration -----------------------------------------------

    def attach_audit_logger(self, audit_logger: AuditLogger) -> None:
        """Attach an :class:`AuditLogger` for request audit trail.

        When attached, every :meth:`dispatch` call writes an :class:`AuditEntry`
        to the audit log after the backend response completes.
        """
        self._audit_logger = audit_logger
        logger.info("AuditLogger attached to BackendDispatcher")

    @property
    def audit_logger(self) -> AuditLogger | None:
        """The attached :class:`AuditLogger`, or *None*."""
        return self._audit_logger

    # -- Remit watcher integration -------------------------------------------

    def attach_remit_watcher(self, watcher: RemitWatcher) -> None:
        """Attach a :class:`RemitWatcher` for dynamic per-backend filtering.

        When attached, :meth:`_in_remit` delegates to the watcher which
        supports allowlist/denylist modes and time-based scheduling.
        """
        self._remit_watcher = watcher
        logger.info("RemitWatcher attached to BackendDispatcher")

    # -- Probe store integration (marketplace health) -------------------------

    def attach_probe_store(self, store: ProbeStore) -> None:
        """Attach a :class:`ProbeStore` for marketplace device health lookups.

        When attached, :meth:`_is_remote_healthy` queries the probe store for
        recent uptime data before forwarding requests to remote backends.
        """
        self._probe_store = store
        logger.info("ProbeStore attached to BackendDispatcher")

    def attach_gpu_monitor(self, monitor: GPUMonitor) -> None:
        """Attach a :class:`GPUMonitor` for GPU Respect Architecture.

        When attached, the dispatcher passively checks GPU availability before
        routing to local backends.  If the user's GPU is busy (training,
        rendering, gaming, etc.), the dispatcher yields and routes to cloud
        instead, setting ``X-LLMHosts-GPU-Yielded: true`` on the response.
        """
        self._gpu_monitor = monitor
        logger.info("GPUMonitor attached to BackendDispatcher (GPU Respect enabled)")

    async def _is_remote_healthy(self, backend: BackendEntry, *, min_uptime_pct: float = 50.0) -> bool:
        """Return *True* if a remote backend is considered healthy.

        Checks:
        1. ``backend.healthy`` flag (set to False on transport errors).
        2. ProbeStore uptime data (when attached): device must have >= *min_uptime_pct*
           uptime over the last 30 days.  If no probe data exists the device is
           assumed healthy (new nodes get a grace period).
        """
        if not backend.healthy:
            return False
        if self._probe_store is None or backend.device_id is None:
            return True
        try:
            uptime = await self._probe_store.get_uptime_pct(backend.device_id, days=30)
        except Exception:
            logger.debug("ProbeStore query failed for device_id=%s, assuming healthy", backend.device_id)
            return True
        if uptime is None:
            # No probe data yet — new node, assume healthy
            return True
        return uptime >= min_uptime_pct

    # -- Billing (Phase 5 metering) ------------------------------------------

    def attach_billing_callback(self, callback: Callable[[dict[str, Any]], None]) -> None:
        """Attach a callback invoked after each successful remote (marketplace) dispatch.

        The callback receives a single dict: request_id, timestamp, api_key_id,
        user_id, device_id, model, prompt_tokens, completion_tokens, total_tokens,
        price_cents (optional), price_cents_per_hour (optional), source ("marketplace").
        If no handler is attached, no-op.
        """
        self._billing_callback = callback
        logger.info("Billing callback attached to BackendDispatcher")

    def attach_billing_store(self, store: BillingStore) -> None:
        """Attach a :class:`BillingStore` to append billing events for remote dispatches.

        When attached, each successful dispatch to a backend with backend_type
        \"remote\" appends a :class:`BillingEvent` to the store for later
        Stripe metered billing consumption.
        """
        self._billing_store = store
        logger.info("BillingStore attached to BackendDispatcher")

    def attach_dashboard_usage_push(self, url: str, secret: str) -> None:
        """Configure fire-and-forget POST of each billing event to the dashboard usage API."""
        self._dashboard_usage_url = url.rstrip("/")
        self._dashboard_usage_secret = secret
        logger.info("Dashboard usage push configured (url=%s)", self._dashboard_usage_url)

    async def _push_usage_to_dashboard(self, event: Any) -> None:
        """POST a billing event to the dashboard internal usage endpoint. Fire-and-forget."""
        url = self._dashboard_usage_url
        secret = self._dashboard_usage_secret
        if not url or not secret:
            return
        payload = {
            "api_key_id": event.api_key_id or "",
            "request_id": event.request_id,
            "device_id": getattr(event, "device_id", None),
            "model": event.model,
            "prompt_tokens": getattr(event, "prompt_tokens", 0),
            "completion_tokens": getattr(event, "completion_tokens", 0),
            "price_cents": getattr(event, "price_cents", None),
            "timestamp": event.timestamp.isoformat() if hasattr(event, "timestamp") and event.timestamp else None,
        }
        try:
            async with httpx.AsyncClient(timeout=5.0) as client:
                r = await client.post(
                    f"{url}/api/internal/marketplace/usage",
                    json=payload,
                    headers={"x-internal-secret": secret},
                )
                if r.status_code >= 400:
                    logger.warning(
                        "Dashboard usage push failed request_id=%s status=%s", event.request_id, r.status_code
                    )
        except Exception:
            logger.warning("Dashboard usage push error request_id=%s", event.request_id, exc_info=True)

    async def _emit_billing_event(
        self,
        *,
        request_id: str,
        api_key_id: str = "",
        user_id: str = "",
        device_id: str | None,
        model: str,
        prompt_tokens: int,
        completion_tokens: int,
        total_tokens: int,
        price_cents: int | None = None,
        price_cents_per_hour: int | None = None,
    ) -> None:
        """Emit a billing event for a completed remote request. No-op if no callback/store."""
        if self._billing_callback is None and self._billing_store is None:
            return
        from llmhosts.billing.models import BillingEvent

        event = BillingEvent(
            request_id=request_id,
            timestamp=datetime.now(timezone.utc),
            api_key_id=api_key_id,
            user_id=user_id,
            device_id=device_id,
            model=model,
            prompt_tokens=prompt_tokens,
            completion_tokens=completion_tokens,
            total_tokens=total_tokens,
            price_cents=price_cents,
            price_cents_per_hour=price_cents_per_hour,
            source="marketplace",
        )
        if self._billing_callback is not None:
            try:
                self._billing_callback(event.model_dump(mode="json"))
            except Exception:
                logger.warning("Billing callback failed for request_id=%s", request_id, exc_info=True)
        if self._billing_store is not None:
            try:
                await self._billing_store.append(event)
            except Exception:
                logger.warning("Billing store append failed for request_id=%s", request_id, exc_info=True)
        if self._dashboard_usage_url and self._dashboard_usage_secret:
            task = asyncio.create_task(self._push_usage_to_dashboard(event))
            self._dashboard_usage_tasks.add(task)
            task.add_done_callback(self._dashboard_usage_tasks.discard)

    def _is_local_backend(self, backend: BackendEntry) -> bool:
        """Return True if this backend is the local node (we contribute when we serve)."""
        return backend.backend_type == "llmhosts-core" or backend.name == "llmhosts-core"

    @staticmethod
    def _compute_hours_from_response(response: UnifiedResponse) -> float:
        """Compute hours for karma from response (simple heuristic)."""
        if response.total_tokens and response.total_tokens > 0:
            return max(COMPUTE_HOURS_PER_REQUEST, response.total_tokens / 1_000_000.0)
        return COMPUTE_HOURS_PER_REQUEST

    async def _record_karma_after_dispatch(
        self,
        backend: BackendEntry,
        current_member_id: str | None,
        compute_hours: float,
    ) -> None:
        """Record contribution (local) or consumption (remote) after successful dispatch."""
        if not self._karma_engine or not current_member_id:
            return
        try:
            if self._is_local_backend(backend):
                await self._karma_engine.record_contribution(current_member_id, compute_hours)
            else:
                await self._karma_engine.record_consumption(current_member_id, compute_hours)
        except Exception:
            logger.warning("Karma update failed", exc_info=True)

    async def _record_audit(
        self,
        *,
        request: UnifiedRequest,
        response: UnifiedResponse | None,
        backend: BackendEntry,
        start_time: float,
        status_code: int = 200,
        error: str | None = None,
        source_ip: str = "",
        request_id: str = "",
        api_key_id: str = "",
        user_id: str = "",
        dialect: str = "openai",
        path: str = "/v1/chat/completions",
    ) -> None:
        """Record an audit entry after a dispatch completes (or fails).

        This is a best-effort operation -- failures are logged but never
        propagated to the caller.
        """
        if self._audit_logger is None:
            return

        from llmhosts.audit.models import AuditEntry

        latency_ms = (time.monotonic() - start_time) * 1000.0
        prompt_tokens = response.prompt_tokens if response else 0
        completion_tokens = response.completion_tokens if response else 0
        model_used = response.model if response else request.model

        entry = AuditEntry(
            request_id=request_id,
            timestamp=datetime.now(timezone.utc),
            source_ip=source_ip,
            method="POST",
            path=path,
            dialect=dialect,
            model_requested=request.model,
            model_used=model_used,
            backend_type=backend.backend_type,
            routing_tier="rule",
            routing_reasoning="",
            routing_confidence=0.0,
            input_tokens=prompt_tokens,
            output_tokens=completion_tokens,
            actual_cost=0.0,
            cloud_equivalent_cost=0.0,
            cached=False,
            latency_ms=round(latency_ms, 2),
            status_code=status_code,
            error=error,
            api_key_id=api_key_id,
            user_id=user_id,
        )
        try:
            await self._audit_logger.log(entry)
        except Exception:
            logger.warning("Failed to write audit entry for request_id=%s", request_id, exc_info=True)

    async def _select_backend_with_privacy(self, request: UnifiedRequest) -> BackendEntry:
        """Select a backend, applying privacy enforcement when a Router is attached.

        If the Router classifies the request as SENSITIVE (e.g. SSN, credit card),
        only local (Core/LAN) backends are eligible.  Falls back to normal
        :meth:`_select_backend` when no router is attached or routing fails.
        When Hive karma is attached, sorts candidates by karma (higher first).

        Raises :class:`BackendUnavailableError` if a SENSITIVE request cannot be
        satisfied by any available local backend.
        """
        priority_order: list[tuple[str, float]] | None = None
        karma_engine = getattr(self, "_karma_engine", None)
        if karma_engine:
            try:
                priority_order = await karma_engine.get_priority_order()
            except Exception:
                logger.debug("Karma get_priority_order failed, skipping karma sort", exc_info=True)

        if self._router is None:
            return self._select_backend(request.model, priority_order=priority_order)

        try:
            decision = await self._router.route(request)
        except Exception:
            logger.warning("Router.route() failed, falling back to rule-based selection", exc_info=True)
            return self._select_backend(request.model, priority_order=priority_order)

        privacy_tier = getattr(decision, "privacy_tier", "public")
        forced_local = getattr(decision, "forced_local", False)

        if forced_local or privacy_tier == "sensitive":
            local_candidates = [
                b
                for b in self._backends
                if b.healthy and b.backend_type == "llmhosts-core" and (request.model in b.models or not b.models)
            ]
            if local_candidates:
                local_candidates.sort(key=lambda b: b.priority)
                logger.info(
                    "Privacy enforcement: routing '%s' to local backend (tier=%s)",
                    request.model,
                    privacy_tier,
                )
                return local_candidates[0]
            # Core engine may handle the model even without a backend entry
            if self._core_engine is not None and self._core_engine.is_loaded:
                logger.info(
                    "Privacy enforcement: routing '%s' to Core engine (tier=%s)",
                    request.model,
                    privacy_tier,
                )
                return BackendEntry(
                    name="llmhosts-core",
                    base_url="local://",
                    backend_type="llmhosts-core",
                    models=[],
                    priority=-1,
                    healthy=True,
                )
            raise BackendUnavailableError(
                f"No local backend available for model '{request.model}'. "
                f"Request contains {privacy_tier}-tier PII and cannot be routed to cloud."
            )

        return self._select_backend(request.model, priority_order=priority_order)

    # -- discovery -----------------------------------------------------------

    async def _discover_lan_nodes(self) -> None:
        """Discover LAN nodes via mDNS and add as remote backends with lower priority than local."""
        try:
            from llmhosts.discovery.mdns import get_discovered_nodes
        except ImportError:
            return

        nodes = get_discovered_nodes()
        for node in nodes:
            if not node.models:
                continue
            # Avoid duplicating an existing backend
            existing_urls = {b.base_url for b in self._backends}
            if node.base_url in existing_urls:
                continue

            entry = BackendEntry(
                name=f"lan-{node.hostname}",
                base_url=node.base_url,
                backend_type="generic",  # LAN nodes expose OpenAI-compatible API
                models=node.models,
                priority=5,  # between local (0) and cloud (10)
                healthy=True,
            )
            before = len(self._backends)
            self.register_backend(entry)
            if len(self._backends) > before:
                logger.info(
                    "Added LAN node %s at %s with %d model(s)",
                    node.hostname,
                    node.base_url,
                    len(node.models),
                )

    async def refresh_lan_nodes(self) -> int:
        """Re-scan LAN nodes and update backend list. Returns count of LAN backends."""
        # Remove existing LAN entries
        self._backends = [b for b in self._backends if not b.name.startswith("lan-")]
        await self._discover_lan_nodes()
        return sum(1 for b in self._backends if b.name.startswith("lan-"))

    # -- remit allowlist (config.remit.remit_allowlist) ------------------------

    def _in_remit(self, entry: BackendEntry) -> bool:
        """True if entry is in remit (allowlist empty = all in remit).

        When a :class:`RemitWatcher` is attached, delegates to its
        ``is_backend_allowed`` method which supports allowlist/denylist
        modes and time-based scheduling.  Otherwise falls back to the
        simple ``remit_allowlist`` from the constructor.

        For remote backends, also applies :meth:`_is_device_in_remit`
        which checks the ``RemitConfig.allowlist`` and ``denylist``
        against the device_id.
        """
        if self._remit_watcher is not None:
            if not self._remit_watcher.is_backend_allowed(entry.name, entry.base_url):
                return False
            # For remote backends, additionally check device-specific remit
            return not (entry.backend_type == "remote" and not self._is_device_in_remit(entry))
        if not self._remit_allowlist:
            # Still check device-specific remit from config for remote backends
            return not (entry.backend_type == "remote" and not self._is_device_in_remit(entry))
        normalized = {s.strip().rstrip("/") for s in self._remit_allowlist if s}
        name_ok = entry.name in normalized
        url_ok = entry.base_url.rstrip("/") in normalized
        device_ok = entry.device_id is not None and entry.device_id in normalized
        return name_ok or url_ok or device_ok

    def _is_device_in_remit(self, entry: BackendEntry) -> bool:
        """Check RemitConfig allowlist/denylist for a remote device.

        Returns *True* if the device is allowed to receive requests. Enforcement:

        * ``mode="all"`` — all devices allowed (default).
        * ``mode="allowlist"`` — only device_ids in ``config.remit.allowlist``.
        * ``mode="denylist"`` — all except device_ids in ``config.remit.denylist``.

        Non-remote backends always pass.
        """
        if entry.backend_type != "remote":
            return True
        remit: RemitConfig = self._config.remit
        if remit.mode == "all":
            return True
        device_id = entry.device_id or ""
        name = entry.name
        identifiers = {device_id, name}
        if remit.mode == "allowlist":
            if not remit.allowlist:
                return True  # empty allowlist = no restriction
            return bool(identifiers & set(remit.allowlist))
        if remit.mode == "denylist":
            if not remit.denylist:
                return True  # empty denylist = no restriction
            return not bool(identifiers & set(remit.denylist))
        return True

    # -- dynamic backend management (used by DiscoveryWatcher) ----------------

    def register_backend(self, entry: BackendEntry) -> None:
        """Add or update a backend entry (idempotent by name).

        If a backend with the same name already exists, it is replaced in-place
        so that the watcher can update models/health without creating duplicates.
        When remit_allowlist is non-empty, only backends whose name, base_url,
        or device_id is in the allowlist are registered.
        """
        if self._remit_allowlist and not self._in_remit(entry):
            logger.debug(
                "Backend '%s' (%s) not in remit_allowlist; skipping registration",
                entry.name,
                entry.base_url or entry.device_id or "no url/id",
            )
            return
        for i, existing in enumerate(self._backends):
            if existing.name == entry.name:
                self._backends[i] = entry
                logger.info(
                    "Updated backend '%s' (%d models)",
                    entry.name,
                    len(entry.models),
                )
                return
        self._backends.append(entry)
        logger.info(
            "Registered backend '%s' at %s (%d models)",
            entry.name,
            entry.base_url,
            len(entry.models),
        )

    def unregister_backend(self, name: str) -> None:
        """Remove a backend by name. No-op if not found."""
        before = len(self._backends)
        self._backends = [b for b in self._backends if b.name != name]
        if len(self._backends) < before:
            logger.info("Unregistered backend '%s'", name)

    # -- remote (Phase 5 marketplace) backends ---------------------------------

    def register_remote_backend(
        self,
        *,
        tunnel_url: str,
        name: str,
        models: list[str],
        device_id: str | None = None,
        priority: int = 7,
    ) -> None:
        """Register a remote device as a backend (Phase 5 marketplace).

        Requests to this backend are forwarded to the device's tunnel URL
        at ``{tunnel_url}/v1/chat/completions`` (OpenAI-compatible proxy).
        *tunnel_url* is normalized (trailing slash removed).

        Parameters
        ----------
        tunnel_url : str
            Base URL of the device's proxy (e.g. https://device-abc.llmhosts.com).
        name : str
            Backend name (e.g. device name or ``remote-{device_id}``).
        models : list[str]
            Model IDs served by this device.
        device_id : str, optional
            Device record ID for audit/metering.
        priority : int
            Lower = preferred; remote backends typically sit after LAN (5) and before cloud (10).
        """
        base_url = (tunnel_url or "").rstrip("/")
        if not base_url:
            logger.warning("register_remote_backend: empty tunnel_url for name=%s, skipping", name)
            return
        entry = BackendEntry(
            name=name,
            base_url=base_url,
            backend_type="remote",
            models=list(models),
            priority=priority,
            healthy=True,
            device_id=device_id,
        )
        self.register_backend(entry)

    def register_remote_backends(
        self,
        devices: list[dict[str, Any]],
        *,
        name_key: str = "name",
        tunnel_url_key: str = "tunnel_url",
        models_key: str = "models",
        device_id_key: str = "id",
        priority_key: str = "priority",
    ) -> int:
        """Register multiple remote devices from a list of dicts (e.g. from /api/devices or marketplace API).

        Each dict must have *tunnel_url* and *name*; *models* can be a list or JSON string.
        Optional *priority* (lower = preferred); default 7 when absent.
        Returns the number of backends registered (after idempotent upsert by name).

        Parameters
        ----------
        devices : list[dict]
            List of device-like dicts with tunnel_url, name, models, optional id, optional priority.
        name_key, tunnel_url_key, models_key, device_id_key, priority_key : str
            Keys to read from each dict (defaults match Next.js /api/devices and marketplace API).
        """
        count = 0
        for d in devices:
            tunnel_url = d.get(tunnel_url_key)
            name = d.get(name_key)
            if not tunnel_url or not name:
                continue
            raw_models = d.get(models_key)
            if isinstance(raw_models, str):
                try:
                    models = json.loads(raw_models) if raw_models else []
                except json.JSONDecodeError:
                    models = []
            else:
                models = list(raw_models) if raw_models else []
            device_id = d.get(device_id_key)
            priority = d.get(priority_key, 7)
            if not isinstance(priority, int):
                priority = 7
            self.register_remote_backend(
                tunnel_url=tunnel_url,
                name=name,
                models=models,
                device_id=str(device_id) if device_id is not None else None,
                priority=priority,
            )
            count += 1
        return count

    def unregister_remote_backends(self) -> int:
        """Remove all backends with backend_type \"remote\". Returns the number removed."""
        before = len(self._backends)
        self._backends = [b for b in self._backends if b.backend_type != "remote"]
        removed = before - len(self._backends)
        if removed:
            logger.info("Unregistered %d remote backend(s)", removed)
        return removed

    def update_backend_models(self, name: str, models: list[str]) -> None:
        """Update the model list for an existing backend. No-op if not found."""
        for backend in self._backends:
            if backend.name == name:
                old_count = len(backend.models)
                backend.models = models
                logger.info(
                    "Updated models for '%s': %d → %d",
                    name,
                    old_count,
                    len(models),
                )
                return

    # -- model listing -------------------------------------------------------

    def list_backends_info(self) -> list[Any]:
        """Return backend info in Router-compatible :class:`~llmhost.router.models.BackendInfo` format."""
        from llmhosts.router.models import BackendInfo as RouterBackendInfo

        result = []
        for b in self._backends:
            if b.backend_type == "litellm":
                continue  # LiteLLM is a cloud aggregator, not a direct backend
            result.append(
                RouterBackendInfo(
                    backend_type=b.backend_type,
                    url=b.base_url,
                    models=list(b.models),
                    is_local=b.backend_type == "llmhosts-core",
                    is_healthy=b.healthy,
                )
            )
        return result

    def list_models(self) -> list[dict[str, str]]:
        """Return a flat list of ``{"id": ..., "owned_by": ...}`` for all backends."""
        result: list[dict[str, str]] = []
        for backend in self._backends:
            if backend.backend_type == "litellm":
                # Cloud models aren't pre-enumerated; skip static listing
                # They're accessible but listed via /v1/models with provider filter
                continue
            for model in backend.models:
                result.append({"id": model, "owned_by": backend.name})
        return result

    # -- at-capacity fallback ------------------------------------------------

    def _is_at_capacity(self) -> bool:
        """Return *True* when local backends appear overloaded.

        Checks two signals:
        1. Active concurrent requests >= queue_depth_threshold
        2. p95 latency over the sliding window > latency_threshold_ms
           (requires at least 10 samples)
        """
        if not self._config.fallback.enabled:
            return False
        if self._active_requests >= self._config.fallback.queue_depth_threshold:
            return True
        if len(self._latency_history) >= 10:
            sorted_latencies = sorted(self._latency_history)
            p95_idx = int(len(sorted_latencies) * 0.95)
            p95_ms = sorted_latencies[p95_idx]
            if p95_ms > self._config.fallback.latency_threshold_ms:
                return True
        return False

    def _get_fallback_model(self, local_model: str) -> str | None:
        """Return the cloud equivalent of *local_model*, or *None*."""
        if not self._config.fallback.enabled:
            return None
        return self._config.fallback.model_mapping.get(local_model)

    # -- backend selection ---------------------------------------------------

    def _select_backend(
        self,
        model: str,
        *,
        priority_order: list[tuple[str, float]] | None = None,
    ) -> BackendEntry:
        """Choose the best healthy backend that serves *model*.

        For cloud models (detected via prefix heuristics), routes to the
        LiteLLM backend entry if available.  Otherwise uses best local backend.
        When *priority_order* is provided (from KarmaEngine.get_priority_order),
        candidates are sorted so higher-karma members are preferred.

        Raises :class:`BackendUnavailableError` if none is found.
        """
        # Check if this looks like a cloud model and we have a cloud backend
        if _detect_cloud_model(model) and self._backend_manager is not None and self._backend_manager.has_cloud:
            cloud_entries = [b for b in self._backends if b.backend_type == "litellm" and b.healthy]
            if cloud_entries:
                return cloud_entries[0]

        # Try exact model match on local backends
        candidates = [b for b in self._backends if b.healthy and model in b.models]
        if not candidates:
            # Fallback: if we have a cloud backend and the model wasn't found locally,
            # route to cloud as a last resort
            if self._backend_manager is not None and self._backend_manager.has_cloud:
                cloud_entries = [b for b in self._backends if b.backend_type == "litellm" and b.healthy]
                if cloud_entries:
                    return cloud_entries[0]
            # Final fallback: any healthy local backend
            candidates = [b for b in self._backends if b.healthy and b.backend_type != "litellm"]
        if not candidates:
            raise BackendUnavailableError(f"No healthy backend available for model '{model}'")
        candidates.sort(key=lambda b: b.priority)
        # Karma-weighted order: prefer backends whose member_id appears first in priority_order
        if priority_order:
            rank_by_member: dict[str, int] = {mid: i for i, (mid, _) in enumerate(priority_order)}
            candidates.sort(
                key=lambda b: rank_by_member.get(b.member_id or "", 9999),
            )
        return candidates[0]

    # -- non-streaming dispatch ----------------------------------------------

    async def dispatch(
        self,
        request: UnifiedRequest,
        *,
        source_ip: str = "",
        request_id: str = "",
        api_key_id: str = "",
        user_id: str = "",
        dialect: str = "openai",
        path: str = "/v1/chat/completions",
    ) -> UnifiedResponse:
        """Send *request* to the best backend and return the result."""
        # Anti-abuse: block when karma below threshold
        hive_manager = getattr(self, "_hive_manager", None)
        karma_engine = getattr(self, "_karma_engine", None)
        if hive_manager and karma_engine:
            current_member_id = hive_manager.get_current_member_id()
            if current_member_id:
                karma = await karma_engine.get_karma(current_member_id)
                if karma < KARMA_THRESHOLD_BLOCK:
                    raise KarmaTooLowError(
                        "Karma too low; contribute more before requesting from Hive.",
                        retry_after_seconds=60,
                    )

        start_time = time.monotonic()
        backend = await self._select_backend_with_privacy(request)
        error_msg: str | None = None
        status_code = 200
        response: UnifiedResponse | None = None

        # GPU Respect: yield to user workloads before at-capacity check
        gpu_yielded = False
        fallback_info: dict[str, str] | None = None
        original_model = request.model
        if backend.backend_type == "llmhosts-core" and self._gpu_monitor is not None:
            try:
                gpu_avail = await self._gpu_monitor.check_availability()
                if gpu_avail and not any(g.available_for_inference for g in gpu_avail):
                    # All GPUs busy with user workloads -- yield to cloud
                    fallback_model = self._get_fallback_model(request.model)
                    if fallback_model and self._backend_manager is not None and self._backend_manager.has_cloud:
                        cloud_entries = [b for b in self._backends if b.backend_type == "litellm" and b.healthy]
                        if cloud_entries:
                            reasons = "; ".join(g.reason for g in gpu_avail if not g.available_for_inference)
                            logger.info(
                                "GPU Respect: yielding %s -> %s (cloud). Reason: %s",
                                request.model,
                                fallback_model,
                                reasons,
                            )
                            backend = cloud_entries[0]
                            request = UnifiedRequest(
                                model=fallback_model,
                                messages=request.messages,
                                stream=request.stream,
                                temperature=request.temperature,
                                max_tokens=request.max_tokens,
                                top_p=request.top_p,
                                stop=request.stop,
                                tools=request.tools,
                                extra=request.extra,
                            )
                            fallback_info = {"original": original_model, "cloud": fallback_model}
                            gpu_yielded = True
            except Exception:
                logger.debug("GPU availability check failed, proceeding without GPU Respect", exc_info=True)

        # At-capacity cloud fallback: reroute to cloud when local is overloaded
        if not gpu_yielded and backend.backend_type == "llmhosts-core" and self._is_at_capacity():
            fallback_model = self._get_fallback_model(request.model)
            if fallback_model and self._backend_manager is not None and self._backend_manager.has_cloud:
                cloud_entries = [b for b in self._backends if b.backend_type == "litellm" and b.healthy]
                if cloud_entries:
                    logger.info(
                        "At capacity — falling back %s -> %s (cloud)",
                        request.model,
                        fallback_model,
                    )
                    backend = cloud_entries[0]
                    request = UnifiedRequest(
                        model=fallback_model,
                        messages=request.messages,
                        stream=request.stream,
                        temperature=request.temperature,
                        max_tokens=request.max_tokens,
                        top_p=request.top_p,
                        stop=request.stop,
                        tools=request.tools,
                        extra=request.extra,
                    )
                    fallback_info = {"original": original_model, "cloud": fallback_model}

        remote_headers: dict[str, str] = {}  # populated when routing to a remote device
        self._active_requests += 1
        try:
            try:
                if self._core_engine is not None and self._core_engine.is_loaded:
                    # Core is the default local engine -- handles ALL local inference
                    messages_raw = [
                        {"role": m.role, "content": m.content if isinstance(m.content, str) else str(m.content)}
                        for m in request.messages
                    ]
                    body, extra_hdrs = await self._dispatch_core(
                        messages_raw,
                        model=request.model,
                        max_tokens=request.max_tokens or 256,
                        temperature=request.temperature or 0.0,
                        top_p=request.top_p or 0.9,
                        request=request.extra,
                    )
                    usage = body.get("usage", {})
                    response = UnifiedResponse(
                        content=body["choices"][0]["message"]["content"],
                        finish_reason=body["choices"][0].get("finish_reason", "stop"),
                        model=body.get("model", request.model),
                        prompt_tokens=usage.get("prompt_tokens", 0),
                        completion_tokens=usage.get("completion_tokens", 0),
                        total_tokens=usage.get("total_tokens", 0),
                        extra={"core_headers": extra_hdrs},
                    )
                elif backend.backend_type == "remote":
                    # Dedicated remote dispatch with retry across healthy candidates
                    response, backend, remote_headers = await self._dispatch_remote_with_retry(
                        request,
                        request_id=request_id,
                        api_key_id=api_key_id,
                        user_id=user_id,
                    )
                elif backend.backend_type == "litellm":
                    response = await self._dispatch_litellm(request)
                elif backend.backend_type in (
                    "openai",
                    "vllm",
                    "llama.cpp",
                    "lmstudio",
                    "localai",
                    "tgi",
                    "generic",
                ):
                    response = await self._dispatch_openai_compat(backend, request)
                else:
                    raise BackendUnavailableError(f"No backend available for model '{request.model}'. Run: llmhosts up")
            except Exception as exc:
                error_msg = str(exc)[:500]
                status_code = 502
                await self._record_audit(
                    request=request,
                    response=None,
                    backend=backend,
                    start_time=start_time,
                    status_code=status_code,
                    error=error_msg,
                    source_ip=source_ip,
                    request_id=request_id,
                    api_key_id=api_key_id,
                    user_id=user_id,
                    dialect=dialect,
                    path=path,
                )
                raise
        finally:
            self._active_requests -= 1
            elapsed_ms = (time.monotonic() - start_time) * 1000
            self._latency_history.append(elapsed_ms)

        # Record karma after successful dispatch
        if hive_manager and karma_engine:
            current_member_id = hive_manager.get_current_member_id()
            compute_hours = self._compute_hours_from_response(response)
            await self._record_karma_after_dispatch(backend, current_member_id, compute_hours)

        # Record cost savings (fire-and-forget — never fails a request)
        if _savings_available and _record_savings is not None and response.total_tokens > 0:
            try:
                _record_savings(response.model, response.prompt_tokens, response.completion_tokens)
            except Exception:
                logger.debug("Savings recording failed (non-critical)", exc_info=True)

        await self._record_audit(
            request=request,
            response=response,
            backend=backend,
            start_time=start_time,
            status_code=200,
            source_ip=source_ip,
            request_id=request_id,
            api_key_id=api_key_id,
            user_id=user_id,
            dialect=dialect,
            path=path,
        )
        if backend.backend_type == "remote":
            await self._emit_billing_event(
                request_id=request_id,
                api_key_id=api_key_id,
                user_id=user_id,
                device_id=backend.device_id,
                model=response.model or request.model,
                prompt_tokens=response.prompt_tokens,
                completion_tokens=response.completion_tokens,
                total_tokens=response.total_tokens,
            )
        if _telemetry_available:
            try:
                elapsed_ms = (time.monotonic() - start_time) * 1000.0
                last_message = request.messages[-1] if request.messages else None
                query_text = ""
                if last_message is not None:
                    if isinstance(last_message.content, str):
                        query_text = last_message.content
                    elif isinstance(last_message.content, list):
                        query_text = " ".join(
                            block.get("text", "") for block in last_message.content if isinstance(block, dict)
                        )
                _emit_telemetry(
                    query=query_text,
                    tier=_TelemetryTier.LOCAL,
                    model=response.model or request.model,
                    latency_ms=elapsed_ms,
                    input_tokens=response.prompt_tokens or 0,
                    output_tokens=response.completion_tokens or 0,
                )
            except Exception:
                pass

        # Attach fallback metadata so route handlers can add the response header
        if fallback_info is not None:
            response.extra = response.extra or {}
            response.extra["fallback_info"] = fallback_info
        # GPU Respect: mark response when we yielded to user workloads
        if gpu_yielded:
            response.extra = response.extra or {}
            response.extra["gpu_yielded"] = True
        # Attach remote routing headers so route handlers can include them in the HTTP response
        if remote_headers:
            response.extra = response.extra or {}
            response.extra["remote_headers"] = remote_headers
        return response

    async def _dispatch_litellm(self, request: UnifiedRequest) -> UnifiedResponse:
        """Forward a non-streaming request through LiteLLM via BackendManager."""
        assert self._backend_manager is not None

        # Convert UnifiedRequest messages to plain dicts for LiteLLM
        messages = [
            {"role": m.role, "content": m.content if isinstance(m.content, str) else (m.content or "")}
            for m in request.messages
        ]

        kwargs: dict[str, Any] = {}
        if request.temperature is not None:
            kwargs["temperature"] = request.temperature
        if request.max_tokens is not None:
            kwargs["max_tokens"] = request.max_tokens
        if request.top_p is not None:
            kwargs["top_p"] = request.top_p
        if request.stop:
            kwargs["stop"] = request.stop
        if request.tools:
            kwargs["tools"] = request.tools

        try:
            result = await self._backend_manager.completion(
                backend_type="litellm",
                model=request.model,
                messages=messages,
                stream=False,
                **kwargs,
            )
            # result is a dict from LiteLLMBackend._normalize_response
            return _litellm_dict_to_unified(result, request.model)  # type: ignore[arg-type]
        except ValueError as exc:
            raise BackendUnavailableError(str(exc)) from exc

    async def _dispatch_openai_compat(self, backend: BackendEntry, request: UnifiedRequest) -> UnifiedResponse:
        """Forward a non-streaming request to an OpenAI-compatible backend (vLLM, llama.cpp, LM Studio, etc.)."""
        client = self._get_or_create_pool(backend)
        messages = [
            {"role": m.role, "content": m.content if isinstance(m.content, str) else (m.content or "")}
            for m in request.messages
        ]
        payload: dict[str, Any] = {"model": request.model, "messages": messages, "stream": False}
        if request.temperature is not None:
            payload["temperature"] = request.temperature
        if request.max_tokens is not None:
            payload["max_tokens"] = request.max_tokens
        if request.top_p is not None:
            payload["top_p"] = request.top_p
        if request.stop:
            payload["stop"] = request.stop
        if request.tools:
            payload["tools"] = request.tools

        url = f"{backend.base_url}/v1/chat/completions"
        try:
            resp = await client.post(url, json=payload)
            resp.raise_for_status()
            data = resp.json()
            choices = data.get("choices", [])
            if not choices:
                return UnifiedResponse(
                    content="",
                    finish_reason="stop",
                    model=data.get("model", request.model),
                    prompt_tokens=data.get("usage", {}).get("prompt_tokens", 0),
                    completion_tokens=0,
                    total_tokens=data.get("usage", {}).get("total_tokens", 0),
                )
            choice = choices[0]
            message = choice.get("message", {})
            usage = data.get("usage", {})
            return UnifiedResponse(
                content=message.get("content", ""),
                finish_reason=choice.get("finish_reason", "stop") or "stop",
                model=data.get("model", request.model),
                prompt_tokens=usage.get("prompt_tokens", 0),
                completion_tokens=usage.get("completion_tokens", 0),
                total_tokens=usage.get("total_tokens", 0),
                tool_calls=message.get("tool_calls"),
            )
        except httpx.TimeoutException as exc:
            backend.healthy = False
            raise BackendTimeoutError(f"Backend at {backend.base_url} timed out") from exc
        except httpx.HTTPStatusError as exc:
            error_text = exc.response.text[:500]
            logger.error("Backend %s returned %s: %s", backend.name, exc.response.status_code, error_text)
            raise BackendUnavailableError(f"Backend error: {error_text}") from exc
        except httpx.ConnectError as exc:
            backend.healthy = False
            raise BackendUnavailableError(f"Cannot reach backend at {backend.base_url}") from exc

    # -- remote (marketplace) dedicated dispatch --------------------------------

    async def _dispatch_remote(
        self,
        backend: BackendEntry,
        request: UnifiedRequest,
        *,
        request_id: str = "",
        api_key_id: str = "",
        user_id: str = "",
    ) -> tuple[UnifiedResponse, dict[str, str]]:
        """Forward a non-streaming request to a remote marketplace device.

        Returns ``(response, extra_headers)`` where *extra_headers* contains
        ``X-LLMHosts-Remote`` and ``X-LLMHosts-Device-Id``.

        On transport error the backend is marked ``healthy=False`` and the
        caller can retry on the next available backend.
        """
        client = self._get_or_create_pool(backend)
        messages = [
            {"role": m.role, "content": m.content if isinstance(m.content, str) else (m.content or "")}
            for m in request.messages
        ]
        payload: dict[str, Any] = {"model": request.model, "messages": messages, "stream": False}
        if request.temperature is not None:
            payload["temperature"] = request.temperature
        if request.max_tokens is not None:
            payload["max_tokens"] = request.max_tokens
        if request.top_p is not None:
            payload["top_p"] = request.top_p
        if request.stop:
            payload["stop"] = request.stop
        if request.tools:
            payload["tools"] = request.tools

        url = f"{backend.base_url}/v1/chat/completions"
        extra_headers: dict[str, str] = {
            "X-LLMHosts-Remote": "true",
            "X-LLMHosts-Device-Id": backend.device_id or "",
        }
        try:
            resp = await client.post(url, json=payload, timeout=self._remote_timeout)
            resp.raise_for_status()
            data = resp.json()
            choices = data.get("choices", [])
            if not choices:
                return (
                    UnifiedResponse(
                        content="",
                        finish_reason="stop",
                        model=data.get("model", request.model),
                        prompt_tokens=data.get("usage", {}).get("prompt_tokens", 0),
                        completion_tokens=0,
                        total_tokens=data.get("usage", {}).get("total_tokens", 0),
                    ),
                    extra_headers,
                )
            choice = choices[0]
            message = choice.get("message", {})
            usage = data.get("usage", {})
            return (
                UnifiedResponse(
                    content=message.get("content", ""),
                    finish_reason=choice.get("finish_reason", "stop") or "stop",
                    model=data.get("model", request.model),
                    prompt_tokens=usage.get("prompt_tokens", 0),
                    completion_tokens=usage.get("completion_tokens", 0),
                    total_tokens=usage.get("total_tokens", 0),
                    tool_calls=message.get("tool_calls"),
                ),
                extra_headers,
            )
        except httpx.TimeoutException as exc:
            backend.healthy = False
            logger.warning("Remote device %s timed out at %s", backend.device_id or backend.name, backend.base_url)
            raise BackendTimeoutError(f"Remote device at {backend.base_url} timed out") from exc
        except httpx.HTTPStatusError as exc:
            error_text = exc.response.text[:500]
            logger.error(
                "Remote device %s returned %s: %s",
                backend.device_id or backend.name,
                exc.response.status_code,
                error_text,
            )
            raise BackendUnavailableError(f"Remote device error: {error_text}") from exc
        except httpx.ConnectError as exc:
            backend.healthy = False
            logger.warning("Cannot reach remote device %s at %s", backend.device_id or backend.name, backend.base_url)
            raise BackendUnavailableError(f"Cannot reach remote device at {backend.base_url}") from exc

    def _get_remote_candidates(self, model: str) -> list[BackendEntry]:
        """Return remote backends that serve *model*, ordered by priority."""
        candidates = [
            b
            for b in self._backends
            if b.backend_type == "remote" and b.healthy and model in b.models and self._is_device_in_remit(b)
        ]
        candidates.sort(key=lambda b: b.priority)
        return candidates

    async def _dispatch_remote_with_retry(
        self,
        request: UnifiedRequest,
        *,
        request_id: str = "",
        api_key_id: str = "",
        user_id: str = "",
    ) -> tuple[UnifiedResponse, BackendEntry, dict[str, str]]:
        """Dispatch to a remote backend with fallback to the next healthy remote.

        Tries each remote candidate serving the requested model in priority
        order. If a remote device fails (timeout, connection error, HTTP error),
        it is marked unhealthy and the next candidate is tried.

        Returns ``(response, used_backend, extra_headers)``.
        Raises :class:`BackendUnavailableError` if all remote candidates fail.
        """
        candidates = self._get_remote_candidates(request.model)
        # Also check probe-based health for each candidate
        healthy_candidates: list[BackendEntry] = []
        for c in candidates:
            if await self._is_remote_healthy(c):
                healthy_candidates.append(c)

        if not healthy_candidates:
            raise BackendUnavailableError(f"No healthy remote device available for model '{request.model}'")

        last_exc: Exception | None = None
        for candidate in healthy_candidates:
            try:
                response, headers = await self._dispatch_remote(
                    candidate,
                    request,
                    request_id=request_id,
                    api_key_id=api_key_id,
                    user_id=user_id,
                )
                return response, candidate, headers
            except (BackendTimeoutError, BackendUnavailableError) as exc:
                last_exc = exc
                logger.warning(
                    "Remote candidate %s failed, trying next (%d remaining)",
                    candidate.name,
                    len(healthy_candidates) - healthy_candidates.index(candidate) - 1,
                )
                continue

        raise BackendUnavailableError(f"All remote devices failed for model '{request.model}': {last_exc}")

    async def _stream_remote(
        self,
        backend: BackendEntry,
        request: UnifiedRequest,
    ) -> AsyncIterator[str]:
        """Stream tokens from a remote marketplace device.

        Uses the remote-specific timeout and marks the backend unhealthy on
        transport errors.
        """
        client = self._get_or_create_pool(backend)
        messages = [
            {"role": m.role, "content": m.content if isinstance(m.content, str) else (m.content or "")}
            for m in request.messages
        ]
        payload: dict[str, Any] = {"model": request.model, "messages": messages, "stream": True}
        if request.temperature is not None:
            payload["temperature"] = request.temperature
        if request.max_tokens is not None:
            payload["max_tokens"] = request.max_tokens
        if request.top_p is not None:
            payload["top_p"] = request.top_p
        if request.stop:
            payload["stop"] = request.stop
        if request.tools:
            payload["tools"] = request.tools

        url = f"{backend.base_url}/v1/chat/completions"
        try:
            async with client.stream("POST", url, json=payload, timeout=self._remote_timeout) as resp:
                resp.raise_for_status()
                async for line in resp.aiter_lines():
                    line = line.strip()
                    if not line or line == "data: [DONE]":
                        continue
                    if line.startswith("data: "):
                        line = line[6:]
                    try:
                        chunk = json.loads(line)
                    except json.JSONDecodeError:
                        continue
                    choices = chunk.get("choices", [])
                    if choices:
                        delta = choices[0].get("delta", {})
                        token = delta.get("content", "")
                        if token:
                            yield token
                        if choices[0].get("finish_reason") is not None:
                            return
        except httpx.TimeoutException as exc:
            backend.healthy = False
            raise BackendTimeoutError(f"Remote device stream timed out at {backend.base_url}") from exc
        except httpx.ConnectError as exc:
            backend.healthy = False
            raise BackendUnavailableError(f"Cannot reach remote device at {backend.base_url}") from exc

    # -- streaming dispatch --------------------------------------------------

    async def dispatch_stream(
        self,
        request: UnifiedRequest,
        *,
        source_ip: str = "",
        request_id: str = "",
        api_key_id: str = "",
        user_id: str = "",
        dialect: str = "openai",
        path: str = "/v1/chat/completions",
    ) -> AsyncIterator[str]:
        """Send *request* to the best backend and yield text tokens."""
        # Anti-abuse: block when karma below threshold
        hive_manager = getattr(self, "_hive_manager", None)
        karma_engine = getattr(self, "_karma_engine", None)
        if hive_manager and karma_engine:
            current_member_id = hive_manager.get_current_member_id()
            if current_member_id:
                karma = await karma_engine.get_karma(current_member_id)
                if karma < KARMA_THRESHOLD_BLOCK:
                    raise KarmaTooLowError(
                        "Karma too low; contribute more before requesting from Hive.",
                        retry_after_seconds=60,
                    )

        start_time = time.monotonic()
        backend = await self._select_backend_with_privacy(request)

        # GPU Respect: yield to user workloads for streaming requests too
        if backend.backend_type == "llmhosts-core" and self._gpu_monitor is not None:
            try:
                gpu_avail = await self._gpu_monitor.check_availability()
                if gpu_avail and not any(g.available_for_inference for g in gpu_avail):
                    fallback_model = self._get_fallback_model(request.model)
                    if fallback_model and self._backend_manager is not None and self._backend_manager.has_cloud:
                        cloud_entries = [b for b in self._backends if b.backend_type == "litellm" and b.healthy]
                        if cloud_entries:
                            reasons = "; ".join(g.reason for g in gpu_avail if not g.available_for_inference)
                            logger.info(
                                "GPU Respect (stream): yielding %s -> %s (cloud). Reason: %s",
                                request.model,
                                fallback_model,
                                reasons,
                            )
                            backend = cloud_entries[0]
                            request = UnifiedRequest(
                                model=fallback_model,
                                messages=request.messages,
                                stream=request.stream,
                                temperature=request.temperature,
                                max_tokens=request.max_tokens,
                                top_p=request.top_p,
                                stop=request.stop,
                                tools=request.tools,
                                extra=request.extra,
                            )
            except Exception:
                logger.debug("GPU availability check failed (stream), proceeding without GPU Respect", exc_info=True)

        if self._core_engine is not None and self._core_engine.is_loaded:
            # Core is the default -- handles all local streaming
            messages_raw = [
                {"role": m.role, "content": m.content if isinstance(m.content, str) else str(m.content)}
                for m in request.messages
            ]
            async for chunk in self._dispatch_core_stream(
                messages_raw,
                model=request.model,
                max_tokens=request.max_tokens or 256,
                temperature=request.temperature or 0.0,
            ):
                yield chunk
            await self._record_audit(
                request=request,
                response=None,
                backend=backend,
                start_time=start_time,
                source_ip=source_ip,
                request_id=request_id,
                api_key_id=api_key_id,
                user_id=user_id,
                dialect=dialect,
                path=path,
            )
            return

        if backend.backend_type == "remote":
            # Use dedicated remote streaming with health-aware routing
            candidates = self._get_remote_candidates(request.model)
            streamed = False
            for candidate in candidates:
                if not await self._is_remote_healthy(candidate):
                    continue
                try:
                    async for token in self._stream_remote(candidate, request):
                        yield token
                    streamed = True
                    backend = candidate  # track which backend was used
                    break
                except (BackendTimeoutError, BackendUnavailableError):
                    logger.warning("Remote stream candidate %s failed, trying next", candidate.name)
                    continue
            if not streamed:
                raise BackendUnavailableError(f"All remote devices failed streaming for model '{request.model}'")
            if hive_manager and karma_engine:
                current_member_id = hive_manager.get_current_member_id()
                await self._record_karma_after_dispatch(backend, current_member_id, COMPUTE_HOURS_PER_REQUEST)
            await self._record_audit(
                request=request,
                response=None,
                backend=backend,
                start_time=start_time,
                source_ip=source_ip,
                request_id=request_id,
                api_key_id=api_key_id,
                user_id=user_id,
                dialect=dialect,
                path=path,
            )
            return

        if backend.backend_type == "litellm":
            async for token in self._stream_litellm(request):
                yield token
            if hive_manager and karma_engine:
                current_member_id = hive_manager.get_current_member_id()
                await self._record_karma_after_dispatch(backend, current_member_id, COMPUTE_HOURS_PER_REQUEST)
            await self._record_audit(
                request=request,
                response=None,
                backend=backend,
                start_time=start_time,
                source_ip=source_ip,
                request_id=request_id,
                api_key_id=api_key_id,
                user_id=user_id,
                dialect=dialect,
                path=path,
            )
            return

        if backend.backend_type in (
            "ollama",
            "openai",
            "vllm",
            "llama.cpp",
            "lmstudio",
            "localai",
            "tgi",
            "generic",
        ):
            async for token in self._stream_openai_compat(backend, request):
                yield token
            if hive_manager and karma_engine:
                current_member_id = hive_manager.get_current_member_id()
                await self._record_karma_after_dispatch(backend, current_member_id, COMPUTE_HOURS_PER_REQUEST)
            await self._record_audit(
                request=request,
                response=None,
                backend=backend,
                start_time=start_time,
                source_ip=source_ip,
                request_id=request_id,
                api_key_id=api_key_id,
                user_id=user_id,
                dialect=dialect,
                path=path,
            )
            return

        raise BackendUnavailableError(f"No backend available for model '{request.model}'. Run: llmhosts up")

    async def _stream_litellm(self, request: UnifiedRequest) -> AsyncIterator[str]:
        """Stream tokens through LiteLLM via BackendManager."""
        assert self._backend_manager is not None

        messages = [
            {"role": m.role, "content": m.content if isinstance(m.content, str) else (m.content or "")}
            for m in request.messages
        ]

        kwargs: dict[str, Any] = {}
        if request.temperature is not None:
            kwargs["temperature"] = request.temperature
        if request.max_tokens is not None:
            kwargs["max_tokens"] = request.max_tokens
        if request.top_p is not None:
            kwargs["top_p"] = request.top_p
        if request.stop:
            kwargs["stop"] = request.stop
        if request.tools:
            kwargs["tools"] = request.tools

        try:
            stream = await self._backend_manager.completion(
                backend_type="litellm",
                model=request.model,
                messages=messages,
                stream=True,
                **kwargs,
            )
            # stream is an AsyncIterator[dict] from LiteLLMBackend
            async for chunk in stream:  # type: ignore[union-attr]
                token = chunk.get("content", "")
                if token:
                    yield token
                if chunk.get("finish_reason") is not None:
                    return
        except ValueError as exc:
            raise BackendUnavailableError(str(exc)) from exc

    async def _stream_openai_compat(self, backend: BackendEntry, request: UnifiedRequest) -> AsyncIterator[str]:
        """Stream tokens from an OpenAI-compatible backend (vLLM, llama.cpp, LM Studio, etc.)."""
        client = self._get_or_create_pool(backend)
        messages = [
            {"role": m.role, "content": m.content if isinstance(m.content, str) else (m.content or "")}
            for m in request.messages
        ]
        payload: dict[str, Any] = {"model": request.model, "messages": messages, "stream": True}
        if request.temperature is not None:
            payload["temperature"] = request.temperature
        if request.max_tokens is not None:
            payload["max_tokens"] = request.max_tokens
        if request.top_p is not None:
            payload["top_p"] = request.top_p
        if request.stop:
            payload["stop"] = request.stop
        if request.tools:
            payload["tools"] = request.tools

        url = f"{backend.base_url}/v1/chat/completions"
        try:
            async with client.stream("POST", url, json=payload) as resp:
                resp.raise_for_status()
                async for line in resp.aiter_lines():
                    line = line.strip()
                    if not line or line == "data: [DONE]":
                        continue
                    if line.startswith("data: "):
                        line = line[6:]
                    try:
                        chunk = json.loads(line)
                    except json.JSONDecodeError:
                        continue
                    choices = chunk.get("choices", [])
                    if choices:
                        delta = choices[0].get("delta", {})
                        token = delta.get("content", "")
                        if token:
                            yield token
                        if choices[0].get("finish_reason") is not None:
                            return
        except httpx.TimeoutException as exc:
            backend.healthy = False
            raise BackendTimeoutError(f"Backend stream timed out at {backend.base_url}") from exc
        except httpx.ConnectError as exc:
            backend.healthy = False
            raise BackendUnavailableError(f"Cannot reach backend at {backend.base_url}") from exc

    # -- LLMHosts Core engine ------------------------------------------------

    def _register_core_model(self) -> None:
        """Register LLMHosts Core's loaded model in the backend/model list."""
        if self._core_engine is None or not self._core_engine.is_loaded:
            return
        # Add a virtual backend entry so list_models() includes it
        core_entry = BackendEntry(
            name="llmhosts-core",
            base_url="local://",
            backend_type="llmhosts-core",
            models=["llmhosts-core"],
            priority=-1,  # Highest priority
            healthy=True,
        )
        # Avoid duplicate registration
        if not any(b.name == "llmhosts-core" for b in self._backends):
            self._backends.append(core_entry)
        logger.info("LLMHosts Core model registered as preferred backend")

    def _format_chat_prompt(self, messages: list[dict[str, str]], model: str) -> str:
        """Format messages into a prompt string using model-appropriate chat template."""
        model_lower = model.lower() if model else ""

        # Qwen/Qwen2 family: <|im_start|>role\ncontent<|im_end|>
        if "qwen" in model_lower:
            parts = []
            for msg in messages:
                role = msg.get("role", "user")
                content = msg.get("content", "")
                parts.append(f"<|im_start|>{role}\n{content}<|im_end|>")
            parts.append("<|im_start|>assistant\n")
            return "\n".join(parts)

        # Llama/Mistral family: [INST] content [/INST]
        if any(x in model_lower for x in ("llama", "mistral", "codellama")):
            parts = []
            for msg in messages:
                role = msg.get("role", "user")
                content = msg.get("content", "")
                if role == "system":
                    parts.append(f"<<SYS>>\n{content}\n<</SYS>>\n\n")
                elif role == "user":
                    parts.append(f"[INST] {content} [/INST]")
                elif role == "assistant":
                    parts.append(content)
            return "".join(parts)

        # Generic fallback
        parts = []
        for msg in messages:
            role = msg.get("role", "user")
            content = msg.get("content", "")
            parts.append(f"<|{role}|>\n{content}")
        parts.append("<|assistant|>\n")
        return "\n".join(parts)

    async def _dispatch_core(
        self,
        messages: list[dict[str, str]],
        model: str,
        max_tokens: int = 256,
        temperature: float = 0.0,
        top_p: float = 0.9,
        request: dict | None = None,
    ) -> tuple[dict, dict[str, str]]:
        """Dispatch a chat completion request to LLMHosts Core engine.

        Returns (response_body, extra_headers) formatted as OpenAI ChatCompletion.
        """
        import uuid

        if self._core_engine is None or not self._core_engine.is_loaded:
            raise RuntimeError("LLMHosts Core engine not available or no model loaded")

        # Format messages using model-appropriate chat template
        prompt = self._format_chat_prompt(messages, model)

        # Extract stop sequences from API request for post-processing
        stop_sequences = (request or {}).get("stop", [])
        if isinstance(stop_sequences, str):
            stop_sequences = [stop_sequences]

        # Run inference
        try:
            result = self._core_engine.generate(prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p)
            generated_text = result["text"]
            completion_tokens = result["token_count"]
        except RuntimeError:
            # No tokenizer -- fall back to raw token generation
            tokens = self._core_engine.generate_tokens([1], max_tokens=max_tokens, temperature=temperature, top_p=top_p)
            generated_text = " ".join(str(t) for t in tokens)
            completion_tokens = len(tokens)

        # Post-process stop sequences
        if stop_sequences and generated_text:
            for stop_seq in stop_sequences:
                idx = generated_text.find(stop_seq)
                if idx != -1:
                    generated_text = generated_text[:idx]

        # Estimate prompt tokens
        prompt_tokens = len(prompt.split()) * 2  # rough fallback

        model_name = model or "llmhosts-core"

        response_body = {
            "id": f"chatcmpl-{uuid.uuid4().hex[:24]}",
            "object": "chat.completion",
            "created": int(time.time()),
            "model": model,
            "choices": [
                {
                    "index": 0,
                    "message": {
                        "role": "assistant",
                        "content": generated_text,
                    },
                    "finish_reason": "stop",
                }
            ],
            "usage": {
                "prompt_tokens": prompt_tokens,
                "completion_tokens": completion_tokens,
                "total_tokens": prompt_tokens + completion_tokens,
            },
        }

        extra_headers = {
            "X-LLMHosts-Backend": "llmhosts-core",
            "X-LLMHosts-Model": model_name,
            "X-LLMHosts-Cost": "0.0",
        }

        # Privacy strict mode header
        if getattr(self._config, "privacy_mode", None) == "strict":
            extra_headers["X-LLMHosts-Privacy"] = "strict"

        return response_body, extra_headers

    async def _dispatch_core_stream(
        self,
        messages: list[dict[str, str]],
        model: str,
        max_tokens: int = 256,
        temperature: float = 0.0,
        top_p: float = 0.9,
        request: dict | None = None,
    ) -> AsyncIterator[str]:
        """Stream LLMHosts Core response as SSE chunks (simulated word-by-word)."""
        import uuid

        if self._core_engine is None or not self._core_engine.is_loaded:
            raise RuntimeError("LLMHosts Core engine not available or no model loaded")

        prompt = self._format_chat_prompt(messages, model)

        stop_sequences = (request or {}).get("stop", [])
        if isinstance(stop_sequences, str):
            stop_sequences = [stop_sequences]

        try:
            result = self._core_engine.generate(prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p)
            text = result["text"]
        except RuntimeError:
            tokens = self._core_engine.generate_tokens([1], max_tokens=max_tokens, temperature=temperature, top_p=top_p)
            text = " ".join(str(t) for t in tokens)

        if stop_sequences and text:
            for stop_seq in stop_sequences:
                idx = text.find(stop_seq)
                if idx != -1:
                    text = text[:idx]

        model_name = model or "llmhosts-core"
        stream_id = f"chatcmpl-{uuid.uuid4().hex[:8]}"

        words = text.split()
        for i, word in enumerate(words):
            content = word + (" " if i < len(words) - 1 else "")
            chunk = {
                "id": stream_id,
                "object": "chat.completion.chunk",
                "created": int(time.time()),
                "model": model_name,
                "choices": [{"index": 0, "delta": {"content": content}, "finish_reason": None}],
            }
            yield f"data: {json.dumps(chunk)}\n\n"

        final_chunk = {
            "id": stream_id,
            "object": "chat.completion.chunk",
            "created": int(time.time()),
            "model": model_name,
            "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
        }
        yield f"data: {json.dumps(final_chunk)}\n\n"
        yield "data: [DONE]\n\n"
