"""Registry client for fetching model and deployment configurations.

This module provides the RegistryClient for communicating with the
Model Registry API to retrieve configuration at runtime.
"""

import logging
import threading
import time
from dataclasses import dataclass
from typing import Any, Dict, Optional, Union, TYPE_CHECKING

if TYPE_CHECKING:
    from ..config.settings import SensitiveString
from urllib.parse import urljoin, urlencode

import requests

# GCP authentication imports (optional)
try:
    import google.auth.transport.requests
    import google.oauth2.id_token

    GCP_AUTH_AVAILABLE = True
except ImportError:
    GCP_AUTH_AVAILABLE = False

from ..buffering.retry import RetryPolicy
from ..config.settings import sanitize_token
from ..utils.exceptions import (
    RegistryError,
    RegistryConnectionError,
    RegistryConfigNotFoundError,
    RegistryAuthError,
)
from .models import ModelConfig, DeploymentConfig
from .telemetry import RegistryTelemetry

logger = logging.getLogger(__name__)

# Constants for default configuration values
DEFAULT_CACHE_TTL_SECS = 600  # 10 minutes
DEFAULT_TIMEOUT_SECS = 5.0
DEFAULT_MAX_RETRIES = 3
DEFAULT_BASE_DELAY = 1.0
DEFAULT_MAX_DELAY = 10.0
DEFAULT_REFRESH_INTERVAL_SECS = 600  # 10 minutes
MIN_REFRESH_INTERVAL_SECS = 30  # Minimum refresh interval to prevent excessive polling

# GCP token management constants
GCP_TOKEN_LIFETIME_SECS = 3600  # GCP access tokens typically last 1 hour
GCP_TOKEN_REFRESH_MARGIN_SECS = 300  # Refresh 5 minutes before expiry


@dataclass
class CacheEntry:
    """Cache entry with expiration time."""

    value: Any  # Can be ModelConfig or DeploymentConfig
    expires_at: float


class RegistryClient:
    """Client for fetching configurations from the Model Registry.

    Provides methods to fetch model and deployment configurations from
    a central registry service with automatic retry, caching, and security.

    Features:
        - Automatic retry with exponential backoff
        - In-memory cache with TTL
        - HTTPS enforcement for non-localhost endpoints
        - Bearer token authentication
        - Request timeout protection

    Example:
        >>> client = RegistryClient(
        ...     url="https://registry.example.com",
        ...     token="secret-token",
        ...     timeout=5.0,
        ...     cache_ttl_secs=600
        ... )
        >>> config = client.fetch_model_config("mdl-001", version="2.0.0")
    """

    def __init__(
        self,
        url: str,
        token: Optional[str] = None,
        timeout: float = DEFAULT_TIMEOUT_SECS,
        cache_ttl_secs: int = DEFAULT_CACHE_TTL_SECS,
        refresh_interval_secs: int = DEFAULT_REFRESH_INTERVAL_SECS,
        meter=None,
        use_gcp_auth: bool = False,
    ):
        """Initialize registry client.

        Args:
            url: Base URL of the registry service
            token: Bearer token for authentication (optional, mutually exclusive with use_gcp_auth)
            timeout: Request timeout in seconds (default: 5.0)
            cache_ttl_secs: Cache TTL in seconds (default: 600 = 10 minutes)
            refresh_interval_secs: Background refresh interval in seconds (default: 600)
                Set to 0 or None to disable background refresh
            meter: OpenTelemetry Meter for telemetry (optional)
            use_gcp_auth: Use GCP ID token authentication via ADC (default: False)

        Raises:
            RegistryError: If URL validation fails or if both token and use_gcp_auth are provided
            ValueError: If refresh_interval_secs is invalid or if GCP auth requested but not available
        """
        self._validate_url(url)

        # Validate authentication parameters
        if token and use_gcp_auth:
            raise RegistryError(
                "Cannot use both token and use_gcp_auth. "
                "Choose either bearer token or GCP authentication."
            )

        if use_gcp_auth and not GCP_AUTH_AVAILABLE:
            raise ValueError(
                "GCP authentication requested but google-auth library not available. "
                "Install with: pip install google-auth"
            )

        # Validate refresh interval
        if refresh_interval_secs is not None and refresh_interval_secs > 0:
            if refresh_interval_secs < MIN_REFRESH_INTERVAL_SECS:
                raise ValueError(
                    f"refresh_interval_secs must be >= {MIN_REFRESH_INTERVAL_SECS} seconds. "
                    f"Got: {refresh_interval_secs}"
                )

        self._url = url.rstrip("/")
        self._use_gcp_auth = use_gcp_auth

        # SECURITY: Auto-wrap plain string tokens in SensitiveString for protection
        # This ensures tokens are protected even if caller passes plain string
        self._token: Optional[Union["SensitiveString", str]] = None
        if token is not None:
            from ..config.settings import SensitiveString

            if not isinstance(token, SensitiveString):
                # Emit deprecation warning for plain string tokens
                import warnings

                warnings.warn(
                    "Passing plain string tokens to RegistryClient is deprecated. "
                    "Please wrap tokens in SensitiveString for enhanced security. "
                    "Auto-wrapping will be applied, but explicit wrapping is recommended.",
                    DeprecationWarning,
                    stacklevel=2,
                )
                self._token = SensitiveString(token)
            else:
                self._token = token
        else:
            self._token = None

        self._timeout = timeout
        self._cache: Dict[str, CacheEntry] = {}
        self._cache_lock = threading.RLock()  # Thread-safe cache access
        self._cache_ttl = cache_ttl_secs
        self._retry_policy = RetryPolicy(
            max_attempts=DEFAULT_MAX_RETRIES,
            base_delay=DEFAULT_BASE_DELAY,
            max_delay=DEFAULT_MAX_DELAY,
        )
        self._session = requests.Session()

        # Initialize telemetry if meter provided
        self._telemetry = RegistryTelemetry(meter) if meter else None

        # GCP authentication state
        self._gcp_token: Optional[str] = None
        self._gcp_token_expiry: float = 0
        self._gcp_token_lock = threading.RLock()
        self._gcp_token_refresh_thread: Optional[threading.Thread] = None
        self._gcp_stop_event = threading.Event()

        # Set up authentication
        if self._use_gcp_auth:
            # Fetch initial GCP token
            self._refresh_gcp_token()
            # Start background token refresh thread
            self._start_gcp_token_refresh_thread()
        elif self._token:
            # SECURITY: Use callback pattern to prevent token from existing in local variables
            # Token only exists inside the lambda, never in this frame's traceback
            # Note: self._token is always SensitiveString at this point (auto-wrapped above)
            auth_header = self._token._SensitiveString__unsafe_process_value(  # type: ignore[attr-defined]
                lambda token_val: {"Authorization": f"Bearer {token_val}"}
            )
            self._session.headers.update(auth_header)

        # Background refresh thread management
        self._refresh_interval = refresh_interval_secs
        self._refresh_thread: Optional[threading.Thread] = None
        self._stop_event = threading.Event()
        self._last_refresh_time: float = 0
        self._refresh_count: int = 0
        self._refresh_errors: int = 0
        self._fetched_models: Dict[str, Optional[str]] = {}  # Track {model_id: version}
        self._max_tracked_models: int = 1000  # Prevent unbounded memory growth

        # Start refresh thread if configured
        if url and refresh_interval_secs and refresh_interval_secs > 0:
            self._start_refresh_thread()

    def _sanitize_message(self, message: str) -> str:
        """Sanitize message to prevent token exposure (defense-in-depth).

        Args:
            message: Error or log message that may contain token

        Returns:
            Sanitized message with token masked
        """
        if self._token is None:
            return message

        # Extract raw token string for sanitization
        from ..config.settings import SensitiveString

        if isinstance(self._token, SensitiveString):
            # Use callback to get raw token without exposing it to calling frame
            return self._token._SensitiveString__unsafe_process_value(  # type: ignore[attr-defined]
                lambda token_val: sanitize_token(message, token_val)
            )
        else:
            # Fallback for plain string (shouldn't happen after auto-wrapping)
            return sanitize_token(message, self._token)

    def _validate_url(self, url: str) -> None:
        """Validate registry URL for security.

        Args:
            url: URL to validate

        Raises:
            RegistryError: If URL is invalid or insecure
        """
        if not url.startswith(("http://", "https://")):
            raise RegistryError(f"Registry URL must start with http:// or https://, got: {url}")

        # SECURITY: Require HTTPS for non-localhost
        if url.startswith("http://"):
            if "localhost" not in url and "127.0.0.1" not in url:
                raise RegistryError(
                    "Registry URL must use HTTPS for non-localhost endpoints. " f"Got: {url}"
                )
            logger.warning(
                f"Using HTTP for registry URL: {url}. "
                "This should only be used for local development."
            )

    def fetch_model_config(self, model_id: str, version: Optional[str] = None) -> ModelConfig:
        """Fetch model configuration from registry.

        Args:
            model_id: Unique model identifier
            version: Model version (optional, defaults to latest)

        Returns:
            ModelConfig with fetched configuration

        Raises:
            RegistryConnectionError: If unable to connect to registry
            RegistryConfigNotFoundError: If model not found
            RegistryAuthError: If authentication fails
            RegistryError: For other registry errors
        """
        cache_key = f"model:{model_id}:{version or 'latest'}"

        # Check cache first
        cached = self._get_cached(cache_key)
        if cached:
            logger.debug(f"Cache hit for {cache_key}")
            return cached

        # Fetch from registry with retry
        logger.info(f"Fetching model config for {model_id} version {version or 'latest'}")
        start_time = time.time()

        # Track this model for background refresh (with bounds check)
        with self._cache_lock:
            if len(self._fetched_models) >= self._max_tracked_models:
                logger.warning(
                    f"Reached maximum tracked models ({self._max_tracked_models}). "
                    f"Not tracking {model_id} for background refresh. "
                    "Use untrack_model() to remove stale models or increase max_tracked_models."
                )
            else:
                self._fetched_models[model_id] = version

        # First try without retry to catch non-retryable errors immediately
        try:
            config = self._fetch_model_config_internal(model_id, version)
            self._set_cache(cache_key, config)

            # Record telemetry success
            if self._telemetry:
                latency = time.time() - start_time
                self._telemetry.record_success(latency)

            return config
        except (RegistryAuthError, RegistryConfigNotFoundError) as e:
            # Don't retry permanent failures (401/403/404) - fail immediately
            if self._telemetry:
                self._telemetry.record_error(type(e).__name__)
            raise
        except (RegistryConnectionError, RegistryError) as e:
            # Retry transient failures
            logger.warning(f"Retrying model config fetch for {model_id} due to: {e}")

            def _fetch_with_retry():
                return self._fetch_model_config_internal(model_id, version)

            try:
                config = self._retry_policy.execute(_fetch_with_retry)
                self._set_cache(cache_key, config)

                # Record telemetry success
                if self._telemetry:
                    latency = time.time() - start_time
                    self._telemetry.record_success(latency)

                return config
            except Exception as retry_e:
                if self._telemetry:
                    self._telemetry.record_error(type(retry_e).__name__)
                logger.error(
                    f"Failed to fetch model config for {model_id} after retries: {retry_e}"
                )
                raise

    def _fetch_model_config_internal(self, model_id: str, version: Optional[str]) -> ModelConfig:
        """Internal method to fetch model config (called by retry policy).

        Args:
            model_id: Model identifier
            version: Model version (optional)

        Returns:
            ModelConfig from registry response

        Raises:
            RegistryConnectionError: On connection failures
            RegistryConfigNotFoundError: If model not found
            RegistryAuthError: On auth failures
            RegistryError: On other errors
        """
        # Build URL - API uses /api/v1/configs/{model_id}
        endpoint = f"/api/v1/configs/{model_id}"
        params: Dict[str, str] = {}
        # Note: API doesn't support version parameter yet - tracked in API team request
        if version:
            logger.warning(
                f"Version parameter '{version}' provided but not yet supported by registry API. "
                "Fetching latest version. See API team request document."
            )

        url = urljoin(self._url + "/", endpoint.lstrip("/"))
        if params:
            url = f"{url}?{urlencode(params)}"

        # Make request
        try:
            # SECURITY: Don't log full URL (may contain query params)
            logger.debug(f"GET {endpoint}")
            response = self._session.get(url, timeout=self._timeout)

            # Handle HTTP errors
            if response.status_code == 404:
                raise RegistryConfigNotFoundError(
                    self._sanitize_message(
                        f"Model config not found: {model_id} version {version or 'latest'}"
                    )
                )
            elif response.status_code in (401, 403):
                raise RegistryAuthError(
                    self._sanitize_message(
                        f"Registry authentication failed: {response.status_code}"
                    )
                )
            elif response.status_code >= 400:
                # SECURITY: Don't log response body (may contain sensitive data)
                # Defense-in-depth: sanitize error message
                raise RegistryError(
                    self._sanitize_message(f"Registry returned error {response.status_code}")
                )

        except requests.exceptions.Timeout as e:
            raise RegistryConnectionError(
                self._sanitize_message(f"Registry request timed out after {self._timeout}s")
            ) from e
        except requests.exceptions.ConnectionError as e:
            # SECURITY: Redact full URL, show only hostname
            hostname = self._url.split("//")[-1].split("/")[0]
            raise RegistryConnectionError(
                self._sanitize_message(f"Failed to connect to registry at {hostname}")
            ) from e
        except (RegistryConfigNotFoundError, RegistryAuthError):
            # Re-raise our custom exceptions (already sanitized above)
            raise
        except requests.exceptions.RequestException as e:
            # Defense-in-depth: sanitize exception message
            raise RegistryConnectionError(
                self._sanitize_message(f"Registry request failed: {e}")
            ) from e

        # Parse response
        try:
            data = response.json()
        except ValueError as e:
            raise RegistryError(f"Invalid JSON response from registry: {e}") from e

        # Validate response is a dictionary
        if not isinstance(data, dict):
            raise RegistryError(
                f"Registry response must be a JSON object, got {type(data).__name__}"
            )

        # Convert to ModelConfig
        try:
            # Two-field taxonomy support with backward compatibility
            from ..utils.routing import migrate_legacy_model_type

            # API returns these fields at top level
            model_category = data.get("model_category")
            model_type = data.get("model_type")

            # BACKWARD COMPATIBILITY: Handle legacy registry responses
            # If model_category is missing but model_type is a legacy value, auto-migrate
            if not model_category and model_type in ["internal", "generative", "vendor"]:
                logger.warning(
                    "Registry response missing model_category field. "
                    "Auto-migrating legacy model_type='%s' to two-field taxonomy. "
                    "Registry should be updated to provide both model_category and model_type.",
                    model_type,
                )
                model_category, model_type = migrate_legacy_model_type(model_type)

            # Validate required fields are present
            if not model_category:
                raise KeyError(
                    "model_category field is required in registry response. "
                    "Registry must provide both model_category and model_type for two-field taxonomy."
                )
            if not model_type:
                raise KeyError("model_type")

            # Extract model_version from TOP LEVEL (API team added this field)
            # API response structure: { "model_version": "2.1.3", ... }
            model_version = data.get("model_version", "unknown")
            if model_version == "unknown":
                logger.warning(
                    f"model_version not found in registry response for {data.get('model_id')}. "
                    "Using 'unknown'. This should not happen with updated API (Feb 2026)."
                )

            # Extract thresholds from TOP LEVEL (not nested in config)
            # API response structure: { "thresholds": {"MAE": 12.5, "RMSE": 18.3}, ... }
            thresholds = data.get("thresholds", {})
            if not isinstance(thresholds, dict):
                logger.warning(
                    f"thresholds is not a dict for {data.get('model_id')}, "
                    f"got {type(thresholds).__name__}. Using empty dict."
                )
                thresholds = {}

            # Extract PHI fields from TOP LEVEL (can be empty dict or nested structure)
            # API response structure:
            #   {} (no PHI) or {"patient_id": {"criticality": "yes"}, ...}
            phi_fields = data.get("phi_fields", {})
            if not isinstance(phi_fields, dict):
                logger.warning(
                    f"phi_fields is not a dict for {data.get('model_id')}, "
                    f"got {type(phi_fields).__name__}. Using empty dict."
                )
                phi_fields = {}

            # Extract config JSONB (can be null)
            # API response structure: null or { "extra_resource": {...}, "association_id_column": "..." }
            config_data = data.get("config")
            extra_resource = {}
            association_id_column = None

            if config_data is not None and isinstance(config_data, dict):
                # Extract extra_resource (nested in config JSONB)
                extra_resource = config_data.get("extra_resource", {})
                if not isinstance(extra_resource, dict):
                    logger.warning(
                        f"extra_resource is not a dict for {data.get('model_id')}, "
                        f"got {type(extra_resource).__name__}. Using empty dict."
                    )
                    extra_resource = {}

                # Extract time-series specific field (optional)
                association_id_column = config_data.get("association_id_column")

            # Extract optional identifiers (can be null for vendor models)
            registry_id = data.get("registry_id")
            deployment_id = data.get("deployment_id")

            # Extract timestamps (optional, for debugging)
            created_at = data.get("created_at")
            updated_at = data.get("updated_at")

            config = ModelConfig(
                service_name=data["service_name"],
                model_id=data["model_id"],
                model_version=model_version,
                team_name=data["team_name"],
                model_category=model_category,
                model_type=model_type,
                registry_id=registry_id,
                deployment_id=deployment_id,
                thresholds=thresholds,
                phi_fields=phi_fields,
                extra_resource=extra_resource,
                association_id_column=association_id_column,
                created_at=created_at,
                updated_at=updated_at,
            )
            return config
        except KeyError as e:
            raise RegistryError(f"Registry response missing required field: {e}") from e
        except (TypeError, ValueError) as e:
            raise RegistryError(f"Invalid registry response format: {e}") from e

    def fetch_deployment_config(self, deployment_id: str) -> DeploymentConfig:
        """Fetch deployment configuration from registry.

        Args:
            deployment_id: Unique deployment identifier

        Returns:
            DeploymentConfig with fetched configuration

        Raises:
            RegistryConnectionError: If unable to connect to registry
            RegistryConfigNotFoundError: If deployment not found
            RegistryAuthError: If authentication fails
            RegistryError: For other registry errors
        """
        # Check cache first
        cache_key = f"deployment:{deployment_id}"
        cached = self._get_cached(cache_key)
        if cached:
            logger.debug(f"Cache hit for {cache_key}")
            return cached

        logger.info(f"Fetching deployment config for {deployment_id}")
        start_time = time.time()

        # First try without retry to catch non-retryable errors immediately
        try:
            config = self._fetch_deployment_config_internal(deployment_id)
            self._set_cache(cache_key, config)

            # Record telemetry success
            if self._telemetry:
                latency = time.time() - start_time
                self._telemetry.record_success(latency)

            return config
        except (RegistryAuthError, RegistryConfigNotFoundError) as e:
            # Don't retry permanent failures (401/403/404) - fail immediately
            if self._telemetry:
                self._telemetry.record_error(type(e).__name__)
            raise
        except (RegistryConnectionError, RegistryError) as e:
            # Retry transient failures
            logger.warning(f"Retrying deployment config fetch for {deployment_id} due to: {e}")

            def _fetch_with_retry():
                return self._fetch_deployment_config_internal(deployment_id)

            try:
                config = self._retry_policy.execute(_fetch_with_retry)
                self._set_cache(cache_key, config)

                # Record telemetry success
                if self._telemetry:
                    latency = time.time() - start_time
                    self._telemetry.record_success(latency)

                return config
            except Exception as retry_e:
                if self._telemetry:
                    self._telemetry.record_error(type(retry_e).__name__)
                logger.error(
                    f"Failed to fetch deployment config for {deployment_id} after retries: {retry_e}"
                )
                raise

    def _fetch_deployment_config_internal(self, deployment_id: str) -> DeploymentConfig:
        """Internal method to fetch deployment config.

        Args:
            deployment_id: Deployment identifier

        Returns:
            DeploymentConfig from registry response

        Raises:
            Similar exceptions as _fetch_model_config_internal
        """
        # Build URL
        endpoint = f"/deployments/{deployment_id}"
        url = urljoin(self._url + "/", endpoint.lstrip("/"))

        # Make request
        try:
            logger.debug(f"GET {endpoint}")
            response = self._session.get(url, timeout=self._timeout)

            if response.status_code == 404:
                raise RegistryConfigNotFoundError(
                    self._sanitize_message(f"Deployment config not found: {deployment_id}")
                )
            elif response.status_code in (401, 403):
                raise RegistryAuthError(
                    self._sanitize_message(
                        f"Registry authentication failed: {response.status_code}"
                    )
                )
            elif response.status_code >= 400:
                # SECURITY: Don't log response body (may contain sensitive data)
                # Defense-in-depth: sanitize error message
                raise RegistryError(
                    self._sanitize_message(f"Registry returned error {response.status_code}")
                )

        except requests.exceptions.Timeout as e:
            raise RegistryConnectionError(
                self._sanitize_message(f"Registry request timed out after {self._timeout}s")
            ) from e
        except requests.exceptions.ConnectionError as e:
            # SECURITY: Redact full URL, show only hostname
            hostname = self._url.split("//")[-1].split("/")[0]
            raise RegistryConnectionError(
                self._sanitize_message(f"Failed to connect to registry at {hostname}")
            ) from e
        except (RegistryConfigNotFoundError, RegistryAuthError):
            # Re-raise our custom exceptions (already sanitized above)
            raise
        except requests.exceptions.RequestException as e:
            # Defense-in-depth: sanitize exception message
            raise RegistryConnectionError(
                self._sanitize_message(f"Registry request failed: {e}")
            ) from e

        # Parse response
        try:
            data = response.json()
        except ValueError as e:
            raise RegistryError(f"Invalid JSON response from registry: {e}") from e

        # Validate response is a dictionary
        if not isinstance(data, dict):
            raise RegistryError(
                f"Registry response must be a JSON object, got {type(data).__name__}"
            )

        # Convert to DeploymentConfig
        try:
            config = DeploymentConfig(
                deployment_id=data["deployment_id"],
                environment=data["environment"],
                region=data["region"],
                resource_overrides=data.get("resource_overrides", {}),
            )
            return config
        except KeyError as e:
            raise RegistryError(f"Registry response missing required field: {e}") from e
        except (TypeError, ValueError) as e:
            raise RegistryError(f"Invalid registry response format: {e}") from e

    def _get_cached(self, key: str) -> Optional[Any]:
        """Get cached config if not expired.

        Args:
            key: Cache key

        Returns:
            Cached config (ModelConfig or DeploymentConfig) or None if not found/expired
        """
        with self._cache_lock:
            entry = self._cache.get(key)
            if entry and time.time() < entry.expires_at:
                return entry.value
            elif entry:
                # Expired, remove from cache
                del self._cache[key]
            return None

    def _set_cache(self, key: str, value: Any) -> None:
        """Cache a config with TTL.

        Args:
            key: Cache key
            value: Config to cache (ModelConfig or DeploymentConfig)
        """
        with self._cache_lock:
            self._cache[key] = CacheEntry(value=value, expires_at=time.time() + self._cache_ttl)

    def clear_cache(self) -> None:
        """Clear all cached configurations."""
        with self._cache_lock:
            self._cache.clear()
            logger.debug("Registry cache cleared")

    def untrack_model(self, model_id: str) -> None:
        """Stop tracking a model for background refresh.

        Use this to prevent memory leaks in long-running applications that
        dynamically fetch many different models over time.

        Args:
            model_id: Model identifier to stop tracking

        Example:
            >>> client.untrack_model("mdl-001")
        """
        with self._cache_lock:
            if model_id in self._fetched_models:
                del self._fetched_models[model_id]
                logger.debug(f"Stopped tracking model {model_id} for background refresh")

    def get_tracked_models(self) -> Dict[str, Optional[str]]:
        """Get list of models currently tracked for background refresh.

        Returns:
            Dictionary mapping model_id to version (or None for latest)
        """
        with self._cache_lock:
            return self._fetched_models.copy()

    def _start_refresh_thread(self) -> None:
        """Start background refresh thread."""
        if self._refresh_thread is not None and self._refresh_thread.is_alive():
            logger.warning("Refresh thread already running")
            return

        self._refresh_thread = threading.Thread(
            target=self._refresh_loop,
            daemon=True,  # CRITICAL: daemon thread prevents blocking shutdown
            name="registry-refresh",
        )
        self._refresh_thread.start()
        logger.info(f"Started registry refresh thread with interval {self._refresh_interval}s")

    def _refresh_loop(self) -> None:
        """Background refresh loop that periodically updates the cache."""
        while not self._stop_event.is_set():
            # Wait for interval or stop signal
            if self._stop_event.wait(timeout=self._refresh_interval):
                break  # Stop signal received

            # Fetch latest config from registry for all tracked models
            logger.debug("Refreshing model config from registry")

            # Get snapshot of tracked models (thread-safe)
            with self._cache_lock:
                models_to_refresh = list(self._fetched_models.items())

            # Refresh each tracked model
            if models_to_refresh:
                successful_refreshes = 0
                for model_id, version in models_to_refresh:
                    try:
                        # Fetch updated config (config stored in cache automatically)
                        _ = self.fetch_model_config(model_id, version)
                        successful_refreshes += 1
                    except RegistryAuthError as e:
                        # Fatal error: stop thread immediately
                        logger.error(
                            f"Registry auth failed for {model_id}: {e}. Stopping refresh thread."
                        )
                        return  # Exit thread entirely
                    except (
                        RegistryConnectionError,
                        RegistryConfigNotFoundError,
                        RegistryError,
                    ) as e:
                        # Transient or model-specific error: log and continue with next model
                        logger.warning(
                            f"Failed to refresh {model_id}: {e}. Continuing with other models."
                        )
                        self._refresh_errors += 1
                    except Exception as e:
                        # Unexpected error: log and continue (defensive)
                        logger.error(f"Unexpected error refreshing {model_id}: {e}", exc_info=True)
                        self._refresh_errors += 1

                if successful_refreshes > 0:
                    self._last_refresh_time = time.time()
                    self._refresh_count += 1
                    logger.info(
                        f"Registry config refreshed successfully ({successful_refreshes}/{len(models_to_refresh)} models)"
                    )
            else:
                # No models to refresh yet
                logger.debug("No models to refresh yet")

    def stop_refresh_thread(self, timeout: float = 5.0) -> None:
        """Stop background refresh thread gracefully.

        Args:
            timeout: Maximum time to wait for thread to stop (default: 5.0 seconds)
        """
        if self._refresh_thread is None or not self._refresh_thread.is_alive():
            return  # Already stopped or never started

        logger.info("Stopping registry refresh thread")
        self._stop_event.set()  # Signal thread to stop
        self._refresh_thread.join(timeout=timeout)  # Wait for thread

        if self._refresh_thread.is_alive():
            logger.warning(f"Refresh thread did not stop within {timeout}s timeout")
        else:
            logger.info("Refresh thread stopped successfully")

    def get_refresh_stats(self) -> Dict[str, Any]:
        """Get refresh statistics for monitoring.

        Returns:
            Dictionary containing:
                - last_refresh_time: Timestamp of last successful refresh
                - refresh_count: Number of successful refreshes
                - refresh_errors: Number of failed refresh attempts
                - is_running: Whether refresh thread is currently running
        """
        return {
            "last_refresh_time": self._last_refresh_time,
            "refresh_count": self._refresh_count,
            "refresh_errors": self._refresh_errors,
            "is_running": self._refresh_thread is not None and self._refresh_thread.is_alive(),
        }

    def _refresh_gcp_token(self) -> None:
        """Fetch a new GCP ID token for the registry URL.

        Raises:
            RegistryAuthError: If token fetch fails
        """
        if not self._use_gcp_auth:
            return

        try:
            auth_req = google.auth.transport.requests.Request()
            token = google.oauth2.id_token.fetch_id_token(auth_req, self._url)

            with self._gcp_token_lock:
                self._gcp_token = token
                # GCP ID tokens typically last 1 hour
                self._gcp_token_expiry = time.time() + GCP_TOKEN_LIFETIME_SECS
                # Update session headers
                self._session.headers.update({"Authorization": f"Bearer {token}"})

            logger.info("GCP ID token refreshed successfully")

        except Exception as e:
            raise RegistryAuthError(f"Failed to fetch GCP ID token: {e}") from e

    def _should_refresh_gcp_token(self) -> bool:
        """Check if GCP token should be refreshed.

        Returns:
            True if token should be refreshed (expired or near expiry)
        """
        if not self._use_gcp_auth:
            return False

        with self._gcp_token_lock:
            # Refresh if token doesn't exist or is near expiry
            return self._gcp_token is None or time.time() >= (
                self._gcp_token_expiry - GCP_TOKEN_REFRESH_MARGIN_SECS
            )

    def _start_gcp_token_refresh_thread(self) -> None:
        """Start background thread for GCP token refresh."""
        if not self._use_gcp_auth:
            return

        if self._gcp_token_refresh_thread is not None and self._gcp_token_refresh_thread.is_alive():
            logger.warning("GCP token refresh thread already running")
            return

        self._gcp_token_refresh_thread = threading.Thread(
            target=self._gcp_token_refresh_loop,
            daemon=True,
            name="gcp-token-refresh",
        )
        self._gcp_token_refresh_thread.start()
        logger.info("Started GCP token refresh thread")

    def _gcp_token_refresh_loop(self) -> None:
        """Background loop for refreshing GCP tokens."""
        while not self._gcp_stop_event.is_set():
            # Check every 60 seconds if token needs refresh
            if self._gcp_stop_event.wait(timeout=60):
                break

            if self._should_refresh_gcp_token():
                try:
                    self._refresh_gcp_token()
                except RegistryAuthError as e:
                    logger.error(f"Failed to refresh GCP token: {e}")
                    # Continue trying - don't stop the thread

    def _stop_gcp_token_refresh_thread(self, timeout: float = 5.0) -> None:
        """Stop GCP token refresh thread gracefully.

        Args:
            timeout: Maximum time to wait for thread to stop
        """
        if self._gcp_token_refresh_thread is None or not self._gcp_token_refresh_thread.is_alive():
            return

        logger.info("Stopping GCP token refresh thread")
        self._gcp_stop_event.set()
        self._gcp_token_refresh_thread.join(timeout=timeout)

        if self._gcp_token_refresh_thread.is_alive():
            logger.warning(f"GCP token refresh thread did not stop within {timeout}s timeout")
        else:
            logger.info("GCP token refresh thread stopped successfully")

    def close(self) -> None:
        """Close the HTTP session and stop all background threads."""
        self.stop_refresh_thread()
        self._stop_gcp_token_refresh_thread()
        self._session.close()
