"""Main MCA SDK client for model instrumentation.

This module provides the MCAClient class, the primary developer-facing API
for instrumenting models with OpenTelemetry.
"""

import contextvars
import functools
import inspect
import logging
import re
import threading
import time
import uuid
from contextlib import contextmanager
from typing import Any, Callable, Dict, Optional

from opentelemetry import trace
from opentelemetry.sdk._logs import LoggingHandler

from ..config import MCAConfig
from .providers import setup_all_providers
from ..validation import validate_resource_attributes, MetricNamingConvention
from ..validation.telemetry_attributes import TelemetryAttributeValidator
from ..registry import RegistryClient, RegistryTelemetry, ModelConfig
from ..utils.exceptions import RegistryConnectionError, RegistryAuthError
from ..utils.version_check import check_version
from ..utils.serialization import serialize_for_telemetry
from .. import __version__

# Context variable for thread-safe prediction ID tracking
_prediction_context: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
    "prediction_context", default=None
)

# Context variable for tracking nested tool call depth
_tool_call_depth: contextvars.ContextVar[int] = contextvars.ContextVar("tool_call_depth", default=0)


# ============================================================================
# DUAL-PURPOSE MASKING: PHI (Healthcare) + CREDENTIALS (SDK Security)
# ============================================================================
#
# This SDK masks TWO types of sensitive data:
#
# 1. PHI (Protected Health Information) - Healthcare compliance
#    - Patient names, SSN, MRN, DOB, addresses, phone, email, etc.
#    - Applications should sanitize PHI BEFORE SDK where possible
#    - SDK provides defense-in-depth for common PHI patterns
#
# 2. CREDENTIALS - SDK security (prevent leaked keys in telemetry)
#    - AWS keys, GCP keys, GitHub tokens, JWT, Bearer tokens, etc.
#    - Prevents SDK from exposing credentials in error messages/traces
#
# CRITICAL: Credentials MUST be masked in input_data and output payloads.
# The adversarial review identified this as a security regression.
# ============================================================================

# Credential patterns (SDK security - ALWAYS mask these)
CREDENTIAL_PATTERNS = [
    # AWS Access Keys: AKIA followed by 16 alphanumeric chars
    ("aws_access_key", r"\bAKIA[0-9A-Z]{16}\b", "[AWS-KEY-REDACTED]"),
    # GCP API Keys: AIzaSy followed by 33 chars
    ("gcp_api_key", r"\bAIzaSy[A-Za-z0-9_-]{33}\b", "[GCP-KEY-REDACTED]"),
    # GitHub Personal Access Tokens: ghp_ prefix
    ("github_pat", r"\bghp_[A-Za-z0-9]{36}\b", "[GITHUB-TOKEN-REDACTED]"),
    # JWT Tokens: three base64 segments separated by dots
    (
        "jwt",
        r"\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b",
        "[JWT-REDACTED]",
    ),
    # Bearer Tokens
    ("bearer_token", r"Bearer\s+[A-Za-z0-9._+/=-]{20,}", "Bearer [TOKEN-REDACTED]"),
    # Basic Auth
    ("basic_auth", r"Basic\s+[A-Za-z0-9+/=_-]{20,}", "Basic [REDACTED]"),
]

# PHI patterns (Healthcare compliance - mask common patterns)
PHI_PATTERNS = [
    # SSN: 3-2-4 digits with dashes, spaces, or no separators
    ("ssn", r"\b\d{3}[-\s]?\d{2}[-\s]?\d{4}\b", "[SSN-REDACTED]"),
    # DOB: MM/DD/YYYY or YYYY-MM-DD
    ("dob", r"\b(?:\d{1,2}/\d{1,2}/\d{4}|\d{4}-\d{2}-\d{2})\b", "[DATE-REDACTED]"),
    # Phone: (555) 123-4567, 555-123-4567, 555.123.4567
    ("phone", r"(?:\(\d{3}\)\s?|\b\d{3}[-.]?)\d{3}[-.]\d{4}\b", "[PHONE-REDACTED]"),
    ("email", r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", "[EMAIL-REDACTED]"),
    (
        "ipv4",
        r"(?<!\d\.)(?<!\d)(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?!\.\d)(?!\d)",
        "[IP-REDACTED]",
    ),
    # ZIP codes (HIPAA: first 3 digits can be PHI in populations < 20,000)
    ("zip", r"\b\d{5}(?:-\d{4})?\b", "[ZIP-REDACTED]"),
    # Common name patterns (First Last, Last First, etc.)
    # NOTE: These are basic patterns and may have false positives
    ("name_first_last", r"\b[A-Z][a-z]+\s+[A-Z][a-z]+\b", "[NAME-REDACTED]"),
    ("name_last_first", r"\b[A-Z][a-z]+,\s*[A-Z][a-z]+\b", "[NAME-REDACTED]"),
    # Street addresses (basic pattern - number + street name)
    (
        "address",
        r"\b\d+\s+[A-Z][a-z]+(\s+[A-Z][a-z]+)*\s+(Street|St|Avenue|Ave|Road|Rd|Boulevard|Blvd|Lane|Ln|Drive|Dr|Court|Ct|Circle|Cir)\b",
        "[ADDRESS-REDACTED]",
    ),
    # Alphanumeric MRNs - negative lookbehind to avoid matching our own replacement strings
    # Matches "MRN" or "Record" followed by alphanumeric ID
    (
        "mrn_explicit",
        r"\b(?:MRN|Patient ID|patient_id|RECORD)[:\s-]*([A-Z0-9-]+)\b",
        "MRN:[REDACTED]",
    ),
    ("mrn", r"(?<!\[)\b[A-Z0-9]{6,12}\b(?!\])", "[REDACTED]"),
]

# Combined patterns: Credentials first (higher priority), then PHI
ALL_SENSITIVE_PATTERNS = CREDENTIAL_PATTERNS + PHI_PATTERNS


def _sanitize_json_credentials(data: dict) -> dict:
    """Safely sanitize credentials in parsed JSON/dict structures.

    This structural approach is safer than regex-based parsing of JSON strings,
    which was identified as "fundamentally flawed" in the adversarial review.

    Args:
        data: Parsed dictionary (from json.loads())

    Returns:
        Dictionary with credential values masked
    """
    credential_keys = {
        "token",
        "secret",
        "password",
        "api_key",
        "apikey",
        "access_key",
        "secret_key",
        "private_key",
        "auth_token",
        "bearer_token",
        "jwt",
        "client_secret",
        "api_secret",
        "aws_access_key_id",
        "aws_secret_access_key",
    }

    sanitized = {}
    for key, value in data.items():
        key_lower = key.lower()

        # Check if key indicates a credential
        is_credential = any(cred_key in key_lower for cred_key in credential_keys)

        if is_credential and isinstance(value, str) and len(value) >= 8:
            # Mask credential value
            sanitized[key] = "[REDACTED]"
        elif isinstance(value, dict):
            # Recursively sanitize nested dicts
            sanitized[key] = _sanitize_json_credentials(value)
        elif isinstance(value, list):
            # Recursively sanitize lists
            sanitized[key] = [
                _sanitize_json_credentials(item) if isinstance(item, dict) else item
                for item in value
            ]
        else:
            # Pass through other values
            sanitized[key] = value

    return sanitized


def _sanitize_error_message(error_message: str, max_length: int = 200) -> str:
    """Sanitize error message to prevent PHI and credential leakage.

    Masks both credentials (AWS keys, GCP keys, tokens) and PHI patterns
    (SSN, MRN, DOB, phone, email, IP) to prevent exposure in logs/traces.

    Args:
        error_message: Raw error message from exception
        max_length: Maximum allowed message length (default: 200 chars)

    Returns:
        Sanitized error message with sensitive data masked and truncated if necessary
    """
    sanitized = error_message

    # Apply ALL sensitive patterns (credentials first, then PHI)
    for pattern_type, regex, replacement in ALL_SENSITIVE_PATTERNS:
        sanitized = re.sub(regex, replacement, sanitized)

    # Truncate if still too long after masking
    if len(sanitized) > max_length:
        return sanitized[:max_length] + "...[truncated]"

    return sanitized


def _sanitize_data_recursive(data: Any, max_value_length: int = 1024) -> Any:
    """Recursively sanitize credentials and PHI in nested data structures.

    CRITICAL: This function masks CREDENTIALS in input_data and output payloads
    to prevent credential leakage (identified as security regression in review).
    It also masks PHI patterns for defense-in-depth.

    Args:
        data: Data to sanitize (dict, list, str, or primitive)
        max_value_length: Maximum string length before rejection

    Returns:
        Sanitized data with credentials and PHI masked

    Raises:
        ValidationError: If string exceeds max_value_length
    """
    from ..utils.exceptions import ValidationError

    if isinstance(data, str):
        # Apply ALL sensitive patterns to strings
        sanitized = data
        for pattern_type, regex, replacement in ALL_SENSITIVE_PATTERNS:
            sanitized = re.sub(regex, replacement, sanitized)

        # Enforce length limit (no silent truncation)
        if len(sanitized) > max_value_length:
            raise ValidationError(
                f"String value exceeds maximum length of {max_value_length} characters "
                f"(got {len(sanitized)} characters). Please reduce data size before instrumentation."
            )

        return sanitized

    elif isinstance(data, dict):
        # PHASE 1.3 FIX: Use structural parsing for credential masking (safer than regex)
        # First, apply structural credential sanitization
        sanitized_dict = _sanitize_json_credentials(data)
        # Then recursively sanitize remaining values (for PHI patterns)
        return {k: _sanitize_data_recursive(v, max_value_length) for k, v in sanitized_dict.items()}

    elif isinstance(data, (list, tuple)):
        # Recursively sanitize list/tuple items (preserve type)
        sanitized_items = [_sanitize_data_recursive(item, max_value_length) for item in data]
        return type(data)(sanitized_items)

    elif isinstance(data, (int, float, bool, type(None))):
        # Primitives pass through unchanged
        return data

    else:
        # For custom objects, attempt conversion to dict
        import logging
        from dataclasses import is_dataclass, asdict

        logger = logging.getLogger("mca_sdk")

        # Try Pydantic v2
        if hasattr(data, "model_dump") and callable(getattr(data, "model_dump")):
            try:
                data_dict = data.model_dump()
                return _sanitize_data_recursive(data_dict, max_value_length)
            except Exception as e:
                logger.warning(f"Failed to convert Pydantic v2 model: {e}")

        # Try Pydantic v1
        if hasattr(data, "dict") and callable(getattr(data, "dict")):
            try:
                data_dict = data.dict()
                return _sanitize_data_recursive(data_dict, max_value_length)
            except Exception as e:
                logger.warning(f"Failed to convert Pydantic v1 model: {e}")

        # Try dataclasses
        if is_dataclass(data):
            try:
                data_dict = asdict(data)
                return _sanitize_data_recursive(data_dict, max_value_length)
            except Exception as e:
                logger.warning(f"Failed to convert dataclass: {e}")

        # Try __dict__
        if hasattr(data, "__dict__"):
            try:
                data_dict = data.__dict__
                return _sanitize_data_recursive(data_dict, max_value_length)
            except Exception as e:
                logger.warning(f"Failed to access __dict__: {e}")

        # Try dict() constructor
        try:
            data_dict = dict(data)
            return _sanitize_data_recursive(data_dict, max_value_length)
        except Exception:
            pass

        # If all conversions fail, raise error
        raise ValidationError(
            f"Cannot sanitize object of type {type(data).__name__}. "
            f"Please convert to dict before instrumentation."
        )


class MCAClient:
    """Main client for MCA SDK instrumentation.

    This class provides a simple API for instrumenting models with metrics,
    logs, and traces. It handles provider setup, validation, and buffering
    automatically.

    IMPORTANT: Configuration is immutable after initialization. Runtime modifications
    to client.config are unsupported and may cause inconsistent behavior. Create a
    new client instance if configuration needs to change.

    Examples:
        Basic usage:
        >>> client = MCAClient(
        ...     service_name="readmission-model",
        ...     model_id="mdl-001",
        ...     team_name="clinical-ai"
        ... )
        >>> def predict(data):
        ...     with client.trace("model.predict"):
        ...         result = model.predict(data)
        ...         client.record_prediction(data, result, latency=0.15)
        ...         return result
        >>> client.shutdown()

        Custom metrics:
        >>> counter = client.meter.create_counter("model.false_positives_total")
        >>> counter.add(1, {"threshold": "0.7"})

        Auto-instrumentation with decorator:
        >>> @client.predict()
        ... def predict(data):
        ...     return model.predict(data)
        >>> result = predict(input_data)
    """

    def __init__(
        self,
        service_name: Optional[str] = None,
        model_id: Optional[str] = None,
        model_version: Optional[str] = None,
        team_name: Optional[str] = None,
        model_type: str = "internal",
        collector_endpoint: str = "http://localhost:4318",
        strict_validation: bool = True,
        auto_sanitize: bool = True,
        data_sanitizer_hook: Optional[Callable[[dict], dict]] = None,
        config: Optional[MCAConfig] = None,
        **kwargs,
    ):
        """Initialize MCA SDK client.

        May take 2-5 seconds due to version check (2s timeout) and optional
        registry hydration (with circuit breaker and timeouts).

        Args:
            service_name: Name of the service (required if config not provided)
            model_id: Unique model identifier
            model_version: Model version string
            team_name: Team responsible for the model
            model_type: Type of model (internal, generative, vendor)
            collector_endpoint: OTLP collector endpoint
            strict_validation: Whether to enforce strict validation
            auto_sanitize: DEPRECATED. Always True. Credential sanitization is now
                always enabled to prevent security regressions. This parameter will
                be removed in the next major version. Applications should still
                sanitize PHI before calling SDK methods.
            data_sanitizer_hook: Optional callback for application-specific PHI masking.
                Function signature: Callable[[dict], dict]. Called on input_data and
                output before SDK sanitization. Allows applications to inject their
                own domain-specific PHI masking logic.
                Example:
                    def sanitize_phi(data):
                        sanitized = data.copy()
                        sanitized.pop('patient_name', None)
                        sanitized.pop('ssn', None)
                        return sanitized
                    client = MCAClient(..., data_sanitizer_hook=sanitize_phi)
            config: MCAConfig instance (if provided, overrides other args)
            **kwargs: Additional configuration options

        Raises:
            ConfigurationError: If required configuration is missing
        """
        # FIXED (Round 13): Initialize ALL state variables FIRST to guarantee safe state
        # This ensures the client is always in a valid state even if later initialization fails
        self._cached_metrics = {}
        self._metrics_lock = threading.Lock()
        self._metric_cardinality = {}
        # Note: Round 14 Issue #2 - removed deque, using attribute tagging instead

        # Safe defaults before config is loaded
        self._max_attr_length = 1024
        self._max_attr_count = 128
        self._max_cached_metrics = 1000
        self._max_cardinality = 2000
        self._debug_mode = False
        self._collector_endpoint = "http://localhost:4318"
        self._metric_export_interval_ms = 60000
        self._strict_validation = True

        # Handle deprecated auto_sanitize parameter
        if not auto_sanitize:
            import warnings

            warnings.warn(
                "DeprecationWarning: auto_sanitize=False is deprecated and will be removed "
                "in v1.0.0. Credential sanitization is now always enabled to prevent "
                "security regressions. This parameter no longer has any effect.",
                DeprecationWarning,
                stacklevel=2,
            )
        # Store for backward compatibility (but always sanitize regardless)
        self._auto_sanitize = True  # Always True now
        self._data_sanitizer_hook = data_sanitizer_hook

        # Now proceed with normal initialization
        self._initialize_logging()
        self._load_configuration(
            config,
            service_name,
            model_id,
            model_version,
            team_name,
            model_type,
            collector_endpoint,
            strict_validation,
            **kwargs,
        )

        # Update state with actual config values after config is loaded
        self._max_attr_length = getattr(self.config, "telemetry_max_attribute_length", 1024)
        self._max_attr_count = getattr(self.config, "telemetry_max_attribute_count", 128)
        self._max_cached_metrics = getattr(self.config, "telemetry_max_cached_metrics", 1000)
        self._max_cardinality = getattr(self.config, "telemetry_max_cardinality", 2000)
        self._debug_mode = self.config.debug_mode
        self._collector_endpoint = self.config.collector_endpoint
        self._metric_export_interval_ms = self.config.metric_export_interval_ms
        self._strict_validation = self.config.strict_validation

        self._initialize_metric_routing()
        self._initialize_registry()
        self._validate_configuration()
        self._initialize_buffering_and_retry()
        self._initialize_providers()
        self._initialize_standard_metrics()
        self._initialize_agentic_tracking()

    def _initialize_logging(self) -> None:
        """Setup SDK logging."""
        sdk_logger = logging.getLogger("mca_sdk")
        self._sdk_logger = sdk_logger

    def _load_configuration(
        self,
        config: Optional[MCAConfig],
        service_name: Optional[str],
        model_id: Optional[str],
        model_version: Optional[str],
        team_name: Optional[str],
        model_type: str,
        collector_endpoint: str,
        strict_validation: bool,
        **kwargs,
    ) -> None:
        """Load and validate configuration."""
        if config:
            self.config = config
        else:
            self.config = MCAConfig.load(
                service_name=service_name,
                model_id=model_id,
                model_version=model_version or __version__,
                team_name=team_name,
                model_type=model_type,
                collector_endpoint=collector_endpoint,
                strict_validation=strict_validation,
                **kwargs,
            )

        # Enable debug logging if debug_mode is set
        if self.config.debug_mode:
            self._sdk_logger.setLevel(logging.DEBUG)
            self._sdk_logger.debug("Debug mode enabled")
            self._sdk_logger.debug(f"MCA SDK version: {__version__}")
            self._sdk_logger.debug(
                f"Configuration loaded - service_name={self.config.service_name}, "
                f"model_type={self.config.model_type}, "
                f"collector_endpoint={self.config.collector_endpoint}"
            )

        # Run version check on startup
        if self.config.debug_mode:
            self._sdk_logger.debug("Running version check...")
        check_version(__version__, timeout=self.config.version_check_timeout)

    def _initialize_metric_routing(self) -> None:
        """Setup metric prefix routing."""
        from ..utils.routing import get_metric_prefix

        self._metric_prefix = get_metric_prefix(self.config.model_category, self.config.model_type)
        if self.config.debug_mode:
            self._sdk_logger.debug(
                f"Metric routing: model_category='{self.config.model_category}', "
                f"model_type='{self.config.model_type}' → prefix='{self._metric_prefix}'"
            )

    def _initialize_registry(self) -> None:
        """Initialize registry client if configured."""
        self._registry_client: Optional[RegistryClient] = None
        self._registry_config: Optional[ModelConfig] = None
        self._thresholds: Dict[str, float] = {}
        self._registry_telemetry: Optional[RegistryTelemetry] = None

        if self.config.registry_url:
            if self.config.debug_mode:
                self._sdk_logger.debug(f"Initializing registry client: {self.config.registry_url}")
            self._hydrate_from_registry()
            if self.config.debug_mode and self._registry_config:
                self._sdk_logger.debug(
                    f"Registry hydration successful - model_id={self._registry_config.model_id}, "
                    f"thresholds={len(self._thresholds)} configured"
                )

    def _validate_configuration(self) -> None:
        """Validate resource attributes."""
        if not self.config.strict_validation:
            return

        # Build resource attributes dict for validation
        resource_attrs = {"service.name": self.config.service_name}

        # Add prefix-specific attributes (determined by routing)
        if self._metric_prefix == "model":  # Traditional ML models
            resource_attrs.update(
                {
                    "model.id": self.config.model_id,
                    "model.version": self.config.model_version,
                    "team.name": self.config.team_name,
                }
            )
        elif self._metric_prefix == "genai":  # AI/LLM models
            resource_attrs.update(
                {
                    "llm.provider": self.config.llm_provider,
                    "llm.model": self.config.llm_model,
                    "team.name": self.config.team_name,
                }
            )
        elif self._metric_prefix == "vendor":  # Vendor models
            resource_attrs.update(
                {
                    "model.type": self.config.model_type,
                    "vendor.name": self.config.vendor_name,
                    "team.name": self.config.team_name,
                }
            )

        validate_resource_attributes(
            resource_attrs,
            model_category=self.config.model_category,
            model_type=self.config.model_type,
            strict=self.config.strict_validation,
        )
        if self.config.debug_mode:
            self._sdk_logger.debug(f"Resource attributes validated: {list(resource_attrs.keys())}")

    def _get_gcp_credentials(self):
        """Get GCP credentials for exporters (Story 6.5 integration)."""
        if not self.config.gcp_logging_enabled:
            return None

        try:
            from .gcp_auth import get_google_credentials

            scopes = ["https://www.googleapis.com/auth/logging.write"]
            return get_google_credentials(self.config, scopes=scopes)
        except Exception as e:
            self._sdk_logger.warning(f"Failed to load GCP credentials: {e}")
            return None

    def _initialize_buffering_and_retry(self) -> None:
        """Setup buffering, retry policy, circuit breaker."""
        self._retry_policy = None
        self._circuit_breaker_retry_policy = None

        if not self.config.buffering_enabled:
            return

        from ..buffering.retry import RetryPolicy

        self._retry_policy = RetryPolicy(
            max_attempts=self.config.retry_attempts,
            base_delay=self.config.base_delay,
            max_delay=self.config.max_delay,
            backoff_multiplier=self.config.backoff_multiplier,
        )

        # Wrap with circuit breaker if enabled (Story 3-3)
        if self.config.circuit_breaker_enabled:
            from ..resilience.circuit_breaker import CircuitBreakerRetryPolicy

            self._circuit_breaker_retry_policy = CircuitBreakerRetryPolicy(
                retry_policy=self._retry_policy,
                fail_max=self.config.circuit_breaker_fail_max,
                reset_timeout=self.config.circuit_breaker_timeout,
                name=f"{self.config.service_name}_circuit_breaker",
            )

            if self.config.debug_mode:
                self._sdk_logger.debug(
                    f"Circuit breaker enabled: "
                    f"fail_max={self.config.circuit_breaker_fail_max}, "
                    f"timeout={self.config.circuit_breaker_timeout}s"
                )
        else:
            self._circuit_breaker_retry_policy = self._retry_policy

        if self.config.debug_mode:
            self._sdk_logger.debug(
                f"Buffering enabled: "
                f"retry_attempts={self.config.retry_attempts}, "
                f"max_delay={self.config.max_delay}s"
            )

        # Queue recovery logging (Story 3-4)
        if self.config.persist_queue and self.config.debug_mode:
            self._sdk_logger.debug(
                f"Queue persistence enabled - queues will be recovered "
                f"during exporter initialization from: {self.config.persist_path}"
            )

    def _initialize_providers(self) -> None:
        """Setup OpenTelemetry providers."""
        extra_resource = None
        if self._registry_config:
            extra_resource = self._registry_config.extra_resource

        if self.config.debug_mode:
            self._sdk_logger.debug("Setting up OpenTelemetry providers...")

        (
            self._meter_provider,
            self._logger_provider,
            self._tracer_provider,
            self._resource,
            self._metric_exporter,
            self._log_exporter,
            self._span_exporter,
            self._prometheus_reader,
            self._prometheus_shutdown,
        ) = setup_all_providers(
            service_name=self.config.service_name,
            model_id=self.config.model_id,
            model_version=self.config.model_version,
            team_name=self.config.team_name,
            model_category=self.config.model_category,
            model_type=self.config.model_type,
            collector_endpoint=self.config.collector_endpoint,
            metric_export_interval_ms=self.config.metric_export_interval_ms,
            log_batch_size=self.config.log_batch_size,
            trace_batch_size=self.config.trace_batch_size,
            extra_resource=extra_resource,
            use_buffering=self.config.buffering_enabled,
            retry_policy=self._circuit_breaker_retry_policy,
            persist_queue=self.config.persist_queue,
            persist_path=self.config.persist_path,
            persist_min_disk_space_mb=self.config.persist_min_disk_space_mb,
            gcp_logging_enabled=self.config.gcp_logging_enabled,
            gcp_project_id=self.config.gcp_project_id,
            gcp_log_name=self.config.gcp_log_name,
            gcp_resource_type=self.config.gcp_resource_type,
            gcp_credentials=self._get_gcp_credentials(),
            gcp_log_timeout=self.config.gcp_log_timeout,
            prometheus_enabled=self.config.prometheus_enabled,
            prometheus_port=self.config.prometheus_port,
            prometheus_host=self.config.prometheus_host,
        )

        if self.config.debug_mode:
            self._sdk_logger.debug(
                f"OpenTelemetry providers initialized - "
                f"endpoint={self.config.collector_endpoint}, "
                f"metric_interval={self.config.metric_export_interval_ms}ms, "
                f"log_batch_size={self.config.log_batch_size}, "
                f"trace_batch_size={self.config.trace_batch_size}"
            )

        # Setup GCP Cloud Trace exporter if enabled
        self._gcp_trace_exporter = None

        if self.config.gcp_trace_enabled:
            from .providers import setup_gcp_trace_exporter
            from opentelemetry.sdk.trace.export import BatchSpanProcessor

            gcp_exporter = setup_gcp_trace_exporter(
                config=self.config, resource=self._resource, batch_size=self.config.trace_batch_size
            )

            if gcp_exporter:
                gcp_processor = BatchSpanProcessor(
                    gcp_exporter, max_export_batch_size=self.config.trace_batch_size
                )
                self._tracer_provider.add_span_processor(gcp_processor)
                self._gcp_trace_exporter = gcp_exporter

                if self.config.debug_mode:
                    self._sdk_logger.debug("GCP Cloud Trace exporter added to tracer provider")

        # Get instances for user access
        self._meter = self._meter_provider.get_meter(f"{self.config.service_name}.meter")
        self._tracer = self._tracer_provider.get_tracer(f"{self.config.service_name}.tracer")

        # Initialize circuit breaker metrics if enabled (Story 3-3)
        # Use Protocol for type-safe meter injection
        from ..buffering.exporters import MeterSettable

        if self.config.circuit_breaker_enabled and self._circuit_breaker_retry_policy:
            if isinstance(self._circuit_breaker_retry_policy, MeterSettable):
                self._circuit_breaker_retry_policy.set_meter(self._meter)

        # Initialize persistence metrics for buffered exporters (Story 3-4)
        for exporter in [self._metric_exporter, self._log_exporter, self._span_exporter]:
            if exporter and isinstance(exporter, MeterSettable):
                exporter.set_meter(self._meter)

        # Setup Python logging integration
        self._setup_logging()

    def _initialize_standard_metrics(self) -> None:
        """Create standard prediction metrics."""
        self._predictions_counter = self._meter.create_counter(
            f"{self._metric_prefix}.predictions_total",
            description="Total number of model predictions",
        )
        self._latency_histogram = self._meter.create_histogram(
            f"{self._metric_prefix}.latency_seconds",
            description="Model prediction latency",
            unit="s",
        )

        # Add dedicated errors counter for dual recording (Issue #10)
        # FIXED (Issue #7): Remove unit="1" for consistency with _predictions_counter
        self._errors_counter = self._meter.create_counter(
            f"emms_{self._metric_prefix}_errors_total",
            description="Total number of prediction errors",
        )

        # FIXED (Round 13): State initialization moved to __init__() to guarantee safe state
        # All _max_* variables, locks, and dicts are now initialized before any complex setup

        if self._debug_mode:
            self._sdk_logger.debug(
                f"Standard metrics created: {self._metric_prefix}.predictions_total (counter), "
                f"{self._metric_prefix}.latency_seconds (histogram)"
            )

        # Initialize metric naming validation
        self._strict_validation = self.config.strict_validation
        self._metric_validator = MetricNamingConvention(
            model_category=self.config.model_category,
            model_type=self.config.model_type,
            strict=self._strict_validation,
        )

        # Initialize registry telemetry
        if self._registry_client:
            self._registry_telemetry = RegistryTelemetry(self._meter)

    def _initialize_agentic_tracking(self) -> None:
        """Setup agentic AI metrics and cleanup thread (eager initialization)."""
        self._active_goals: Dict[str, tuple] = {}
        self._goals_lock = threading.Lock()
        self._cleanup_thread_stop = threading.Event()
        self._cleanup_thread: Optional[threading.Thread] = None

        # Initialize agentic metrics to None first
        self._goals_started_counter = None
        self._goals_completed_counter = None
        self._goals_failed_counter = None
        self._goal_duration_histogram = None
        self._tool_calls_counter = None
        self._tool_latency_histogram = None
        self._tool_errors_counter = None
        self._human_interventions_counter = None
        self._intervention_wait_histogram = None
        self._goal_cleanup_errors_counter = None
        self._reasoning_steps_counter = None

        # Create metrics eagerly
        self._initialize_agentic_metrics()

        # Start background cleanup thread
        self._start_cleanup_thread()

    def _setup_logging(self):
        """Setup Python logging integration with OpenTelemetry."""
        self._logger = logging.getLogger(f"{self.config.service_name}.logger")
        self._logger.setLevel(logging.INFO)

        # Add OTel handler and store reference for cleanup
        self._otel_handler = LoggingHandler(logger_provider=self._logger_provider)
        self._logger.addHandler(self._otel_handler)

    def _initialize_agentic_metrics(self) -> None:
        """Initialize agentic AI metrics (eager initialization).

        Creates all agent.* metrics at client initialization. This ensures
        consistent metric instances are available when agentic features are used.
        """
        if self._goals_started_counter is not None:
            return  # Already initialized

        self._goals_started_counter = self._meter.create_counter(
            "agent.goals_started_total", description="Total number of agent goals started", unit="1"
        )
        self._goals_completed_counter = self._meter.create_counter(
            "agent.goals_completed_total",
            description="Total number of agent goals completed successfully",
            unit="1",
        )
        self._goals_failed_counter = self._meter.create_counter(
            "agent.goals_failed_total",
            description="Total number of agent goals that failed",
            unit="1",
        )
        self._goal_duration_histogram = self._meter.create_histogram(
            "agent.goal_duration_seconds", description="Duration of agent goal execution", unit="s"
        )
        self._tool_calls_counter = self._meter.create_counter(
            "agent.tool_calls_total",
            description="Total number of tool calls made by agent",
            unit="1",
        )
        self._tool_latency_histogram = self._meter.create_histogram(
            "agent.tool_latency_seconds", description="Latency of tool executions", unit="s"
        )
        self._tool_errors_counter = self._meter.create_counter(
            "agent.tool_errors_total", description="Total number of tool execution errors", unit="1"
        )
        self._human_interventions_counter = self._meter.create_counter(
            "agent.human_interventions_total",
            description="Total number of human interventions requested",
            unit="1",
        )
        self._intervention_wait_histogram = self._meter.create_histogram(
            "agent.intervention_wait_time_seconds",
            description="Time spent waiting for human intervention",
            unit="s",
        )
        self._goal_cleanup_errors_counter = self._meter.create_counter(
            "agent.goal_cleanup_errors_total",
            description="Total number of errors during stale goal cleanup",
            unit="1",
        )
        self._reasoning_steps_counter = self._meter.create_counter(
            "agent.reasoning_steps_total",
            description="Total number of reasoning steps executed by agent",
            unit="1",
        )

    def _start_cleanup_thread(self) -> None:
        """Start background thread for periodic stale goal cleanup.

        The cleanup thread wakes every 300 seconds (5 minutes) to check for
        and remove stale goals, ensuring memory doesn't leak even during
        idle periods.
        """
        self._cleanup_thread = threading.Thread(
            target=self._background_cleanup_loop, daemon=True, name="mca-sdk-cleanup"
        )
        self._cleanup_thread.start()

    def _background_cleanup_loop(self) -> None:
        """Background thread loop that periodically cleans up stale goals.

        Runs independently of agent activity to ensure cleanup happens
        even during idle periods. Wakes every cleanup_thread_poll_interval
        seconds (from config) or when stop event is set.
        """
        while not self._cleanup_thread_stop.wait(timeout=self.config.cleanup_thread_poll_interval):
            try:
                self._cleanup_stale_goals()
            except Exception as e:
                # Log and record metric, but don't crash the cleanup thread
                sdk_logger = logging.getLogger("mca_sdk")
                sdk_logger.error(f"Error in background cleanup thread: {e}")
                # Record cleanup failure for monitoring/alerting
                self._goal_cleanup_errors_counter.add(
                    1, attributes={"error_type": type(e).__name__}
                )

    def _stop_cleanup_thread(self, timeout: float = 5.0) -> None:
        """Stop the background cleanup thread gracefully.

        Args:
            timeout: Maximum time to wait for thread to stop (seconds)
        """
        if self._cleanup_thread and self._cleanup_thread.is_alive():
            self._cleanup_thread_stop.set()
            self._cleanup_thread.join(timeout=timeout)
            if self._cleanup_thread.is_alive():
                sdk_logger = logging.getLogger("mca_sdk")
                sdk_logger.warning("Cleanup thread did not stop within timeout")

    def _ensure_cleanup_thread_alive(self) -> None:
        """Restart cleanup thread if dead (Finding 9 fix).

        Called periodically to detect and recover from thread failures.
        """
        if not self._cleanup_thread or not self._cleanup_thread.is_alive():
            sdk_logger = logging.getLogger("mca_sdk")
            sdk_logger.warning("Cleanup thread died, restarting...")
            self._start_cleanup_thread()

    def _cleanup_stale_goals(self, max_age_seconds: float = 3600) -> None:
        """Clean up stale goals that were never completed to prevent memory leaks.

        This method is called periodically by the background cleanup thread to remove
        goals older than max_age_seconds from the _active_goals dictionary.

        Args:
            max_age_seconds: Maximum age of a goal in seconds before cleanup (default: 1 hour)
        """
        current_time = time.perf_counter()
        stale_goal_ids = []

        # Find stale goals (acquire lock for thread safety)
        with self._goals_lock:
            for goal_id, (span, start_time, context_token) in self._active_goals.items():
                age = current_time - start_time
                if age > max_age_seconds:
                    stale_goal_ids.append(goal_id)

        # Clean up stale goals (acquire lock again to remove)
        if not stale_goal_ids:
            return

        from opentelemetry import context as otel_context
        from opentelemetry.trace import Status, StatusCode

        with self._goals_lock:
            for goal_id in stale_goal_ids:
                if goal_id not in self._active_goals:
                    continue  # Already completed, skip

                span, start_time, context_token = self._active_goals.pop(goal_id)

                try:
                    # End the stale span with a warning
                    duration = current_time - start_time
                    span.set_attribute("goal_status", "timeout")
                    span.set_attribute("goal_duration_seconds", duration)
                    span.set_attribute("goal_cleanup_reason", "stale_goal_timeout")

                    span.set_status(
                        Status(StatusCode.ERROR, f"Goal timed out after {duration:.1f}s")
                    )
                    span.end()

                    # Record as failed goal
                    # Timeout is considered a failure (AC: 2)
                    self._goals_failed_counter.add(1, attributes={"status": "timeout"})
                    self._goal_duration_histogram.record(duration, attributes={"status": "timeout"})

                    if self.config.debug_mode:
                        sdk_logger = logging.getLogger("mca_sdk")
                        sdk_logger.warning(
                            f"[{goal_id}] Stale goal cleaned up after {duration:.1f}s "
                            f"(max_age={max_age_seconds}s)"
                        )
                finally:
                    # Always detach context even if span operations fail
                    otel_context.detach(context_token)

    @property
    def meter(self):
        """Get the OpenTelemetry Meter for creating custom metrics.

        Returns:
            Meter instance for creating counters, histograms, gauges
        """
        return self._meter

    @property
    def logger(self):
        """Get the Python logger with OpenTelemetry integration.

        Returns:
            Logger instance for structured logging
        """
        return self._logger

    @property
    def tracer(self):
        """Get the OpenTelemetry Tracer for creating custom spans.

        Returns:
            Tracer instance for distributed tracing
        """
        return self._tracer

    @property
    def thresholds(self) -> Dict[str, float]:
        """Get current thresholds from registry.

        Returns:
            Dictionary of threshold values from registry (refreshed by background thread)
        """
        # Get latest thresholds from registry client's cache (updated by refresh thread)
        if self._registry_client and self._registry_config:
            try:
                # fetch_model_config returns from cache if fresh (updated by refresh thread)
                current_config = self._registry_client.fetch_model_config(
                    model_id=self._registry_config.model_id,
                    version=self._registry_config.model_version,
                )
                return current_config.thresholds.copy()
            except Exception:
                # Fallback to initial config if fetch fails
                return self._registry_config.thresholds.copy()

        # No registry configured
        return {}

    @property
    def prometheus_endpoint(self) -> Optional[str]:
        """Get Prometheus metrics endpoint URL if enabled.

        Returns:
            URL string if Prometheus exporter is running, None otherwise
        """
        if self._prometheus_reader and self.config.prometheus_enabled:
            return f"http://{self.config.prometheus_host}:{self.config.prometheus_port}/metrics"
        return None

    def buffer_stats(self) -> Dict[str, any]:
        """Get telemetry buffer statistics aggregated from all exporters.

        Returns:
            Dictionary with aggregated buffer statistics from all three exporters
            (metrics, logs, spans), including total queue sizes and export metrics.
            Returns None if buffering is not enabled.
        """
        if not self.config.buffering_enabled:
            return None

        # Aggregate stats from all three buffered exporters
        stats = {
            "enabled": True,
            "per_signal_stats": {},
            "total_queued_items": 0,
            "total_export_attempts": 0,
            "total_export_successes": 0,
            "total_export_failures": 0,
        }

        # Get stats from metrics exporter
        if self._metric_exporter:
            metric_stats = self._metric_exporter.get_stats()
            stats["per_signal_stats"]["metrics"] = metric_stats
            stats["total_queued_items"] += metric_stats.get("items_queued", 0)
            stats["total_export_attempts"] += metric_stats.get("export_attempts", 0)
            stats["total_export_successes"] += metric_stats.get("export_successes", 0)
            stats["total_export_failures"] += metric_stats.get("export_failures", 0)

        # Get stats from log exporter
        if self._log_exporter:
            log_stats = self._log_exporter.get_stats()
            stats["per_signal_stats"]["logs"] = log_stats
            stats["total_queued_items"] += log_stats.get("items_queued", 0)
            stats["total_export_attempts"] += log_stats.get("export_attempts", 0)
            stats["total_export_successes"] += log_stats.get("export_successes", 0)
            stats["total_export_failures"] += log_stats.get("export_failures", 0)

        # Get stats from span exporter
        if self._span_exporter:
            span_stats = self._span_exporter.get_stats()
            stats["per_signal_stats"]["spans"] = span_stats
            stats["total_queued_items"] += span_stats.get("items_queued", 0)
            stats["total_export_attempts"] += span_stats.get("export_attempts", 0)
            stats["total_export_successes"] += span_stats.get("export_successes", 0)
            stats["total_export_failures"] += span_stats.get("export_failures", 0)

        # Get circuit breaker stats (Story 3-3)
        if self._circuit_breaker_retry_policy and hasattr(
            self._circuit_breaker_retry_policy, "get_stats"
        ):
            stats["circuit_breaker"] = self._circuit_breaker_retry_policy.get_stats()
        else:
            stats["circuit_breaker"] = None

        # Get GCP Cloud Trace exporter stats if enabled (Story 6-3 fix)
        if self._gcp_trace_exporter and hasattr(self._gcp_trace_exporter, "get_metrics"):
            gcp_stats = self._gcp_trace_exporter.get_metrics()
            stats["per_signal_stats"]["gcp_trace"] = gcp_stats
            # Add GCP-specific metrics to totals
            stats["total_export_attempts"] += gcp_stats.get("export_attempts_total", 0)
            # GCP exporter doesn't have explicit success counter, infer from spans_exported
            stats["total_export_failures"] += gcp_stats.get("export_failures_total", 0)

        # Add cleanup thread health monitoring (Finding 9)
        stats["cleanup_thread_alive"] = (
            self._cleanup_thread.is_alive() if self._cleanup_thread else False
        )

        return stats

    def _hydrate_from_registry(self) -> None:
        """Fetch config from registry and merge with local config."""
        try:
            self._registry_client = RegistryClient(
                url=self.config.registry_url,
                token=self.config.registry_token,
                timeout=5.0,
                cache_ttl_secs=600,  # 10 minutes cache TTL
                refresh_interval_secs=self.config.refresh_interval_secs,
            )

            # Fetch model or deployment config
            if self.config.deployment_id:
                # Deployment indirection: fetch deployment config first
                deployment_config = self._registry_client.fetch_deployment_config(
                    self.config.deployment_id
                )
                # Apply deployment resource overrides
                if deployment_config.resource_overrides:
                    logging.info(f"Applied deployment config for {self.config.deployment_id}")

            # Fetch model config
            registry_config = self._registry_client.fetch_model_config(
                model_id=self.config.model_id or self.config.service_name,
                version=self.config.model_version,
            )
            self._registry_config = registry_config

            # Apply registry config based on prefer_registry setting
            if self.config.prefer_registry:
                self._apply_registry_config(registry_config)

            # Store thresholds
            self._thresholds = registry_config.thresholds

            logging.info(
                f"Successfully hydrated config from registry for model {registry_config.model_id}"
            )

        except RegistryAuthError as e:
            # Auth errors should fail fast
            if self.config.debug_mode:
                self._sdk_logger.debug(f"Registry authentication failed with auth error: {e}")
            logging.error(f"Registry authentication failed: {e}")
            raise
        except RegistryConnectionError as e:
            # Log warning but continue with local config
            if self.config.debug_mode:
                self._sdk_logger.debug(
                    f"Registry connection failed, falling back to local config: {e}"
                )
            logging.warning(
                f"Registry unavailable, using local config: {e}. "
                "Telemetry will continue with local configuration."
            )
        except Exception as e:
            # Unexpected errors - log and continue
            if self.config.debug_mode:
                self._sdk_logger.debug(f"Unexpected registry error: {type(e).__name__}: {e}")
            logging.warning(f"Unexpected error during registry hydration: {e}")

    def _apply_registry_config(self, registry_config: ModelConfig) -> None:
        """Apply registry config to local config (mutable fields only).

        Args:
            registry_config: Configuration from registry
        """
        # Only override if local value is default/None and registry has a value
        if not self.config.team_name and registry_config.team_name:
            self.config.team_name = registry_config.team_name

        # Check for identity field changes (warn only)
        if registry_config.service_name != self.config.service_name:
            logging.warning(
                f"service_name in registry ({registry_config.service_name}) differs "
                f"from local ({self.config.service_name}). Using local value. "
                "Restart required to apply registry service_name."
            )

    def record_prediction(
        self,
        input_data: Any = None,
        output: Any = None,
        latency: Optional[float] = None,
        prediction_id: Optional[str] = None,
        **attributes,
    ):
        """Record a model prediction with standard metrics.

        This is a convenience method that automatically creates:
        - model.predictions_total counter (incremented)
        - model.latency_seconds histogram (if latency provided)
        - Structured log with prediction details

        Args:
            input_data: Input data for the prediction
            output: Output from the model
            latency: Prediction latency in seconds
            prediction_id: Optional prediction ID for trace correlation (auto-generated if None)
            **attributes: Additional attributes to attach to metrics/logs
        """
        # FIXED (Round 11 Issue #1): Removed dead imports (sys, ValidationError)
        # These imports are unused and add unnecessary overhead

        # FIXED (Round 11 Issue #3): Accept external prediction_id for trace correlation
        if prediction_id is None:
            prediction_id = str(uuid.uuid4())

        # Apply application-specific PHI sanitization hook FIRST
        # This allows applications to inject domain-specific PHI masking
        if self._data_sanitizer_hook and callable(self._data_sanitizer_hook):
            if input_data is not None and isinstance(input_data, dict):
                try:
                    input_data = self._data_sanitizer_hook(input_data)
                except Exception as e:
                    self._sdk_logger.warning(
                        f"[{prediction_id}] data_sanitizer_hook failed on input_data: {e}"
                    )

            if output is not None and isinstance(output, dict):
                try:
                    output = self._data_sanitizer_hook(output)
                except Exception as e:
                    self._sdk_logger.warning(
                        f"[{prediction_id}] data_sanitizer_hook failed on output: {e}"
                    )

        # CRITICAL SECURITY FIX: Sanitize credentials in input_data and output
        if input_data is not None:
            try:
                input_data = _sanitize_data_recursive(input_data)
            except Exception as e:
                self._sdk_logger.warning(f"[{prediction_id}] Failed to sanitize input_data: {e}")

        if output is not None:
            try:
                output = _sanitize_data_recursive(output)
            except Exception as e:
                self._sdk_logger.warning(f"[{prediction_id}] Failed to sanitize output: {e}")

        # FIXED (Round 7/8): Use built-in validator with proper strict mode handling
        # CRITICAL: Assign return value - validator sanitizes in permissive mode
        if attributes:
            attributes = TelemetryAttributeValidator.validate(
                attributes,
                strict=self._strict_validation,
                max_value_length=self._max_attr_length,
                max_attribute_count=self._max_attr_count,
            )

        if self._debug_mode:
            self._sdk_logger.debug(
                f"[{prediction_id}] Recording prediction - latency={latency}, attributes={attributes}"
            )

            # Log attribute validation
            attr_count = len(attributes)
            attr_keys = list(attributes.keys())
            self._sdk_logger.debug(
                f"[{prediction_id}] Attribute validation: {attr_count} attributes provided: {attr_keys}"
            )

            # FIXED (Round 11 Issue #9): Removed dead validation loop
            # TelemetryAttributeValidator already truncated strings, this check was always false

        # FIXED (Round 14 Issue #1): Inject prediction_id for trace correlation
        # Without this, metrics are stripped of prediction_id, breaking correlation
        attributes["prediction_id"] = prediction_id

        # Increment predictions counter
        self._predictions_counter.add(1, attributes=attributes)

        if self._debug_mode:
            self._sdk_logger.debug(
                f"[{prediction_id}] Metric recorded: model.predictions_total += 1"
            )
            self._sdk_logger.debug(
                f"[{prediction_id}] Export attempt: Metrics will be exported to {self._collector_endpoint}"
            )

        # Record latency if provided
        if latency is not None:
            self._latency_histogram.record(latency, attributes=attributes)
            if self._debug_mode:
                self._sdk_logger.debug(
                    f"[{prediction_id}] Metric recorded: model.latency_seconds = {latency}s"
                )

        # Log prediction completion for production auditability
        # Always include prediction_id for trace correlation
        # Build log extra fields with input_data and output if provided (debug mode only)
        # FIXED (Round 11 Issue #8): Strip system keys from user attributes to prevent spoofing
        safe_attributes = {
            k: v
            for k, v in attributes.items()
            if k not in ("prediction_id", "latency", "input_data", "output")
        }
        log_extra = {**safe_attributes, "prediction_id": prediction_id, "latency": latency}
        if self._debug_mode:
            # FIXED (Round 14 Issue #7): Serialize to prevent JSON crashes in logging exporter
            # Raw objects (DataFrames, custom classes, NumPy arrays) crash OTel logging
            if input_data is not None:
                log_extra["input_data"] = serialize_for_telemetry(input_data)
            if output is not None:
                log_extra["output"] = serialize_for_telemetry(output)

        # Log prediction metadata (applications should sanitize sensitive data before logging)
        # Always log at INFO level for production auditability
        self._logger.info(
            "Prediction completed",
            extra=log_extra,
        )

        if self._debug_mode:
            # Log export information
            metric_count = 2 if latency is not None else 1
            self._sdk_logger.debug(
                f"[{prediction_id}] Export queued: {metric_count} metrics + 1 log to {self._collector_endpoint}/v1/metrics, /v1/logs"
            )
            self._sdk_logger.debug(
                f"[{prediction_id}] Prediction recording completed - metrics will be exported via batch processor (interval: {self._metric_export_interval_ms}ms)"
            )

    def record_error(
        self,
        error: Exception,
        latency: Optional[float] = None,
        prediction_id: Optional[str] = None,
        **attributes,
    ) -> str:
        """Record a prediction error with dual metrics.

        Records to both:
        - predictions_total{status="error"} for consistency
        - errors_total for dedicated error tracking

        Args:
            error: The exception that occurred
            latency: Optional prediction latency in seconds
            prediction_id: Optional prediction ID for trace correlation (auto-generated if None)
            **attributes: Additional attributes (e.g., function)

        Returns:
            Sanitized error message (for reuse in span attributes, avoiding redundant sanitization)

        Raises:
            ValidationError: If attributes fail validation in strict mode

        Example:
            >>> try:
            ...     result = model.predict(data)
            ... except Exception as e:
            ...     client.record_error(e, latency=0.5, function="predict")
            ...     raise
        """
        # FIXED (Round 14 Issue #2): Stop using id() - memory addresses get reused after GC
        # Use attribute tagging instead, with safe handling for read-only exceptions
        if getattr(error, "_mca_recorded", False):
            return _sanitize_error_message(str(error))  # Already recorded

        try:
            # Mark error as recorded to prevent duplicates
            setattr(error, "_mca_recorded", True)
        except Exception:
            # Read-only exception (e.g., built-in exceptions) - accept risk of duplicate
            # This is safer than memory leak from id() reuse
            pass

        # FIXED (Issue #9): Truncate exception class name to prevent unbounded labels
        error_type = type(error).__name__[:64]

        # FIXED (Issue #8): Sanitize once and return for reuse
        error_message = _sanitize_error_message(str(error))

        # FIXED (Round 11 Issue #3): Accept external prediction_id for correlation
        if prediction_id is None:
            prediction_id = attributes.get("prediction_id")
        if not prediction_id:
            prediction_id = str(uuid.uuid4())

        # FIXED (Round 14 Issue #9): Don't let safe_limit fall to 0
        # If _max_attr_count is tiny (e.g., 1), setting safe_limit to 0 drops ALL user attrs
        # Just use the global limit - system attributes will naturally overwrite if needed
        safe_limit = self._max_attr_count

        # Validate and sanitize user attributes
        safe_user_attributes = {}
        if attributes:
            # CRITICAL: Assign return value - validator sanitizes in permissive mode
            safe_user_attributes = TelemetryAttributeValidator.validate(
                attributes,
                strict=self._strict_validation,  # Let validator handle strict vs permissive
                max_value_length=self._max_attr_length,
                max_attribute_count=safe_limit,
            )

        # FIXED (Re-Review Issue #1): System attributes LAST to prevent user spoofing
        # User cannot override status="error" or error_type
        # FIXED (Round 14 Issue #1): Inject prediction_id for trace correlation
        error_attributes = {
            **safe_user_attributes,  # User attributes first
            "status": "error",  # System attribute - cannot be overridden
            "error_type": error_type,  # System attribute - cannot be overridden
            "prediction_id": prediction_id,  # For trace-to-metric correlation
        }

        # Dual recording: predictions_total AND errors_total
        self._predictions_counter.add(1, attributes=error_attributes)
        self._errors_counter.add(1, attributes=error_attributes)

        # Record latency if provided
        if latency is not None:
            self._latency_histogram.record(latency, attributes=error_attributes)

        # FIXED (Round 7 Issue #1): Use validated safe_user_attributes, not raw attributes
        # This prevents logging bypass that crashes the logging pipeline
        self._logger.error(
            f"Prediction error: {error_type}",
            extra={
                **safe_user_attributes,  # MUST be validated attributes!
                # System keys last to overwrite any conflicts
                "prediction_id": prediction_id,
                "error_type": error_type,
                "error_message": error_message,
                "latency": latency,
            },
        )

        # Return sanitized message for reuse (Issue #8)
        return error_message

    def _handle_prediction_success(
        self,
        span: trace.Span,
        input_data: Optional[Dict[str, Any]],
        output_data: Any,
        latency: float,
        prediction_id: str,
        metric_attributes: Dict[str, Any],
    ) -> None:
        """Handle successful prediction - record metrics and set span attributes.

        Args:
            span: Active trace span
            input_data: Serialized input data
            output_data: Serialized output data
            latency: Prediction latency in seconds
            prediction_id: Unique prediction identifier
            metric_attributes: Additional metric attributes
        """
        self.record_prediction(
            input_data=input_data,
            output=output_data,
            latency=latency,
            prediction_id=prediction_id,
            **metric_attributes,
        )
        span.set_attribute("prediction.id", prediction_id)
        span.set_attribute("prediction.latency_ms", latency * 1000)

    def _handle_prediction_error(
        self,
        span: trace.Span,
        error: Exception,
        latency: float,
        prediction_id: str,
        metric_attributes: Dict[str, Any],
    ) -> None:
        """Handle prediction error - record error metrics, set span attributes.

        Args:
            span: Active trace span
            error: Exception that occurred
            latency: Prediction latency in seconds
            prediction_id: Unique prediction identifier
            metric_attributes: Additional metric attributes
        """
        # FIXED (Issue #8): record_error() now returns sanitized message to avoid double sanitization
        error_message = self.record_error(
            error, latency=latency, prediction_id=prediction_id, **metric_attributes
        )

        # Set span error attributes (reuse sanitized message from record_error)
        error_type = type(error).__name__[:64]  # Consistent with record_error truncation
        span.set_attribute("error", True)
        span.set_attribute("error.type", error_type)
        span.set_attribute("error.message", error_message)
        span.set_attribute("prediction.id", prediction_id)
        span.set_attribute("prediction.latency_ms", latency * 1000)

        # Logging now handled by record_error(), don't duplicate

    def _setup_prediction_context(self) -> tuple:
        """Setup prediction context with ID and token.

        Returns:
            Tuple of (prediction_id, context_token)
        """
        prediction_id = str(uuid.uuid4())
        token = _prediction_context.set(prediction_id)
        return prediction_id, token

    def _start_prediction_timer(self) -> float:
        """Start prediction timing.

        Returns:
            Start time for latency calculation
        """
        return time.perf_counter()

    def _capture_prediction_data(
        self, capture_input: bool, capture_output: bool, args: tuple, kwargs: dict, result: Any
    ) -> tuple:
        """Capture and serialize input/output data for telemetry.

        Args:
            capture_input: Whether to capture inputs
            capture_output: Whether to capture outputs
            args: Function positional arguments
            kwargs: Function keyword arguments
            result: Function return value

        Returns:
            Tuple of (input_data, output_data)
        """
        input_data = None
        output_data = None

        if capture_input and (args or kwargs):
            input_data = serialize_for_telemetry({"args": args, "kwargs": kwargs})

        if capture_output:
            output_data = serialize_for_telemetry(result)

        return input_data, output_data

    def predict(
        self,
        capture_input: bool = False,
        capture_output: bool = False,
        span_name: Optional[str] = None,
        **metric_attributes,
    ) -> Callable:
        """Decorator for auto-instrumentation of prediction functions.

        Automatically tracks latency, generates prediction_id, and records metrics.
        Supports both sync and async functions with thread-safe operation.

        Args:
            capture_input: Whether to capture function inputs (default: False)
            capture_output: Whether to capture function output (default: False)
            span_name: Custom span name (default: function.__name__)
            **metric_attributes: Additional metric attributes to include

        Returns:
            Decorator function that wraps the prediction function

        Examples:
            Basic usage (no data capture):
            >>> @client.predict()
            ... def predict(data):
            ...     return model.predict(data)

            Explicit data capture (requires PHI sanitization):
            >>> @client.predict(capture_input=True, capture_output=True)
            ... async def predict_async(data):
            ...     # WARNING: Ensure data is sanitized before calling this function
            ...     return await model.predict_async(data)

            Custom span name and attributes:
            >>> @client.predict(span_name="readmission_v2", model_version="2.0")
            ... def predict(data):
            ...     return model.predict(data)

        Warning:
            HIPAA COMPLIANCE: Data capture is DISABLED by default to prevent
            accidental PHI exposure. If you enable capture_input=True or
            capture_output=True, you MUST ensure ALL data passed to the
            decorated function has been sanitized to remove PHI (patient names,
            SSNs, addresses, etc.). The SDK provides NO automatic PHI masking
            for captured data. Failure to sanitize may result in HIPAA violations.
        """

        def decorator(func: Callable) -> Callable:
            name = span_name or func.__name__

            # Detect if function is async or sync
            if inspect.iscoroutinefunction(func):
                # Async wrapper
                @functools.wraps(func)
                async def async_wrapper(*args, **kwargs):
                    prediction_id, token = self._setup_prediction_context()
                    start_time = self._start_prediction_timer()

                    with self.trace(name) as span:
                        try:
                            result = await func(*args, **kwargs)
                            latency = time.perf_counter() - start_time
                            input_data, output_data = self._capture_prediction_data(
                                capture_input, capture_output, args, kwargs, result
                            )
                            self._handle_prediction_success(
                                span,
                                input_data,
                                output_data,
                                latency,
                                prediction_id,
                                metric_attributes,
                            )
                            return result
                        except Exception as e:
                            latency = time.perf_counter() - start_time
                            self._handle_prediction_error(
                                span, e, latency, prediction_id, metric_attributes
                            )
                            raise
                        finally:
                            _prediction_context.reset(token)

                return async_wrapper
            else:
                # Sync wrapper
                @functools.wraps(func)
                def sync_wrapper(*args, **kwargs):
                    prediction_id, token = self._setup_prediction_context()
                    start_time = self._start_prediction_timer()

                    with self.trace(name) as span:
                        try:
                            result = func(*args, **kwargs)
                            latency = time.perf_counter() - start_time
                            input_data, output_data = self._capture_prediction_data(
                                capture_input, capture_output, args, kwargs, result
                            )
                            self._handle_prediction_success(
                                span,
                                input_data,
                                output_data,
                                latency,
                                prediction_id,
                                metric_attributes,
                            )
                            return result
                        except Exception as e:
                            latency = time.perf_counter() - start_time
                            self._handle_prediction_error(
                                span, e, latency, prediction_id, metric_attributes
                            )
                            raise
                        finally:
                            _prediction_context.reset(token)

                return sync_wrapper

        return decorator

    def record_metric(self, name: str, value: float, **attributes):
        """Record a custom metric value.

        This is a generic method for recording any metric. For common patterns
        like predictions and latency, use record_prediction() instead.

        IMPORTANT: This method is for one-time metric recording. For counters,
        histograms, or gauges that are updated frequently, use client.meter
        to create the metric once and reuse it.

        Args:
            name: Metric name (will be validated if strict_validation=True)
            value: Metric value
            **attributes: Attributes to attach to the metric

        Example:
            >>> # For one-time recording:
            >>> client.record_metric("model.accuracy_ratio", 0.92, dataset="test")
            >>>
            >>> # For frequent updates, create metric once:
            >>> accuracy_gauge = client.meter.create_observable_gauge("model.accuracy")
        """
        # Validate metric name and structure (AC #5 - Protobuf schema validation)
        metric_data = {"name": name, "value": value, "attributes": attributes}

        try:
            # Validate both name and structure
            self._metric_validator.validate(name, metric_data=metric_data)
        except (ValueError, Exception) as e:
            if self._strict_validation:
                # Strict mode: raise immediately to fail fast
                raise
            else:
                # Non-strict mode: log warning and continue
                self._sdk_logger.warning(
                    f"Metric validation failed for '{name}': {e}. "
                    "Recording anyway (non-strict mode)."
                )
                # Continue to record metric

        # FIXED (Round 14 Issue #10): Protect attribute validation with try/except
        # Validation can raise in strict mode, must be caught like name validation
        if attributes:
            try:
                # CRITICAL: Assign return value - validator sanitizes in permissive mode
                attributes = TelemetryAttributeValidator.validate(
                    attributes,
                    strict=self._strict_validation,
                    max_value_length=self._max_attr_length,
                    max_attribute_count=self._max_attr_count,
                )
            except Exception as e:
                if self._strict_validation:
                    raise  # Re-raise in strict mode
                else:
                    # Permissive mode: drop invalid attributes and continue
                    self._sdk_logger.warning(
                        f"Attribute validation failed for metric '{name}': {e}. "
                        "Dropping invalid attributes (non-strict mode)."
                    )
                    attributes = {}

        # FIXED (Round 14 Issue #6): Strip memory addresses from cardinality signature
        # Objects with __repr__ containing memory addresses cause unique signatures
        if attributes:
            # Normalize values to prevent memory address leaks
            clean_attrs = {
                k: str(v) if not isinstance(v, (int, float, bool, str)) else v
                for k, v in attributes.items()
            }
            attr_sig = str(sorted(clean_attrs.items()))
        else:
            attr_sig = ""

        # FIXED (Round 11 Issue #10): Use cached cardinality limit
        MAX_CARDINALITY = self._max_cardinality
        with self._metrics_lock:
            if name not in self._metric_cardinality:
                self._metric_cardinality[name] = set()

            if attr_sig not in self._metric_cardinality[name]:
                if len(self._metric_cardinality[name]) >= MAX_CARDINALITY:
                    self._sdk_logger.warning(
                        f"Cardinality limit ({MAX_CARDINALITY}) reached for metric '{name}'. "
                        "Using overflow label to prevent explosion."
                    )
                    # FIXED (Round 9 Issue #2): Preserve context with overflow label
                    attributes = {"mca.cardinality_overflow": True}
                    # FIXED (Round 12 Issue #5): DO NOT add overflow signature to set
                    # Just use overflow attributes and stop growing the set
                else:
                    # Only add to set if under the limit
                    self._metric_cardinality[name].add(attr_sig)

        # FIXED (Round 14 Issue #3): Ensure lock covers both check and creation
        # Multiple threads creating instruments causes leaks. Use single lock.
        with self._metrics_lock:
            instrument = self._cached_metrics.get(name)
            if not instrument:
                instrument = self._meter.create_histogram(
                    name, description=f"Custom metric: {name}"
                )
                self._cached_metrics[name] = instrument

        # Record outside the lock for better performance
        instrument.record(value, attributes=attributes)

    def record_gauge(self, name: str, value: float, **attributes):
        """Record a gauge metric (current value that can go up or down).

        Args:
            name: Metric name
            value: Current gauge value
            **attributes: Attributes to attach to the metric

        Example:
            >>> # FIXED (Round 8 Issue #10): Use correct naming convention
            >>> client.record_gauge("emms_model_accuracy", 0.92, dataset="test")
        """
        self.record_metric(name, value, **attributes)

    def record_counter(self, name: str, value: float, **attributes):
        """Record a counter metric (cumulative value that only increases).

        Args:
            name: Metric name
            value: Counter value to add
            **attributes: Attributes to attach to the metric

        Example:
            >>> client.record_counter("model.predictions_total", 100, model_version="1.0")
        """
        self.record_metric(name, value, **attributes)

    @contextmanager
    def trace(self, span_name: str, **attributes):
        """Context manager for creating a trace span.

        Args:
            span_name: Name of the span
            **attributes: Attributes to attach to the span

        Yields:
            Span instance

        Example:
            >>> with client.trace("model.predict", model_version="1.0"):
            ...     result = model.predict(data)
        """
        with self._tracer.start_as_current_span(span_name) as span:
            for key, value in attributes.items():
                span.set_attribute(key, value)
            yield span

    # Agentic AI tracking methods (Story 1.11)

    def record_goal_started(
        self, goal_description: str, goal_type: str = "general", **attributes
    ) -> str:
        """Start tracking an agent goal.

        Creates a new goal with a unique ID, starts a parent span, and
        increments the goals_started counter. The goal must be completed
        by calling record_goal_completed() with the returned goal_id.

        Args:
            goal_description: Human-readable description of the goal
            goal_type: Category of goal (e.g., "research", "analysis", "general")
            **attributes: Additional attributes to attach to the goal span

        Returns:
            Unique goal_id (UUID string) for tracking and completion

        Warning:
            HIPAA COMPLIANCE: goal_description is truncated but NOT sanitized for PHI.
            Users MUST NOT include patient identifiers, SSNs, names, or other PHI
            in goal descriptions. Use generic descriptions only (e.g., "Research
            treatment options" NOT "Research for patient John Doe SSN 123-45-6789").

        Example:
            >>> goal_id = client.record_goal_started(
            ...     "Research diabetes treatments",
            ...     goal_type="research"
            ... )
            >>> # ... execute goal steps ...
            >>> client.record_goal_completed(goal_id, status="success")
        """
        # Ensure cleanup thread is alive (Finding 9 fix)
        self._ensure_cleanup_thread_alive()

        # Sanitize inputs to prevent high-cardinality issues
        goal_description_sanitized = (
            goal_description[:200] if goal_description else "No description"
        )
        goal_type_sanitized = goal_type[:50] if goal_type else "general"

        goal_id = str(uuid.uuid4())
        start_time = time.perf_counter()

        # Start goal span
        span = self._tracer.start_span(
            "agent.goal",
            attributes={
                "goal_id": goal_id,
                "goal_type": goal_type_sanitized,
                "goal_description": goal_description_sanitized,
                **attributes,
            },
        )

        # Make this span the current context
        context_token = trace.use_span(span, end_on_exit=False)

        # Store for later completion (cleanup handled by background thread)
        with self._goals_lock:
            self._active_goals[goal_id] = (span, start_time, context_token)

        # Increment counter
        self._goals_started_counter.add(1, attributes={"goal_type": goal_type_sanitized})

        if self.config.debug_mode:
            self._sdk_logger.debug(
                f"[{goal_id}] Goal started: {goal_description_sanitized[:50]}... "
                f"(type={goal_type_sanitized})"
            )

        return goal_id

    def record_goal_completed(self, goal_id: str, status: str = "success", **attributes) -> None:
        """Complete tracking of an agent goal.

        Ends the goal span, records duration metrics, and increments the
        goals_completed counter with the specified status.

        Args:
            goal_id: The goal ID returned from record_goal_started()
            status: Completion status - must be one of: "success", "failure", "partial"
            **attributes: Additional attributes to attach to the goal span

        Raises:
            ValueError: If goal_id is not found or status is invalid
        """
        valid_statuses = {"success", "failure", "partial"}
        if status not in valid_statuses:
            raise ValueError(
                f"Invalid goal status: '{status}'. "
                f"Must be one of: {', '.join(sorted(valid_statuses))}"
            )

        with self._goals_lock:
            if goal_id not in self._active_goals:
                raise ValueError(
                    f"Unknown goal_id: '{goal_id}'. "
                    f"Goal may have already been completed or was never started."
                )
            span, start_time, context_token = self._active_goals.pop(goal_id)

        duration = time.perf_counter() - start_time

        # Set span attributes and status
        span.set_attribute("goal_status", status)
        span.set_attribute("goal_duration_seconds", duration)
        for key, value in attributes.items():
            span.set_attribute(key, value)

        from opentelemetry.trace import Status, StatusCode

        if status == "success":
            span.set_status(Status(StatusCode.OK))
        elif status == "failure":
            span.set_status(Status(StatusCode.ERROR, "Goal failed"))
        else:  # partial
            span.set_status(Status(StatusCode.OK, "Goal partially completed"))

        span.end()

        # Detach context
        from opentelemetry import context as otel_context

        otel_context.detach(context_token)

        # Record metrics - use separate counters for success vs failure (AC: 2)
        if status == "failure":
            self._goals_failed_counter.add(1, attributes={"status": status})
        else:
            # Count success and partial as completed
            self._goals_completed_counter.add(1, attributes={"status": status})
        self._goal_duration_histogram.record(duration, attributes={"status": status})

        if self.config.debug_mode:
            self._sdk_logger.debug(
                f"[{goal_id}] Goal completed: status={status}, duration={duration:.3f}s"
            )

    @contextmanager
    def track_tool(self, tool_name: str, goal_id: Optional[str] = None, **attributes):
        """Context manager for tracking tool execution.

        NOTE: Token tracking (tool.input_tokens, tool.output_tokens) should be
        implemented when integrating with LLM-based tools. For mock tools or
        non-LLM tools, token attributes are not applicable.

        Automatically captures tool latency, records metrics, and handles
        errors. When used within a goal, the tool span becomes a child of
        the goal span.

        Nested tool calls are supported: only the outermost call records metrics
        to prevent double-counting, but all calls create spans for visibility.

        Args:
            tool_name: Name of the tool being executed
            goal_id: Optional goal ID to correlate this tool call with a specific goal
            **attributes: Additional attributes to attach to the tool span

        Yields:
            The tool span for adding custom attributes

        Example:
            >>> goal_id = client.record_goal_started("Research question")
            >>> with client.track_tool("pubmed_search", goal_id=goal_id, query="diabetes") as span:
            ...     results = pubmed.search("diabetes")
            ...     span.set_attribute("results_count", len(results))
            >>> client.record_goal_completed(goal_id)
        """
        # Track nesting depth using contextvars (thread-safe)
        current_depth = _tool_call_depth.get()
        _tool_call_depth.set(current_depth + 1)

        # Sanitize tool_name to prevent high-cardinality issues
        tool_name_sanitized = tool_name[:100] if tool_name else "unknown"
        start_time = time.perf_counter()

        # Add goal_id to span attributes if provided for correlation
        span_attributes = {
            "tool_name": tool_name_sanitized,
            "tool_depth": current_depth + 1,
            **attributes,
        }
        if goal_id:
            span_attributes["goal_id"] = goal_id

        try:
            with self._tracer.start_as_current_span(
                f"agent.tool.{tool_name_sanitized}", attributes=span_attributes
            ) as span:
                try:
                    yield span

                    # Calculate latency for successful tool execution
                    latency = time.perf_counter() - start_time

                    # Record metrics for ALL tool calls (AC: 3)
                    # Add goal_id to metrics for correlation
                    metric_attrs = {"tool_name": tool_name_sanitized, "status": "success"}
                    if goal_id:
                        metric_attrs["goal_id"] = goal_id

                    # Latency histogram attributes
                    latency_attrs = {"tool_name": tool_name_sanitized}
                    if goal_id:
                        latency_attrs["goal_id"] = goal_id

                    self._tool_calls_counter.add(1, attributes=metric_attrs)
                    self._tool_latency_histogram.record(latency, attributes=latency_attrs)

                    if self.config.debug_mode:
                        goal_info = f" (goal={goal_id[:8]}...)" if goal_id else ""
                        depth_info = f" [depth={current_depth + 1}]" if current_depth > 0 else ""
                        self._sdk_logger.debug(
                            f"Tool '{tool_name_sanitized}' completed: "
                            f"latency={latency:.3f}s{goal_info}{depth_info}"
                        )

                except Exception as e:
                    latency = time.perf_counter() - start_time
                    error_type = type(e).__name__
                    error_message = _sanitize_error_message(str(e), max_length=200)

                    from opentelemetry.trace import Status, StatusCode

                    span.set_status(Status(StatusCode.ERROR, error_message))
                    # NOTE: NOT using span.record_exception(e) to prevent PHI leakage
                    # Only sanitized error information is recorded
                    span.set_attribute("tool_latency_seconds", latency)
                    span.set_attribute("error_type", error_type)
                    span.set_attribute("error_message", error_message)

                    # Record metrics for ALL tool calls (AC: 3)
                    # Add goal_id and error details to metrics
                    metric_attrs = {"tool_name": tool_name_sanitized, "status": "error"}
                    if goal_id:
                        metric_attrs["goal_id"] = goal_id

                    error_attrs = {"tool_name": tool_name_sanitized, "error_type": error_type}
                    if goal_id:
                        error_attrs["goal_id"] = goal_id

                    # Latency histogram attributes
                    latency_attrs = {"tool_name": tool_name_sanitized}
                    if goal_id:
                        latency_attrs["goal_id"] = goal_id

                    self._tool_calls_counter.add(1, attributes=metric_attrs)
                    self._tool_errors_counter.add(1, attributes=error_attrs)
                    self._tool_latency_histogram.record(latency, attributes=latency_attrs)

                    if self.config.debug_mode:
                        goal_info = f" (goal={goal_id[:8]}...)" if goal_id else ""
                        depth_info = f" [depth={current_depth + 1}]" if current_depth > 0 else ""
                        self._sdk_logger.debug(
                            f"Tool '{tool_name_sanitized}' failed: error={error_type}, "
                            f"message={error_message[:50]}, latency={latency:.3f}s{goal_info}{depth_info}"
                        )

                    raise
        finally:
            # Always restore depth, even if exception occurred
            _tool_call_depth.set(current_depth)

    def record_human_intervention(
        self,
        reason: str,
        wait_time: Optional[float] = None,
        intervention_type: str = "review",
        **attributes,
    ) -> None:
        """Record a human intervention event.

        Tracks when the agent requests or requires human intervention,
        such as approval, review, or clarification.

        Args:
            reason: Description of why intervention was needed
            wait_time: Optional time spent waiting for human response (seconds)
            intervention_type: Type of intervention (e.g., "review", "approval",
                              "clarification", "escalation")
            **attributes: Additional attributes to attach
        """
        # Sanitize inputs to prevent high-cardinality issues
        reason_sanitized = reason[:200] if reason else "No reason provided"
        intervention_type_sanitized = intervention_type[:50] if intervention_type else "review"

        with self._tracer.start_as_current_span(
            "agent.human_intervention",
            attributes={
                "intervention_reason": reason_sanitized,
                "intervention_type": intervention_type_sanitized,
                **attributes,
            },
        ) as span:
            self._human_interventions_counter.add(
                1,
                attributes={
                    "reason": reason_sanitized,
                    "intervention_type": intervention_type_sanitized,
                },
            )

            if wait_time is not None:
                span.set_attribute("intervention_wait_time_seconds", wait_time)
                self._intervention_wait_histogram.record(
                    wait_time,
                    attributes={
                        "reason": reason_sanitized,
                        "intervention_type": intervention_type_sanitized,
                    },
                )

        if self.config.debug_mode:
            wait_info = f", wait_time={wait_time:.3f}s" if wait_time else ""
            self._sdk_logger.debug(
                f"Human intervention recorded: type={intervention_type_sanitized}, "
                f"reason={reason_sanitized[:50]}...{wait_info}"
            )

    @contextmanager
    def reasoning_step(
        self,
        step_type: str,
        step_description: Optional[str] = None,
        llm_model: Optional[str] = None,
        **attributes,
    ):
        """Context manager for tracking an agent reasoning step.

        Tracks duration and metadata of reasoning steps in an agent's decision-making
        process. This allows visibility into planning, evaluation, tool selection,
        and other cognitive processes.

        NOTE: For LLM-based reasoning, add tokens_used attribute within the context:
            >>> with client.reasoning_step("planning") as span:
            ...     response = llm.complete(prompt)
            ...     span.set_attribute("tokens_used", response.usage.total_tokens)

        Args:
            step_type: Type of reasoning step (e.g., "planning", "tool_call",
                      "evaluation", "synthesis")
            step_description: Optional human-readable description of the step
            llm_model: Optional LLM model used for this reasoning step
            **attributes: Additional attributes to attach to the step

        Yields:
            The reasoning step span for adding custom attributes

        Example:
            >>> with client.reasoning_step("planning", "Analyzing query"):
            ...     plan = planner.create_plan(query)
        """
        # Sanitize inputs to prevent high-cardinality issues
        step_type_sanitized = step_type[:50] if step_type else "unknown"
        step_description_sanitized = step_description[:200] if step_description else None
        llm_model_sanitized = llm_model[:50] if llm_model else None
        start_time = time.perf_counter()

        # Build span attributes
        span_attributes = {"reasoning_step_type": step_type_sanitized, **attributes}
        if step_description_sanitized:
            span_attributes["reasoning_step_description"] = step_description_sanitized
        if llm_model_sanitized:
            span_attributes["llm_model"] = llm_model_sanitized

        # Create span for reasoning step
        with self._tracer.start_as_current_span(
            "agent.reasoning_step", attributes=span_attributes
        ) as span:
            try:
                yield span

                # Record successful completion
                self._reasoning_steps_counter.add(
                    1, attributes={"step_type": step_type_sanitized, "status": "success"}
                )

                if self.config.debug_mode:
                    duration = time.perf_counter() - start_time
                    model_info = f", model={llm_model_sanitized}" if llm_model_sanitized else ""
                    self._sdk_logger.debug(
                        f"Reasoning step completed: type={step_type_sanitized}{model_info}, "
                        f"duration={duration:.3f}s"
                    )

            except Exception:
                # Record failure
                self._reasoning_steps_counter.add(
                    1, attributes={"step_type": step_type_sanitized, "status": "error"}
                )
                raise

    def shutdown(self, timeout_millis: int = 30000):
        """Gracefully shutdown the client and flush all telemetry.

        This should be called when the application is shutting down to ensure
        all metrics, logs, and traces are exported.

        Args:
            timeout_millis: Maximum time to wait for flush in milliseconds
        """
        self._logger.info("Shutting down MCA SDK client")

        # Stop background cleanup thread for stale goals
        self._stop_cleanup_thread(timeout=5.0)

        # Stop background refresh thread (if registry client exists)
        if self._registry_client:
            self._registry_client.stop_refresh_thread(timeout=5.0)
            self._registry_client.close()
            logging.debug("Registry client closed")

        # Shutdown Prometheus server if running
        if hasattr(self, "_prometheus_shutdown") and self._prometheus_shutdown:
            self._prometheus_shutdown()

        # Force flush all providers BEFORE shutting down exporters
        # This ensures BatchSpanProcessor drains its queue first
        self._meter_provider.force_flush(timeout_millis=timeout_millis)
        self._logger_provider.force_flush(timeout_millis=timeout_millis)
        self._tracer_provider.force_flush(timeout_millis=timeout_millis)

        # Remove logging handler to prevent resource leak
        if hasattr(self, "_otel_handler"):
            self._logger.removeHandler(self._otel_handler)

        # Shutdown providers (this will automatically shut down all registered processors and their exporters)
        # NOTE: GCP Cloud Trace exporter will be shut down by tracer_provider.shutdown()
        # via BatchSpanProcessor.shutdown(), so we don't call exporter.shutdown() directly
        self._meter_provider.shutdown()
        self._logger_provider.shutdown()
        self._tracer_provider.shutdown()

    def __enter__(self):
        """Support for context manager protocol."""
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        """Ensure shutdown is called when exiting context."""
        self.shutdown()
        return False
