"""Integration tests for OpenTelemetry Collector attributes processor.

Tests attributes processor enrichment behavior including:
- Region attribute addition (gcp.region=us-central1) - AC#1
- Environment attribute addition (environment=prototype) - AC#2
- Non-overwriting behavior (insert action preserves existing) - AC#3
- Enrichment across all telemetry types (metrics, logs, traces)
- Enrichment from different SDK sources

These tests require the OTel Collector to be running.
Start with: docker-compose up -d otel-collector

Story: 4.3 - Implement Attributes Processor for Enrichment

OTel Version Dependency:
- Tested with OpenTelemetry Collector Contrib v0.91.0
- Debug exporter output format assumptions:
  * Uses "ResourceMetrics", "ResourceLog" (singular), "ResourceSpans"
  * Format: "-> key: Type(value)" or "key: value"
  * If format changes in future versions, update verify_attribute_in_logs regex
"""

import pytest
import time
import requests
import subprocess
import logging
import os
from typing import List, Dict, Any
import re

from opentelemetry import metrics, trace
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor

from mca_sdk import MCAClient


# Test configuration with validation
def _validate_url(url: str, name: str) -> str:
    """Validate URL format."""
    if not url.startswith(("http://", "https://")):
        raise ValueError(f"{name} must start with http:// or https://, got: {url}")
    return url


def _validate_timeout(timeout_str: str, name: str) -> int:
    """Validate timeout is positive integer."""
    try:
        timeout = int(timeout_str)
        if timeout <= 0:
            raise ValueError(f"{name} must be positive, got: {timeout}")
        return timeout
    except ValueError as e:
        raise ValueError(f"{name} must be a positive integer, got: {timeout_str}") from e


def _validate_container_name(name: str) -> str:
    """Validate container name contains only safe characters."""
    if not name or not all(c.isalnum() or c in "-_" for c in name):
        raise ValueError(
            f"Container name must contain only alphanumeric, dash, or underscore characters, got: {name}"
        )
    return name


COLLECTOR_URL = _validate_url(os.getenv("COLLECTOR_URL", "http://localhost:4318"), "COLLECTOR_URL")
HEALTH_ENDPOINT = _validate_url(
    os.getenv("HEALTH_ENDPOINT", "http://localhost:13133"), "HEALTH_ENDPOINT"
)
# Batch timeout: 7s = collector's 5s timeout + 2s buffer for processing
# Updated to match actual otel-collector-config.yaml (batch timeout: 5s)
BATCH_TIMEOUT = _validate_timeout(os.getenv("BATCH_TIMEOUT", "7"), "BATCH_TIMEOUT")
COLLECTOR_CONTAINER = _validate_container_name(
    os.getenv("COLLECTOR_CONTAINER", "mca-otel-collector")
)


@pytest.fixture(scope="module")
def collector_running():
    """Verify collector is running and healthy before tests.

    This fixture runs once per test module and verifies the OTel Collector
    is accessible before any tests execute.
    """
    # Setup: Verify collector is healthy
    try:
        response = requests.get(HEALTH_ENDPOINT, timeout=2)
        if response.status_code != 200:
            pytest.skip("OTel Collector not running. Start with: docker-compose up -d")
    except requests.exceptions.RequestException:
        pytest.skip("OTel Collector not running. Start with: docker-compose up -d")

    logging.info(f"OTel Collector health check passed at {HEALTH_ENDPOINT}")

    yield

    # Teardown: Log completion (collector lifecycle managed externally)
    logging.info("Test module completed - collector remains running for next test module")


def get_collector_logs(since_seconds: int = 5) -> str:
    """Capture recent collector container logs."""
    try:
        result = subprocess.run(
            ["docker", "logs", "--since", f"{since_seconds}s", COLLECTOR_CONTAINER],
            shell=False,
            capture_output=True,
            text=True,
            timeout=10,
        )
        return result.stdout + result.stderr
    except Exception as e:
        pytest.skip(f"Could not capture collector logs: {e}")


def safe_shutdown(provider, provider_name: str = "provider") -> None:
    """Safely shutdown OTel provider with error logging.

    Args:
        provider: The OTel provider to shutdown
        provider_name: Name of provider for logging
    """
    try:
        result = provider.shutdown()
        # Some providers return bool, others return None
        if result is False:
            logging.warning(f"{provider_name} shutdown returned False - cleanup may be incomplete")
    except Exception as e:
        logging.error(f"{provider_name} shutdown failed: {e}")
        # Re-raise to avoid masking critical issues in tests
        raise


def verify_attribute_in_logs(logs: str, key: str, value: str) -> bool:
    """Parse collector debug logs to verify attribute key-value pair exists.

    Debug exporter outputs resource attributes in this format:
        ResourceMetrics #0
        Resource attributes:
             -> service.name: Str(test-service)
             -> gcp.region: Str(us-central1)
             -> environment: Str(prototype)

    This function parses that format and verifies key=value exists.

    Args:
        logs: Collector logs from debug exporter
        key: Attribute key to find
        value: Expected attribute value

    Returns:
        True if attribute with key=value found, False otherwise
    """
    # Pattern to find attribute in debug output
    # Account for both "Key: Value" and "-> key: Type(value)" formats
    pattern = rf"[->\s]*{re.escape(key)}:\s*(?:Str\()?{re.escape(value)}\)?"
    return bool(re.search(pattern, logs, re.MULTILINE))


def wait_for_attribute_in_logs(
    key: str, value: str, timeout: int = 15, poll_interval: float = 0.5
) -> str:
    """Poll collector logs until attribute appears or timeout.

    Uses exponential backoff for better performance on fast systems while
    maintaining reliability on slow CI systems.

    Args:
        key: Attribute key to find
        value: Expected attribute value
        timeout: Maximum seconds to wait
        poll_interval: Initial seconds between polls (will increase exponentially)

    Returns:
        Collector logs containing the attribute

    Raises:
        TimeoutError: If attribute not found within timeout
    """
    start_time = time.time()
    # Capture logs from just before this test started (timeout + 5s buffer)
    # This improves test isolation by reducing overlap with previous tests
    log_window = int(timeout) + 5

    current_interval = poll_interval
    max_interval = 2.0  # Cap backoff at 2 seconds

    while time.time() - start_time < timeout:
        logs = get_collector_logs(since_seconds=log_window)
        if verify_attribute_in_logs(logs, key, value):
            return logs

        # Exponential backoff: 0.5s -> 0.75s -> 1.125s -> 1.6875s -> 2s (capped)
        time.sleep(current_interval)
        current_interval = min(current_interval * 1.5, max_interval)

    # Final check with full logs
    logs = get_collector_logs(since_seconds=log_window)
    if verify_attribute_in_logs(logs, key, value):
        return logs

    raise TimeoutError(f"Attribute {key}={value} not found in collector logs after {timeout}s")


def verify_attribute_not_in_resource(logs: str, resource_id: str, key: str, value: str) -> bool:
    """Verify specific value is NOT in a resource block (for non-overwriting tests).

    This function parses OTel debug exporter output to find a specific resource
    (identified by resource_id appearing in its attributes) and verifies that
    a specific key=value pair is NOT present in that resource's attributes.

    Args:
        logs: Collector logs from debug exporter
        resource_id: Resource identifier string (e.g., service.name value like "test-service")
        key: Attribute key to check (e.g., "gcp.region")
        value: Value that should NOT be present (e.g., "us-central1")

    Returns:
        True if the value is NOT found in that resource's attributes, False if found

    Example logs format:
        ResourceMetrics #0
        Resource attributes:
             -> service.name: Str(test-preserve-region)
             -> gcp.region: Str(eu-west1)
        ScopeMetrics #0
        ...

    Example usage:
        >>> logs = '''
        ... ResourceMetrics #0
        ... Resource attributes:
        ...      -> service.name: Str(test-service)
        ...      -> gcp.region: Str(eu-west1)
        ... '''
        >>> # Should return True (us-central1 NOT in this resource)
        >>> verify_attribute_not_in_resource(logs, "test-service", "gcp.region", "us-central1")
        True
        >>> # Should return False (eu-west1 IS in this resource)
        >>> verify_attribute_not_in_resource(logs, "test-service", "gcp.region", "eu-west1")
        False
    """
    # Find the resource block containing resource_id
    lines = logs.split("\n")
    in_target_resource = False
    resource_block = []

    for line in lines:
        # Check if we're entering a new resource block
        if (
            "Resource attributes:" in line
            or "ResourceMetrics #" in line
            or "ResourceLogs #" in line
            or "ResourceSpans #" in line
        ):
            # If we were in target resource, we're done
            if in_target_resource and resource_block:
                break
            in_target_resource = False
            resource_block = []

        # Check if this resource contains our identifier
        if resource_id in line:
            in_target_resource = True

        # Collect lines from target resource
        if in_target_resource:
            resource_block.append(line)

    # Check that the value is NOT in the resource block
    resource_text = "\n".join(resource_block)
    pattern = rf"{re.escape(key)}:\s*(?:Str\()?{re.escape(value)}\)?"
    return not bool(re.search(pattern, resource_text, re.MULTILINE))


@pytest.mark.integration
class TestRegionEnrichment:
    """Test gcp.region attribute addition (AC#1).

    AC#1: Given telemetry is received without gcp.region attribute
          When attributes processor runs
          Then gcp.region=us-central1 is added to all telemetry
    """

    def test_region_added_to_metrics_without_region(self, collector_running):
        """AC#1: Metrics without gcp.region receive gcp.region=us-central1."""
        print("\n--- Testing Region Added to Metrics ---")

        # Create resource WITHOUT gcp.region
        resource = Resource.create(
            {"service.name": "test-region-metrics", "test.id": "region-metrics-001"}
        )

        # Send metric via OTLP
        exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.attributes.region")
            counter = meter.create_counter("test_region_counter")
            counter.add(1, {"test": "region"})

            print("Sent metric without gcp.region")

            # Force flush and wait for data to appear in logs
            provider.force_flush(timeout_millis=5000)

            # Poll for attribute (much faster than fixed sleep)
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            # Verify gcp.region=us-central1 was added
            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-central1"
            ), "Collector should add gcp.region=us-central1 to metrics without region"

            print("[PASS] Collector added gcp.region=us-central1 to metrics")

        finally:
            safe_shutdown(provider, "MeterProvider")

    def test_region_added_to_logs_without_region(self, collector_running):
        """AC#1: Logs without gcp.region receive gcp.region=us-central1."""
        print("\n--- Testing Region Added to Logs ---")

        # Create resource WITHOUT gcp.region
        resource = Resource.create(
            {"service.name": "test-region-logs", "test.id": "region-logs-001"}
        )

        # Send log via OTLP
        exporter = OTLPLogExporter(endpoint=f"{COLLECTOR_URL}/v1/logs")
        provider = LoggerProvider(resource=resource)
        provider.add_log_record_processor(BatchLogRecordProcessor(exporter))

        try:
            handler = LoggingHandler(logger_provider=provider)
            logger = logging.getLogger("test.attributes.region.logs")
            logger.setLevel(logging.INFO)
            logger.addHandler(handler)

            logger.info("Test log for region enrichment")

            print("Sent log without gcp.region")

            # Force flush and poll for data
            provider.force_flush()
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            # Verify gcp.region=us-central1 was added
            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-central1"
            ), "Collector should add gcp.region=us-central1 to logs without region"

            print("[PASS] Collector added gcp.region=us-central1 to logs")

        finally:
            provider.shutdown()

    def test_region_added_to_traces_without_region(self, collector_running):
        """AC#1: Traces without gcp.region receive gcp.region=us-central1."""
        print("\n--- Testing Region Added to Traces ---")

        # Create resource WITHOUT gcp.region
        resource = Resource.create(
            {"service.name": "test-region-traces", "test.id": "region-traces-001"}
        )

        # Send trace via OTLP
        exporter = OTLPSpanExporter(endpoint=f"{COLLECTOR_URL}/v1/traces")
        provider = TracerProvider(resource=resource)
        provider.add_span_processor(BatchSpanProcessor(exporter))

        try:
            tracer = provider.get_tracer("test.attributes.region.traces")

            with tracer.start_as_current_span("test_region_span") as span:
                span.set_attribute("test", "region")

            print("Sent trace without gcp.region")

            # Force flush and poll for data
            provider.force_flush()
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            # Verify gcp.region=us-central1 was added
            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-central1"
            ), "Collector should add gcp.region=us-central1 to traces without region"

            print("[PASS] Collector added gcp.region=us-central1 to traces")

        finally:
            provider.shutdown()

    def test_region_attribute_format_validation(self, collector_running):
        """AC#1: Verify gcp.region attribute has exact format and value."""
        print("\n--- Testing Region Attribute Format ---")

        resource = Resource.create(
            {"service.name": "test-region-format", "test.id": "region-format-001"}
        )

        exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.attributes.region.format")
            counter = meter.create_counter("test_region_format_counter")
            counter.add(1)

            provider.force_flush(timeout_millis=5000)
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            # Verify exact format: gcp.region: Str(us-central1)
            assert "gcp.region" in logs, "Attribute key should be exactly 'gcp.region'"
            assert "us-central1" in logs, "Attribute value should be exactly 'us-central1'"

            # Verify it's in the resource attributes section
            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-central1"
            ), "gcp.region attribute should be in resource attributes with correct format"

            print("[PASS] gcp.region attribute has correct format and value")

        finally:
            provider.shutdown()


@pytest.mark.integration
class TestEnvironmentEnrichment:
    """Test environment attribute addition (AC#2).

    AC#2: Given telemetry is received
          When attributes processor runs
          Then environment=prototype attribute is added
    """

    def test_environment_added_to_metrics(self, collector_running):
        """AC#2: Metrics receive environment=prototype attribute."""
        print("\n--- Testing Environment Added to Metrics ---")

        resource = Resource.create(
            {"service.name": "test-environment-metrics", "test.id": "env-metrics-001"}
        )

        exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.attributes.environment")
            counter = meter.create_counter("test_environment_counter")
            counter.add(1, {"test": "environment"})

            print("Sent metric without environment attribute")

            provider.force_flush(timeout_millis=5000)
            logs = wait_for_attribute_in_logs("environment", "prototype", timeout=BATCH_TIMEOUT)

            # Verify environment=prototype was added
            assert verify_attribute_in_logs(
                logs, "environment", "prototype"
            ), "Collector should add environment=prototype to metrics"

            print("[PASS] Collector added environment=prototype to metrics")

        finally:
            safe_shutdown(provider, "MeterProvider")

    def test_environment_added_to_logs(self, collector_running):
        """AC#2: Logs receive environment=prototype attribute."""
        print("\n--- Testing Environment Added to Logs ---")

        resource = Resource.create(
            {"service.name": "test-environment-logs", "test.id": "env-logs-001"}
        )

        exporter = OTLPLogExporter(endpoint=f"{COLLECTOR_URL}/v1/logs")
        provider = LoggerProvider(resource=resource)
        provider.add_log_record_processor(BatchLogRecordProcessor(exporter))

        try:
            handler = LoggingHandler(logger_provider=provider)
            logger = logging.getLogger("test.attributes.environment.logs")
            logger.setLevel(logging.INFO)
            logger.addHandler(handler)

            logger.info("Test log for environment enrichment")

            print("Sent log without environment attribute")

            provider.force_flush()
            logs = wait_for_attribute_in_logs("environment", "prototype", timeout=BATCH_TIMEOUT)

            assert verify_attribute_in_logs(
                logs, "environment", "prototype"
            ), "Collector should add environment=prototype to logs"

            print("[PASS] Collector added environment=prototype to logs")

        finally:
            provider.shutdown()

    def test_environment_added_to_traces(self, collector_running):
        """AC#2: Traces receive environment=prototype attribute."""
        print("\n--- Testing Environment Added to Traces ---")

        resource = Resource.create(
            {"service.name": "test-environment-traces", "test.id": "env-traces-001"}
        )

        exporter = OTLPSpanExporter(endpoint=f"{COLLECTOR_URL}/v1/traces")
        provider = TracerProvider(resource=resource)
        provider.add_span_processor(BatchSpanProcessor(exporter))

        try:
            tracer = provider.get_tracer("test.attributes.environment.traces")

            with tracer.start_as_current_span("test_environment_span") as span:
                span.set_attribute("test", "environment")

            print("Sent trace without environment attribute")

            provider.force_flush()
            logs = wait_for_attribute_in_logs("environment", "prototype", timeout=BATCH_TIMEOUT)

            assert verify_attribute_in_logs(
                logs, "environment", "prototype"
            ), "Collector should add environment=prototype to traces"

            print("[PASS] Collector added environment=prototype to traces")

        finally:
            provider.shutdown()

    def test_environment_attribute_format_validation(self, collector_running):
        """AC#2: Verify environment attribute has exact format and value."""
        print("\n--- Testing Environment Attribute Format ---")

        resource = Resource.create(
            {"service.name": "test-environment-format", "test.id": "env-format-001"}
        )

        exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.attributes.environment.format")
            counter = meter.create_counter("test_environment_format_counter")
            counter.add(1)

            provider.force_flush(timeout_millis=5000)
            logs = wait_for_attribute_in_logs("environment", "prototype", timeout=BATCH_TIMEOUT)

            # Verify exact format: environment: Str(prototype)
            assert "environment" in logs, "Attribute key should be exactly 'environment'"
            assert "prototype" in logs, "Attribute value should be exactly 'prototype'"

            assert verify_attribute_in_logs(
                logs, "environment", "prototype"
            ), "environment attribute should be in resource attributes with correct format"

            print("[PASS] environment attribute has correct format and value")

        finally:
            provider.shutdown()


@pytest.mark.integration
class TestNonOverwritingBehavior:
    """Test insert action preserves existing attributes (AC#3).

    AC#3: Given telemetry already has gcp.region attribute
          When attributes processor runs
          Then existing attribute is preserved (not overwritten)

    This is CRITICAL for ensuring SDK-provided values are not lost.
    """

    def test_existing_region_preserved_metrics(self, collector_running):
        """AC#3: Metrics with existing gcp.region preserve original value."""
        print("\n--- Testing Existing Region Preserved in Metrics ---")

        # Create resource WITH pre-existing gcp.region
        resource = Resource.create(
            {
                "service.name": "test-preserve-region-metrics",
                "gcp.region": "eu-west1",  # Original value - should be preserved
                "test.id": "preserve-region-001",
            }
        )

        exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.attributes.preserve.region")
            counter = meter.create_counter("test_preserve_region_counter")
            counter.add(1, {"test": "preserve"})

            print("Sent metric WITH gcp.region=eu-west1")

            provider.force_flush(timeout_millis=5000)
            logs = wait_for_attribute_in_logs("gcp.region", "eu-west1", timeout=BATCH_TIMEOUT)

            # CRITICAL: Verify original value is preserved
            assert verify_attribute_in_logs(
                logs, "gcp.region", "eu-west1"
            ), "Original gcp.region=eu-west1 should be preserved, not overwritten"

            # CRITICAL: Verify us-central1 was NOT added to this resource
            assert verify_attribute_not_in_resource(
                logs, "test-preserve-region-metrics", "gcp.region", "us-central1"
            ), "Collector should NOT overwrite existing gcp.region with us-central1"

            print("[PASS] Original gcp.region=eu-west1 preserved (not overwritten)")

        finally:
            safe_shutdown(provider, "MeterProvider")

    def test_existing_region_preserved_logs(self, collector_running):
        """AC#3: Logs with existing gcp.region preserve original value."""
        print("\n--- Testing Existing Region Preserved in Logs ---")

        # Create resource WITH pre-existing gcp.region
        resource = Resource.create(
            {
                "service.name": "test-preserve-region-logs",
                "gcp.region": "asia-east1",  # Original value - should be preserved
                "test.id": "preserve-region-logs-001",
            }
        )

        exporter = OTLPLogExporter(endpoint=f"{COLLECTOR_URL}/v1/logs")
        provider = LoggerProvider(resource=resource)
        provider.add_log_record_processor(BatchLogRecordProcessor(exporter))

        try:
            handler = LoggingHandler(logger_provider=provider)
            logger = logging.getLogger("test.attributes.preserve.region.logs")
            logger.setLevel(logging.INFO)
            logger.addHandler(handler)

            logger.info("Test log with existing region")

            print("Sent log WITH gcp.region=asia-east1")

            provider.force_flush()
            logs = wait_for_attribute_in_logs("gcp.region", "asia-east1", timeout=BATCH_TIMEOUT)

            # Verify original value preserved
            assert verify_attribute_in_logs(
                logs, "gcp.region", "asia-east1"
            ), "Original gcp.region=asia-east1 should be preserved in logs"

            # Verify us-central1 not added
            assert verify_attribute_not_in_resource(
                logs, "test-preserve-region-logs", "gcp.region", "us-central1"
            ), "Collector should NOT overwrite existing gcp.region in logs"

            print("[PASS] Original gcp.region=asia-east1 preserved in logs")

        finally:
            provider.shutdown()

    def test_existing_region_preserved_traces(self, collector_running):
        """AC#3: Traces with existing gcp.region preserve original value."""
        print("\n--- Testing Existing Region Preserved in Traces ---")

        # Create resource WITH pre-existing gcp.region
        resource = Resource.create(
            {
                "service.name": "test-preserve-region-traces",
                "gcp.region": "us-west1",  # Original value - should be preserved
                "test.id": "preserve-region-traces-001",
            }
        )

        exporter = OTLPSpanExporter(endpoint=f"{COLLECTOR_URL}/v1/traces")
        provider = TracerProvider(resource=resource)
        provider.add_span_processor(BatchSpanProcessor(exporter))

        try:
            tracer = provider.get_tracer("test.attributes.preserve.region.traces")

            with tracer.start_as_current_span("test_preserve_region_span") as span:
                span.set_attribute("test", "preserve")

            print("Sent trace WITH gcp.region=us-west1")

            provider.force_flush()
            logs = wait_for_attribute_in_logs("gcp.region", "us-west1", timeout=BATCH_TIMEOUT)

            # Verify original value preserved
            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-west1"
            ), "Original gcp.region=us-west1 should be preserved in traces"

            # Verify us-central1 not added
            assert verify_attribute_not_in_resource(
                logs, "test-preserve-region-traces", "gcp.region", "us-central1"
            ), "Collector should NOT overwrite existing gcp.region in traces"

            print("[PASS] Original gcp.region=us-west1 preserved in traces")

        finally:
            provider.shutdown()


@pytest.mark.integration
class TestAllTelemetryTypes:
    """Test enrichment across all telemetry types.

    Verifies that both attributes (gcp.region AND environment) are added
    to all three telemetry types and survive batch processing.
    """

    def test_both_attributes_added_to_metrics(self, collector_running):
        """Verify both gcp.region AND environment added to metrics."""
        print("\n--- Testing Both Attributes in Metrics ---")

        resource = Resource.create(
            {"service.name": "test-both-attrs-metrics", "test.id": "both-attrs-001"}
        )

        exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.attributes.both")
            counter = meter.create_counter("test_both_attrs_counter")
            counter.add(1)

            provider.force_flush(timeout_millis=5000)
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            # Verify BOTH attributes present
            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-central1"
            ), "gcp.region should be added to metrics"
            assert verify_attribute_in_logs(
                logs, "environment", "prototype"
            ), "environment should be added to metrics"

            print("[PASS] Both gcp.region and environment added to metrics")

        finally:
            provider.shutdown()

    def test_both_attributes_added_to_logs(self, collector_running):
        """Verify both gcp.region AND environment added to logs."""
        print("\n--- Testing Both Attributes in Logs ---")

        resource = Resource.create(
            {"service.name": "test-both-attrs-logs", "test.id": "both-attrs-logs-001"}
        )

        exporter = OTLPLogExporter(endpoint=f"{COLLECTOR_URL}/v1/logs")
        provider = LoggerProvider(resource=resource)
        provider.add_log_record_processor(BatchLogRecordProcessor(exporter))

        try:
            handler = LoggingHandler(logger_provider=provider)
            logger = logging.getLogger("test.attributes.both.logs")
            logger.setLevel(logging.INFO)
            logger.addHandler(handler)

            logger.info("Test log for both attributes")

            provider.force_flush()
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-central1"
            ), "gcp.region should be added to logs"
            assert verify_attribute_in_logs(
                logs, "environment", "prototype"
            ), "environment should be added to logs"

            print("[PASS] Both gcp.region and environment added to logs")

        finally:
            provider.shutdown()

    def test_both_attributes_added_to_traces(self, collector_running):
        """Verify both gcp.region AND environment added to traces."""
        print("\n--- Testing Both Attributes in Traces ---")

        resource = Resource.create(
            {
                "service.name": "test-both-attrs-traces",
                "test.id": "both-attrs-traces-001",
            }
        )

        exporter = OTLPSpanExporter(endpoint=f"{COLLECTOR_URL}/v1/traces")
        provider = TracerProvider(resource=resource)
        provider.add_span_processor(BatchSpanProcessor(exporter))

        try:
            tracer = provider.get_tracer("test.attributes.both.traces")

            with tracer.start_as_current_span("test_both_attrs_span"):
                pass

            provider.force_flush()
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-central1"
            ), "gcp.region should be added to traces"
            assert verify_attribute_in_logs(
                logs, "environment", "prototype"
            ), "environment should be added to traces"

            print("[PASS] Both gcp.region and environment added to traces")

        finally:
            provider.shutdown()

    def test_attributes_in_all_pipelines_simultaneously(self, collector_running):
        """Test concurrent enrichment across all 3 pipelines."""
        print("\n--- Testing Concurrent Pipeline Enrichment ---")

        # Send all three types simultaneously
        resource = Resource.create(
            {"service.name": "test-concurrent-pipelines", "test.id": "concurrent-001"}
        )

        # Metrics
        metric_exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        metric_reader = PeriodicExportingMetricReader(metric_exporter, export_interval_millis=5000)
        metric_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])

        # Logs
        log_exporter = OTLPLogExporter(endpoint=f"{COLLECTOR_URL}/v1/logs")
        log_provider = LoggerProvider(resource=resource)
        log_provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter))

        # Traces
        trace_exporter = OTLPSpanExporter(endpoint=f"{COLLECTOR_URL}/v1/traces")
        trace_provider = TracerProvider(resource=resource)
        trace_provider.add_span_processor(BatchSpanProcessor(trace_exporter))

        try:
            # Send metric
            meter = metric_provider.get_meter("test.concurrent")
            counter = meter.create_counter("test_concurrent_counter")
            counter.add(1)

            # Send log
            handler = LoggingHandler(logger_provider=log_provider)
            logger = logging.getLogger("test.concurrent.logs")
            logger.setLevel(logging.INFO)
            logger.addHandler(handler)
            logger.info("Concurrent test log")

            # Send trace
            tracer = trace_provider.get_tracer("test.concurrent.traces")
            with tracer.start_as_current_span("test_concurrent_span"):
                pass

            # Flush all
            metric_provider.force_flush(timeout_millis=5000)
            log_provider.force_flush()
            trace_provider.force_flush()

            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            # Verify all three types have enrichment
            # Should see ResourceMetrics, ResourceLog(s), ResourceSpans all with attributes
            # Note: OTel v0.91.0 uses singular "ResourceLog", but check for both forms for compatibility
            assert "ResourceMetrics" in logs, "Metrics should be present"
            assert "ResourceLog" in logs or "ResourceLogs" in logs, "Logs should be present"
            assert "ResourceSpans" in logs, "Traces should be present"

            # Verify enrichment on all
            assert verify_attribute_in_logs(logs, "gcp.region", "us-central1")
            assert verify_attribute_in_logs(logs, "environment", "prototype")

            print("[PASS] All 3 pipelines enriched concurrently")

        finally:
            safe_shutdown(metric_provider, "MeterProvider")
            safe_shutdown(log_provider, "LoggerProvider")
            safe_shutdown(trace_provider, "TracerProvider")

    def test_attributes_survive_batch_processing(self, collector_running):
        """Verify attributes persist through batch processor."""
        print("\n--- Testing Attributes Survive Batching ---")

        resource = Resource.create({"service.name": "test-batch-survival", "test.id": "batch-001"})

        exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.batch.survival")
            counter = meter.create_counter("test_batch_survival_counter")

            # Send multiple metrics to trigger batching
            for i in range(10):
                counter.add(1, {"batch.item": str(i)})

            provider.force_flush(timeout_millis=5000)
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            # Verify attributes present in batched output
            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-central1"
            ), "gcp.region should survive batch processing"
            assert verify_attribute_in_logs(
                logs, "environment", "prototype"
            ), "environment should survive batch processing"

            # Verify we see batched metrics
            assert "Metric #" in logs, "Batched metrics should be present"

            print("[PASS] Attributes survive batch processing")

        finally:
            provider.shutdown()

    def test_attributes_visible_in_debug_exporter(self, collector_running):
        """Verify enriched attributes visible in debug exporter output."""
        print("\n--- Testing Debug Exporter Visibility ---")

        resource = Resource.create(
            {"service.name": "test-debug-visibility", "test.id": "debug-001"}
        )

        exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.debug.visibility")
            counter = meter.create_counter("test_debug_visibility_counter")
            counter.add(1)

            provider.force_flush(timeout_millis=5000)
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            # Verify debug exporter shows resource attributes
            assert (
                "Resource attributes:" in logs
            ), "Debug exporter should show resource attributes section"

            # Verify our enriched attributes are in that section
            assert verify_attribute_in_logs(logs, "gcp.region", "us-central1")
            assert verify_attribute_in_logs(logs, "environment", "prototype")

            # Verify it's in the right format
            assert "Str(" in logs, "Debug exporter should show attribute types"

            print("[PASS] Enriched attributes visible in debug exporter")

        finally:
            provider.shutdown()


@pytest.mark.integration
class TestDifferentSDKSources:
    """Test enrichment works regardless of SDK source.

    Verifies that enrichment applies to:
    - MCAClient SDK telemetry
    - Raw OTLP exporter telemetry
    - Multiple services sending telemetry
    """

    def test_attributes_from_mca_client(self, collector_running):
        """Test enrichment works with MCAClient SDK."""
        print("\n--- Testing Enrichment with MCAClient SDK ---")

        client = MCAClient(
            service_name="test-mca-client-enrichment",
            model_id="mdl-enrichment-001",
            model_version="1.0.0",
            model_category="internal",
            model_type="regression",
            team_name="test-team",
            collector_endpoint=COLLECTOR_URL,
            buffering_enabled=False,
        )

        try:
            # Send telemetry via MCAClient
            client.record_prediction(
                input_data={"test": "enrichment"},
                output={"result": "success"},
                latency=0.05,
            )

            client.shutdown()
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            # Verify enrichment applied to MCAClient telemetry
            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-central1"
            ), "Enrichment should work with MCAClient SDK"
            assert verify_attribute_in_logs(
                logs, "environment", "prototype"
            ), "Environment attribute should be added to MCAClient telemetry"

            print("[PASS] MCAClient telemetry enriched successfully")

        finally:
            pass  # Already shutdown

    def test_attributes_from_raw_otlp_exporter(self, collector_running):
        """Test enrichment works with raw OTLP exporters."""
        print("\n--- Testing Enrichment with Raw OTLP Exporters ---")

        # Use raw OTLP exporter (not MCAClient)
        resource = Resource.create({"service.name": "test-raw-otlp", "test.id": "raw-otlp-001"})

        exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.raw.otlp")
            counter = meter.create_counter("test_raw_otlp_counter")
            counter.add(1)

            provider.force_flush(timeout_millis=5000)
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            # Verify enrichment works with raw OTLP
            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-central1"
            ), "Enrichment should work with raw OTLP exporters"
            assert verify_attribute_in_logs(logs, "environment", "prototype")

            print("[PASS] Raw OTLP telemetry enriched successfully")

        finally:
            provider.shutdown()

    def test_attributes_from_multiple_services(self, collector_running):
        """Test enrichment applies independently to multiple services."""
        print("\n--- Testing Multiple Services Enrichment ---")

        # Create two different services
        resource1 = Resource.create(
            {"service.name": "test-service-a", "test.id": "multi-service-a"}
        )

        resource2 = Resource.create(
            {"service.name": "test-service-b", "test.id": "multi-service-b"}
        )

        # Service A
        exporter1 = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader1 = PeriodicExportingMetricReader(exporter1, export_interval_millis=5000)
        provider1 = MeterProvider(resource=resource1, metric_readers=[reader1])

        # Service B
        exporter2 = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader2 = PeriodicExportingMetricReader(exporter2, export_interval_millis=5000)
        provider2 = MeterProvider(resource=resource2, metric_readers=[reader2])

        try:
            # Send from both services (send multiple to ensure visibility in logs)
            meter1 = provider1.get_meter("test.service.a")
            counter1 = meter1.create_counter("test_service_a_counter")
            # Send 10 metrics from service A
            for i in range(10):
                counter1.add(1, {"iteration": str(i)})

            meter2 = provider2.get_meter("test.service.b")
            counter2 = meter2.create_counter("test_service_b_counter")
            # Send 10 metrics from service B
            for i in range(10):
                counter2.add(1, {"iteration": str(i)})

            # Force flush both providers before waiting
            provider1.force_flush(timeout_millis=10000)
            provider2.force_flush(timeout_millis=10000)

            # Shutdown providers to ensure all data is sent
            provider1.shutdown()
            provider2.shutdown()

            # Wait longer for batch timeout (5s) plus processing buffer
            # Use a custom wait that checks for specific service names
            timeout = BATCH_TIMEOUT + 5
            start_time = time.time()
            log_window = int(timeout) + 5

            logs = None
            while time.time() - start_time < timeout:
                logs = get_collector_logs(since_seconds=log_window)
                if "test-service-a" in logs and "test-service-b" in logs:
                    break
                time.sleep(1)

            if logs is None or "test-service-a" not in logs or "test-service-b" not in logs:
                raise TimeoutError(
                    f"Services not found in logs within {timeout}s. "
                    f"Found service-a: {'test-service-a' in logs if logs else False}, "
                    f"Found service-b: {'test-service-b' in logs if logs else False}"
                )

            # Verify both services got enrichment
            assert "test-service-a" in logs, "Service A telemetry should be present"
            assert "test-service-b" in logs, "Service B telemetry should be present"

            # Verify enrichment applied to both
            assert verify_attribute_in_logs(logs, "gcp.region", "us-central1")
            assert verify_attribute_in_logs(logs, "environment", "prototype")

            print("[PASS] Multiple services enriched independently")

        except Exception:
            # Ensure cleanup even on error
            try:
                provider1.shutdown()
            except:
                pass
            try:
                provider2.shutdown()
            except:
                pass
            raise


@pytest.mark.integration
class TestCollectorConfiguration:
    """Test collector configuration verification.

    Verifies that the OTel Collector is running with the expected
    resource processor configuration to prevent false positives.
    """

    def test_collector_has_resource_processor_configured(self, collector_running):
        """Verify collector config has resource processor with correct attributes."""
        print("\n--- Testing Collector Configuration ---")

        # Read the collector config file that should be mounted
        config_path = os.path.join("config", "otel-collector-config.yaml")

        if not os.path.exists(config_path):
            pytest.skip(f"Config file not found at {config_path}")

        try:
            import yaml

            with open(config_path, "r") as f:
                config = yaml.safe_load(f)

            # Verify resource processor exists
            assert "processors" in config, "Config should have processors section"
            assert "resource" in config["processors"], "Config should have resource processor"

            # Verify resource processor has correct attributes
            resource_config = config["processors"]["resource"]
            assert "attributes" in resource_config, "Resource processor should have attributes"

            attributes = resource_config["attributes"]

            # Check for gcp.region attribute
            region_attr = next((a for a in attributes if a.get("key") == "gcp.region"), None)
            assert region_attr is not None, "Resource processor should have gcp.region attribute"
            assert region_attr.get("value") == "us-central1", "gcp.region should be us-central1"
            assert region_attr.get("action") == "insert", "gcp.region action should be insert"

            # Check for environment attribute
            env_attr = next((a for a in attributes if a.get("key") == "environment"), None)
            assert env_attr is not None, "Resource processor should have environment attribute"
            assert env_attr.get("value") == "prototype", "environment should be prototype"
            assert env_attr.get("action") == "insert", "environment action should be insert"

            # Verify pipelines use resource processor
            assert "service" in config, "Config should have service section"
            assert "pipelines" in config["service"], "Service should have pipelines"

            pipelines = config["service"]["pipelines"]
            for pipeline_name in ["metrics", "logs", "traces"]:
                assert pipeline_name in pipelines, f"Should have {pipeline_name} pipeline"
                assert "resource" in pipelines[pipeline_name].get(
                    "processors", []
                ), f"{pipeline_name} pipeline should use resource processor"

            print("[PASS] Collector configuration verified correctly")

        except ImportError:
            pytest.skip("PyYAML not installed - cannot verify config")
        except Exception as e:
            pytest.fail(f"Config verification failed: {e}")


@pytest.mark.integration
class TestErrorScenarios:
    """Test error handling and edge cases.

    Validates that enrichment processor handles:
    - Collector unreachable scenarios
    - Invalid/malformed attribute values
    - Network errors during export
    - Resource-level errors
    """

    def test_enrichment_with_collector_unreachable(self):
        """Test SDK behavior when collector is unreachable."""
        print("\n--- Testing Collector Unreachable Scenario ---")

        # Use non-existent endpoint
        bad_endpoint = "http://localhost:9999"

        resource = Resource.create({"service.name": "test-collector-down", "test.id": "error-001"})

        exporter = OTLPMetricExporter(endpoint=f"{bad_endpoint}/v1/metrics", timeout=2)
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.error.unreachable")
            counter = meter.create_counter("test_unreachable_counter")
            counter.add(1)

            # Should not raise exception (SDK should handle gracefully)
            provider.force_flush(timeout_millis=3000)

            print("[PASS] SDK handles unreachable collector gracefully")

        except Exception as e:
            pytest.fail(f"SDK should handle unreachable collector gracefully, but raised: {e}")
        finally:
            provider.shutdown()

    def test_enrichment_with_empty_resource_attributes(self, collector_running):
        """Test enrichment when resource has no custom attributes."""
        print("\n--- Testing Empty Resource Attributes ---")

        # Minimal resource (only service.name)
        resource = Resource.create({"service.name": "test-empty-resource"})

        exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.error.empty")
            counter = meter.create_counter("test_empty_resource_counter")
            counter.add(1)

            provider.force_flush(timeout_millis=5000)
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            # Enrichment should still work with minimal resource
            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-central1"
            ), "Enrichment should work even with minimal resource attributes"
            assert verify_attribute_in_logs(logs, "environment", "prototype")

            print("[PASS] Enrichment works with minimal resource attributes")

        finally:
            provider.shutdown()

    def test_enrichment_with_very_long_attribute_values(self, collector_running):
        """Test enrichment doesn't interfere with long SDK attribute values."""
        print("\n--- Testing Long Attribute Values ---")

        # Create resource with very long attribute value
        long_value = "x" * 2000  # 2000 characters
        resource = Resource.create(
            {
                "service.name": "test-long-attributes",
                "custom.long.attribute": long_value,
                "test.id": "error-003",
            }
        )

        exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.error.long")
            counter = meter.create_counter("test_long_attr_counter")
            counter.add(1)

            provider.force_flush(timeout_millis=5000)
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            # Enrichment should still apply
            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-central1"
            ), "Enrichment should work even with long SDK attributes"
            assert verify_attribute_in_logs(logs, "environment", "prototype")

            print("[PASS] Enrichment works with long attribute values")

        finally:
            provider.shutdown()

    def test_enrichment_with_special_characters_in_attributes(self, collector_running):
        """Test enrichment with special characters in SDK attributes."""
        print("\n--- Testing Special Characters in Attributes ---")

        # Resource with special characters
        resource = Resource.create(
            {
                "service.name": "test-special-chars",
                "custom.emoji": "🔥🚀✅",
                "custom.unicode": "测试中文",
                "custom.symbols": "!@#$%^&*()",
                "test.id": "error-004",
            }
        )

        exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.error.special")
            counter = meter.create_counter("test_special_chars_counter")
            counter.add(1)

            provider.force_flush(timeout_millis=5000)
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=BATCH_TIMEOUT)

            # Enrichment should not be affected by special characters
            assert verify_attribute_in_logs(logs, "gcp.region", "us-central1")
            assert verify_attribute_in_logs(logs, "environment", "prototype")

            print("[PASS] Enrichment works with special characters in attributes")

        finally:
            provider.shutdown()

    def test_collector_handles_high_volume_burst(self, collector_running):
        """Test enrichment under high-volume burst scenario."""
        print("\n--- Testing High-Volume Burst ---")

        resource = Resource.create({"service.name": "test-high-volume", "test.id": "error-005"})

        exporter = OTLPMetricExporter(endpoint=f"{COLLECTOR_URL}/v1/metrics")
        reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
        provider = MeterProvider(resource=resource, metric_readers=[reader])

        try:
            meter = provider.get_meter("test.error.volume")
            counter = meter.create_counter("test_volume_counter")

            # Send burst of 1000 metrics
            for i in range(1000):
                counter.add(1, {"iteration": str(i)})

            provider.force_flush(timeout_millis=10000)
            # Wait longer for high volume processing
            logs = wait_for_attribute_in_logs("gcp.region", "us-central1", timeout=20)

            # Verify enrichment still applied
            assert verify_attribute_in_logs(
                logs, "gcp.region", "us-central1"
            ), "Enrichment should work even with high-volume burst"
            assert verify_attribute_in_logs(logs, "environment", "prototype")

            print("[PASS] Enrichment survives high-volume burst")

        finally:
            provider.shutdown()
