"""
REM Settings and Configuration.

Pydantic settings with environment variable support:
- Nested settings with env_prefix for organization
- Environment variables use double underscore delimiter (ENV__NESTED__VAR)
- Sensitive defaults (auth disabled, OTEL disabled for local dev)
- Global settings singleton

Example .env file:
    # API Server
    API__HOST=0.0.0.0
    API__PORT=8000
    API__RELOAD=true
    API__LOG_LEVEL=info

    # LLM
    LLM__DEFAULT_MODEL=anthropic:claude-sonnet-4-5-20250929
    LLM__DEFAULT_TEMPERATURE=0.5
    LLM__MAX_RETRIES=10
    LLM__OPENAI_API_KEY=sk-...
    LLM__ANTHROPIC_API_KEY=sk-ant-...

    # Database (port 5050 for Docker Compose)
    POSTGRES__CONNECTION_STRING=postgresql://rem:rem@localhost:5050/rem
    POSTGRES__POOL_MIN_SIZE=5
    POSTGRES__POOL_MAX_SIZE=20
    POSTGRES__STATEMENT_TIMEOUT=30000

    # Auth (disabled by default)
    AUTH__ENABLED=false
    AUTH__OIDC_ISSUER_URL=https://accounts.google.com
    AUTH__OIDC_CLIENT_ID=your-client-id
    AUTH__SESSION_SECRET=your-secret-key

    # OpenTelemetry (disabled by default)
    OTEL__ENABLED=false
    OTEL__SERVICE_NAME=rem-api
    OTEL__COLLECTOR_ENDPOINT=http://localhost:4318
    OTEL__PROTOCOL=http

    # Arize Phoenix (disabled by default)
    PHOENIX__ENABLED=false
    PHOENIX__COLLECTOR_ENDPOINT=http://localhost:6006/v1/traces
    PHOENIX__PROJECT_NAME=rem

    # S3 Storage
    S3__BUCKET_NAME=rem-storage
    S3__REGION=us-east-1
    S3__ENDPOINT_URL=http://localhost:9000  # For MinIO
    S3__ACCESS_KEY_ID=minioadmin
    S3__SECRET_ACCESS_KEY=minioadmin

    # Environment
    ENVIRONMENT=development
    TEAM=rem
"""

import os
import hashlib
from pydantic import Field, field_validator, FieldValidationInfo
from pydantic_settings import BaseSettings, SettingsConfigDict
from loguru import logger


class LLMSettings(BaseSettings):
    """
    LLM provider settings for Pydantic AI agents.

    Environment variables (accepts both prefixed and unprefixed):
        LLM__DEFAULT_MODEL or DEFAULT_MODEL - Default model (format: provider:model-id)
        LLM__DEFAULT_TEMPERATURE or DEFAULT_TEMPERATURE - Temperature for generation
        LLM__MAX_RETRIES or MAX_RETRIES - Max agent request retries
        LLM__EVALUATOR_MODEL or EVALUATOR_MODEL - Model for LLM-as-judge evaluation
        LLM__OPENAI_API_KEY or OPENAI_API_KEY - OpenAI API key
        LLM__ANTHROPIC_API_KEY or ANTHROPIC_API_KEY - Anthropic API key
        LLM__EMBEDDING_PROVIDER or EMBEDDING_PROVIDER - Default embedding provider (openai)
        LLM__EMBEDDING_MODEL or EMBEDDING_MODEL - Default embedding model name
    """

    model_config = SettingsConfigDict(
        env_prefix="LLM__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    default_model: str = Field(
        default="anthropic:claude-sonnet-4-5-20250929",
        description="Default LLM model (format: provider:model-id)",
    )

    default_temperature: float = Field(
        default=0.5,
        ge=0.0,
        le=1.0,
        description="Default temperature (0.0-0.3: analytical, 0.7-1.0: creative)",
    )

    max_retries: int = Field(
        default=10,
        description="Maximum agent request retries (prevents infinite loops from tool errors)",
    )

    default_max_iterations: int = Field(
        default=20,
        description="Default max iterations for agentic calls (limits total LLM requests per agent.run())",
    )

    evaluator_model: str = Field(
        default="gpt-4.1",
        description="Model for LLM-as-judge evaluators (separate from generation model)",
    )

    query_agent_model: str = Field(
        default="cerebras:qwen-3-32b",
        description="Model for REM Query Agent (natural language to REM query). Cerebras Qwen 3-32B provides ultra-fast inference (1.2s reasoning, 2400 tok/s). Alternative: cerebras:llama-3.3-70b, gpt-4o-mini, or claude-sonnet-4.5",
    )

    openai_api_key: str | None = Field(
        default=None,
        description="OpenAI API key for GPT models (reads from LLM__OPENAI_API_KEY or OPENAI_API_KEY)",
    )

    anthropic_api_key: str | None = Field(
        default=None,
        description="Anthropic API key for Claude models (reads from LLM__ANTHROPIC_API_KEY or ANTHROPIC_API_KEY)",
    )

    embedding_provider: str = Field(
        default="openai",
        description="Default embedding provider (currently only openai supported)",
    )

    embedding_model: str = Field(
        default="text-embedding-3-small",
        description="Default embedding model (provider-specific model name)",
    )

    @field_validator("openai_api_key", mode="before")
    @classmethod
    def validate_openai_api_key(cls, v):
        """Fallback to OPENAI_API_KEY if LLM__OPENAI_API_KEY not set (LLM__ takes precedence)."""
        if v is None:
            return os.getenv("OPENAI_API_KEY")
        return v

    @field_validator("anthropic_api_key", mode="before")
    @classmethod
    def validate_anthropic_api_key(cls, v):
        """Fallback to ANTHROPIC_API_KEY if LLM__ANTHROPIC_API_KEY not set (LLM__ takes precedence)."""
        if v is None:
            return os.getenv("ANTHROPIC_API_KEY")
        return v


class MCPSettings(BaseSettings):
    """
    MCP server settings.

    MCP server is mounted at /api/v1/mcp with FastMCP.
    Can be accessed via:
    - HTTP transport (production): /api/v1/mcp
    - SSE transport (compatible with Claude Desktop)

    Environment variables:
        MCP_SERVER_{NAME} - Server URLs for MCP client connections
    """

    model_config = SettingsConfigDict(
        env_prefix="MCP__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    @staticmethod
    def get_server_url(server_name: str) -> str | None:
        """
        Get MCP server URL from environment variable.

        Args:
            server_name: Server name (e.g., "test", "prod")

        Returns:
            Server URL or None if not configured

        Example:
            MCP_SERVER_TEST=http://localhost:8000/api/v1/mcp
        """
        import os

        env_key = f"MCP_SERVER_{server_name.upper()}"
        return os.getenv(env_key)


class OTELSettings(BaseSettings):
    """
    OpenTelemetry observability settings.

    Integrates with OpenTelemetry Collector for distributed tracing.
    Uses OTLP protocol to export to Arize Phoenix or other OTLP backends.

    Environment variables:
        OTEL__ENABLED - Enable instrumentation (default: false for local dev)
        OTEL__SERVICE_NAME - Service name for traces
        OTEL__COLLECTOR_ENDPOINT - OTLP endpoint (gRPC: 4317, HTTP: 4318)
        OTEL__PROTOCOL - Protocol to use (grpc or http)
        OTEL__EXPORT_TIMEOUT - Export timeout in milliseconds
    """

    model_config = SettingsConfigDict(
        env_prefix="OTEL__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    enabled: bool = Field(
        default=False,
        description="Enable OpenTelemetry instrumentation (disabled by default for local dev)",
    )

    service_name: str = Field(
        default="rem-api",
        description="Service name for traces",
    )

    collector_endpoint: str = Field(
        default="http://localhost:4318",
        description="OTLP collector endpoint (HTTP: 4318, gRPC: 4317)",
    )

    protocol: str = Field(
        default="http",
        description="OTLP protocol (http or grpc)",
    )

    export_timeout: int = Field(
        default=10000,
        description="Export timeout in milliseconds",
    )


class PhoenixSettings(BaseSettings):
    """
    Arize Phoenix settings for LLM observability and evaluation.

    Phoenix provides:
    - OpenTelemetry-based LLM tracing (OpenInference conventions)
    - Experiment tracking
    - Evaluation feedback

    Environment variables:
        PHOENIX__ENABLED - Enable Phoenix integration
        PHOENIX__BASE_URL - Phoenix base URL (for client connections)
        PHOENIX__API_KEY - Phoenix API key (cloud instances)
        PHOENIX__COLLECTOR_ENDPOINT - Phoenix OTLP endpoint
        PHOENIX__PROJECT_NAME - Phoenix project name for trace organization
    """

    model_config = SettingsConfigDict(
        env_prefix="PHOENIX__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    enabled: bool = Field(
        default=False,
        description="Enable Phoenix integration (disabled by default for local dev)",
    )

    base_url: str = Field(
        default="http://localhost:6006",
        description="Phoenix base URL for client connections (default local Phoenix port)",
    )

    api_key: str | None = Field(
        default=None,
        description="Arize Phoenix API key for cloud instances",
    )

    collector_endpoint: str = Field(
        default="http://localhost:6006/v1/traces",
        description="Phoenix OTLP endpoint for traces (default local Phoenix port)",
    )

    project_name: str = Field(
        default="rem",
        description="Phoenix project name for trace organization",
    )


class GoogleOAuthSettings(BaseSettings):
    """
    Google OAuth settings.

    Environment variables:
        AUTH__GOOGLE__CLIENT_ID - Google OAuth client ID
        AUTH__GOOGLE__CLIENT_SECRET - Google OAuth client secret
        AUTH__GOOGLE__REDIRECT_URI - OAuth callback URL
        AUTH__GOOGLE__HOSTED_DOMAIN - Restrict to Google Workspace domain
    """

    model_config = SettingsConfigDict(
        env_prefix="AUTH__GOOGLE__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    client_id: str = Field(default="", description="Google OAuth client ID")
    client_secret: str = Field(default="", description="Google OAuth client secret")
    redirect_uri: str = Field(
        default="http://localhost:8000/api/auth/google/callback",
        description="OAuth redirect URI",
    )
    hosted_domain: str | None = Field(
        default=None, description="Restrict to Google Workspace domain (e.g., example.com)"
    )


class MicrosoftOAuthSettings(BaseSettings):
    """
    Microsoft Entra ID OAuth settings.

    Environment variables:
        AUTH__MICROSOFT__CLIENT_ID - Application (client) ID
        AUTH__MICROSOFT__CLIENT_SECRET - Client secret
        AUTH__MICROSOFT__REDIRECT_URI - OAuth callback URL
        AUTH__MICROSOFT__TENANT - Tenant ID or common/organizations/consumers
    """

    model_config = SettingsConfigDict(
        env_prefix="AUTH__MICROSOFT__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    client_id: str = Field(default="", description="Microsoft Application ID")
    client_secret: str = Field(default="", description="Microsoft client secret")
    redirect_uri: str = Field(
        default="http://localhost:8000/api/auth/microsoft/callback",
        description="OAuth redirect URI",
    )
    tenant: str = Field(
        default="common",
        description="Tenant ID or common/organizations/consumers",
    )


class AuthSettings(BaseSettings):
    """
    Authentication settings for OAuth 2.1 / OIDC.

    Supports multiple providers:
    - Google OAuth
    - Microsoft Entra ID
    - Custom OIDC provider

    Environment variables:
        AUTH__ENABLED - Enable authentication (default: true)
        AUTH__ALLOW_ANONYMOUS - Allow rate-limited anonymous access (default: true)
        AUTH__SESSION_SECRET - Secret for session cookie signing
        AUTH__GOOGLE__* - Google OAuth settings
        AUTH__MICROSOFT__* - Microsoft OAuth settings

    Access modes:
    - enabled=true, allow_anonymous=true: Auth available, anonymous gets rate-limited access
    - enabled=true, allow_anonymous=false: Auth required for all requests
    - enabled=false: No auth, all requests treated as default user (dev mode)
    """

    model_config = SettingsConfigDict(
        env_prefix="AUTH__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    enabled: bool = Field(
        default=True,
        description="Enable authentication (OAuth endpoints and middleware)",
    )

    allow_anonymous: bool = Field(
        default=True,
        description=(
            "Allow anonymous (unauthenticated) access with rate limits. "
            "When true, requests without auth get ANONYMOUS tier rate limits. "
            "When false, all requests require authentication."
        ),
    )

    session_secret: str = Field(
        default="",
        description="Secret key for session cookie signing (generate with secrets.token_hex(32))",
    )

    # OAuth provider settings
    google: GoogleOAuthSettings = Field(default_factory=GoogleOAuthSettings)
    microsoft: MicrosoftOAuthSettings = Field(default_factory=MicrosoftOAuthSettings)

    @field_validator("session_secret", mode="before")
    @classmethod
    def generate_dev_secret(cls, v: str | None, info: FieldValidationInfo) -> str:
        # Only generate if not already set and not in production
        if not v and info.data.get("environment") != "production":
            # Deterministic secret for development
            seed_string = f"{info.data.get('team', 'rem')}-{info.data.get('environment', 'development')}-auth-secret-salt"
            logger.warning(
                "AUTH__SESSION_SECRET not set. Generating deterministic secret for non-production environment. "
                "DO NOT use in production."
            )
            return hashlib.sha256(seed_string.encode()).hexdigest()
        elif not v and info.data.get("environment") == "production":
            raise ValueError("AUTH__SESSION_SECRET must be set in production environment.")
        return v


class PostgresSettings(BaseSettings):
    """
    PostgreSQL settings for CloudNativePG.

    Connects to PostgreSQL 18 with pgvector extension running on CloudNativePG.

    Environment variables:
        POSTGRES__ENABLED - Enable database connection (default: true)
        POSTGRES__CONNECTION_STRING - PostgreSQL connection string
        POSTGRES__POOL_SIZE - Connection pool size
        POSTGRES__POOL_MIN_SIZE - Minimum pool size
        POSTGRES__POOL_MAX_SIZE - Maximum pool size
        POSTGRES__POOL_TIMEOUT - Connection timeout in seconds
        POSTGRES__STATEMENT_TIMEOUT - Statement timeout in milliseconds
    """

    model_config = SettingsConfigDict(
        env_prefix="POSTGRES__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    enabled: bool = Field(
        default=True,
        description="Enable database connection (set to false for testing without DB)",
    )

    connection_string: str = Field(
        default="postgresql://rem:rem@localhost:5050/rem",
        description="PostgreSQL connection string (default uses Docker Compose port 5050)",
    )

    pool_size: int = Field(
        default=10,
        description="Connection pool size (deprecated, use pool_min_size/pool_max_size)",
    )

    pool_min_size: int = Field(
        default=5,
        description="Minimum number of connections in pool",
    )

    pool_max_size: int = Field(
        default=20,
        description="Maximum number of connections in pool",
    )

    pool_timeout: int = Field(
        default=30,
        description="Connection timeout in seconds",
    )

    statement_timeout: int = Field(
        default=30000,
        description="Statement timeout in milliseconds (30 seconds default)",
    )

    @property
    def user(self) -> str:
        from urllib.parse import urlparse
        return urlparse(self.connection_string).username or "postgres"

    @property
    def password(self) -> str | None:
        from urllib.parse import urlparse
        return urlparse(self.connection_string).password

    @property
    def database(self) -> str:
        from urllib.parse import urlparse
        return urlparse(self.connection_string).path.lstrip("/")

    @property
    def host(self) -> str:
        from urllib.parse import urlparse
        return urlparse(self.connection_string).hostname or "localhost"

    @property
    def port(self) -> int:
        from urllib.parse import urlparse
        return urlparse(self.connection_string).port or 5432


class MigrationSettings(BaseSettings):
    """
    Migration settings.

    Environment variables:
        MIGRATION__AUTO_UPGRADE - Automatically run migrations on startup
        MIGRATION__MODE - Migration safety mode (permissive, additive, strict)
        MIGRATION__ALLOW_DROP_COLUMNS - Allow DROP COLUMN operations
        MIGRATION__ALLOW_DROP_TABLES - Allow DROP TABLE operations
        MIGRATION__ALLOW_ALTER_COLUMNS - Allow ALTER COLUMN TYPE operations
        MIGRATION__ALLOW_RENAME_COLUMNS - Allow RENAME COLUMN operations
        MIGRATION__ALLOW_RENAME_TABLES - Allow RENAME TABLE operations
        MIGRATION__UNSAFE_ALTER_WARNING - Warn on unsafe ALTER operations
    """

    model_config = SettingsConfigDict(
        env_prefix="MIGRATION__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    auto_upgrade: bool = Field(
        default=True,
        description="Automatically run database migrations on startup",
    )
    
    mode: str = Field(
        default="permissive",
        description="Migration safety mode: permissive, additive, strict",
    )

    allow_drop_columns: bool = Field(
        default=False,
        description="Allow DROP COLUMN operations (unsafe)",
    )

    allow_drop_tables: bool = Field(
        default=False,
        description="Allow DROP TABLE operations (unsafe)",
    )

    allow_alter_columns: bool = Field(
        default=True,
        description="Allow ALTER COLUMN TYPE operations (can be unsafe)",
    )

    allow_rename_columns: bool = Field(
        default=True,
        description="Allow RENAME COLUMN operations (can be unsafe)",
    )

    allow_rename_tables: bool = Field(
        default=True,
        description="Allow RENAME TABLE operations (can be unsafe)",
    )

    unsafe_alter_warning: bool = Field(
        default=True,
        description="Emit warning on potentially unsafe ALTER operations",
    )


class StorageSettings(BaseSettings):
    """
    Storage provider settings.

    Controls which storage backend to use for file uploads and artifacts.

    Environment variables:
        STORAGE__PROVIDER - Storage provider (local or s3, default: local)
        STORAGE__BASE_PATH - Base path for local filesystem storage (default: ~/.rem/fs)
    """

    model_config = SettingsConfigDict(
        env_prefix="STORAGE__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    provider: str = Field(
        default="local",
        description="Storage provider: 'local' for filesystem, 's3' for AWS S3",
    )

    base_path: str = Field(
        default="~/.rem/fs",
        description="Base path for local filesystem storage (only used when provider='local')",
    )


class S3Settings(BaseSettings):
    """
    S3 storage settings for file uploads and artifacts.

    Uses IRSA (IAM Roles for Service Accounts) for AWS permissions in EKS.
    For local development, can use MinIO or provide access keys.

    Bucket Naming Convention:
        - Default: rem-io-{environment} (e.g., rem-io-development, rem-io-staging, rem-io-production)
        - Matches Kubernetes manifest convention for consistency
        - Override with S3__BUCKET_NAME environment variable

    Path Convention:
        Uploads: s3://{bucket}/{version}/uploads/{user_id}/{yyyy}/{mm}/{dd}/{filename}
        Parsed:  s3://{bucket}/{version}/parsed/{user_id}/{yyyy}/{mm}/{dd}/{filename}/{resource}

    Environment variables:
        S3__BUCKET_NAME - S3 bucket name (default: rem-io-development)
        S3__VERSION - API version for paths (default: v1)
        S3__UPLOADS_PREFIX - Uploads directory prefix (default: uploads)
        S3__PARSED_PREFIX - Parsed content directory prefix (default: parsed)
        S3__REGION - AWS region
        S3__ENDPOINT_URL - Custom endpoint (for MinIO, LocalStack)
        S3__ACCESS_KEY_ID - AWS access key (not needed with IRSA)
        S3__SECRET_ACCESS_KEY - AWS secret key (not needed with IRSA)
        S3__USE_SSL - Use SSL for connections (default: true)
    """

    model_config = SettingsConfigDict(
        env_prefix="S3__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    bucket_name: str = Field(
        default="rem-io-development",
        description="S3 bucket name (convention: rem-io-{environment})",
    )

    version: str = Field(
        default="v1",
        description="API version for S3 path structure",
    )

    uploads_prefix: str = Field(
        default="uploads",
        description="Prefix for uploaded files (e.g., 'uploads' -> bucket/v1/uploads/...)",
    )

    parsed_prefix: str = Field(
        default="parsed",
        description="Prefix for parsed content (e.g., 'parsed' -> bucket/v1/parsed/...)",
    )

    region: str = Field(
        default="us-east-1",
        description="AWS region",
    )

    endpoint_url: str | None = Field(
        default=None,
        description="Custom S3 endpoint (for MinIO, LocalStack)",
    )

    access_key_id: str | None = Field(
        default=None,
        description="AWS access key ID (not needed with IRSA in EKS)",
    )

    secret_access_key: str | None = Field(
        default=None,
        description="AWS secret access key (not needed with IRSA in EKS)",
    )

    use_ssl: bool = Field(
        default=True,
        description="Use SSL for S3 connections",
    )


class ChunkingSettings(BaseSettings):
    """
    Document chunking settings for semantic text splitting.

    Uses semchunk for semantic-aware text chunking that respects document structure.
    Generous chunk sizes (couple paragraphs) with reasonable overlaps for context.

    Environment variables:
        CHUNKING__CHUNK_SIZE - Target chunk size in characters
        CHUNKING__OVERLAP - Overlap between chunks in characters
        CHUNKING__MIN_CHUNK_SIZE - Minimum chunk size (avoid tiny chunks)
        CHUNKING__MAX_CHUNK_SIZE - Maximum chunk size (hard limit)
    """

    model_config = SettingsConfigDict(
        env_prefix="CHUNKING__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    chunk_size: int = Field(
        default=1500,
        description="Target chunk size in characters (couple paragraphs, ~300-400 words)",
    )

    overlap: int = Field(
        default=200,
        description="Overlap between chunks in characters for context preservation",
    )

    min_chunk_size: int = Field(
        default=100,
        description="Minimum chunk size to avoid tiny fragments",
    )

    max_chunk_size: int = Field(
        default=2500,
        description="Maximum chunk size (hard limit, prevents oversized chunks)",
    )


class ContentSettings(BaseSettings):
    """
    Content provider settings for file processing.

    Defines supported file types for each provider type.
    Allows override of specific extensions via register_provider().

    Environment variables:
        CONTENT__SUPPORTED_TEXT_TYPES - Comma-separated text extensions
        CONTENT__SUPPORTED_DOC_TYPES - Comma-separated document extensions
        CONTENT__SUPPORTED_AUDIO_TYPES - Comma-separated audio extensions
        CONTENT__SUPPORTED_IMAGE_TYPES - Comma-separated image extensions
        CONTENT__IMAGE_VLLM_SAMPLE_RATE - Sampling rate for vision LLM analysis (0.0-1.0)
        CONTENT__IMAGE_VLLM_PROVIDER - Vision provider (anthropic, gemini, openai)
        CONTENT__IMAGE_VLLM_MODEL - Vision model name (provider default if not set)
        CONTENT__CLIP_PROVIDER - CLIP embedding provider (jina, self-hosted)
        CONTENT__CLIP_MODEL - CLIP model name (jina-clip-v1, jina-clip-v2)
        CONTENT__JINA_API_KEY - Jina AI API key for CLIP embeddings
    """

    model_config = SettingsConfigDict(
        env_prefix="CONTENT__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    supported_text_types: list[str] = Field(
        default_factory=lambda: [
            # Plain text
            ".txt",
            ".md",
            ".markdown",
            # Data formats
            ".json",
            ".yaml",
            ".yml",
            ".csv",
            ".tsv",
            ".log",
            # Code files
            ".py",
            ".js",
            ".ts",
            ".tsx",
            ".jsx",
            ".java",
            ".c",
            ".cpp",
            ".h",
            ".rs",
            ".go",
            ".rb",
            ".php",
            ".sh",
            ".bash",
            ".sql",
            # Web files
            ".html",
            ".css",
            ".xml",
        ],
        description="File extensions handled by TextProvider (plain text, code, data files)",
    )

    supported_doc_types: list[str] = Field(
        default_factory=lambda: [
            # Documents
            ".pdf",
            ".docx",
            ".pptx",
            ".xlsx",
            # Images (OCR text extraction)
            ".png",
            ".jpg",
            ".jpeg",
        ],
        description="File extensions handled by DocProvider (Kreuzberg: PDFs, Office docs, images with OCR)",
    )

    supported_audio_types: list[str] = Field(
        default_factory=lambda: [
            ".wav",
            ".mp3",
            ".m4a",
            ".flac",
            ".ogg",
        ],
        description="File extensions handled by AudioProvider (Whisper API transcription)",
    )

    supported_image_types: list[str] = Field(
        default_factory=lambda: [
            ".png",
            ".jpg",
            ".jpeg",
            ".gif",
            ".webp",
        ],
        description="File extensions handled by ImageProvider (vision LLM + CLIP embeddings)",
    )

    image_vllm_sample_rate: float = Field(
        default=0.0,
        ge=0.0,
        le=1.0,
        description="Sampling rate for vision LLM analysis (0.0 = never, 1.0 = always, 0.1 = 10% of images). Gold tier users always get vision analysis.",
    )

    image_vllm_provider: str = Field(
        default="anthropic",
        description="Vision LLM provider: anthropic, gemini, or openai",
    )

    image_vllm_model: str | None = Field(
        default=None,
        description="Vision model name (uses provider default if None)",
    )

    clip_provider: str = Field(
        default="jina",
        description="CLIP embedding provider (jina for API, self-hosted for future KEDA pods)",
    )

    clip_model: str = Field(
        default="jina-clip-v2",
        description="CLIP model for image embeddings (jina-clip-v1, jina-clip-v2, or custom)",
    )

    jina_api_key: str | None = Field(
        default=None,
        description="Jina AI API key for CLIP embeddings (https://jina.ai/embeddings/)",
    )


class SQSSettings(BaseSettings):
    """
    SQS queue settings for file processing.

    Uses IRSA (IAM Roles for Service Accounts) for AWS permissions in EKS.
    For local development, can use access keys.

    Environment variables:
        SQS__QUEUE_URL - SQS queue URL (from Pulumi output)
        SQS__REGION - AWS region
        SQS__MAX_MESSAGES - Max messages per receive (1-10)
        SQS__WAIT_TIME_SECONDS - Long polling wait time
        SQS__VISIBILITY_TIMEOUT - Message visibility timeout
    """

    model_config = SettingsConfigDict(
        env_prefix="SQS__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    queue_url: str = Field(
        default="",
        description="SQS queue URL for file processing events",
    )

    region: str = Field(
        default="us-east-1",
        description="AWS region",
    )

    max_messages: int = Field(
        default=10,
        ge=1,
        le=10,
        description="Maximum messages to receive per batch (1-10)",
    )

    wait_time_seconds: int = Field(
        default=20,
        ge=0,
        le=20,
        description="Long polling wait time in seconds (0-20, 20 recommended)",
    )

    visibility_timeout: int = Field(
        default=300,
        description="Visibility timeout in seconds (should match processing time)",
    )


class ChatSettings(BaseSettings):
    """
    Chat and session context settings.

    Environment variables:
        CHAT__AUTO_INJECT_USER_CONTEXT - Automatically inject user profile into every request (default: false)

    Design Philosophy:
    - Session history is ALWAYS loaded (required for multi-turn conversations)
    - Compression with REM LOOKUP hints keeps session history efficient
    - User context is on-demand by default (agents receive REM LOOKUP hints)
    - When auto_inject_user_context enabled, user profile is loaded and injected

    Session History (always loaded with compression):
    - Each chat request is a single message, so history MUST be recovered
    - Long assistant responses stored as separate Message entities
    - Compressed versions include REM LOOKUP hints: "... [REM LOOKUP session-{id}-msg-{index}] ..."
    - Agent can retrieve full content on-demand using REM LOOKUP
    - Prevents context window bloat while maintaining conversation continuity

    User Context (on-demand by default):
    - Agent system prompt includes: "User ID: {user_id}. To load user profile: Use REM LOOKUP users/{user_id}"
    - Agent decides whether to load profile based on query
    - More efficient for queries that don't need personalization

    User Context (auto-inject when enabled):
    - Set CHAT__AUTO_INJECT_USER_CONTEXT=true
    - User profile automatically loaded and injected into system message
    - Simpler for basic chatbots that always need context
    """

    model_config = SettingsConfigDict(
        env_prefix="CHAT__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    auto_inject_user_context: bool = Field(
        default=False,
        description="Automatically inject user profile into every request (default: false, use REM LOOKUP hint instead)",
    )


class APISettings(BaseSettings):
    """
    API server settings.

    Environment variables:
        API__HOST - Host to bind to (0.0.0.0 for Docker, 127.0.0.1 for local)
        API__PORT - Port to listen on
        API__RELOAD - Enable auto-reload for development
        API__WORKERS - Number of worker processes (production)
        API__LOG_LEVEL - Logging level (debug, info, warning, error)
    """

    model_config = SettingsConfigDict(
        env_prefix="API__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    host: str = Field(
        default="0.0.0.0",
        description="Host to bind to (0.0.0.0 for Docker, 127.0.0.1 for local only)",
    )

    port: int = Field(
        default=8000,
        description="Port to listen on",
    )

    reload: bool = Field(
        default=True,
        description="Enable auto-reload for development (disable in production)",
    )

    workers: int = Field(
        default=1,
        description="Number of worker processes (use >1 in production)",
    )

    log_level: str = Field(
        default="info",
        description="Logging level (debug, info, warning, error, critical)",
    )


class SchemaSettings(BaseSettings):
    """
    Schema search path settings for agent and evaluator schemas.

    Allows extending REM's schema search with custom directories.
    Custom paths are searched BEFORE built-in package schemas.

    Environment variables:
        SCHEMA__PATHS - Semicolon-separated list of directories to search
                        Example: "/app/schemas;/shared/agents;./local-schemas"

    Search Order:
    1. Exact path (if file exists)
    2. Custom paths from SCHEMA__PATHS (in order)
    3. Built-in package schemas (schemas/agents/, schemas/evaluators/, etc.)
    4. Database LOOKUP (if enabled)

    Example:
        # In .env or environment
        SCHEMA__PATHS=/app/custom-agents;/shared/evaluators

        # Then in code
        from rem.utils.schema_loader import load_agent_schema
        schema = load_agent_schema("my-custom-agent")  # Found in /app/custom-agents/
    """

    model_config = SettingsConfigDict(
        env_prefix="SCHEMA__",
        extra="ignore",
    )

    paths: str = Field(
        default="",
        description=(
            "Semicolon-separated list of directories to search for schemas. "
            "These paths are searched BEFORE built-in package schemas. "
            "Example: '/app/schemas;/shared/agents'"
        ),
    )

    @property
    def path_list(self) -> list[str]:
        """Get paths as a list, filtering empty strings."""
        if not self.paths:
            return []
        return [p.strip() for p in self.paths.split(";") if p.strip()]


class GitSettings(BaseSettings):
    """
    Git repository provider settings for versioned schema/experiment syncing.

    Enables syncing of agent schemas, evaluators, and experiments from Git repositories
    using either SSH or HTTPS authentication. Designed for cluster environments where
    secrets are provided via Kubernetes Secrets or similar mechanisms.

    **Use Cases**:
    - Sync agent schemas from versioned repos (repo/schemas/)
    - Sync experiments and evaluation datasets (repo/experiments/)
    - Clone specific tags/releases for reproducible evaluations
    - Support multi-tenancy with per-tenant repo configurations

    **Authentication Methods**:
    1. **SSH** (recommended for production):
       - Uses SSH keys from filesystem or Kubernetes Secrets
       - Path specified via GIT__SSH_KEY_PATH or mounted at /etc/git-secret/ssh
       - Known hosts file at /etc/git-secret/known_hosts

    2. **HTTPS with Personal Access Token** (PAT):
       - GitHub: 5,000 API requests/hour per authenticated user
       - GitLab: Similar rate limits
       - Store PAT in GIT__PERSONAL_ACCESS_TOKEN environment variable

    **Kubernetes Deployment Pattern** (git-sync sidecar):
    ```yaml
    # Secret creation (one-time setup)
    kubectl create secret generic git-creds \\
      --from-file=ssh=$HOME/.ssh/id_rsa \\
      --from-file=known_hosts=$HOME/.ssh/known_hosts

    # Pod spec with secret mounting
    volumes:
      - name: git-secret
        secret:
          secretName: git-creds
          defaultMode: 0400  # Read-only for owner
    containers:
      - name: rem-api
        volumeMounts:
          - name: git-secret
            mountPath: /etc/git-secret
            readOnly: true
        securityContext:
          fsGroup: 65533  # Make secrets readable by git user
    ```

    **Path Conventions**:
    - Agent schemas: {repo_root}/schemas/
    - Experiments: {repo_root}/experiments/
    - Evaluators: {repo_root}/schemas/evaluators/

    **Performance & Caching**:
    - Clones cached locally in {cache_dir}/{repo_hash}/
    - Supports shallow clones (--depth=1) for faster syncing
    - Periodic refresh via cron jobs or git-sync sidecar

    Environment variables:
        GIT__ENABLED - Enable Git provider (default: False)
        GIT__DEFAULT_REPO_URL - Default Git repository URL (ssh:// or https://)
        GIT__DEFAULT_BRANCH - Default branch to clone (default: main)
        GIT__SSH_KEY_PATH - Path to SSH private key (default: /etc/git-secret/ssh)
        GIT__KNOWN_HOSTS_PATH - Path to known_hosts file (default: /etc/git-secret/known_hosts)
        GIT__PERSONAL_ACCESS_TOKEN - GitHub/GitLab PAT for HTTPS auth
        GIT__CACHE_DIR - Local cache directory for cloned repos
        GIT__SHALLOW_CLONE - Use shallow clone (--depth=1) for faster sync
        GIT__VERIFY_SSL - Verify SSL certificates for HTTPS (default: True)

    **Security Best Practices**:
    - Store SSH keys in Kubernetes Secrets, never in environment variables
    - Use read-only SSH keys (deploy keys) with minimal permissions
    - Enable known_hosts verification to prevent MITM attacks
    - Rotate PATs regularly (90-day expiration recommended)
    - Use IRSA/Workload Identity for cloud-provider Git services
    """

    model_config = SettingsConfigDict(
        env_prefix="GIT__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    enabled: bool = Field(
        default=False,
        description="Enable Git provider for syncing schemas/experiments from Git repos",
    )

    default_repo_url: str | None = Field(
        default=None,
        description="Default Git repository URL (ssh://git@github.com/org/repo.git or https://github.com/org/repo.git)",
    )

    default_branch: str = Field(
        default="main",
        description="Default branch to clone/checkout (main, master, develop, etc.)",
    )

    ssh_key_path: str = Field(
        default="/etc/git-secret/ssh",
        description="Path to SSH private key (Kubernetes Secret mount point or local path)",
    )

    known_hosts_path: str = Field(
        default="/etc/git-secret/known_hosts",
        description="Path to known_hosts file for SSH host verification",
    )

    personal_access_token: str | None = Field(
        default=None,
        description="Personal Access Token (PAT) for HTTPS authentication (GitHub, GitLab, etc.)",
    )

    cache_dir: str = Field(
        default="/tmp/rem-git-cache",
        description="Local cache directory for cloned repositories",
    )

    shallow_clone: bool = Field(
        default=True,
        description="Use shallow clone (--depth=1) for faster syncing (recommended for large repos)",
    )

    verify_ssl: bool = Field(
        default=True,
        description="Verify SSL certificates for HTTPS connections (disable for self-signed certs)",
    )

    sync_interval: int = Field(
        default=300,
        description="Sync interval in seconds for git-sync sidecar pattern (default: 5 minutes)",
    )


class TestSettings(BaseSettings):
    """
    Test environment settings.

    Environment variables:
        TEST__USER_EMAIL - Test user email (default: test@rem.ai)
        TEST__USER_ID - Test user UUID (auto-generated from email if not provided)

    The user_id is a deterministic UUID v5 generated from the email address.
    This ensures consistent IDs across test runs and allows tests to use both
    email and UUID interchangeably.
    """

    model_config = SettingsConfigDict(
        env_prefix="TEST__",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    user_email: str = Field(
        default="test@rem.ai",
        description="Test user email address",
    )

    user_id: str | None = Field(
        default=None,
        description="Test user UUID (auto-generated from email if not provided)",
    )

    @property
    def effective_user_id(self) -> str:
        """
        Get the effective user ID (either explicit or generated from email).

        Returns a deterministic UUID v5 based on the email address if user_id
        is not explicitly set. This ensures consistent test data across runs.
        """
        if self.user_id:
            return self.user_id

        # Generate deterministic UUID v5 from email
        # Using DNS namespace as the base (standard practice for email-based UUIDs)
        import uuid
        return str(uuid.uuid5(uuid.NAMESPACE_DNS, self.user_email))


class Settings(BaseSettings):
    """
    Global application settings.

    Aggregates all nested settings groups with environment variable support.
    Uses double underscore delimiter for nested variables (LLM__DEFAULT_MODEL).

    Environment variables:
        TEAM - Team/project name for observability
        ENVIRONMENT - Environment (development, staging, production)
        DOMAIN - Public domain for OAuth discovery
        ROOT_PATH - Root path for reverse proxy (e.g., /rem for ALB routing)
        TEST__USER_ID - Default user ID for integration tests
    """

    model_config = SettingsConfigDict(
        env_file=".env",
        env_file_encoding="utf-8",
        env_nested_delimiter="__",
        extra="ignore",
    )

    app_name: str = Field(
        default="REM",
        description="Application/API name used in docs, titles, and user-facing text",
    )

    team: str = Field(
        default="rem",
        description="Team or project name for observability",
    )

    environment: str = Field(
        default="development",
        description="Environment (development, staging, production)",
    )

    domain: str | None = Field(
        default=None,
        description="Public domain for OAuth discovery (e.g., https://api.example.com)",
    )

    root_path: str = Field(
        default="",
        description="Root path for reverse proxy (e.g., /rem for ALB routing)",
    )

    sql_dir: str = Field(
        default="src/rem/sql",
        description="Directory for SQL files and migrations",
    )

    # Nested settings groups
    api: APISettings = Field(default_factory=APISettings)
    chat: ChatSettings = Field(default_factory=ChatSettings)
    llm: LLMSettings = Field(default_factory=LLMSettings)
    mcp: MCPSettings = Field(default_factory=MCPSettings)
    otel: OTELSettings = Field(default_factory=OTELSettings)
    phoenix: PhoenixSettings = Field(default_factory=PhoenixSettings)
    auth: AuthSettings = Field(default_factory=AuthSettings)
    postgres: PostgresSettings = Field(default_factory=PostgresSettings)
    migration: MigrationSettings = Field(default_factory=MigrationSettings)
    storage: StorageSettings = Field(default_factory=StorageSettings)
    s3: S3Settings = Field(default_factory=S3Settings)
    git: GitSettings = Field(default_factory=GitSettings)
    sqs: SQSSettings = Field(default_factory=SQSSettings)
    chunking: ChunkingSettings = Field(default_factory=ChunkingSettings)
    content: ContentSettings = Field(default_factory=ContentSettings)
    schema_search: SchemaSettings = Field(default_factory=SchemaSettings)
    test: TestSettings = Field(default_factory=TestSettings)


# Load configuration from ~/.rem/config.yaml before initializing settings
# This allows user configuration to be merged with environment variables
# Set REM_SKIP_CONFIG_FILE=true to disable (useful for development with .env)
if not os.getenv("REM_SKIP_CONFIG_FILE", "").lower() in ("true", "1", "yes"):
    try:
        from rem.config import load_config, merge_config_to_env

        _config = load_config()
        if _config:
            merge_config_to_env(_config)
    except ImportError:
        # config module not available (e.g., during initial setup)
        pass

# Global settings singleton
settings = Settings()

# Sync API keys to environment for pydantic-ai
# Pydantic AI providers check environment directly, so we need to ensure
# API keys from settings (LLM__*_API_KEY) are also available without prefix
if settings.llm.openai_api_key and not os.getenv("OPENAI_API_KEY"):
    os.environ["OPENAI_API_KEY"] = settings.llm.openai_api_key

if settings.llm.anthropic_api_key and not os.getenv("ANTHROPIC_API_KEY"):
    os.environ["ANTHROPIC_API_KEY"] = settings.llm.anthropic_api_key
