"""
ETL Generator - Modern ETL pipeline framework.

Two APIs:
=========
1. Programmatic API (Pipeline with | operator):
   >>> pipeline = Pipeline(HTTPExtractor(url="...")) | Rename(...) | PostgresLoader(...)
   >>> result = await pipeline.run()

2. Config-driven API (YAML files):
   >>> # From explicit files (most flexible)
   >>> pipeline = Pipeline.from_config_files(
   ...     extract="configs/extract.yaml",
   ...     load="configs/load.yaml",
   ...     variables={"API_KEY": "secret"}
   ... )
   >>>
   >>> # From directory (expects extract.yaml, transform.yaml, load.yaml)
   >>> pipeline = Pipeline.from_config_dir("pipelines/users/")
   >>>
   >>> # From single file (pipeline.yaml)
   >>> pipeline = Pipeline.from_config_file("pipelines/users/pipeline.yaml")

Quick Start:
============
    from pycharter.etl_generator import (
        Pipeline, HTTPExtractor, PostgresLoader,
        Rename, AddField, Filter
    )

    # Programmatic
    pipeline = (
        Pipeline(HTTPExtractor(url="https://api.example.com/users"))
        | Rename({"user_name": "name"})
        | AddField("full_name", "${first_name} ${last_name}")
        | PostgresLoader(connection_string="...", table="users")
    )
    result = await pipeline.run()

    # Config-driven (explicit files)
    pipeline = Pipeline.from_config_files(
        extract="pipelines/users/extract.yaml",
        load="pipelines/users/load.yaml",
        variables={"DATA_DIR": "./data"}
    )
    result = await pipeline.run()

Config Format:
==============
    Single-file (pipeline.yaml):
        name: users_pipeline
        extract:
          type: http  # Required: http | file | database | cloud_storage
          url: https://api.example.com/users
        transform:
          - rename: {userId: user_id}
          - add:
              full_name: "${first_name} ${last_name}"
        load:
          type: postgres  # Required: postgres | sqlite | file | cloud_storage
          table: users
          database:
            url: ${DATABASE_URL}
"""

from __future__ import annotations

from typing import Any

# DLQ sync wrappers
from pycharter.etl_generator._dlq_sync import (
    add_batch_sync,
    add_record_sync,
    retry_record_sync,
)
from pycharter.etl_generator.builder import PipelineBuilder

# Config models (Pydantic)
from pycharter.etl_generator.config_models import (
    DLQConfig,
    ExtractConfig,
    ExtractValidationConfig,
    LineageConfig,
    LoadConfig,
    LoadValidationConfig,
    SettingsConfig,
    TransformConfig,
    parse_extract_config,
    parse_load_config,
    parse_settings_config,
    parse_transform_config,
)
from pycharter.etl_generator.config_validator import (
    ConfigValidationError,
    ConfigValidator,
    validate_config,
)
from pycharter.etl_generator.context import PipelineContext

# Expression evaluation
from pycharter.etl_generator.expression import (
    ExpressionError,
    ExpressionEvaluator,
    evaluate_expression,
    is_expression,
)

# Extractors
from pycharter.etl_generator.extractors import (
    BaseExtractor,
    CloudStorageExtractor,
    DatabaseExtractor,
    ExtractorFactory,
    FileExtractor,
    HTTPExtractor,
)
from pycharter.etl_generator.factory import PipelineFactory

# Lineage (OpenLineage integration)
from pycharter.etl_generator.lineage import LineageEmitter, create_lineage_emitter

# Loaders
from pycharter.etl_generator.loaders import (
    BaseLoader,
    CloudStorageLoader,
    DatabaseLoader,
    FileLoader,
    LoaderFactory,
    PostgresLoader,
)

# Orchestrator and pipeline discovery (config-driven ETL entry points)
from pycharter.etl_generator.orchestrator import ETLOrchestrator

# Core pipeline classes
from pycharter.etl_generator.pipeline import Pipeline
from pycharter.etl_generator.protocols import (
    AckableExtractor,
    Extractor,
    Loader,
    Transformer,
)

# Quality checks
from pycharter.etl_generator.quality import (
    PostLoadChecker,
    QualityChecker,
    QualityCheckResult,
    QualityCheckSeverity,
    QualityCheckStatus,
    QualityReport,
    create_quality_checker,
)
from pycharter.etl_generator.rate_limiter import AsyncRateLimiter
from pycharter.etl_generator.result import BatchResult, LoadResult, PipelineResult

# State persistence (incremental extraction)
from pycharter.etl_generator.state import (
    FileStateStore,
    PipelineState,
    SqliteStateStore,
    StateStore,
    create_state_store,
)

# Testing framework
from pycharter.etl_generator.testing import (
    MockExtractor,
    MockLoader,
    PipelineTestHarness,
    TestFixture,
    assert_field_values,
    assert_fields_present,
    assert_no_field,
    assert_record_count,
    assert_records_match,
    assert_schema_shape,
    load_fixture,
    load_test_fixture,
    validate_pipeline_config,
)

# Transformers
from pycharter.etl_generator.transformers import (
    AddField,
    BaseTransformer,
    Convert,
    CustomFunction,
    Default,
    Drop,
    Filter,
    FlatMap,
    Map,
    Rename,
    Select,
    TransformerChain,
)

# ETL Validation
from pycharter.etl_generator.validation import (
    ETLValidationError,
    ETLValidator,
    create_dlq,
    create_etl_validator,
    resolve_contract,
    resolve_schema,
)

# Config validation and errors
from pycharter.shared.errors import ConfigLoadError


def create_orchestrator(
    contract_dir: str | None = None, **kwargs: Any
) -> ETLOrchestrator:
    """Factory helper that returns ETLOrchestrator(contract_dir=contract_dir, **kwargs)."""
    return ETLOrchestrator(contract_dir=contract_dir, **kwargs)


__all__ = [
    # Core
    "Pipeline",
    "PipelineBuilder",
    "PipelineContext",
    "PipelineResult",
    "BatchResult",
    "LoadResult",
    # Protocols
    "Extractor",
    "AckableExtractor",
    "Transformer",
    "Loader",
    # Config validation
    "ConfigLoadError",
    "ConfigValidator",
    "ConfigValidationError",
    "validate_config",
    # Config models
    "ExtractConfig",
    "LoadConfig",
    "TransformConfig",
    "SettingsConfig",
    "ExtractValidationConfig",
    "LoadValidationConfig",
    "DLQConfig",
    "LineageConfig",
    "parse_extract_config",
    "parse_load_config",
    "parse_transform_config",
    "parse_settings_config",
    # ETL Validation
    "ETLValidator",
    "ETLValidationError",
    "resolve_schema",
    "resolve_contract",
    "create_dlq",
    "create_etl_validator",
    # Expressions
    "ExpressionEvaluator",
    "ExpressionError",
    "evaluate_expression",
    "is_expression",
    # Extractors
    "BaseExtractor",
    "HTTPExtractor",
    "FileExtractor",
    "DatabaseExtractor",
    "CloudStorageExtractor",
    "KafkaExtractor",
    "RabbitMQExtractor",
    "SQSExtractor",
    "ExtractorFactory",
    # Transformers
    "BaseTransformer",
    "TransformerChain",
    "Rename",
    "AddField",
    "Drop",
    "Select",
    "Filter",
    "Convert",
    "Default",
    "Map",
    "FlatMap",
    "CustomFunction",
    # Loaders
    "BaseLoader",
    "PostgresLoader",
    "DatabaseLoader",
    "FileLoader",
    "CloudStorageLoader",
    "LoaderFactory",
    # Orchestrator and pipeline discovery
    "ETLOrchestrator",
    "PipelineFactory",
    "AsyncRateLimiter",
    "create_orchestrator",
    # Lineage
    "LineageEmitter",
    "create_lineage_emitter",
    # State persistence
    "PipelineState",
    "StateStore",
    "FileStateStore",
    "SqliteStateStore",
    "create_state_store",
    # Quality checks
    "QualityChecker",
    "PostLoadChecker",
    "QualityCheckResult",
    "QualityCheckSeverity",
    "QualityCheckStatus",
    "QualityReport",
    "create_quality_checker",
    # DLQ sync wrappers
    "add_record_sync",
    "add_batch_sync",
    "retry_record_sync",
    # Testing framework
    "MockExtractor",
    "MockLoader",
    "PipelineTestHarness",
    "TestFixture",
    "assert_records_match",
    "assert_record_count",
    "assert_fields_present",
    "assert_no_field",
    "assert_field_values",
    "assert_schema_shape",
    "load_fixture",
    "load_test_fixture",
    "validate_pipeline_config",
]
