"""TPC-DI batch processing logic for handling incremental data loads.

Copyright 2026 Joe Harris / BenchBox Project

TPC Benchmark™ DI (TPC-DI) - Copyright © Transaction Processing Performance Council
This implementation is based on the TPC-DI specification.

Licensed under the MIT License. See LICENSE file in the project root for details.

This module provides comprehensive ETL (Extract, Transform, Load) capabilities for TPC-DI
batch processing, including:

1. Historical Load (Batch 1): Initial data warehouse population
2. Incremental Load (Batch 2): Daily updates without SCD changes
3. SCD Batch Load (Batch 3): Incremental load with Slowly Changing Dimension processing

Key Features:
- Complete ETL pipeline with Extract → Transform → Load → Validate operations
- Support for multiple source data formats (CSV, XML, pipe-delimited, fixed-width)
- TPC-DI specific business rules and data transformations
- SCD Type 2 processing for dimension tables
- Comprehensive data quality validation and monitoring
- Parallel processing support for large datasets
- Data lineage tracking for audit trails
- Performance metrics and monitoring
- Dependency management between batch operations

Example Usage:
    # Initialize batch processor
    processor = BatchProcessor(
        scale_factor=1.0,
        parallel_processing=True
    )

    # Configure ETL operations
    extract_op = ExtractOperation(
        source_dir=Path("/data/tpcdi/sources"),
        file_patterns={
            "DimCustomer": "Customer_{batch_id}.txt",
            "DimAccount": "Account_{batch_id}.txt"
        }
    )

    transform_op = TransformOperation(
        transformation_rules={
            "DimCustomer": ["apply_scd", "validate_customer_data"],
            "DimAccount": ["apply_business_rules"]
        }
    )

    load_op = LoadOperation(
        connection_config={"host": "localhost", "database": "tpcdi"},
        load_strategies={
            "DimCustomer": "scd2",
            "DimAccount": "upsert"
        }
    )

    validate_op = ValidationOperation(
        validation_rules={
            "DimCustomer": ["completeness", "referential_integrity"],
            "DimAccount": ["data_quality"]
        }
    )

    # Include operations in pipeline
    processor.add_operation(extract_op)
    processor.add_operation(transform_op)
    processor.add_operation(load_op)
    processor.add_operation(validate_op)

    # Process historical load
    batch_status = processor.process_historical_load(
        source_config={"data_dir": "/data/tpcdi"},
        target_config={"database": "tpcdi_warehouse"}
    )

    # Process incremental loads
    batch_status = processor.process_incremental_batch(
        batch_number=2,
        batch_date=date(2024, 1, 1),
        source_config={"data_dir": "/data/tpcdi/incremental"},
        target_config={"database": "tpcdi_warehouse"}
    )

    # Get processing statistics
    stats = processor.get_processing_statistics()
    emit(f"Success rate: {stats['success_rate']:.2%}")
"""

import contextlib
import gc
import logging
import multiprocessing
import queue
import threading
import xml.etree.ElementTree as ET
from abc import ABC, abstractmethod
from collections.abc import Generator, Iterator
from concurrent.futures import (
    as_completed,
)
from dataclasses import dataclass, field
from datetime import date, datetime
from enum import Enum
from pathlib import Path
from typing import (
    Any,
    Optional,
)

import pandas as pd

from benchbox.utils.clock import elapsed_seconds, mono_time

# Configure logging
logger = logging.getLogger(__name__)


class SimpleMemoryMonitor:
    """Simple memory monitor placeholder."""

    def check_memory_usage(self) -> bool:
        """Check if memory usage is high."""
        return False

    def get_current_usage_mb(self) -> float:
        """Get current memory usage in MB."""
        return 0.0


class SimpleProgressTracker:
    """Simple progress tracker placeholder."""

    def __init__(self):
        self.start_time = mono_time()

    def update_progress(self, processed: int, count: int, description: str = "") -> None:
        """Update progress."""

    def get_elapsed_time(self) -> float:
        """Get elapsed time."""
        return elapsed_seconds(self.start_time)


class BatchType(Enum):
    """Types of TPC-DI batch processing."""

    HISTORICAL = "historical"  # Initial data warehouse population
    INCREMENTAL = "incremental"  # Daily incremental updates
    SCD_BATCH = "scd_batch"  # Incremental with SCD changes


class OperationType(Enum):
    """Types of ETL operations."""

    EXTRACT = "extract"
    TRANSFORM = "transform"
    LOAD = "load"
    VALIDATE = "validate"


@dataclass
class BatchMetrics:
    """Comprehensive metrics for batch processing."""

    rows_processed: dict[str, int] = field(default_factory=dict)
    rows_inserted: dict[str, int] = field(default_factory=dict)
    rows_updated: dict[str, int] = field(default_factory=dict)
    rows_rejected: dict[str, int] = field(default_factory=dict)
    processing_times: dict[str, float] = field(default_factory=dict)
    data_quality_scores: dict[str, float] = field(default_factory=dict)
    error_counts: dict[str, int] = field(default_factory=dict)

    def add_processed(self, table: str, count: int) -> None:
        """Add processed records for a table."""
        self.rows_processed[table] = self.rows_processed.get(table, 0) + count

    def add_inserted(self, table: str, count: int) -> None:
        """Add inserted records for a table."""
        self.rows_inserted[table] = self.rows_inserted.get(table, 0) + count

    def add_updated(self, table: str, count: int) -> None:
        """Add updated records for a table."""
        self.rows_updated[table] = self.rows_updated.get(table, 0) + count

    def add_rejected(self, table: str, count: int) -> None:
        """Add rejected records for a table."""
        self.rows_rejected[table] = self.rows_rejected.get(table, 0) + count

    def add_processing_time(self, operation: str, time_seconds: float) -> None:
        """Add processing time for an operation."""
        self.processing_times[operation] = time_seconds

    def get_total_processed(self) -> int:
        """Get total records processed across all tables."""
        return sum(self.rows_processed.values())

    def get_total_errors(self) -> int:
        """Get total error count across all operations."""
        return sum(self.error_counts.values())


@dataclass
class LineageRecord:
    """Data lineage tracking record."""

    source_file: str
    target_table: str
    transformation_applied: str
    records_affected: int
    batch_id: int
    timestamp: datetime
    checksum: str


class BatchStatus:
    """Enhanced status tracking for batch processing operations."""

    def __init__(self, batch_id: int, batch_type: BatchType, start_time: datetime) -> None:
        """Initialize batch status.

        Args:
            batch_id: Unique identifier for the batch
            batch_type: Type of batch processing
            start_time: When batch processing started
        """
        self.batch_id = batch_id
        self.batch_type = batch_type
        self.start_time = start_time
        self.end_time: Optional[datetime] = None
        self.status: str = "started"
        self.metrics = BatchMetrics()
        self.error_messages: list[str] = []
        self.warnings: list[str] = []
        self.lineage_records: list[LineageRecord] = []
        self.dependencies_satisfied: dict[str, bool] = {}

    def mark_completed(self, end_time: datetime) -> None:
        """Mark batch as completed.

        Args:
            end_time: When batch processing completed
        """
        self.end_time = end_time
        self.status = "completed"
        logger.info(f"Batch {self.batch_id} completed")

    def mark_failed(self, end_time: datetime, error_message: str) -> None:
        """Mark batch as failed.

        Args:
            end_time: When batch processing failed
            error_message: Description of the failure
        """
        self.end_time = end_time
        self.status = "failed"
        self.error_messages.append(error_message)
        logger.error(f"Batch {self.batch_id} failed: {error_message}")

    def add_lineage(self, lineage: LineageRecord) -> None:
        """Add data lineage tracking record.

        Args:
            lineage: Lineage record to add
        """
        self.lineage_records.append(lineage)

    def get_duration(self) -> Optional[float]:
        """Get batch processing duration in seconds.

        Returns:
            Duration in seconds, or None if batch hasn't ended
        """
        if self.end_time:
            return (self.end_time - self.start_time).total_seconds()
        return None

    def get_summary(self) -> dict[str, Any]:
        """Get comprehensive batch processing summary.

        Returns:
            Dictionary containing batch summary information
        """
        return {
            "batch_id": self.batch_id,
            "batch_type": self.batch_type.value,
            "status": self.status,
            "duration_seconds": self.get_duration(),
            "total_processed": self.metrics.get_total_processed(),
            "total_errors": self.metrics.get_total_errors(),
            "error_messages": self.error_messages,
            "warnings": self.warnings,
            "lineage_records": len(self.lineage_records),
        }


class BatchOperation(ABC):
    """Abstract base class for batch operations."""

    @abstractmethod
    def execute(
        self,
        batch_data: dict[str, pd.DataFrame],
        batch_id: int,
        context: dict[str, Any],
    ) -> dict[str, Any]:
        """Execute the batch operation.

        Args:
            batch_data: Dictionary of dataframes to process
            batch_id: Unique identifier for the batch
            context: Additional context for the operation

        Returns:
            Dictionary containing operation results
        """
        ...


class ExtractOperation(BatchOperation):
    """Handles extraction of data from various source file formats."""

    def __init__(
        self,
        source_dir: Path,
        file_patterns: dict[str, str],
        supported_formats: Optional[list[str]] = None,
    ):
        """Initialize extract operation.

        Args:
            source_dir: Directory containing source files
            file_patterns: Dictionary mapping table names to file patterns
            supported_formats: List of supported file formats (csv, xml, txt, pipe)
        """
        self.source_dir = source_dir
        self.file_patterns = file_patterns
        self.supported_formats = supported_formats or ["csv", "xml", "txt", "pipe"]
        self.extraction_stats: dict[str, dict[str, Any]] = {}

    def execute(
        self,
        batch_data: dict[str, pd.DataFrame],
        batch_id: int,
        context: dict[str, Any],
    ) -> dict[str, Any]:
        """Extract data from source files.

        Args:
            batch_data: Dictionary to populate with extracted data
            batch_id: Unique identifier for the batch
            context: Additional context for extraction

        Returns:
            Dictionary containing extraction results
        """
        start_time = mono_time()
        results: dict[str, Any] = {
            "operation": "extract",
            "batch_id": batch_id,
            "tables_processed": [],
            "total_records": 0,
            "errors": [],
        }

        try:
            logger.info(f"Starting data extraction for batch {batch_id}")

            for table_name, pattern in self.file_patterns.items():
                try:
                    df = self.extract_table(table_name, batch_id, context)
                    batch_data[table_name] = df

                    record_count = len(df)
                    results["tables_processed"].append(table_name)
                    results["total_records"] += record_count

                    # Create lineage record
                    lineage = LineageRecord(
                        source_file=str(self.source_dir / pattern),
                        target_table=table_name,
                        transformation_applied="raw_extraction",
                        records_affected=record_count,
                        batch_id=batch_id,
                        timestamp=datetime.now(),
                        checksum=self._calculate_checksum(df),
                    )
                    context.setdefault("lineage", []).append(lineage)

                    logger.info(f"Extracted {record_count} records from {table_name}")

                except Exception as e:
                    error_msg = f"Failed to extract {table_name}: {str(e)}"
                    results["errors"].append(error_msg)
                    logger.error(error_msg)

            processing_time = elapsed_seconds(start_time)
            results["processing_time"] = processing_time
            results["success"] = len(results["errors"]) == 0

            logger.info(f"Extraction completed in {processing_time:.2f}s")
            return results

        except Exception as e:
            error_msg = f"Extract operation failed: {str(e)}"
            results["errors"].append(error_msg)
            results["success"] = False
            logger.error(error_msg)
            return results

    def extract_table(self, table_name: str, batch_id: int, context: Optional[dict[str, Any]] = None) -> pd.DataFrame:
        """Extract data for a specific table.

        Args:
            table_name: Name of the table to extract
            batch_id: Batch identifier
            context: Additional extraction context

        Returns:
            Dataframe containing extracted data
        """
        pattern = self.file_patterns.get(table_name)
        if not pattern:
            raise ValueError(f"No file pattern defined for table {table_name}")

        # Find matching files
        file_path = self.source_dir / pattern.format(batch_id=batch_id)

        if not file_path.exists():
            # Try without batch_id for historical loads
            file_path = self.source_dir / pattern

        if not file_path.exists():
            raise FileNotFoundError(f"Source file not found: {file_path}")

        # Determine file format and extract accordingly
        file_ext = file_path.suffix.lower()

        if file_ext == ".csv":
            return self._extract_csv(file_path)
        elif file_ext == ".xml":
            return self._extract_xml(file_path, table_name)
        elif file_ext == ".txt":
            return self._extract_pipe_delimited(file_path)
        else:
            raise ValueError(f"Unsupported file format: {file_ext}")

    def _extract_csv(self, file_path: Path) -> pd.DataFrame:
        """Extract data from CSV file."""
        try:
            return pd.read_csv(file_path, low_memory=False)
        except Exception as e:
            raise RuntimeError(f"Failed to read CSV file {file_path}: {str(e)}")

    def _extract_xml(self, file_path: Path, table_name: str) -> pd.DataFrame:
        """Extract data from XML file."""
        try:
            tree = ET.parse(file_path)
            root = tree.getroot()

            records = []
            for record_elem in root.findall(".//Record"):
                record = {}
                for elem in record_elem:
                    record[elem.tag] = elem.text
                records.append(record)

            return pd.DataFrame(records)

        except Exception as e:
            raise RuntimeError(f"Failed to read XML file {file_path}: {str(e)}")

    def _extract_pipe_delimited(self, file_path: Path) -> pd.DataFrame:
        """Extract data from pipe-delimited file."""
        try:
            return pd.read_csv(file_path, delimiter="|", low_memory=False)
        except Exception as e:
            raise RuntimeError(f"Failed to read pipe-delimited file {file_path}: {str(e)}")

    def _calculate_checksum(self, df: pd.DataFrame) -> str:
        """Calculate checksum for data lineage."""
        import hashlib

        data_str = df.to_string()
        return hashlib.md5(data_str.encode()).hexdigest()[:16]

    def execute_streaming(
        self,
        batch_id: int,
        context: dict[str, Any],
        memory_limit_mb: Optional[int] = None,
    ) -> Iterator[tuple[str, Iterator[pd.DataFrame]]]:
        """Extract data from source files in streaming mode.

        Args:
            batch_id: Unique identifier for the batch
            context: Additional context for extraction
            memory_limit_mb: Maximum memory usage in MB

        Yields:
            Tuples of (table_name, chunk_iterator) for each extracted table
        """
        try:
            logger.info(f"Starting streaming data extraction for batch {batch_id}")

            # Import streaming parsers (assuming they're available)
            try:
                from ..tools.file_parsers import MultiFormatParser
            except ImportError:
                logger.error("Streaming parsers not available")
                return

            parser = MultiFormatParser()

            for table_name, pattern in self.file_patterns.items():
                try:
                    # Find the source file
                    file_path = self.source_dir / pattern.format(batch_id=batch_id)
                    if not file_path.exists():
                        file_path = self.source_dir / pattern

                    if not file_path.exists():
                        logger.error(f"Source file not found: {file_path}")
                        continue

                    logger.info(f"Streaming extraction for {table_name} from {file_path}")

                    # Create streaming iterator for this table
                    def chunk_generator() -> Generator[Any, None, None]:
                        chunk_num = 0
                        total_records = 0

                        try:
                            for chunk_result in parser.parse_file_streaming(
                                file_path,
                                memory_limit_mb=memory_limit_mb,
                                progress_callback=lambda info: logger.debug(
                                    f"Chunk {info.get('chunk_number', 0)} processed"
                                ),
                            ):
                                chunk_num += 1
                                chunk_records = len(chunk_result.data)
                                total_records += chunk_records

                                # Create lineage record for this chunk
                                lineage = LineageRecord(
                                    source_file=str(file_path),
                                    target_table=table_name,
                                    transformation_applied="streaming_extraction",
                                    records_affected=chunk_records,
                                    batch_id=batch_id,
                                    timestamp=datetime.now(),
                                    checksum=self._calculate_checksum(chunk_result.data),
                                )
                                context.setdefault("lineage", []).append(lineage)

                                yield chunk_result.data

                        except Exception as e:
                            logger.error(f"Error in streaming extraction for {table_name}: {e}")

                        logger.info(
                            f"Completed streaming extraction for {table_name}: {total_records} records in {chunk_num} chunks"
                        )

                    yield table_name, chunk_generator()

                except Exception as e:
                    logger.error(f"Failed to setup streaming extraction for {table_name}: {e}")

        except Exception as e:
            logger.error(f"Streaming extract operation failed: {e}")
            raise


class TransformOperation(BatchOperation):
    """Handles transformation of extracted data with TPC-DI business rules."""

    def __init__(self, transformation_rules: dict[str, Any]) -> None:
        """Initialize transform operation.

        Args:
            transformation_rules: Dictionary of transformation rules per table
        """
        self.transformation_rules = transformation_rules
        self.data_quality_thresholds = {
            "completeness": 0.95,
            "accuracy": 0.98,
            "consistency": 0.99,
        }

    def execute(
        self,
        batch_data: dict[str, pd.DataFrame],
        batch_id: int,
        context: dict[str, Any],
    ) -> dict[str, Any]:
        """Transform extracted data.

        Args:
            batch_data: Dictionary of dataframes to transform
            batch_id: Unique identifier for the batch
            context: Additional context for transformation

        Returns:
            Dictionary containing transformation results
        """
        start_time = mono_time()
        results: dict[str, Any] = {
            "operation": "transform",
            "batch_id": batch_id,
            "tables_processed": [],
            "quality_scores": {},
            "transformations_applied": {},
            "errors": [],
        }

        try:
            logger.info(f"Starting data transformation for batch {batch_id}")

            for table_name, df in batch_data.items():
                try:
                    logger.info(f"Transforming table {table_name} with {len(df)} records")

                    # Apply table-specific transformations
                    transformed_df = self._apply_transformations(df, table_name, batch_id, context)

                    # Calculate data quality metrics
                    quality_score = self._calculate_quality_score(transformed_df)
                    results["quality_scores"][table_name] = quality_score

                    # Configure the dataframe in batch_data
                    batch_data[table_name] = transformed_df

                    results["tables_processed"].append(table_name)
                    results["transformations_applied"][table_name] = self.transformation_rules.get(table_name, [])

                    # Create lineage record
                    lineage = LineageRecord(
                        source_file=f"transform_{table_name}",
                        target_table=table_name,
                        transformation_applied=f"business_rules_{table_name}",
                        records_affected=len(transformed_df),
                        batch_id=batch_id,
                        timestamp=datetime.now(),
                        checksum=self._calculate_checksum(transformed_df),
                    )
                    context.setdefault("lineage", []).append(lineage)

                    logger.info(
                        f"Transformed {table_name}: {len(transformed_df)} records, quality score: {quality_score:.3f}"
                    )

                except Exception as e:
                    error_msg = f"Failed to transform {table_name}: {str(e)}"
                    results["errors"].append(error_msg)
                    logger.error(error_msg)

            processing_time = elapsed_seconds(start_time)
            results["processing_time"] = processing_time
            results["success"] = len(results["errors"]) == 0

            logger.info(f"Transformation completed in {processing_time:.2f}s")
            return results

        except Exception as e:
            error_msg = f"Transform operation failed: {str(e)}"
            results["errors"].append(error_msg)
            results["success"] = False
            logger.error(error_msg)
            return results

    def _apply_transformations(
        self, df: pd.DataFrame, table_name: str, batch_id: int, context: dict[str, Any]
    ) -> pd.DataFrame:
        """Apply table-specific transformations."""
        transformed_df = df.copy()

        # Apply TPC-DI specific transformations based on table type
        if table_name.startswith("Dim"):
            transformed_df = self._apply_dimension_transformations(transformed_df, table_name, batch_id, context)
        elif table_name.startswith("Fact"):
            transformed_df = self._apply_fact_transformations(transformed_df, table_name, batch_id)

        # Apply common transformations
        transformed_df = self._apply_common_transformations(transformed_df)

        return transformed_df

    def _apply_dimension_transformations(
        self, df: pd.DataFrame, table_name: str, batch_id: int, context: dict[str, Any]
    ) -> pd.DataFrame:
        """Apply SCD (Slowly Changing Dimension) transformations."""
        # Add standard dimension columns
        if "BatchID" not in df.columns:
            df["BatchID"] = batch_id

        if "IsCurrent" not in df.columns:
            df["IsCurrent"] = True

        if "EffectiveDate" not in df.columns:
            df["EffectiveDate"] = datetime.now().date()

        if "EndDate" not in df.columns:
            df["EndDate"] = date(9999, 12, 31)

        # Generate surrogate keys with batch-specific offsets to avoid conflicts
        batch_type = context.get("batch_type", "historical")

        # Define unique surrogate key ranges for different batch types
        sk_offsets = {"historical": 0, "incremental": 1000000, "scd": 2000000}
        account_offsets = {"historical": 0, "incremental": 5000000, "scd": 6000000}
        security_offsets = {"historical": 0, "incremental": 3000000, "scd": 4000000}
        company_offsets = {"historical": 0, "incremental": 7000000, "scd": 8000000}

        batch_offset = sk_offsets.get(batch_type, 0)

        # Generate surrogate keys if needed
        if table_name == "DimCustomer" and "SK_CustomerID" not in df.columns:
            df["SK_CustomerID"] = range(batch_offset + 1, batch_offset + len(df) + 1)
        elif table_name == "DimAccount" and "SK_AccountID" not in df.columns:
            account_offset = account_offsets.get(batch_type, 0)
            df["SK_AccountID"] = range(account_offset + 1, account_offset + len(df) + 1)
        elif table_name == "DimSecurity" and "SK_SecurityID" not in df.columns:
            security_offset = security_offsets.get(batch_type, 0)
            df["SK_SecurityID"] = range(security_offset + 1, security_offset + len(df) + 1)
        elif table_name == "DimCompany" and "SK_CompanyID" not in df.columns:
            company_offset = company_offsets.get(batch_type, 0)
            df["SK_CompanyID"] = range(company_offset + 1, company_offset + len(df) + 1)

        return df

    def _apply_fact_transformations(self, df: pd.DataFrame, table_name: str, batch_id: int) -> pd.DataFrame:
        """Apply fact table transformations."""
        # Add batch tracking
        if "BatchID" not in df.columns:
            df["BatchID"] = batch_id

        # Calculate derived measures for FactTrade
        if table_name == "FactTrade":
            if "TradePrice" in df.columns and "Quantity" in df.columns:
                df["TradeValue"] = df["TradePrice"] * df["Quantity"]

            if "Fee" in df.columns and "Commission" in df.columns:
                df["TotalCost"] = df["Fee"] + df["Commission"]

        return df

    def _apply_common_transformations(self, df: pd.DataFrame) -> pd.DataFrame:
        """Apply common data transformations."""
        # Handle data type conversions
        for col in df.columns:
            if df[col].dtype == "object":
                # Clean string columns
                df[col] = df[col].astype(str).str.strip()
                # Replace empty strings with None
                df[col] = df[col].replace("", None)

        # Handle date columns
        date_columns = [col for col in df.columns if "Date" in col or "date" in col.lower()]
        for col in date_columns:
            if col in df.columns:
                with contextlib.suppress(Exception):
                    df[col] = pd.to_datetime(df[col], errors="coerce")

        return df

    def _calculate_quality_score(self, df: pd.DataFrame) -> float:
        """Calculate data quality score."""
        if df.empty:
            return 0.0

        # Completeness: percentage of non-null values
        total_cells = df.size
        non_null_cells = df.count().sum()
        completeness = non_null_cells / total_cells if total_cells > 0 else 0

        # Consistency: percentage of consistent data types
        consistency = 1.0  # Simplified for now

        # Accuracy: percentage of valid values (simplified)
        accuracy = 0.95  # Simplified for now

        # Weighted average
        quality_score = completeness * 0.4 + consistency * 0.3 + accuracy * 0.3
        return min(quality_score, 1.0)

    def _calculate_checksum(self, df: pd.DataFrame) -> str:
        """Calculate checksum for transformed data."""
        import hashlib

        data_str = df.to_string()
        return hashlib.md5(data_str.encode()).hexdigest()[:16]

    def execute_streaming(
        self,
        source_data_chunks: dict[str, Iterator[pd.DataFrame]],
        batch_id: int,
        context: dict[str, Any],
        memory_limit_mb: Optional[int] = None,
    ) -> Iterator[tuple[str, pd.DataFrame]]:
        """Transform extracted data in streaming mode.

        Args:
            source_data_chunks: Dictionary of iterators yielding dataframe chunks keyed by table name
            batch_id: Unique identifier for the batch
            context: Additional context for transformation
            memory_limit_mb: Maximum memory usage in MB

        Yields:
            Tuples of (table_name, transformed_chunk) ready for loading
        """
        try:
            logger.info(f"Starting streaming data transformation for batch {batch_id}")

            # Import streaming transformation engine
            try:
                from .transformations import TransformationEngine
            except ImportError:
                logger.error("Streaming transformation engine not available")
                return

            # Initialize transformation engine
            transformation_engine = TransformationEngine()

            # Use streaming transformation
            for (
                table_name,
                transformed_chunk,
            ) in transformation_engine.transform_batch_streaming(
                source_data_chunks,
                batch_id,
                memory_limit_mb=memory_limit_mb,
                progress_callback=lambda info: logger.debug(
                    f"Transformed {info.get('table_name')} chunk {info.get('chunk_number')}"
                ),
            ):
                # Create lineage record
                lineage = LineageRecord(
                    source_file=f"transform_{table_name}",
                    target_table=table_name,
                    transformation_applied=f"streaming_business_rules_{table_name}",
                    records_affected=len(transformed_chunk),
                    batch_id=batch_id,
                    timestamp=datetime.now(),
                    checksum=self._calculate_checksum(transformed_chunk),
                )
                context.setdefault("lineage", []).append(lineage)

                yield table_name, transformed_chunk

            logger.info(f"Completed streaming transformation for batch {batch_id}")

        except Exception as e:
            logger.error(f"Streaming transform operation failed: {e}")
            raise


class LoadOperation(BatchOperation):
    """Handles loading of transformed data into target database with SCD support."""

    def __init__(
        self,
        connection_config: dict[str, Any],
        load_strategies: Optional[dict[str, str]] = None,
    ) -> None:
        """Initialize load operation.

        Args:
            connection_config: Database connection configuration
            load_strategies: Dictionary mapping table names to load strategies
        """
        self.connection_config = connection_config
        self.load_strategies = load_strategies or {}
        self.connection = None
        self._load_stats: dict[str, dict[str, int]] = {}

    def execute(
        self,
        batch_data: dict[str, pd.DataFrame],
        batch_id: int,
        context: dict[str, Any],
    ) -> dict[str, Any]:
        """Load transformed data into target database.

        Args:
            batch_data: Dictionary of dataframes to load
            batch_id: Unique identifier for the batch
            context: Additional context for loading

        Returns:
            Dictionary containing load results
        """
        start_time = mono_time()
        results: dict[str, Any] = {
            "operation": "load",
            "batch_id": batch_id,
            "tables_loaded": [],
            "load_statistics": {},
            "errors": [],
        }

        try:
            logger.info(f"Starting data load for batch {batch_id}")
            self._establish_connection()

            # Load tables in dependency order
            load_order = self._determine_load_order(list(batch_data.keys()))

            for table_name in load_order:
                if table_name not in batch_data:
                    continue

                try:
                    df = batch_data[table_name]
                    logger.info(f"Loading table {table_name} with {len(df)} records")

                    load_strategy = self.load_strategies.get(table_name, "append")
                    load_stats = self.load_table(table_name, df, load_strategy)

                    results["tables_loaded"].append(table_name)
                    results["load_statistics"][table_name] = load_stats

                    # Create lineage record
                    lineage = LineageRecord(
                        source_file=f"transform_{table_name}",
                        target_table=table_name,
                        transformation_applied=f"load_{load_strategy}",
                        records_affected=load_stats.get("inserted", 0) + load_stats.get("updated", 0),
                        batch_id=batch_id,
                        timestamp=datetime.now(),
                        checksum=self._calculate_checksum(df),
                    )
                    context.setdefault("lineage", []).append(lineage)

                    logger.info(f"Loaded {table_name}: {load_stats}")

                except Exception as e:
                    error_msg = f"Failed to load {table_name}: {str(e)}"
                    results["errors"].append(error_msg)
                    logger.error(error_msg)
                    # Continue with other tables unless critical error

            if self.connection:
                self.connection.commit()

            processing_time = elapsed_seconds(start_time)
            results["processing_time"] = processing_time
            results["success"] = len(results["errors"]) == 0

            logger.info(f"Load completed in {processing_time:.2f}s")
            return results

        except Exception as e:
            if self.connection:
                self.connection.rollback()
            error_msg = f"Load operation failed: {str(e)}"
            results["errors"].append(error_msg)
            results["success"] = False
            logger.error(error_msg)
            return results
        finally:
            self._close_connection()

    def load_table(self, table_name: str, data: pd.DataFrame, load_type: str = "append") -> dict[str, int]:
        """Load data for a specific table with appropriate strategy.

        Args:
            table_name: Name of the target table
            data: Dataframe to load
            load_type: Type of load operation (append, replace, upsert, scd2)

        Returns:
            Dictionary with load statistics
        """
        if data.empty:
            return {"inserted": 0, "updated": 0, "rejected": 0}

        try:
            if load_type == "append":
                return self._load_append(table_name, data)
            elif load_type == "replace":
                return self._load_replace(table_name, data)
            elif load_type == "upsert":
                return self._load_upsert(table_name, data)
            elif load_type == "scd2":
                return self._load_scd2(table_name, data)
            else:
                raise ValueError(f"Unsupported load type: {load_type}")

        except Exception as e:
            logger.error(f"Failed to load table {table_name}: {str(e)}")
            return {"inserted": 0, "updated": 0, "rejected": len(data)}

    def _establish_connection(self) -> None:
        """Establish database connection."""
        # This would be implemented based on the specific database type
        # For now, we'll simulate a connection
        logger.info("Establishing database connection")
        self.connection = {"status": "connected"}

    def _close_connection(self) -> None:
        """Close database connection."""
        if self.connection:
            logger.info("Closing database connection")
            self.connection = None

    def _determine_load_order(self, table_names: list[str]) -> list[str]:
        """Determine optimal load order based on dependencies."""
        # Define dependency order for TPC-DI tables
        priority_order = [
            "DimDate",
            "DimTime",
            "DimCompany",
            "DimSecurity",
            "DimCustomer",
            "DimAccount",
            "DimBroker",
            "FactTrade",
            "FactCashBalances",
            "FactHoldings",
            "FactMarketHistory",
        ]

        # Sort tables based on priority
        ordered_tables = []
        for table in priority_order:
            if table in table_names:
                ordered_tables.append(table)

        # Add any remaining tables
        for table in table_names:
            if table not in ordered_tables:
                ordered_tables.append(table)

        return ordered_tables

    def _load_append(self, table_name: str, data: pd.DataFrame) -> dict[str, int]:
        """Load data using append strategy."""
        # Simulate append load
        logger.debug(f"Appending {len(data)} records to {table_name}")
        return {"inserted": len(data), "updated": 0, "rejected": 0}

    def _load_replace(self, table_name: str, data: pd.DataFrame) -> dict[str, int]:
        """Load data using replace strategy."""
        # Simulate replace load
        logger.debug(f"Replacing table {table_name} with {len(data)} records")
        return {"inserted": len(data), "updated": 0, "rejected": 0}

    def _load_upsert(self, table_name: str, data: pd.DataFrame) -> dict[str, int]:
        """Load data using upsert strategy."""
        # Simulate upsert load
        insert_count = int(len(data) * 0.7)  # Assume 70% are inserts
        update_count = len(data) - insert_count
        logger.debug(f"Upserting to {table_name}: {insert_count} inserts, {update_count} updates")
        return {"inserted": insert_count, "updated": update_count, "rejected": 0}

    def _load_scd2(self, table_name: str, data: pd.DataFrame) -> dict[str, int]:
        """Load data using SCD Type 2 strategy."""
        # Simulate SCD2 load with historical tracking
        new_records = int(len(data) * 0.6)  # 60% new records
        changed_records = int(len(data) * 0.3)  # 30% changed (expire old, insert new)
        unchanged_records = len(data) - new_records - changed_records

        logger.debug(
            f"SCD2 load to {table_name}: {new_records} new, {changed_records} changed, {unchanged_records} unchanged"
        )
        return {
            "inserted": new_records + changed_records,  # New records + new versions
            "updated": changed_records,  # Records expired
            "rejected": 0,
        }

    def execute_streaming(
        self,
        source_data_chunks: dict[str, Iterator[pd.DataFrame]],
        batch_id: int,
        context: dict[str, Any],
        memory_limit_mb: Optional[int] = None,
    ) -> Iterator[dict[str, Any]]:
        """Execute streaming load operation with memory-efficient processing.

        Args:
            source_data_chunks: Dictionary mapping table names to iterators of dataframe chunks
            batch_id: Unique identifier for the batch
            context: Additional context for loading operation
            memory_limit_mb: Optional memory limit in MB for load operations

        Yields:
            Load result dictionaries for each chunk processed
        """
        try:
            logger.info(f"Starting data load for batch {batch_id}")

            # Initialize monitoring components
            memory_monitor = SimpleMemoryMonitor()
            progress_tracker = SimpleProgressTracker()

            # Simple progress tracking without memory monitoring

            # Establish connection once for all streaming loads
            self._establish_connection()

            # Determine load order for dependencies
            table_names = list(source_data_chunks.keys())
            load_order = self._determine_load_order(table_names)

            total_chunks_processed = 0
            total_records_loaded = 0

            for table_name in load_order:
                if table_name not in source_data_chunks:
                    continue

                logger.info(f"Starting streaming load for table {table_name}")
                table_start_time = mono_time()
                table_chunks_processed = 0
                table_records_loaded = 0

                load_strategy = self.load_strategies.get(table_name, "append")

                # Process chunks for this table
                for chunk_idx, chunk_df in enumerate(source_data_chunks[table_name]):
                    try:
                        chunk_start_time = mono_time()

                        # Memory check before processing
                        if memory_monitor.check_memory_usage():
                            logger.warning(f"High memory usage detected before loading chunk {chunk_idx}")
                            gc.collect()

                        # Load the chunk
                        load_stats: dict[str, int] = {"inserted": 0, "updated": 0, "rejected": 0}
                        if not chunk_df.empty:
                            load_stats = self.load_table(table_name, chunk_df, load_strategy)

                            table_records_loaded += load_stats.get("inserted", 0) + load_stats.get("updated", 0)
                            total_records_loaded += table_records_loaded

                            # Create lineage record for this chunk
                            lineage = LineageRecord(
                                source_file=f"streaming_load_{table_name}_chunk_{chunk_idx}",
                                target_table=table_name,
                                transformation_applied=f"load_{load_strategy}",
                                records_affected=load_stats.get("inserted", 0) + load_stats.get("updated", 0),
                                batch_id=batch_id,
                                timestamp=datetime.now(),
                                checksum=self._calculate_checksum(chunk_df),
                            )
                            context.setdefault("lineage", []).append(lineage)

                        chunk_time = elapsed_seconds(chunk_start_time)
                        table_chunks_processed += 1
                        total_chunks_processed += 1

                        # Progress tracking
                        progress_tracker.update_progress(
                            table_chunks_processed,
                            chunk_df.shape[0] if not chunk_df.empty else 0,
                            f"Loading {table_name}",
                        )

                        # Yield result for this chunk
                        chunk_result: dict[str, Any] = {
                            "operation": "streaming_load",
                            "batch_id": batch_id,
                            "table_name": table_name,
                            "chunk_index": chunk_idx,
                            "records_in_chunk": len(chunk_df) if not chunk_df.empty else 0,
                            "load_statistics": load_stats,
                            "processing_time": chunk_time,
                            "memory_usage_mb": memory_monitor.get_current_usage_mb(),
                            "success": True,
                        }

                        yield chunk_result

                        # Memory cleanup after each chunk
                        del chunk_df
                        gc.collect()

                    except Exception as e:
                        error_msg = f"Failed to load chunk {chunk_idx} for table {table_name}: {str(e)}"
                        logger.error(error_msg)

                        yield {
                            "operation": "streaming_load",
                            "batch_id": batch_id,
                            "table_name": table_name,
                            "chunk_index": chunk_idx,
                            "error": error_msg,
                            "success": False,
                        }
                        # Continue with next chunk

                table_time = elapsed_seconds(table_start_time)
                logger.info(
                    f"Completed streaming load for {table_name}: {table_chunks_processed} chunks, "
                    f"{table_records_loaded} records in {table_time:.2f}s"
                )

            # Commit all changes
            if self.connection:
                self.connection.commit()

            # Final summary
            total_time = progress_tracker.get_elapsed_time()
            logger.info(
                f"Streaming load completed: {total_chunks_processed} chunks, "
                f"{total_records_loaded} total records in {total_time:.2f}s"
            )

        except Exception as e:
            logger.error(f"Streaming load operation failed: {e}")
            if self.connection:
                self.connection.rollback()
            raise
        finally:
            self._close_connection()

    def load_table_streaming(
        self,
        table_name: str,
        data_chunks: Iterator[pd.DataFrame],
        load_type: str = "append",
        memory_limit_mb: Optional[int] = None,
    ) -> Iterator[dict[str, int]]:
        """Load data for a specific table using streaming chunks.

        Args:
            table_name: Name of the target table
            data_chunks: Iterator of dataframe chunks to load
            load_type: Type of load operation (append, replace, upsert, scd2)
            memory_limit_mb: Optional memory limit in MB

        Yields:
            Load statistics for each chunk processed
        """
        # Initialize monitoring components
        memory_monitor = SimpleMemoryMonitor()

        try:
            chunk_count = 0
            total_records = 0

            for chunk_df in data_chunks:
                try:
                    # Memory check
                    if memory_monitor.check_memory_usage():
                        logger.warning(f"High memory usage during streaming load of {table_name}")
                        gc.collect()

                    if not chunk_df.empty:
                        load_stats = self.load_table(table_name, chunk_df, load_type)
                        total_records += load_stats.get("inserted", 0) + load_stats.get("updated", 0)
                        chunk_count += 1

                        yield load_stats

                    # Clean up chunk
                    del chunk_df
                    gc.collect()

                except Exception as e:
                    logger.error(f"Error loading chunk {chunk_count} for table {table_name}: {e}")
                    yield {"inserted": 0, "updated": 0, "rejected": 1}

            logger.info(f"Completed streaming load for {table_name}: {chunk_count} chunks, {total_records} records")

        except Exception as e:
            logger.error(f"Streaming load failed for table {table_name}: {e}")
            raise

    def _calculate_checksum(self, df: pd.DataFrame) -> str:
        """Calculate checksum for loaded data."""
        import hashlib

        data_str = df.to_string()
        return hashlib.md5(data_str.encode()).hexdigest()[:16]


class ValidationOperation(BatchOperation):
    """Handles comprehensive validation of data during batch processing."""

    def __init__(
        self,
        validation_rules: dict[str, list[str]],
        quality_thresholds: Optional[dict[str, float]] = None,
    ) -> None:
        """Initialize validation operation.

        Args:
            validation_rules: Dictionary mapping table names to validation rule lists
            quality_thresholds: Quality score thresholds for each validation type
        """
        self.validation_rules = validation_rules
        self.quality_thresholds = quality_thresholds or {
            "completeness": 0.95,
            "consistency": 0.90,
            "accuracy": 0.85,
            "referential_integrity": 0.98,
        }

    def execute(
        self,
        batch_data: dict[str, pd.DataFrame],
        batch_id: int,
        context: dict[str, Any],
    ) -> dict[str, Any]:
        """Validate batch data against quality rules.

        Args:
            batch_data: Dictionary of dataframes to validate
            batch_id: Unique identifier for the batch
            context: Additional context for validation

        Returns:
            Dictionary containing validation results
        """
        start_time = mono_time()
        results: dict[str, Any] = {
            "operation": "validate",
            "batch_id": batch_id,
            "tables_validated": [],
            "validation_results": {},
            "quality_scores": {},
            "failed_validations": [],
            "warnings": [],
            "errors": [],
        }

        try:
            logger.info(f"Starting data validation for batch {batch_id}")

            for table_name, df in batch_data.items():
                try:
                    logger.info(f"Validating table {table_name} with {len(df)} records")

                    # Run validation rules for this table
                    table_validation = self._validate_table(df, table_name, batch_id)
                    results["validation_results"][table_name] = table_validation

                    # Calculate overall quality score
                    quality_score = self._calculate_overall_quality_score(table_validation)
                    results["quality_scores"][table_name] = quality_score

                    # Check if validation passed thresholds
                    if not self._check_quality_thresholds(table_validation):
                        results["failed_validations"].append(table_name)

                    results["tables_validated"].append(table_name)

                    # Create lineage record
                    lineage = LineageRecord(
                        source_file=f"validate_{table_name}",
                        target_table=table_name,
                        transformation_applied="data_quality_validation",
                        records_affected=len(df),
                        batch_id=batch_id,
                        timestamp=datetime.now(),
                        checksum=self._calculate_checksum(df),
                    )
                    context.setdefault("lineage", []).append(lineage)

                    logger.info(f"Validated {table_name}: quality score {quality_score:.3f}")

                except Exception as e:
                    error_msg = f"Failed to validate {table_name}: {str(e)}"
                    results["errors"].append(error_msg)
                    logger.error(error_msg)

            # Generate validation summary
            results["summary"] = self._generate_validation_summary(results)

            processing_time = elapsed_seconds(start_time)
            results["processing_time"] = processing_time
            results["success"] = len(results["errors"]) == 0 and len(results["failed_validations"]) == 0

            logger.info(f"Validation completed in {processing_time:.2f}s")
            return results

        except Exception as e:
            error_msg = f"Validation operation failed: {str(e)}"
            results["errors"].append(error_msg)
            results["success"] = False
            logger.error(error_msg)
            return results

    def _validate_table(self, df: pd.DataFrame, table_name: str, batch_id: int) -> dict[str, Any]:
        """Run all validation rules for a specific table."""
        validation_result = {
            "table_name": table_name,
            "record_count": len(df),
            "completeness_score": 0.0,
            "consistency_score": 0.0,
            "accuracy_score": 0.0,
            "referential_integrity_score": 0.0,
            "rule_results": {},
            "data_issues": [],
        }

        if df.empty:
            validation_result["data_issues"].append("Table is empty")
            return validation_result

        # Run TPC-DI specific validations
        validation_result["completeness_score"] = self._check_completeness(df)
        validation_result["consistency_score"] = self._check_consistency(df, table_name)
        validation_result["accuracy_score"] = self._check_accuracy(df, table_name)
        validation_result["referential_integrity_score"] = self._check_referential_integrity(df, table_name)

        # Run custom validation rules if defined
        table_rules = self.validation_rules.get(table_name, [])
        for rule in table_rules:
            try:
                rule_result = self._apply_validation_rule(df, rule)
                validation_result["rule_results"][rule] = rule_result
            except Exception as e:
                validation_result["data_issues"].append(f"Rule '{rule}' failed: {str(e)}")

        return validation_result

    def _check_completeness(self, df: pd.DataFrame) -> float:
        """Check data completeness (non-null percentage)."""
        if df.empty:
            return 0.0

        total_cells = df.size
        non_null_cells = df.count().sum()
        return non_null_cells / total_cells if total_cells > 0 else 0.0

    def _check_consistency(self, df: pd.DataFrame, table_name: str) -> float:
        """Check data consistency across records."""
        consistency_score = 1.0

        # Check for duplicate primary keys
        if table_name.startswith("Dim") and "SK_" in "".join(df.columns):
            sk_columns = [col for col in df.columns if col.startswith("SK_")]
            for sk_col in sk_columns:
                if sk_col in df.columns:
                    duplicates = df[sk_col].duplicated().sum()
                    if duplicates > 0:
                        consistency_score *= 1 - duplicates / len(df)

        # Check date consistency (EffectiveDate <= EndDate)
        if "EffectiveDate" in df.columns and "EndDate" in df.columns:
            try:
                df_dates = df.dropna(subset=["EffectiveDate", "EndDate"])
                if not df_dates.empty:
                    invalid_dates = (
                        pd.to_datetime(df_dates["EffectiveDate"]) > pd.to_datetime(df_dates["EndDate"])
                    ).sum()
                    if invalid_dates > 0:
                        consistency_score *= 1 - invalid_dates / len(df_dates)
            except Exception:
                pass

        return max(consistency_score, 0.0)

    def _check_accuracy(self, df: pd.DataFrame, table_name: str) -> float:
        """Check data accuracy based on business rules."""
        accuracy_score = 1.0

        # Check for valid status values
        if "Status" in df.columns:
            valid_statuses = ["ACTV", "INACT", "PNDG", "SBMT", "CNCL"]
            invalid_status = (~df["Status"].isin(valid_statuses)).sum()
            if invalid_status > 0:
                accuracy_score *= 1 - invalid_status / len(df)

        # Check for valid credit ratings (1-10)
        if "CreditRating" in df.columns:
            try:
                credit_ratings = pd.to_numeric(df["CreditRating"], errors="coerce")
                invalid_ratings = ((credit_ratings < 1) | (credit_ratings > 10)).sum()
                if invalid_ratings > 0:
                    accuracy_score *= 1 - invalid_ratings / len(df)
            except Exception:
                pass

        # Check for valid email formats
        email_columns = [col for col in df.columns if "Email" in col]
        for email_col in email_columns:
            if email_col in df.columns:
                # Simple email validation
                email_pattern = r"^[\w\.-]+@[\w\.-]+\.[a-zA-Z]{2,}$"
                valid_emails = df[email_col].dropna().str.match(email_pattern, na=False)
                if len(valid_emails) > 0:
                    invalid_emails = (~valid_emails).sum()
                    accuracy_score *= 1 - invalid_emails / len(valid_emails)

        return max(accuracy_score, 0.0)

    def _check_referential_integrity(self, df: pd.DataFrame, table_name: str) -> float:
        """Check referential integrity constraints."""
        # Simplified referential integrity check
        # In a real implementation, this would check foreign key constraints
        return 0.95  # Assume 95% referential integrity for now

    def _apply_validation_rule(self, df: pd.DataFrame, rule: str) -> dict[str, Any]:
        """Apply a custom validation rule."""
        # This would implement custom business rule validation
        # For now, return a placeholder result
        return {"rule": rule, "passed": True, "violations": 0, "score": 1.0}

    def _calculate_overall_quality_score(self, table_validation: dict[str, Any]) -> float:
        """Calculate overall quality score for a table."""
        scores = [
            table_validation["completeness_score"],
            table_validation["consistency_score"],
            table_validation["accuracy_score"],
            table_validation["referential_integrity_score"],
        ]

        # Weighted average
        weights = [0.3, 0.25, 0.25, 0.2]
        weighted_score = sum(score * weight for score, weight in zip(scores, weights))

        return min(weighted_score, 1.0)

    def _check_quality_thresholds(self, table_validation: dict[str, Any]) -> bool:
        """Check if validation results meet quality thresholds."""
        checks = [
            table_validation["completeness_score"] >= self.quality_thresholds["completeness"],
            table_validation["consistency_score"] >= self.quality_thresholds["consistency"],
            table_validation["accuracy_score"] >= self.quality_thresholds["accuracy"],
            table_validation["referential_integrity_score"] >= self.quality_thresholds["referential_integrity"],
        ]

        return all(checks)

    def _generate_validation_summary(self, results: dict[str, Any]) -> dict[str, Any]:
        """Generate validation summary."""
        total_tables = len(results["tables_validated"])
        failed_tables = len(results["failed_validations"])

        avg_quality_score = 0.0
        if results["quality_scores"]:
            avg_quality_score = sum(results["quality_scores"].values()) / len(results["quality_scores"])

        return {
            "total_tables_validated": total_tables,
            "tables_passed": total_tables - failed_tables,
            "tables_failed": failed_tables,
            "average_quality_score": avg_quality_score,
            "validation_passed": failed_tables == 0 and len(results["errors"]) == 0,
        }

    def execute_streaming(
        self,
        source_data_chunks: dict[str, Iterator[pd.DataFrame]],
        batch_id: int,
        context: dict[str, Any],
        memory_limit_mb: Optional[int] = None,
    ) -> Iterator[dict[str, Any]]:
        """Execute streaming validation operation with memory-efficient processing.

        Args:
            source_data_chunks: Dictionary mapping table names to iterators of dataframe chunks
            batch_id: Unique identifier for the batch
            context: Additional context for validation operation
            memory_limit_mb: Optional memory limit in MB for validation operations

        Yields:
            Validation result dictionaries for each chunk processed
        """
        try:
            logger.info(f"Starting data validation for batch {batch_id}")

            # Initialize monitoring components
            memory_monitor = SimpleMemoryMonitor()
            progress_tracker = SimpleProgressTracker()

            total_chunks_processed = 0
            total_records_validated = 0
            table_quality_scores = {}
            table_validation_summaries = {}

            for table_name, data_chunks in source_data_chunks.items():
                logger.info(f"Starting streaming validation for table {table_name}")
                table_start_time = mono_time()
                table_chunks_processed = 0
                table_records_validated = 0
                chunk_validations = []

                # Process chunks for this table
                for chunk_idx, chunk_df in enumerate(data_chunks):
                    try:
                        chunk_start_time = mono_time()

                        # Memory check before processing
                        if memory_monitor.check_memory_usage():
                            logger.warning(f"High memory usage detected before validating chunk {chunk_idx}")
                            gc.collect()

                        # Validate the chunk
                        if not chunk_df.empty:
                            chunk_validation = self._validate_table(chunk_df, table_name, batch_id)
                            chunk_validations.append(chunk_validation)

                            table_records_validated += len(chunk_df)
                            total_records_validated += len(chunk_df)

                            # Calculate chunk quality score
                            chunk_quality_score = self._calculate_overall_quality_score(chunk_validation)

                            # Create lineage record for this chunk
                            lineage = LineageRecord(
                                source_file=f"streaming_validation_{table_name}_chunk_{chunk_idx}",
                                target_table=table_name,
                                transformation_applied="data_quality_validation",
                                records_affected=len(chunk_df),
                                batch_id=batch_id,
                                timestamp=datetime.now(),
                                checksum=self._calculate_checksum(chunk_df),
                            )
                            context.setdefault("lineage", []).append(lineage)

                        else:
                            chunk_validation = {
                                "table_name": table_name,
                                "record_count": 0,
                                "completeness_score": 0.0,
                                "consistency_score": 0.0,
                                "accuracy_score": 0.0,
                                "referential_integrity_score": 0.0,
                                "rule_results": {},
                                "data_issues": ["Empty chunk"],
                            }
                            chunk_quality_score = 0.0

                        chunk_time = elapsed_seconds(chunk_start_time)
                        table_chunks_processed += 1
                        total_chunks_processed += 1

                        # Progress tracking
                        progress_tracker.update_progress(
                            table_chunks_processed,
                            chunk_df.shape[0] if not chunk_df.empty else 0,
                            f"Validating {table_name}",
                        )

                        # Check if chunk validation passed quality thresholds
                        chunk_passed = self._check_quality_thresholds(chunk_validation) if not chunk_df.empty else False

                        # Yield result for this chunk
                        chunk_result = {
                            "operation": "streaming_validation",
                            "batch_id": batch_id,
                            "table_name": table_name,
                            "chunk_index": chunk_idx,
                            "records_in_chunk": len(chunk_df) if not chunk_df.empty else 0,
                            "validation_results": chunk_validation,
                            "quality_score": chunk_quality_score,
                            "passed_thresholds": chunk_passed,
                            "processing_time": chunk_time,
                            "memory_usage_mb": memory_monitor.get_current_usage_mb(),
                            "success": True,
                        }

                        yield chunk_result

                        # Memory cleanup after each chunk
                        del chunk_df
                        gc.collect()

                    except Exception as e:
                        error_msg = f"Failed to validate chunk {chunk_idx} for table {table_name}: {str(e)}"
                        logger.error(error_msg)

                        yield {
                            "operation": "streaming_validation",
                            "batch_id": batch_id,
                            "table_name": table_name,
                            "chunk_index": chunk_idx,
                            "error": error_msg,
                            "success": False,
                        }
                        # Continue with next chunk

                # Aggregate table-level validation results
                if chunk_validations:
                    table_validation_summary = self._aggregate_chunk_validations(chunk_validations, table_name)
                    table_quality_scores[table_name] = self._calculate_overall_quality_score(table_validation_summary)
                    table_validation_summaries[table_name] = table_validation_summary

                table_time = elapsed_seconds(table_start_time)
                logger.info(
                    f"Completed streaming validation for {table_name}: {table_chunks_processed} chunks, "
                    f"{table_records_validated} records in {table_time:.2f}s"
                )

            # Final summary
            total_time = progress_tracker.get_elapsed_time()
            logger.info(
                f"Streaming validation completed: {total_chunks_processed} chunks, "
                f"{total_records_validated} total records in {total_time:.2f}s"
            )

            # Yield final summary
            yield {
                "operation": "streaming_validation_summary",
                "batch_id": batch_id,
                "total_chunks_processed": total_chunks_processed,
                "total_records_validated": total_records_validated,
                "table_quality_scores": table_quality_scores,
                "table_validation_summaries": table_validation_summaries,
                "processing_time": total_time,
                "success": True,
            }

        except Exception as e:
            logger.error(f"Streaming validation operation failed: {e}")
            raise

    def validate_table_streaming(
        self,
        table_name: str,
        data_chunks: Iterator[pd.DataFrame],
        memory_limit_mb: Optional[int] = None,
    ) -> Iterator[dict[str, Any]]:
        """Validate data for a specific table using streaming chunks.

        Args:
            table_name: Name of the table to validate
            data_chunks: Iterator of dataframe chunks to validate
            memory_limit_mb: Optional memory limit in MB

        Yields:
            Validation results for each chunk processed
        """
        # Initialize monitoring components
        memory_monitor = SimpleMemoryMonitor()

        try:
            chunk_count = 0
            total_records = 0
            all_validations = []

            for chunk_df in data_chunks:
                try:
                    # Memory check
                    if memory_monitor.check_memory_usage():
                        logger.warning(f"High memory usage during streaming validation of {table_name}")
                        gc.collect()

                    if not chunk_df.empty:
                        validation_result = self._validate_table(chunk_df, table_name, 1)
                        all_validations.append(validation_result)
                        total_records += len(chunk_df)
                        chunk_count += 1

                        yield validation_result

                    # Clean up chunk
                    del chunk_df
                    gc.collect()

                except Exception as e:
                    logger.error(f"Error validating chunk {chunk_count} for table {table_name}: {e}")
                    yield {
                        "table_name": table_name,
                        "record_count": 0,
                        "completeness_score": 0.0,
                        "consistency_score": 0.0,
                        "accuracy_score": 0.0,
                        "referential_integrity_score": 0.0,
                        "rule_results": {},
                        "data_issues": [f"Validation error: {str(e)}"],
                    }

            logger.info(
                f"Completed streaming validation for {table_name}: {chunk_count} chunks, {total_records} records"
            )

        except Exception as e:
            logger.error(f"Streaming validation failed for table {table_name}: {e}")
            raise

    def _aggregate_chunk_validations(self, chunk_validations: list[dict[str, Any]], table_name: str) -> dict[str, Any]:
        """Aggregate validation results from multiple chunks into table-level summary.

        Args:
            chunk_validations: List of validation results from individual chunks
            table_name: Name of the table being validated

        Returns:
            Aggregated validation result for the table
        """
        if not chunk_validations:
            return {
                "table_name": table_name,
                "record_count": 0,
                "completeness_score": 0.0,
                "consistency_score": 0.0,
                "accuracy_score": 0.0,
                "referential_integrity_score": 0.0,
                "rule_results": {},
                "data_issues": ["No chunks processed"],
            }

        # Aggregate metrics across chunks
        total_records = sum(cv["record_count"] for cv in chunk_validations)

        # Calculate weighted averages based on record counts
        if total_records > 0:
            completeness_score = (
                sum(cv["completeness_score"] * cv["record_count"] for cv in chunk_validations) / total_records
            )
            consistency_score = (
                sum(cv["consistency_score"] * cv["record_count"] for cv in chunk_validations) / total_records
            )
            accuracy_score = sum(cv["accuracy_score"] * cv["record_count"] for cv in chunk_validations) / total_records
            referential_integrity_score = (
                sum(cv["referential_integrity_score"] * cv["record_count"] for cv in chunk_validations) / total_records
            )
        else:
            completeness_score = consistency_score = accuracy_score = referential_integrity_score = 0.0

        # Aggregate rule results
        all_rule_results = {}
        for cv in chunk_validations:
            for rule, result in cv["rule_results"].items():
                if rule not in all_rule_results:
                    all_rule_results[rule] = []
                all_rule_results[rule].append(result)

        # Aggregate data issues
        all_issues = []
        for cv in chunk_validations:
            all_issues.extend(cv["data_issues"])

        return {
            "table_name": table_name,
            "record_count": total_records,
            "completeness_score": completeness_score,
            "consistency_score": consistency_score,
            "accuracy_score": accuracy_score,
            "referential_integrity_score": referential_integrity_score,
            "rule_results": all_rule_results,
            "data_issues": all_issues,
        }

    def _calculate_checksum(self, df: pd.DataFrame) -> str:
        """Calculate checksum for validation tracking."""
        import hashlib

        data_str = df.to_string()
        return hashlib.md5(data_str.encode()).hexdigest()[:16]


class ParallelConfig:
    """Simplified configuration for parallel processing capabilities."""

    def __init__(
        self,
        max_workers: Optional[int] = None,
        chunk_size: int = 1000,
        enable_parallel: bool = False,  # Parallel processing is opt-in
        worker_timeout: float = 300.0,
    ):
        """Initialize simplified parallel configuration.

        Args:
            max_workers: Maximum number of worker threads/processes
            chunk_size: Size of data chunks for parallel processing
            enable_parallel: Whether to enable parallel processing
            worker_timeout: Timeout for worker operations in seconds
        """
        self.max_workers = max_workers or min(8, (multiprocessing.cpu_count() or 1) + 2)
        self.chunk_size = chunk_size
        self.enable_parallel = enable_parallel
        self.worker_timeout = worker_timeout

        # Validate configuration
        if self.max_workers < 1 or self.max_workers > 16:
            raise ValueError("max_workers must be between 1 and 16")
        if self.chunk_size < 1:
            raise ValueError("chunk_size must be positive")
        if self.worker_timeout <= 0:
            raise ValueError("worker_timeout must be positive")

    @classmethod
    def from_tpcdi_config(cls, tpcdi_config: Any) -> "ParallelConfig":
        """Create ParallelConfig from TPCDIConfig."""
        batch_params = tpcdi_config.get_batch_config()
        return cls(
            max_workers=batch_params["max_workers"],
            chunk_size=batch_params["chunk_size"],
            enable_parallel=batch_params["parallel_processing"],
            worker_timeout=batch_params.get("worker_timeout", 300.0),
        )


class ParallelExecutionContext:
    """Context for parallel execution with thread-safe operations."""

    def __init__(self, config: ParallelConfig) -> None:
        """Initialize parallel execution context.

        Args:
            config: Parallel processing configuration
        """
        self.config = config
        self.lock = threading.RLock()
        self.results_queue = queue.Queue()
        self.error_queue = queue.Queue()
        self.performance_metrics = {
            "extract_times": [],
            "transform_times": [],
            "load_times": [],
            "validate_times": [],
            "parallel_efficiency": [],
        }
        self.active_workers = set()
        self.completed_tasks = 0
        self.failed_tasks = 0

    def add_worker(self, worker_id: str) -> None:
        """Add active worker to tracking."""
        with self.lock:
            self.active_workers.add(worker_id)

    def remove_worker(self, worker_id: str) -> None:
        """Remove worker from tracking."""
        with self.lock:
            self.active_workers.discard(worker_id)

    def increment_completed(self) -> None:
        """Increment completed task counter."""
        with self.lock:
            self.completed_tasks += 1

    def increment_failed(self) -> None:
        """Increment failed task counter."""
        with self.lock:
            self.failed_tasks += 1

    def add_performance_metric(self, metric_type: str, value: float) -> None:
        """Add performance metric in thread-safe manner."""
        with self.lock:
            if metric_type in self.performance_metrics:
                self.performance_metrics[metric_type].append(value)

    def get_active_worker_count(self) -> int:
        """Get current active worker count."""
        with self.lock:
            return len(self.active_workers)

    def get_task_summary(self) -> dict[str, int]:
        """Get task execution summary."""
        with self.lock:
            return {
                "completed": self.completed_tasks,
                "failed": self.failed_tasks,
                "active_workers": len(self.active_workers),
            }


class BatchProcessor:
    """Main processor for TPC-DI batch operations with parallel processing support."""

    def __init__(
        self,
        scale_factor: float = 1.0,
        parallel_processing: bool = False,
        parallel_config: Optional[ParallelConfig] = None,
    ):
        """Initialize batch processor.

        Args:
            scale_factor: TPC-DI scale factor
            parallel_processing: Whether to enable parallel processing
            parallel_config: Configuration for parallel processing
        """
        self.scale_factor = scale_factor
        self.parallel_processing = parallel_processing
        self.parallel_config = parallel_config or ParallelConfig()

        # Core components
        self.operations: list[BatchOperation] = []
        self.batch_history: dict[int, BatchStatus] = {}

        # Parallel processing components
        if self.parallel_processing:
            self.execution_context = ParallelExecutionContext(self.parallel_config)
            # Worker pools are now created using context managers when needed

        # Thread safety
        self._lock = threading.RLock()

        # Performance tracking
        self.performance_metrics = {
            "extract_times": [],
            "transform_times": [],
            "load_times": [],
            "validate_times": [],
            "parallel_efficiency": [],
            "worker_utilization": [],
        }

        # Dependency management
        self.dependencies: dict[str, list[str]] = {}
        self.dependency_graph: dict[str, set[str]] = {}

        # Initialize logging
        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")

    def _get_worker_pool_config(self) -> dict[str, Any]:
        """Get simple worker pool configuration."""
        return {
            "max_workers": self.parallel_config.max_workers,
            "enable_parallel": self.parallel_processing,
            "use_threads_only": True,  # Simplified to use only ThreadPoolExecutor
        }

    def add_operation(self, operation: BatchOperation) -> None:
        """Add a batch operation to the processing pipeline.

        Args:
            operation: Batch operation to add
        """
        with self._lock:
            if operation not in self.operations:
                self.operations.append(operation)
                self.logger.info(f"Added operation: {type(operation).__name__}")

    def remove_operation(self, operation: BatchOperation) -> None:
        """Remove a batch operation from the processing pipeline.

        Args:
            operation: Batch operation to remove
        """
        with self._lock:
            if operation in self.operations:
                self.operations.remove(operation)
                self.logger.info(f"Removed operation: {type(operation).__name__}")

    def add_dependency(self, dependent: str, prerequisite: str) -> None:
        """Add a dependency between batch operations.

        Args:
            dependent: Name of the dependent operation
            prerequisite: Name of the prerequisite operation
        """
        with self._lock:
            if dependent not in self.dependencies:
                self.dependencies[dependent] = []
            if prerequisite not in self.dependencies[dependent]:
                self.dependencies[dependent].append(prerequisite)

            # Configure dependency graph
            if dependent not in self.dependency_graph:
                self.dependency_graph[dependent] = set()
            self.dependency_graph[dependent].add(prerequisite)

    def process_batch(
        self,
        batch_id: int,
        batch_type: BatchType,
        source_config: dict[str, Any],
        target_config: dict[str, Any],
    ) -> BatchStatus:
        """Process a complete batch through the ETL pipeline.

        Args:
            batch_id: Unique identifier for the batch
            batch_type: Type of batch processing
            source_config: Configuration for source data
            target_config: Configuration for target database

        Returns:
            BatchStatus object with processing results
        """
        start_time = datetime.now()
        start_mono = mono_time()
        batch_status = BatchStatus(batch_id, batch_type, start_time)

        with self._lock:
            self.batch_history[batch_id] = batch_status

        try:
            self.logger.info(f"Starting batch processing for batch {batch_id} (type: {batch_type.value})")

            # Validate dependencies
            if not self.validate_batch_dependencies(batch_id):
                raise RuntimeError(f"Dependencies not satisfied for batch {batch_id}")

            # Initialize data containers
            batch_data: dict[str, pd.DataFrame] = {}
            context: dict[str, Any] = {
                "batch_id": batch_id,
                "batch_type": batch_type,
                "source_config": source_config,
                "target_config": target_config,
                "lineage": [],
            }

            # Execute pipeline
            if self.parallel_processing and len(self.operations) > 1:
                success = self._execute_parallel_pipeline(batch_data, batch_status, context)
            else:
                success = self._execute_sequential_pipeline(batch_data, batch_status, context)

            # Complete batch processing
            end_time = datetime.now()
            if success:
                batch_status.mark_completed(end_time)
                self.logger.info(f"Batch {batch_id} completed in {elapsed_seconds(start_mono):.2f}s")
            else:
                batch_status.mark_failed(end_time, "Pipeline execution failed")

            return batch_status

        except Exception as e:
            error_msg = f"Batch processing failed: {str(e)}"
            batch_status.mark_failed(datetime.now(), error_msg)

            self.logger.error(f"Batch {batch_id} failed: {error_msg}")
            raise

    def process_historical_load(self, source_config: dict[str, Any], target_config: dict[str, Any]) -> BatchStatus:
        """Process the initial historical data load.

        This is TPC-DI Batch 1 - the initial load that populates the data warehouse
        with historical data before incremental updates begin.

        Args:
            source_config: Configuration for source data
            target_config: Configuration for target database

        Returns:
            BatchStatus object with processing results
        """
        logger.info("Starting TPC-DI historical load (Batch 1)")

        # Configure for historical load
        historical_config = source_config.copy()
        historical_config.update(
            {
                "load_type": "historical",
                "full_refresh": True,
                "scd_processing": False,  # No SCD in initial load
                "batch_date": None,  # Not applicable for historical
            }
        )

        return self.process_batch(
            batch_id=1,
            batch_type=BatchType.HISTORICAL,
            source_config=historical_config,
            target_config=target_config,
        )

    def process_incremental_batch(
        self,
        batch_number: int,
        batch_date: date,
        source_config: dict[str, Any],
        target_config: dict[str, Any],
        include_scd: bool = False,
    ) -> BatchStatus:
        """Process an incremental batch load.

        Args:
            batch_number: Sequential batch number (2 or 3)
            batch_date: Business date for the batch
            source_config: Configuration for source data
            target_config: Configuration for target database
            include_scd: Whether to include SCD processing (Batch 3)

        Returns:
            BatchStatus object with processing results
        """
        if batch_number == 2:
            logger.info(f"Starting TPC-DI incremental load (Batch 2) for {batch_date}")
            batch_type = BatchType.INCREMENTAL
        elif batch_number == 3:
            logger.info(f"Starting TPC-DI incremental load with SCD (Batch 3) for {batch_date}")
            batch_type = BatchType.SCD_BATCH
        else:
            raise ValueError(f"Invalid batch number: {batch_number}. Must be 2 or 3.")

        # Configure for incremental load
        incremental_config = source_config.copy()
        incremental_config.update(
            {
                "load_type": "incremental",
                "batch_date": batch_date,
                "scd_processing": include_scd,
                "full_refresh": False,
            }
        )

        return self.process_batch(
            batch_id=batch_number,
            batch_type=batch_type,
            source_config=incremental_config,
            target_config=target_config,
        )

    def get_batch_status(self, batch_id: int) -> Optional[BatchStatus]:
        """Get status information for a specific batch.

        Args:
            batch_id: Unique identifier for the batch

        Returns:
            BatchStatus object or None if batch not found
        """
        with self._lock:
            return self.batch_history.get(batch_id)

    def get_processing_statistics(self) -> dict[str, Any]:
        """Get overall processing statistics across all batches.

        Returns:
            Dictionary containing processing statistics
        """
        with self._lock:
            total_batches = len(self.batch_history)
            completed_batches = sum(1 for batch in self.batch_history.values() if batch.status == "completed")
            failed_batches = sum(1 for batch in self.batch_history.values() if batch.status == "failed")

            total_processing_time = sum(batch.get_duration() or 0 for batch in self.batch_history.values())

            avg_processing_time = total_processing_time / total_batches if total_batches > 0 else 0

            # Calculate performance metrics
            performance_stats = {}
            for metric_name, times in self.performance_metrics.items():
                if times:
                    performance_stats[metric_name] = {
                        "avg": sum(times) / len(times),
                        "min": min(times),
                        "max": max(times),
                        "count": len(times),
                    }

            return {
                "total_batches": total_batches,
                "completed_batches": completed_batches,
                "failed_batches": failed_batches,
                "success_rate": completed_batches / total_batches if total_batches > 0 else 0,
                "total_processing_time": total_processing_time,
                "average_processing_time": avg_processing_time,
                "performance_metrics": performance_stats,
                "scale_factor": self.scale_factor,
                "parallel_processing": self.parallel_processing,
            }

    def cleanup_batch_resources(self, batch_id: int) -> None:
        """Clean up temporary resources for a specific batch.

        Args:
            batch_id: Unique identifier for the batch
        """
        try:
            # Clean up resources

            # Clean up temporary data files if any
            temp_dir = Path(f"temp_batch_{batch_id}")
            if temp_dir.exists():
                import shutil

                shutil.rmtree(temp_dir)
                logger.info(f"Cleaned up temporary files for batch {batch_id}")

        except Exception as e:
            logger.warning(f"Failed to cleanup resources for batch {batch_id}: {str(e)}")

    def validate_batch_dependencies(self, batch_id: int) -> bool:
        """Validate that all dependencies for a batch are satisfied.

        Args:
            batch_id: Unique identifier for the batch

        Returns:
            True if dependencies are satisfied, False otherwise
        """
        # Check if previous batches completed
        if batch_id > 1:
            for prev_batch_id in range(1, batch_id):
                prev_batch = self.get_batch_status(prev_batch_id)
                if not prev_batch or prev_batch.status != "completed":
                    logger.error(f"Dependency not satisfied: Batch {prev_batch_id} not completed")
                    return False

        # Check custom dependencies
        batch_name = f"batch_{batch_id}"
        if batch_name in self.dependencies:
            for prerequisite in self.dependencies[batch_name]:
                # Check if prerequisite is satisfied
                # This would be implemented based on specific requirements
                logger.debug(f"Checking dependency: {prerequisite}")

        return True

    def _execute_sequential_pipeline(
        self,
        batch_data: dict[str, pd.DataFrame],
        batch_status: BatchStatus,
        context: dict[str, Any],
    ) -> bool:
        """Execute ETL operations sequentially."""
        try:
            for operation in self.operations:
                operation_name = type(operation).__name__
                logger.info(f"Executing {operation_name}")

                start_time = mono_time()
                result = operation.execute(batch_data, batch_status.batch_id, context)
                processing_time = elapsed_seconds(start_time)

                # Track performance metrics
                if isinstance(operation, ExtractOperation):
                    self.performance_metrics["extract_times"].append(processing_time)
                elif isinstance(operation, TransformOperation):
                    self.performance_metrics["transform_times"].append(processing_time)
                elif isinstance(operation, LoadOperation):
                    self.performance_metrics["load_times"].append(processing_time)
                elif isinstance(operation, ValidationOperation):
                    self.performance_metrics["validate_times"].append(processing_time)

                # Configure batch metrics
                if result.get("success", False):
                    batch_status.metrics.add_processing_time(operation_name, processing_time)

                    # Configure lineage
                    if "lineage" in context:
                        for lineage in context["lineage"]:
                            batch_status.add_lineage(lineage)
                else:
                    error_msg = f"{operation_name} failed: {result.get('errors', [])}"
                    batch_status.error_messages.extend(result.get("errors", []))
                    logger.error(error_msg)
                    return False

            return True

        except Exception as e:
            logger.error(f"Sequential pipeline execution failed: {str(e)}")
            return False

    def _execute_parallel_pipeline(
        self,
        batch_data: dict[str, pd.DataFrame],
        batch_status: BatchStatus,
        context: dict[str, Any],
    ) -> bool:
        """Execute ETL operations in parallel where possible."""
        try:
            self.logger.info(f"Starting parallel pipeline execution with {self.parallel_config.max_workers} workers")

            # Group operations by type for parallel execution
            operation_groups = {
                "extract": [op for op in self.operations if isinstance(op, ExtractOperation)],
                "transform": [op for op in self.operations if isinstance(op, TransformOperation)],
                "load": [op for op in self.operations if isinstance(op, LoadOperation)],
                "validate": [op for op in self.operations if isinstance(op, ValidationOperation)],
            }

            # Execute each group sequentially, but operations within group in parallel
            for group_name, operations in operation_groups.items():
                if not operations:
                    continue

                self.logger.info(f"Executing {len(operations)} {group_name} operations in parallel")

                # Determine appropriate executor
                executor = self._get_executor_for_operation_type(group_name)

                if not executor:
                    # Fallback to sequential execution
                    for operation in operations:
                        success = self._execute_single_operation(operation, batch_data, batch_status, context)
                        if not success:
                            return False
                    continue

                # Execute operations in parallel
                futures = {}
                worker_id_counter = 0

                for operation in operations:
                    worker_id = f"{group_name}_worker_{worker_id_counter}"
                    worker_id_counter += 1

                    if self.parallel_processing and hasattr(self, "execution_context"):
                        self.execution_context.add_worker(worker_id)

                    future = executor.submit(
                        self._execute_operation_with_monitoring,
                        operation,
                        batch_data,
                        batch_status.batch_id,
                        context,
                        worker_id,
                    )
                    futures[future] = (operation, worker_id)

                # Wait for completion with timeout
                for future in as_completed(futures, timeout=self.parallel_config.worker_timeout):
                    operation, worker_id = futures[future]
                    operation_name = type(operation).__name__

                    if self.parallel_processing and hasattr(self, "execution_context"):
                        self.execution_context.remove_worker(worker_id)

                    try:
                        result = future.result()

                        if result.get("success", False):
                            # Track performance metrics
                            if "processing_time" in result:
                                metric_type = f"{group_name}_times"
                                if hasattr(self, "execution_context"):
                                    self.execution_context.add_performance_metric(
                                        metric_type, result["processing_time"]
                                    )
                                else:
                                    self.performance_metrics.get(metric_type, []).append(result["processing_time"])

                            if hasattr(self, "execution_context"):
                                self.execution_context.increment_completed()

                            self.logger.info(f"{operation_name} completed in {result.get('processing_time', 0):.2f}s")
                        else:
                            error_msg = f"{operation_name} failed: {result.get('errors', [])}"
                            batch_status.error_messages.extend(result.get("errors", []))

                            if hasattr(self, "execution_context"):
                                self.execution_context.increment_failed()

                            self.logger.error(error_msg)
                            return False

                    except Exception as e:
                        error_msg = f"{operation_name} failed with exception: {str(e)}"
                        batch_status.error_messages.append(error_msg)

                        if hasattr(self, "execution_context"):
                            self.execution_context.increment_failed()

                        self.logger.error(error_msg)
                        return False

            # Calculate parallel efficiency
            if hasattr(self, "execution_context"):
                task_summary = self.execution_context.get_task_summary()
                total_tasks = task_summary["completed"] + task_summary["failed"]
                if total_tasks > 0:
                    efficiency = task_summary["completed"] / total_tasks
                    self.execution_context.add_performance_metric("parallel_efficiency", efficiency)

            self.logger.info("Parallel pipeline execution completed")
            return True

        except Exception as e:
            self.logger.error(f"Parallel pipeline execution failed: {str(e)}")
            return False

    def _should_use_parallel_processing(self, operation_type: str) -> bool:
        """Check if operation should use parallel processing."""
        if not self.parallel_processing:
            return False

        return bool(
            operation_type in ["extract", "load"]
            and self.parallel_config.enable_concurrent_io
            or operation_type == "transform"
            and self.parallel_config.enable_parallel_transform
            or operation_type == "validate"
        )

    def get_parallel_performance_report(self) -> dict[str, Any]:
        """Generate detailed parallel processing performance report.

        Returns:
            Comprehensive performance analysis
        """
        if not self.parallel_processing or not hasattr(self, "execution_context"):
            return {"error": "Parallel processing not enabled or no execution context"}

        metrics = self.execution_context.performance_metrics
        task_summary = self.execution_context.get_task_summary()

        report = {
            "configuration": {
                "max_workers": self.parallel_config.max_workers,
                "use_process_pool": False,  # Simplified to use only ThreadPoolExecutor
                "chunk_size": self.parallel_config.chunk_size,
                "worker_timeout": self.parallel_config.worker_timeout,
            },
            "execution_summary": task_summary,
            "performance_metrics": {},
        }

        # Analyze performance by operation type
        for metric_type, times in metrics.items():
            if times and metric_type.endswith("_times"):
                operation_type = metric_type.replace("_times", "")
                report["performance_metrics"][operation_type] = {
                    "total_operations": len(times),
                    "total_time": sum(times),
                    "average_time": sum(times) / len(times),
                    "min_time": min(times),
                    "max_time": max(times),
                    "throughput": len(times) / sum(times) if sum(times) > 0 else 0,
                }

        # Calculate overall efficiency
        if task_summary["completed"] + task_summary["failed"] > 0:
            report["overall_efficiency"] = {
                "success_rate": task_summary["completed"] / (task_summary["completed"] + task_summary["failed"]),
                "parallel_efficiency": sum(metrics.get("parallel_efficiency", []))
                / len(metrics.get("parallel_efficiency", [1])),
                "worker_utilization": task_summary["completed"] / self.parallel_config.max_workers
                if self.parallel_config.max_workers > 0
                else 0,
            }

        return report

    def _execute_operation_with_monitoring(
        self,
        operation: BatchOperation,
        batch_data: dict[str, pd.DataFrame],
        batch_id: int,
        context: dict[str, Any],
        worker_id: str,
    ) -> dict[str, Any]:
        """Execute operation with performance monitoring."""
        start_time = mono_time()

        try:
            self.logger.debug(f"Worker {worker_id} starting {type(operation).__name__}")

            # Execute the operation
            result = operation.execute(batch_data, batch_id, context)

            # Add timing information
            processing_time = elapsed_seconds(start_time)
            result["processing_time"] = processing_time
            result["worker_id"] = worker_id

            self.logger.debug(f"Worker {worker_id} completed {type(operation).__name__} in {processing_time:.2f}s")

            return result

        except Exception as e:
            processing_time = elapsed_seconds(start_time)
            self.logger.error(
                f"Worker {worker_id} failed {type(operation).__name__} after {processing_time:.2f}s: {str(e)}"
            )

            return {
                "success": False,
                "errors": [str(e)],
                "processing_time": processing_time,
                "worker_id": worker_id,
            }

    def _execute_single_operation(
        self,
        operation: BatchOperation,
        batch_data: dict[str, pd.DataFrame],
        batch_status: BatchStatus,
        context: dict[str, Any],
    ) -> bool:
        """Execute a single operation (fallback for non-parallel execution)."""
        operation_name = type(operation).__name__

        try:
            start_time = mono_time()
            result = operation.execute(batch_data, batch_status.batch_id, context)
            processing_time = elapsed_seconds(start_time)

            if result.get("success", False):
                batch_status.metrics.add_processing_time(operation_name, processing_time)

                # Configure lineage
                if "lineage" in context:
                    for lineage in context.get("lineage", []):
                        batch_status.add_lineage(lineage)

                self.logger.info(f"{operation_name} completed")
                return True
            else:
                error_msg = f"{operation_name} failed: {result.get('errors', [])}"
                batch_status.error_messages.extend(result.get("errors", []))
                self.logger.error(error_msg)
                return False

        except Exception as e:
            error_msg = f"{operation_name} failed with exception: {str(e)}"
            batch_status.error_messages.append(error_msg)
            self.logger.error(error_msg)
            return False

    def _calculate_checksum(self, df: pd.DataFrame) -> str:
        """Calculate checksum for data integrity verification."""
        import hashlib

        data_str = df.to_string()
        return hashlib.md5(data_str.encode()).hexdigest()[:16]

    def get_lineage_report(self, batch_id: int) -> dict[str, Any]:
        """Generate data lineage report for a batch."""
        batch_status = self.get_batch_status(batch_id)
        if not batch_status:
            return {}

        return {
            "batch_id": batch_id,
            "lineage_records": [
                {
                    "source_file": lineage.source_file,
                    "target_table": lineage.target_table,
                    "transformation": lineage.transformation_applied,
                    "records_affected": lineage.records_affected,
                    "timestamp": lineage.timestamp.isoformat(),
                    "checksum": lineage.checksum,
                }
                for lineage in batch_status.lineage_records
            ],
            "total_transformations": len(batch_status.lineage_records),
            "batch_status": batch_status.status,
        }
