"""
Comprehensive test suite for pipeline.py module.

Tests cover:
- NormalizationPipeline initialization
- Pipeline execution with various data sources
- CSV parsing
- Step orchestration and chaining
- Report generation and metrics
- Error handling and strict mode
- Iterator behavior and lazy evaluation
"""

import pytest
import io
from typing import Iterator, Dict, Any, Optional
from atomingest.normalization.pipeline import NormalizationPipeline
from atomingest.normalization.steps import (
    BaseNormalizerStep,
    register_step,
    _STEP_REGISTRY,
)
from atomingest.normalization.core import (
    NormalizationConfig,
    NormalizationStepConfig,
    NormalizationReport,
)


@pytest.fixture
def sample_csv_string():
    """Provide a sample CSV string for testing."""
    return """id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com
3,Charlie,charlie@example.com"""


@pytest.fixture
def sample_csv_file(tmp_path):
    """Create a sample CSV file for testing."""
    csv_file = tmp_path / "test.csv"
    csv_file.write_text("""id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com
3,Charlie,charlie@example.com""")
    return csv_file


@pytest.fixture
def empty_config():
    """Provide an empty normalization configuration."""
    return NormalizationConfig(strict=False, steps=[])


@pytest.fixture(autouse=True)
def cleanup_registry():
    """Clean up step registry before and after each test."""
    initial_registry = _STEP_REGISTRY.copy()
    yield
    _STEP_REGISTRY.clear()
    _STEP_REGISTRY.update(initial_registry)


class TestNormalizationPipelineInitialization:
    """Test NormalizationPipeline initialization."""

    def test_init_with_empty_config(self, empty_config):
        """Test initialization with empty configuration."""
        pipeline = NormalizationPipeline(config=empty_config)

        assert pipeline.config is empty_config
        assert pipeline.config.strict is False
        assert len(pipeline.config.steps) == 0

    def test_init_with_strict_mode(self):
        """Test initialization with strict mode enabled."""
        config = NormalizationConfig(strict=True, steps=[])
        pipeline = NormalizationPipeline(config=config)

        assert pipeline.config.strict is True

    def test_init_with_steps(self):
        """Test initialization with configured steps."""
        steps = [
            NormalizationStepConfig(step="Step1", config={}, enabled=True),
            NormalizationStepConfig(
                step="Step2", config={"key": "value"}, enabled=True
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        assert len(pipeline.config.steps) == 2
        assert pipeline.config.steps[0].step == "Step1"
        assert pipeline.config.steps[1].step == "Step2"


class TestPipelineRunWithDifferentSources:
    """Test pipeline execution with different data sources."""

    def test_run_with_string_list_source(self, empty_config):
        """Test running pipeline with a list of strings source."""
        pipeline = NormalizationPipeline(config=empty_config)
        csv_lines = ["id,name", "1,Alice", "2,Bob"]

        rows, report = pipeline.run(csv_lines)
        result = list(rows)

        assert len(result) == 2
        assert result[0] == {"id": "1", "name": "Alice"}
        assert result[1] == {"id": "2", "name": "Bob"}

    def test_run_with_path_source(self, empty_config, sample_csv_file):
        """Test running pipeline with a Path source."""
        pipeline = NormalizationPipeline(config=empty_config)

        rows, report = pipeline.run(sample_csv_file)
        result = list(rows)

        assert len(result) == 3
        assert result[0]["name"] == "Alice"
        assert result[1]["name"] == "Bob"
        assert result[2]["name"] == "Charlie"

    def test_run_with_string_path_source(self, empty_config, sample_csv_file):
        """Test running pipeline with a string path source."""
        pipeline = NormalizationPipeline(config=empty_config)

        rows, report = pipeline.run(str(sample_csv_file))
        result = list(rows)

        assert len(result) == 3

    def test_run_with_textio_source(self, empty_config):
        """Test running pipeline with a TextIO source."""
        pipeline = NormalizationPipeline(config=empty_config)
        csv_data = io.StringIO("id,name\n1,Alice\n2,Bob")

        rows, report = pipeline.run(csv_data)
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"

    def test_run_with_iterable_source(self, empty_config):
        """Test running pipeline with an iterable source."""
        pipeline = NormalizationPipeline(config=empty_config)
        lines = ["id,name", "1,Alice", "2,Bob"]

        rows, report = pipeline.run(lines)
        result = list(rows)

        assert len(result) == 2

    def test_run_with_empty_source(self, empty_config):
        """Test running pipeline with empty source."""
        pipeline = NormalizationPipeline(config=empty_config)

        rows, report = pipeline.run([])
        result = list(rows)

        assert result == []

    def test_run_with_header_only_source(self, empty_config):
        """Test running pipeline with only header."""
        pipeline = NormalizationPipeline(config=empty_config)

        rows, report = pipeline.run(["id,name,email"])
        result = list(rows)

        assert result == []


class TestPipelineReportGeneration:
    """Test report generation and metrics."""

    def test_report_counts_total_lines(self, empty_config):
        """Test that report correctly counts total lines read."""
        pipeline = NormalizationPipeline(config=empty_config)
        csv_data = io.StringIO("id,name\n1,Alice\n2,Bob\n3,Charlie")

        rows, report = pipeline.run(csv_data)
        list(rows)  # Consume the iterator

        assert report.total_lines_read == 4  # Header + 3 data rows

    def test_report_counts_rows_emitted(self, empty_config):
        """Test that report correctly counts rows emitted."""
        pipeline = NormalizationPipeline(config=empty_config)
        csv_data = io.StringIO("id,name\n1,Alice\n2,Bob\n3,Charlie")

        rows, report = pipeline.run(csv_data)
        list(rows)  # Consume the iterator

        assert report.rows_emitted == 3

    def test_report_with_no_data_rows(self, empty_config):
        """Test report when only header is present."""
        pipeline = NormalizationPipeline(config=empty_config)
        csv_data = io.StringIO("id,name,email")

        rows, report = pipeline.run(csv_data)
        list(rows)

        assert report.total_lines_read == 1
        assert report.rows_emitted == 0

    def test_report_is_returned(self, empty_config):
        """Test that run returns a NormalizationReport instance."""
        pipeline = NormalizationPipeline(config=empty_config)

        rows, report = pipeline.run(io.StringIO("id,name\n1,Alice"))

        assert isinstance(report, NormalizationReport)

    def test_report_initialized_with_default_values(self, empty_config):
        """Test that report is initialized with correct default values."""
        pipeline = NormalizationPipeline(config=empty_config)

        rows, report = pipeline.run(io.StringIO("id,name\n1,Alice"))

        assert report.metadata_lines_skipped == 0
        assert report.trailer_lines_skipped == 0
        assert report.duplicate_headers_removed == 0
        assert len(report.warnings) == 0


class TestPipelineWithSteps:
    """Test pipeline execution with various normalization steps."""

    def test_run_with_disabled_step(self):
        """Test that disabled steps are not executed."""

        @register_step("DisabledTestStep")
        class DisabledTestStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                row["modified"] = True
                return row

        steps = [
            NormalizationStepConfig(step="DisabledTestStep", enabled=False),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO("id,name\n1,Alice"))
        result = list(rows)

        assert "modified" not in result[0]

    def test_run_with_enabled_step(self):
        """Test that enabled steps are executed."""

        @register_step("EnabledTestStep")
        class EnabledTestStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                row["modified"] = True
                return row

        steps = [
            NormalizationStepConfig(step="EnabledTestStep", enabled=True),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO("id,name\n1,Alice"))
        result = list(rows)

        assert result[0]["modified"] is True

    def test_run_with_multiple_steps_in_sequence(self):
        """Test that multiple steps are executed in sequence."""

        @register_step("AddFieldStep")
        class AddFieldStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                row["step1"] = "done"
                return row

        @register_step("ModifyFieldStep")
        class ModifyFieldStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                row["step2"] = "done"
                return row

        steps = [
            NormalizationStepConfig(step="AddFieldStep", enabled=True),
            NormalizationStepConfig(step="ModifyFieldStep", enabled=True),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO("id,name\n1,Alice"))
        result = list(rows)

        assert result[0]["step1"] == "done"
        assert result[0]["step2"] == "done"

    def test_run_with_line_processing_step(self):
        """Test step that processes lines."""

        @register_step("UppercaseLineStep")
        class UppercaseLineStep(BaseNormalizerStep):
            def process_lines(self, lines: Iterator[str]) -> Iterator[str]:
                for line in lines:
                    yield line.upper()

        steps = [
            NormalizationStepConfig(step="UppercaseLineStep", enabled=True),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO("id,name\n1,alice"))
        result = list(rows)

        assert result[0]["ID"] == "1"
        assert result[0]["NAME"] == "ALICE"

    def test_run_with_row_filtering_step(self):
        """Test step that filters rows by returning None."""

        @register_step("FilterOddIdStep")
        class FilterOddIdStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                if int(row.get("id", 0)) % 2 == 1:
                    return None  # Filter out odd IDs
                return row

        steps = [
            NormalizationStepConfig(step="FilterOddIdStep", enabled=True),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(
            io.StringIO("id,name\n1,Alice\n2,Bob\n3,Charlie\n4,David")
        )
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "2"
        assert result[1]["id"] == "4"
        assert report.rows_emitted == 2

    def test_run_with_step_that_uses_config(self):
        """Test step that uses configuration."""

        @register_step("ConfigurableStep")
        class ConfigurableStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                prefix = self.config.get("prefix", "")
                row["name"] = prefix + row.get("name", "")
                return row

        steps = [
            NormalizationStepConfig(
                step="ConfigurableStep", config={"prefix": "Mr. "}, enabled=True
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO("id,name\n1,Alice\n2,Bob"))
        result = list(rows)

        assert result[0]["name"] == "Mr. Alice"
        assert result[1]["name"] == "Mr. Bob"


class TestPipelineErrorHandling:
    """Test error handling in the pipeline."""

    def test_run_with_step_error_in_lenient_mode(self):
        """Test that errors in steps are caught in lenient mode."""

        @register_step("ErrorStep")
        class ErrorStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                if row.get("id") == "2":
                    raise ValueError("Simulated error")
                return row

        steps = [
            NormalizationStepConfig(step="ErrorStep", enabled=True),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO("id,name\n1,Alice\n2,Bob\n3,Charlie"))
        result = list(rows)

        # Row 2 should be dropped due to error
        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[1]["id"] == "3"
        assert len(report.warnings) == 1
        assert "Simulated error" in report.warnings[0]

    def test_run_with_step_error_in_strict_mode(self):
        """Test that errors in steps are raised in strict mode."""

        @register_step("StrictErrorStep")
        class StrictErrorStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                if row.get("id") == "2":
                    raise ValueError("Simulated error in strict mode")
                return row

        steps = [
            NormalizationStepConfig(step="StrictErrorStep", enabled=True),
        ]
        config = NormalizationConfig(strict=True, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO("id,name\n1,Alice\n2,Bob\n3,Charlie"))

        with pytest.raises(ValueError, match="Simulated error in strict mode"):
            list(rows)

    def test_run_adds_warning_with_step_name(self):
        """Test that warnings include the step class name."""

        @register_step("NamedErrorStep")
        class NamedErrorStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                raise ValueError("Test error")

        steps = [
            NormalizationStepConfig(step="NamedErrorStep", enabled=True),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO("id,name\n1,Alice"))
        list(rows)

        assert len(report.warnings) == 1
        assert "NamedErrorStep" in report.warnings[0]


class TestPipelineLazyEvaluation:
    """Test that pipeline uses lazy evaluation."""

    def test_run_returns_iterator_not_list(self, empty_config):
        """Test that run returns an iterator, not a materialized list."""
        pipeline = NormalizationPipeline(config=empty_config)

        rows, report = pipeline.run(io.StringIO("id,name\n1,Alice\n2,Bob"))

        # Should be an iterator
        assert hasattr(rows, "__iter__")
        assert hasattr(rows, "__next__")

    def test_rows_not_processed_until_consumed(self):
        """Test that rows are not processed until the iterator is consumed."""
        call_count = {"count": 0}

        @register_step("CountingStep")
        class CountingStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                call_count["count"] += 1
                return row

        steps = [
            NormalizationStepConfig(step="CountingStep", enabled=True),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO("id,name\n1,Alice\n2,Bob\n3,Charlie"))

        # Before consuming iterator, no rows processed
        assert call_count["count"] == 0

        # Consume one row
        next(rows)
        assert call_count["count"] == 1

        # Consume remaining rows
        list(rows)
        assert call_count["count"] == 3

    def test_can_partially_consume_iterator(self, empty_config):
        """Test that iterator can be partially consumed."""
        pipeline = NormalizationPipeline(config=empty_config)

        rows, report = pipeline.run(io.StringIO("id,name\n1,Alice\n2,Bob\n3,Charlie"))

        first_row = next(rows)
        assert first_row["id"] == "1"

        second_row = next(rows)
        assert second_row["id"] == "2"

        # Don't consume the third row


class TestPipelineCSVParsing:
    """Test CSV parsing behavior."""

    def test_parse_csv_with_different_delimiters(self, empty_config):
        """Test parsing CSV (note: default csv.DictReader uses comma)."""
        pipeline = NormalizationPipeline(config=empty_config)

        rows, report = pipeline.run(
            io.StringIO("id,name,email\n1,Alice,alice@example.com")
        )
        result = list(rows)

        assert result[0]["id"] == "1"
        assert result[0]["name"] == "Alice"
        assert result[0]["email"] == "alice@example.com"

    def test_parse_csv_with_quoted_fields(self, empty_config):
        """Test parsing CSV with quoted fields."""
        pipeline = NormalizationPipeline(config=empty_config)
        csv_data = 'id,name,description\n1,"Alice","A person"\n2,"Bob","Another person"'

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert result[0]["name"] == "Alice"
        assert result[0]["description"] == "A person"

    def test_parse_csv_with_quoted_commas(self, empty_config):
        """Test parsing CSV with commas inside quoted fields."""
        pipeline = NormalizationPipeline(config=empty_config)
        csv_data = 'id,name,description\n1,Alice,"Hello, World"'

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert result[0]["description"] == "Hello, World"

    def test_parse_csv_with_empty_fields(self, empty_config):
        """Test parsing CSV with empty fields."""
        pipeline = NormalizationPipeline(config=empty_config)
        csv_data = "id,name,email\n1,,\n2,Bob,"

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert result[0]["name"] == ""
        assert result[0]["email"] == ""
        assert result[1]["name"] == "Bob"
        assert result[1]["email"] == ""

    def test_parse_csv_with_extra_columns(self, empty_config):
        """Test parsing CSV where rows have more columns than header."""
        pipeline = NormalizationPipeline(config=empty_config)
        # CSV with extra column in data row
        csv_data = "id,name\n1,Alice,extra"

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        # csv.DictReader typically handles this by adding None keys
        assert result[0]["id"] == "1"
        assert result[0]["name"] == "Alice"


class TestPipelineIntegration:
    """Integration tests for complete pipeline scenarios."""

    def test_complete_pipeline_with_multiple_steps(self):
        """Test a complete pipeline with line and row processing steps."""

        @register_step("RemoveCommentsStep")
        class RemoveCommentsStep(BaseNormalizerStep):
            def process_lines(self, lines: Iterator[str]) -> Iterator[str]:
                for line in lines:
                    if not line.strip().startswith("#"):
                        yield line

        @register_step("AddTimestampStep")
        class AddTimestampStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                row["processed"] = "yes"
                return row

        steps = [
            NormalizationStepConfig(step="RemoveCommentsStep", enabled=True),
            NormalizationStepConfig(step="AddTimestampStep", enabled=True),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        csv_data = """# This is a comment
id,name
1,Alice
# Another comment
2,Bob"""

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[0]["processed"] == "yes"
        assert result[1]["id"] == "2"
        assert result[1]["processed"] == "yes"

    def test_pipeline_with_filtering_and_transformation(self):
        """Test pipeline that filters and transforms data."""

        @register_step("FilterNullNamesStep")
        class FilterNullNamesStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                if not row.get("name") or row["name"] == "":
                    return None
                return row

        @register_step("NormalizeEmailStep")
        class NormalizeEmailStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                if "email" in row:
                    row["email"] = row["email"].lower()
                return row

        steps = [
            NormalizationStepConfig(step="FilterNullNamesStep", enabled=True),
            NormalizationStepConfig(step="NormalizeEmailStep", enabled=True),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        csv_data = """id,name,email
1,Alice,ALICE@EXAMPLE.COM
2,,BOB@EXAMPLE.COM
3,Charlie,CHARLIE@EXAMPLE.COM"""

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["name"] == "Alice"
        assert result[0]["email"] == "alice@example.com"
        assert result[1]["name"] == "Charlie"
        assert result[1]["email"] == "charlie@example.com"
        assert report.rows_emitted == 2

    def test_pipeline_reports_metrics_correctly(self):
        """Test that pipeline reports correct metrics throughout processing."""

        @register_step("MetricsTestStep")
        class MetricsTestStep(BaseNormalizerStep):
            def process_row(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
                # Filter out rows with id == "2"
                if row.get("id") == "2":
                    return None
                return row

        steps = [
            NormalizationStepConfig(step="MetricsTestStep", enabled=True),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        csv_data = """id,name
1,Alice
2,Bob
3,Charlie
4,David"""

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert report.total_lines_read == 5  # Header + 4 data rows
        assert report.rows_emitted == 3  # 4 rows - 1 filtered
        assert len(result) == 3
