"""
Test suite for standard normalization steps.
Tests SkipUntilHeaderStep, RemoveBOMStep, and other standard steps.
"""

import pytest
import io
from atomingest.normalization.pipeline import NormalizationPipeline
from atomingest.normalization.core import NormalizationConfig, NormalizationStepConfig
from atomingest.normalization.steps import _STEP_REGISTRY

# Import standard_steps to register the steps
from atomingest.normalization import standard_steps  # noqa: F401


@pytest.fixture(autouse=True)
def cleanup_registry():
    """Clean up step registry before and after each test."""
    initial_registry = _STEP_REGISTRY.copy()
    yield
    _STEP_REGISTRY.clear()
    _STEP_REGISTRY.update(initial_registry)


class TestSkipUntilHeaderStep:
    """Test suite for SkipUntilHeaderStep."""

    def test_skip_metadata_with_header_regex(self):
        """Test skipping metadata lines using header_regex."""
        csv_data = """REPORT GENERATED: 2024-01-15
SOURCE SYSTEM: Legacy DB
EXTRACT DATE: 2024-01-15 08:30:00
id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={"header_regex": r"^id,name,email$"},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[0]["name"] == "Alice"
        assert report.metadata_lines_skipped == 3

    def test_skip_metadata_with_expected_columns(self):
        """Test skipping metadata lines using expected_columns."""
        csv_data = """REPORT GENERATED: 2024-01-15
SOURCE SYSTEM: Legacy DB
EXTRACT DATE: 2024-01-15 08:30:00
id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={"expected_columns": ["id", "name", "email"]},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[1]["name"] == "Bob"
        assert report.metadata_lines_skipped == 3

    def test_skip_with_expected_columns_subset_match(self):
        """Test that expected_columns matches when it's a subset of actual columns."""
        csv_data = """METADATA LINE
id,name,email,status,created_date
1,Alice,alice@example.com,active,2024-01-15
2,Bob,bob@example.com,inactive,2024-01-16"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={"expected_columns": ["id", "name", "email"]},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert report.metadata_lines_skipped == 1

    def test_no_metadata_to_skip(self):
        """Test when header is on first line (no metadata to skip)."""
        csv_data = """id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={"expected_columns": ["id", "name", "email"]},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert report.metadata_lines_skipped == 0

    def test_max_skip_limit_reached_strict_mode(self):
        """Test that max_skip limit raises error in strict mode."""
        csv_data = """METADATA LINE 1
METADATA LINE 2
METADATA LINE 3
METADATA LINE 4
METADATA LINE 5
id,name,email
1,Alice,alice@example.com"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={
                    "expected_columns": ["id", "name", "email"],
                    "max_skip": 3,
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=True, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        with pytest.raises(ValueError) as exc_info:
            rows, report = pipeline.run(io.StringIO(csv_data))
            list(rows)  # Consume iterator to trigger error

        assert "Header not found after skipping" in str(exc_info.value)
        assert "max_skip=3" in str(exc_info.value)

    def test_max_skip_limit_reached_non_strict_mode(self):
        """Test that max_skip limit adds warning in non-strict mode and yields remaining lines."""
        csv_data = """METADATA LINE 1
METADATA LINE 2
METADATA LINE 3
METADATA LINE 4
METADATA LINE 5
id,name,email
1,Alice,alice@example.com"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={
                    "expected_columns": ["id", "name", "email"],
                    "max_skip": 3,
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        # In non-strict mode, should yield remaining lines after max_skip
        # Note: Skips lines 1-3, then on 4th line exceeds max_skip, so 4 lines skipped total
        assert len(result) > 0
        assert report.metadata_lines_skipped == 4  # Skipped 4 lines before giving up
        assert any("Header not found after skipping" in w for w in report.warnings)

    def test_max_skip_exact_limit(self):
        """Test that max_skip works correctly when limit is exactly reached."""
        csv_data = """METADATA LINE 1
METADATA LINE 2
METADATA LINE 3
id,name,email
1,Alice,alice@example.com"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={
                    "expected_columns": ["id", "name", "email"],
                    "max_skip": 3,
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=True, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        assert result[0]["id"] == "1"
        assert report.metadata_lines_skipped == 3

    def test_header_not_found_strict_mode(self):
        """Test that missing header raises error in strict mode."""
        csv_data = """METADATA LINE 1
METADATA LINE 2
METADATA LINE 3"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={"expected_columns": ["id", "name", "email"]},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=True, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        with pytest.raises(ValueError) as exc_info:
            rows, report = pipeline.run(io.StringIO(csv_data))
            list(rows)

        assert "Header not found in file" in str(exc_info.value)

    def test_header_not_found_non_strict_mode(self):
        """Test that missing header adds warning in non-strict mode."""
        csv_data = """METADATA LINE 1
METADATA LINE 2
METADATA LINE 3"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={"expected_columns": ["id", "name", "email"]},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 0
        assert report.metadata_lines_skipped == 3
        assert any("Header not found in file" in w for w in report.warnings)

    def test_no_config_yields_all_lines(self):
        """Test that when no header_regex or expected_columns provided, all lines pass through."""
        csv_data = """METADATA LINE 1
id,name,email
1,Alice,alice@example.com"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        # Should process all lines since no criteria defined
        assert len(result) == 2
        assert report.metadata_lines_skipped == 0

    def test_custom_separator(self):
        """Test using custom separator for column detection."""
        csv_data = """METADATA LINE
id|name|email
1|Alice|alice@example.com
2|Bob|bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={
                    "expected_columns": ["id", "name", "email"],
                    "separator": "|",
                },
                enabled=True,
            ),
        ]
        # Configure pipeline with pipe-delimited CSV
        config = NormalizationConfig(strict=False, steps=steps)
        # Set source configuration as attributes
        config.source_type = "csv"
        config.source_config = {"delimiter": "|"}

        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert report.metadata_lines_skipped == 1

    def test_regex_priority_over_expected_columns(self):
        """Test that header_regex takes priority when both are provided."""
        csv_data = """id,name,email
product_id,product_name,price
1,Widget,9.99
2,Gadget,19.99"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={
                    "header_regex": r"^product_id,",
                    "expected_columns": ["id", "name", "email"],
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        # Should match regex, not expected_columns
        assert len(result) == 2
        assert "product_id" in result[0]
        assert result[0]["product_id"] == "1"
        assert report.metadata_lines_skipped == 1

    def test_empty_file(self):
        """Test handling of empty file."""
        csv_data = ""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={"expected_columns": ["id", "name", "email"]},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 0
        assert report.metadata_lines_skipped == 0
        # Should not warn about empty file
        assert len(report.warnings) == 0

    def test_whitespace_handling_in_columns(self):
        """Test that column names with whitespace are properly trimmed."""
        csv_data = """METADATA
  id  ,  name  ,  email
1,Alice,alice@example.com"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={"expected_columns": ["id", "name", "email"]},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        assert report.metadata_lines_skipped == 1


class TestRemoveBOMStep:
    """Test suite for RemoveBOMStep."""

    def test_remove_bom_from_first_line(self):
        """Test removing BOM from first line only."""
        csv_data = (
            "\ufeffid,name,email\n1,Alice,alice@example.com\n2,Bob,bob@example.com"
        )

        steps = [
            NormalizationStepConfig(
                step="RemoveBOMStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert any("BOM detected and removed" in w for w in report.warnings)

    def test_remove_bom_from_all_lines(self):
        """Test removing BOM from all lines when check_all_lines=True."""
        csv_data = "\ufeffid,name,email\n\ufeff1,Alice,alice@example.com\n\ufeff2,Bob,bob@example.com"

        steps = [
            NormalizationStepConfig(
                step="RemoveBOMStep",
                config={"check_all_lines": True},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[1]["id"] == "2"
        # Should have 3 warnings (one for each line with BOM)
        bom_warnings = [w for w in report.warnings if "BOM detected" in w]
        assert len(bom_warnings) == 3

    def test_no_bom_present(self):
        """Test when no BOM is present."""
        csv_data = "id,name,email\n1,Alice,alice@example.com"

        steps = [
            NormalizationStepConfig(
                step="RemoveBOMStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        assert len(report.warnings) == 0

    def test_combined_bom_and_skip_until_header(self):
        """Test combining RemoveBOMStep with SkipUntilHeaderStep."""
        csv_data = "\ufeffMETADATA LINE\n\ufeffid,name,email\n1,Alice,alice@example.com"

        steps = [
            NormalizationStepConfig(
                step="RemoveBOMStep",
                config={"check_all_lines": True},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={"expected_columns": ["id", "name", "email"]},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        assert result[0]["id"] == "1"
        assert report.metadata_lines_skipped == 1
        bom_warnings = [w for w in report.warnings if "BOM detected" in w]
        assert len(bom_warnings) == 2


class TestDropRepeatedHeaderRowsStep:
    """Test suite for DropRepeatedHeaderRowsStep."""

    def test_drop_single_repeated_header(self):
        """Test dropping a single repeated header row mid-file."""
        csv_data = """id,name,email
1,Alice,alice@example.com
id,name,email
2,Bob,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[0]["name"] == "Alice"
        assert result[1]["id"] == "2"
        assert result[1]["name"] == "Bob"
        assert report.duplicate_headers_removed == 1

    def test_drop_multiple_repeated_headers(self):
        """Test dropping multiple repeated header rows (concatenated extracts)."""
        csv_data = """id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com
id,name,email
3,Charlie,charlie@example.com
id,name,email
4,Diana,diana@example.com"""

        steps = [
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 4
        assert result[0]["id"] == "1"
        assert result[1]["id"] == "2"
        assert result[2]["id"] == "3"
        assert result[3]["id"] == "4"
        assert report.duplicate_headers_removed == 2

    def test_no_repeated_headers(self):
        """Test when there are no repeated headers."""
        csv_data = """id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com
3,Charlie,charlie@example.com"""

        steps = [
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 3
        assert result[0]["id"] == "1"
        assert result[1]["id"] == "2"
        assert result[2]["id"] == "3"
        assert report.duplicate_headers_removed == 0

    def test_data_row_with_header_like_values(self):
        """Test that rows are only dropped when ALL values match their column names."""
        # This test verifies the step only drops rows where every value equals its key
        csv_data = """field_type,field_name,field_value
id,name,email
field_type,field_name,field_value
text,address,location"""

        steps = [
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        # Only the third row should be dropped (it matches all keys)
        assert len(result) == 2
        assert result[0]["field_type"] == "id"
        assert result[0]["field_name"] == "name"
        assert result[1]["field_type"] == "text"
        assert report.duplicate_headers_removed == 1

    def test_empty_file(self):
        """Test with an empty file."""
        csv_data = ""

        steps = [
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 0
        assert report.duplicate_headers_removed == 0

    def test_only_header_no_data(self):
        """Test with only a header row and no data."""
        csv_data = """id,name,email"""

        steps = [
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 0
        assert report.duplicate_headers_removed == 0

    def test_combined_with_skip_until_header(self):
        """Test combining DropRepeatedHeaderRowsStep with SkipUntilHeaderStep."""
        csv_data = """METADATA LINE
METADATA LINE 2
id,name,email
1,Alice,alice@example.com
id,name,email
2,Bob,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={"expected_columns": ["id", "name", "email"]},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[1]["id"] == "2"
        assert report.metadata_lines_skipped == 2
        assert report.duplicate_headers_removed == 1

    def test_partial_header_match_not_dropped(self):
        """Test that rows with only partial matches to headers are not dropped."""
        csv_data = """id,name,email
1,Alice,alice@example.com
id,Bob,bob@example.com
3,Charlie,charlie@example.com"""

        steps = [
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        # The row "id,Bob,bob@example.com" should NOT be dropped because
        # not all values match their keys (name != "Bob", email != "bob@example.com")
        assert len(result) == 3
        assert result[0]["id"] == "1"
        assert result[1]["id"] == "id"
        assert result[1]["name"] == "Bob"
        assert result[2]["id"] == "3"
        assert report.duplicate_headers_removed == 0

    def test_with_different_column_names(self):
        """Test with different column names."""
        csv_data = """user_id,full_name,contact_email,status
1,Alice Smith,alice@example.com,active
user_id,full_name,contact_email,status
2,Bob Jones,bob@example.com,inactive"""

        steps = [
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["user_id"] == "1"
        assert result[1]["user_id"] == "2"
        assert report.duplicate_headers_removed == 1

    def test_consecutive_repeated_headers(self):
        """Test with consecutive repeated header rows."""
        csv_data = """id,name,email
id,name,email
id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[1]["id"] == "2"
        assert report.duplicate_headers_removed == 2


class TestTrimWhitespaceStep:
    """Test suite for TrimWhitespaceStep."""

    def test_basic_whitespace_trimming(self):
        """Test basic leading and trailing whitespace trimming."""
        csv_data = """id,name,email
1,  Alice  ,  alice@example.com
2,  Bob  ,  bob@example.com  """

        steps = [
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["name"] == "Alice"
        assert result[0]["email"] == "alice@example.com"
        assert result[1]["name"] == "Bob"
        assert result[1]["email"] == "bob@example.com"

    def test_trim_with_quotes_disabled(self):
        """Test that the step handles data without quote trimming when disabled."""
        csv_data = """id,name,email
1, Alice Smith , alice@example.com
2, Bob Jones , bob@example.com """

        steps = [
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={"trim_quotes": False},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        # Only whitespace is trimmed, quotes in data would remain if present
        assert result[0]["name"] == "Alice Smith"
        assert result[0]["email"] == "alice@example.com"
        assert result[1]["name"] == "Bob Jones"
        assert result[1]["email"] == "bob@example.com"

    def test_trim_with_quotes_enabled(self):
        """Test that quotes are removed when trim_quotes=True.
        This is useful when data has extra quotes that weren't handled by CSV parser."""
        # Using a custom delimiter so CSV parser doesn't strip the quotes
        csv_data = """id|name|email
1| "Alice Smith" | "alice@example.com"
2| 'Bob Jones' | 'bob@example.com' """

        # First, we need to parse with the correct delimiter
        import csv
        from io import StringIO

        # Manually parse with pipe delimiter
        reader = csv.DictReader(StringIO(csv_data), delimiter="|")
        rows_list = list(reader)

        # Now test the normalization step directly
        from atomingest.normalization.core import NormalizationReport
        from atomingest.normalization.standard_steps import TrimWhitespaceStep

        step = TrimWhitespaceStep(
            config={"trim_quotes": True}, report=NormalizationReport(), strict=False
        )

        result = []
        for row in rows_list:
            processed = step.process_row(row)
            if processed:
                result.append(processed)

        assert len(result) == 2
        assert result[0]["name"] == "Alice Smith"
        assert result[0]["email"] == "alice@example.com"
        assert result[1]["name"] == "Bob Jones"
        assert result[1]["email"] == "bob@example.com"

    def test_collapse_spaces_disabled(self):
        """Test that multiple spaces are preserved when collapse_spaces=False (default)."""
        csv_data = """id,name,description
1,Alice,This  has   multiple    spaces
2,Bob,Another  text   with    spaces"""

        steps = [
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={"collapse_spaces": False},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["description"] == "This  has   multiple    spaces"
        assert result[1]["description"] == "Another  text   with    spaces"

    def test_collapse_spaces_enabled(self):
        """Test that multiple spaces are collapsed to single space when collapse_spaces=True."""
        csv_data = """id,name,description
1,Alice,This  has   multiple    spaces
2,Bob,Another  text   with    spaces"""

        steps = [
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={"collapse_spaces": True},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["description"] == "This has multiple spaces"
        assert result[1]["description"] == "Another text with spaces"

    def test_collapse_spaces_with_newlines_and_tabs(self):
        """Test that collapse_spaces also handles tabs and newlines."""
        csv_data = """id,description
1,Text\twith\ttabs
2,Text  with\tmixed   spaces"""

        steps = [
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={"collapse_spaces": True},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["description"] == "Text with tabs"
        assert result[1]["description"] == "Text with mixed spaces"

    def test_combined_trim_quotes_and_collapse_spaces(self):
        """Test using both trim_quotes and collapse_spaces together."""
        csv_data = """id,name,description
1, "Alice  Smith" , "This   has   spaces"
2, 'Bob   Jones' , 'More    spaces   here' """

        steps = [
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={"trim_quotes": True, "collapse_spaces": True},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["name"] == "Alice Smith"
        assert result[0]["description"] == "This has spaces"
        assert result[1]["name"] == "Bob Jones"
        assert result[1]["description"] == "More spaces here"

    def test_non_string_values_unchanged(self):
        """Test that non-string values are passed through unchanged."""
        # Note: In CSV, all values are strings by default, but this tests the isinstance check
        from atomingest.normalization.pipeline import NormalizationPipeline
        from atomingest.normalization.steps import BaseNormalizerStep

        # Create a custom step that generates non-string values
        class InjectNonStringStep(BaseNormalizerStep):
            def process_row(self, row):
                # Convert id to int for testing
                if "id" in row:
                    row["id"] = int(row["id"]) if row["id"].isdigit() else row["id"]
                return row

        from atomingest.normalization.steps import _STEP_REGISTRY

        _STEP_REGISTRY["InjectNonStringStep"] = InjectNonStringStep

        csv_data = """id,name,email
1,  Alice  ,  alice@example.com
2,  Bob  ,  bob@example.com  """

        steps = [
            NormalizationStepConfig(
                step="InjectNonStringStep",
                config={},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == 1  # Should remain as int
        assert result[0]["name"] == "Alice"
        assert result[1]["id"] == 2  # Should remain as int
        assert result[1]["name"] == "Bob"

    def test_empty_strings_remain_empty(self):
        """Test that empty strings are handled correctly."""
        csv_data = """id,name,email
1,,alice@example.com
2,Bob,"""

        steps = [
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["name"] == ""
        assert result[1]["email"] == ""

    def test_whitespace_only_strings_become_empty(self):
        """Test that strings with only whitespace become empty after trimming."""
        csv_data = """id,name,email
1,   ,alice@example.com
2,Bob,   """

        steps = [
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["name"] == ""
        assert result[1]["email"] == ""

    def test_mixed_quotes_single_and_double(self):
        """Test handling of mixed single and double quotes."""
        csv_data = """id,name,description
1,\"Alice\",Text with 'single' quotes inside
2,'Bob',Text with \"double\" quotes inside"""

        steps = [
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={"trim_quotes": True},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["name"] == "Alice"
        assert result[0]["description"] == "Text with 'single' quotes inside"
        assert result[1]["name"] == "Bob"
        assert result[1]["description"] == 'Text with "double" quotes inside'

    def test_nested_quotes(self):
        """Test handling of nested quotes."""
        csv_data = """id,name
1,\"'Alice'\"
2,'\"Bob\"'"""

        steps = [
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={"trim_quotes": True},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        # strip("'\"") removes all leading/trailing quotes
        assert result[0]["name"] == "Alice"
        assert result[1]["name"] == "Bob"

    def test_combined_with_other_steps(self):
        """Test TrimWhitespaceStep combined with DropRepeatedHeaderRowsStep."""
        csv_data = """id,name,email
1,  Alice  ,alice@example.com
id,name,email
2,  Bob  ,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["name"] == "Alice"
        assert result[1]["name"] == "Bob"
        assert report.duplicate_headers_removed == 1

    def test_unicode_whitespace(self):
        """Test handling of various Unicode whitespace characters."""
        csv_data = """id,name,email
1,\u00a0Alice\u00a0,alice@example.com
2,\u2003Bob\u2003,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        # Python's strip() handles Unicode whitespace
        assert result[0]["name"] == "Alice"
        assert result[1]["name"] == "Bob"


class TestNullTokenNormalizationStep:
    """Test suite for NullTokenNormalizationStep."""

    def test_default_null_tokens(self):
        """Test that default null tokens are converted to None."""
        csv_data = """id,name,status,value,comment
1,Alice,active,100,Good
2,,NA,,-
3,Bob,NULL,nan,"""

        steps = [
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 3
        assert result[0]["name"] == "Alice"
        assert result[0]["status"] == "active"
        assert result[1]["name"] is None  # Empty string -> None
        assert result[1]["status"] is None  # "NA" -> None
        assert result[1]["value"] is None  # Empty string -> None
        assert result[1]["comment"] is None  # "-" -> None
        assert result[2]["status"] is None  # "NULL" -> None
        assert result[2]["value"] is None  # "nan" -> None
        assert result[2]["comment"] is None  # Empty string -> None

    def test_custom_null_tokens(self):
        """Test using custom null tokens."""
        csv_data = """id,name,status
1,Alice,active
2,N/A,unknown
3,Bob,n/a"""

        steps = [
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={"null_tokens": ["N/A", "unknown", "n/a"]},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 3
        assert result[0]["name"] == "Alice"
        assert result[0]["status"] == "active"
        assert result[1]["name"] is None  # "N/A" -> None
        assert result[1]["status"] is None  # "unknown" -> None
        assert result[2]["status"] is None  # "n/a" -> None

    def test_case_sensitivity(self):
        """Test that null token matching is case-sensitive."""
        csv_data = """id,value1,value2,value3
1,NULL,null,Null
2,NA,na,Na"""

        steps = [
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={"null_tokens": ["NULL", "NA"]},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["value1"] is None  # "NULL" matches
        assert result[0]["value2"] == "null"  # Case doesn't match
        assert result[0]["value3"] == "Null"  # Case doesn't match
        assert result[1]["value1"] is None  # "NA" matches
        assert result[1]["value2"] == "na"  # Case doesn't match
        assert result[1]["value3"] == "Na"  # Case doesn't match

    def test_no_null_values(self):
        """Test when there are no null values to convert."""
        csv_data = """id,name,status
1,Alice,active
2,Bob,inactive
3,Charlie,pending"""

        steps = [
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 3
        assert result[0]["name"] == "Alice"
        assert result[1]["name"] == "Bob"
        assert result[2]["name"] == "Charlie"
        # All values should remain unchanged
        assert all(v is not None for row in result for v in row.values())

    def test_all_null_values(self):
        """Test when all values in a row are null."""
        csv_data = """id,name,status,value
1,Alice,active,100
2,NA,NULL,-"""

        steps = [
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[0]["name"] == "Alice"
        assert result[1]["id"] == "2"
        assert result[1]["name"] is None
        assert result[1]["status"] is None
        assert result[1]["value"] is None

    def test_empty_string_handling(self):
        """Test that empty strings are converted to None by default."""
        csv_data = """id,name,email
1,Alice,alice@example.com
2,,
3,Bob,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 3
        assert result[1]["name"] is None
        assert result[1]["email"] is None

    def test_exclude_empty_string_from_nulls(self):
        """Test that empty strings can be excluded from null conversion."""
        csv_data = """id,name,status
1,Alice,active
2,,NA
3,Bob,NULL"""

        steps = [
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={"null_tokens": ["NA", "NULL"]},  # Empty string not included
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 3
        assert result[1]["name"] == ""  # Empty string preserved
        assert result[1]["status"] is None  # "NA" -> None
        assert result[2]["status"] is None  # "NULL" -> None

    def test_numeric_string_values_preserved(self):
        """Test that numeric strings are not converted to None."""
        csv_data = """id,value,amount
1,0,0.0
2,100,99.99
3,-1,-10.5"""

        steps = [
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 3
        assert result[0]["value"] == "0"
        assert result[0]["amount"] == "0.0"
        assert result[1]["value"] == "100"
        assert result[2]["value"] == "-1"
        assert result[2]["amount"] == "-10.5"

    def test_whitespace_variations(self):
        """Test that whitespace variations are not treated as null unless specified."""
        csv_data = """id,value1,value2,value3
1, , NA ,test"""

        steps = [
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        # " " (space) is not in default null tokens
        assert result[0]["value1"] == " "
        # " NA " has spaces, doesn't match "NA"
        assert result[0]["value2"] == " NA "
        # "test" is not a null token
        assert result[0]["value3"] == "test"

    def test_combined_with_trim_whitespace(self):
        """Test NullTokenNormalizationStep combined with TrimWhitespaceStep."""
        csv_data = """id,name,status
1,Alice,active
2, NA , NULL
3,Bob,"""

        steps = [
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 3
        assert result[0]["name"] == "Alice"
        # After trim, " NA " becomes "NA", then converted to None
        assert result[1]["name"] is None
        # After trim, " NULL " becomes "NULL", then converted to None
        assert result[1]["status"] is None
        # Empty string after trim -> None
        assert result[2]["status"] is None

    def test_special_characters_in_null_tokens(self):
        """Test null tokens with special characters."""
        csv_data = """id,value1,value2,value3
1,#N/A,<null>,[empty]
2,Alice,Bob,Charlie"""

        steps = [
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={"null_tokens": ["#N/A", "<null>", "[empty]"]},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["value1"] is None
        assert result[0]["value2"] is None
        assert result[0]["value3"] is None
        assert result[1]["value1"] == "Alice"
        assert result[1]["value2"] == "Bob"
        assert result[1]["value3"] == "Charlie"

    def test_single_dash_as_null(self):
        """Test that single dash is treated as null by default."""
        csv_data = """id,value1,value2,value3
1,100,-,200
2,-,-,-"""

        steps = [
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["value1"] == "100"
        assert result[0]["value2"] is None  # "-" -> None
        assert result[0]["value3"] == "200"
        assert result[1]["value1"] is None
        assert result[1]["value2"] is None
        assert result[1]["value3"] is None

    def test_non_string_values_unchanged(self):
        """Test that non-string values are passed through unchanged."""
        # Create a custom step that generates non-string values
        from atomingest.normalization.steps import BaseNormalizerStep, _STEP_REGISTRY

        class InjectNonStringStep(BaseNormalizerStep):
            def process_row(self, row):
                # Convert id to int for testing
                if "id" in row and row["id"].isdigit():
                    row["id"] = int(row["id"])
                if "value" in row and row["value"] not in ["NA", "NULL"]:
                    try:
                        row["value"] = float(row["value"])
                    except ValueError:
                        pass
                return row

        _STEP_REGISTRY["InjectNonStringStep"] = InjectNonStringStep

        csv_data = """id,name,value
1,Alice,100.5
2,NA,NA
3,Bob,200.0"""

        steps = [
            NormalizationStepConfig(
                step="InjectNonStringStep",
                config={},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 3
        assert result[0]["id"] == 1  # Should remain as int
        assert result[0]["value"] == 100.5  # Should remain as float
        assert result[1]["name"] is None  # "NA" string -> None
        assert result[1]["value"] is None  # "NA" string -> None
        assert result[2]["id"] == 3  # Should remain as int
        assert result[2]["value"] == 200.0  # Should remain as float

    def test_empty_config(self):
        """Test that step works with empty configuration."""
        csv_data = """id,name,status
1,Alice,active
2,NA,NULL"""

        steps = [
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["name"] == "Alice"
        assert result[1]["name"] is None
        assert result[1]["status"] is None

    def test_multiple_rows_with_mixed_nulls(self):
        """Test processing multiple rows with mixed null and non-null values."""
        csv_data = """id,name,email,status,score
1,Alice,alice@example.com,active,100
2,Bob,,NA,90
3,,charlie@example.com,NULL,
4,Diana,diana@example.com,-,nan"""

        steps = [
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 4
        # Row 1: All valid values
        assert result[0]["name"] == "Alice"
        assert result[0]["score"] == "100"
        # Row 2: Some nulls
        assert result[1]["email"] is None
        assert result[1]["status"] is None
        # Row 3: Some nulls
        assert result[2]["name"] is None
        assert result[2]["status"] is None
        assert result[2]["score"] is None
        # Row 4: Some nulls
        assert result[3]["status"] is None  # "-"
        assert result[3]["score"] is None  # "nan"


class TestColumnNameNormalizationStep:
    """Test suite for ColumnNameNormalizationStep."""

    def test_default_normalization_lowercase(self):
        """Test default column name normalization (lowercase, replace spaces with underscores)."""
        csv_data = """User ID,Full Name,Email Address,Status Flag
1,Alice,alice@example.com,active
2,Bob,bob@example.com,inactive"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        # Check normalized column names
        assert "user_id" in result[0]
        assert "full_name" in result[0]
        assert "email_address" in result[0]
        assert "status_flag" in result[0]
        assert result[0]["user_id"] == "1"
        assert result[0]["full_name"] == "Alice"

    def test_uppercase_normalization(self):
        """Test uppercase column name normalization."""
        csv_data = """user id,full name,email address
1,Alice,alice@example.com
2,Bob,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={"case": "upper"},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert "USER_ID" in result[0]
        assert "FULL_NAME" in result[0]
        assert "EMAIL_ADDRESS" in result[0]

    def test_no_case_transformation(self):
        """Test with case transformation disabled."""
        csv_data = """User ID,Full Name,Email Address
1,Alice,alice@example.com
2,Bob,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={"case": None},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert "User_ID" in result[0]
        assert "Full_Name" in result[0]
        assert "Email_Address" in result[0]

    def test_custom_space_replacement(self):
        """Test custom space replacement character."""
        csv_data = """user id,full name,email address
1,Alice,alice@example.com"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={"replace_spaces": "-", "remove_non_alphanumeric": False},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        assert "user-id" in result[0]
        assert "full-name" in result[0]
        assert "email-address" in result[0]

    def test_no_space_replacement(self):
        """Test with space replacement disabled."""
        csv_data = """user id,full name
1,Alice"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={"replace_spaces": None, "remove_non_alphanumeric": False},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        assert "user id" in result[0]
        assert "full name" in result[0]

    def test_remove_non_alphanumeric(self):
        """Test removal of special characters from column names."""
        csv_data = """user@id,full-name!,email#address,status%
1,Alice,alice@example.com,active"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        assert "userid" in result[0]
        assert "fullname" in result[0]
        assert "emailaddress" in result[0]
        assert "status" in result[0]

    def test_keep_non_alphanumeric(self):
        """Test keeping special characters when remove_non_alphanumeric is False."""
        csv_data = """user@id,full-name!
1,Alice"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={"remove_non_alphanumeric": False},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        # Spaces replaced but special chars kept
        assert "user@id" in result[0]
        assert "full-name!" in result[0]

    def test_alias_map_basic(self):
        """Test basic alias mapping functionality."""
        csv_data = """usr_id,usr_nm,usr_email
1,Alice,alice@example.com
2,Bob,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={
                    "alias_map": {
                        "usr_id": "user_id",
                        "usr_nm": "user_name",
                        "usr_email": "email",
                    }
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert "user_id" in result[0]
        assert "user_name" in result[0]
        assert "email" in result[0]
        assert result[0]["user_id"] == "1"
        assert result[0]["user_name"] == "Alice"

    def test_alias_map_with_unknown_columns(self):
        """Test that unknown columns trigger warnings when alias_map is provided."""
        csv_data = """usr_id,usr_nm,unknown_col
1,Alice,test
2,Bob,test2"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={
                    "alias_map": {
                        "usr_id": "user_id",
                        "usr_nm": "user_name",
                    }
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert "user_id" in result[0]
        assert "user_name" in result[0]
        # Unknown column should be normalized
        assert "unknown_col" in result[0]
        # Should have warning about unknown column
        assert any(
            "Unknown column" in w and "unknown_col" in w for w in report.warnings
        )

    def test_alias_map_warning_only_once_per_column(self):
        """Test that warning for unknown column is only issued once."""
        csv_data = """usr_id,unknown_col
1,test1
2,test2
3,test3"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={"alias_map": {"usr_id": "user_id"}},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 3
        # Only one warning should be issued for unknown_col
        unknown_warnings = [w for w in report.warnings if "unknown_col" in w]
        assert len(unknown_warnings) == 1

    def test_leading_trailing_whitespace_in_columns(self):
        """Test that leading/trailing whitespace in column names is stripped."""
        csv_data = """ user id ,  full name  , email
1,Alice,alice@example.com"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        assert "user_id" in result[0]
        assert "full_name" in result[0]
        assert "email" in result[0]

    def test_numeric_column_names(self):
        """Test handling of numeric column names."""
        csv_data = """123,456,789
value1,value2,value3"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        assert "123" in result[0]
        assert "456" in result[0]
        assert "789" in result[0]

    def test_empty_column_names(self):
        """Test handling of empty column names."""
        csv_data = """id,,name
1,value,Alice"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        assert "id" in result[0]
        assert "" in result[0]  # Empty string column name
        assert "name" in result[0]

    def test_unicode_column_names(self):
        """Test handling of Unicode characters in column names."""
        csv_data = """user_id,名前,email
1,Alice,alice@example.com"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        assert "user_id" in result[0]
        # Unicode chars removed by remove_non_alphanumeric
        assert "" in result[0] or any(
            k != "user_id" and k != "email" for k in result[0].keys()
        )

    def test_column_name_with_multiple_spaces(self):
        """Test column names with multiple consecutive spaces."""
        csv_data = """user    id,full     name
1,Alice"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        # Multiple spaces all replaced with underscores
        assert "user____id" in result[0]
        assert "full_____name" in result[0]

    def test_combined_with_trim_whitespace(self):
        """Test ColumnNameNormalizationStep combined with TrimWhitespaceStep."""
        csv_data = """User ID,Full Name
1,  Alice
2,  Bob  """

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert "user_id" in result[0]
        assert "full_name" in result[0]
        assert result[0]["full_name"] == "Alice"
        assert result[1]["full_name"] == "Bob"

    def test_alias_map_skips_further_normalization(self):
        """Test that aliased columns skip further normalization."""
        csv_data = """User ID,Full Name
1,Alice"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={
                    "alias_map": {
                        "User ID": "UserID",  # Preserved as-is
                        "Full Name": "FullName",  # Preserved as-is
                    }
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        # Should keep the alias format (not lowercased or spaces replaced)
        assert "UserID" in result[0]
        assert "FullName" in result[0]
        assert result[0]["UserID"] == "1"

    def test_complex_normalization_scenario(self):
        """Test complex scenario with multiple transformations."""
        csv_data = """User@ID#,Full-Name (Legacy),  Email Address  ,Status%Flag!
1,Alice Smith,alice@example.com,active
2,Bob Jones,bob@example.com,inactive"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={
                    "case": "lower",
                    "replace_spaces": "_",
                    "remove_non_alphanumeric": True,
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert "userid" in result[0]
        assert "fullname_legacy" in result[0]
        assert "email_address" in result[0]
        assert "statusflag" in result[0]

    def test_duplicate_column_names_after_normalization(self):
        """Test behavior when normalization creates duplicate column names."""
        csv_data = """User ID,User-ID,user_id
1,2,3"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        # All three columns normalize to "user_id"
        # The last one will overwrite previous ones
        assert "user_id" in result[0]
        # Value should be from the last column
        assert result[0]["user_id"] == "3"

    def test_underscore_preservation(self):
        """Test that underscores are preserved in column names."""
        csv_data = """user_id,user__name,_status
1,Alice,active"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 1
        assert "user_id" in result[0]
        assert "user__name" in result[0]
        assert "_status" in result[0]


class TestDropTrailerLinesStep:
    """Test suite for DropTrailerLinesStep."""

    def test_drop_simple_trailer(self):
        """Test dropping a simple trailer line."""
        csv_data = """id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com
END OF REPORT"""

        steps = [
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={"trailer_regex": r"^END OF REPORT"},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[1]["name"] == "Bob"
        assert report.trailer_lines_skipped == 1

    def test_extract_row_count_from_trailer(self):
        """Test extracting row count from trailer line."""
        csv_data = """id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com
Total Records: 2"""

        steps = [
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={
                    "trailer_regex": r"^Total Records:",
                    "row_count_regex": r"Total Records:\s*(\d+)",
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert report.trailer_lines_skipped == 1
        assert report.row_count_trailer_value == 2

    def test_multiple_trailers_same_count(self):
        """Test multiple trailer lines with same row count."""
        csv_data = """id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com
Total: 2
Records: 2"""

        steps = [
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={
                    "trailer_regex": r"^(Total|Records):",
                    "row_count_regex": r":\s*(\d+)",
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert report.trailer_lines_skipped == 2
        assert report.row_count_trailer_value == 2
        # No warning since counts match
        count_warnings = [w for w in report.warnings if "different row counts" in w]
        assert len(count_warnings) == 0

    def test_multiple_trailers_different_count_warns(self):
        """Test warning when multiple trailers have different row counts."""
        csv_data = """id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com
Total: 2
Records: 3"""

        steps = [
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={
                    "trailer_regex": r"^(Total|Records):",
                    "row_count_regex": r":\s*(\d+)",
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert report.trailer_lines_skipped == 2
        assert report.row_count_trailer_value == 3  # Last value stored
        # Should warn about different counts
        count_warnings = [w for w in report.warnings if "different row counts" in w]
        assert len(count_warnings) == 1
        assert "previous=2" in count_warnings[0]
        assert "current=3" in count_warnings[0]

    def test_no_trailer_present(self):
        """Test when no trailer is present."""
        csv_data = """id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={"trailer_regex": r"^END OF REPORT"},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert report.trailer_lines_skipped == 0

    def test_trailer_in_middle_anywhere_mode(self):
        """Test trailer in middle of file with 'anywhere' mode."""
        csv_data = """id,name,email
1,Alice,alice@example.com
SUMMARY: Some info
2,Bob,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={
                    "trailer_regex": r"^SUMMARY:",
                    "trailer_position": "anywhere",
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        # Should have 2 rows (trailer in middle removed)
        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[1]["id"] == "2"
        assert report.trailer_lines_skipped == 1

    def test_trailer_at_end_only_mode(self):
        """Test trailer removal with 'end-only' mode."""
        csv_data = """id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com
END OF REPORT
Total Records: 2"""

        steps = [
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={
                    "trailer_regex": r"^(END OF REPORT|Total Records:)",
                    "trailer_position": "end-only",
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[1]["name"] == "Bob"
        assert report.trailer_lines_skipped == 2

    def test_end_only_mode_with_max_trailer_lines(self):
        """Test end-only mode respects max_trailer_lines."""
        csv_data = """id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com
3,Charlie,charlie@example.com
Footer Line 1
Footer Line 2
Footer Line 3"""

        steps = [
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={
                    "trailer_regex": r"^Footer Line",
                    "trailer_position": "end-only",
                    "max_trailer_lines": 2,  # Only check last 2 lines
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        # Should only remove last 2 trailer lines (within max_trailer_lines)
        assert len(result) == 4  # 3 data rows + 1 footer line not checked
        assert report.trailer_lines_skipped == 2

    def test_end_only_stops_at_non_trailer(self):
        """Test end-only mode stops at first non-trailer from end."""
        csv_data = """id,name,email
1,Alice,alice@example.com
Footer Line 1
2,Bob,bob@example.com
Footer Line 2"""

        steps = [
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={
                    "trailer_regex": r"^Footer Line",
                    "trailer_position": "end-only",
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        # Should only remove last Footer Line 2 (stops at Bob line)
        assert len(result) == 3  # Alice, Footer Line 1, Bob
        assert report.trailer_lines_skipped == 1

    def test_invalid_row_count_warns(self):
        """Test warning when row count cannot be parsed."""
        csv_data = """id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com
Total Records: invalid"""

        steps = [
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={
                    "trailer_regex": r"^Total Records:",
                    "row_count_regex": r"Total Records:\s*(\w+)",
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert report.trailer_lines_skipped == 1
        # Should warn about failed parse
        parse_warnings = [
            w for w in report.warnings if "Failed to parse row count" in w
        ]
        assert len(parse_warnings) == 1

    def test_no_trailer_regex_passes_through(self):
        """Test that no trailer_regex passes all lines through."""
        csv_data = """id,name,email
1,Alice,alice@example.com
END OF REPORT"""

        steps = [
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        # Should process all lines including "END OF REPORT"
        assert len(result) == 2
        assert report.trailer_lines_skipped == 0

    def test_empty_file(self):
        """Test handling of empty file."""
        csv_data = ""

        steps = [
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={"trailer_regex": r"^END"},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 0
        assert report.trailer_lines_skipped == 0

    def test_combined_skip_header_and_drop_trailer(self):
        """Test combining SkipUntilHeaderStep with DropTrailerLinesStep."""
        csv_data = """REPORT METADATA
GENERATED: 2024-01-15
id,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com
END OF REPORT
Total Records: 2"""

        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={"expected_columns": ["id", "name", "email"]},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={"trailer_regex": r"^(END OF REPORT|Total Records:)"},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[1]["name"] == "Bob"
        assert report.metadata_lines_skipped == 2
        assert report.trailer_lines_skipped == 2
