"""
Integration tests with comprehensive fixtures for normalization pipeline.
Tests various real-world scenarios with sample data files.
"""

import pytest
import io
from atomingest.normalization.pipeline import NormalizationPipeline
from atomingest.normalization.core import NormalizationConfig, NormalizationStepConfig

# Import standard_steps to register them
import atomingest.normalization.standard_steps  # noqa: F401


@pytest.fixture
def metadata_pre_header_csv():
    """CSV with metadata lines before the header."""
    return """REPORT GENERATED: 2024-01-15 10:30:00
SOURCE SYSTEM: Legacy Mainframe
EXTRACT DATE: 2024-01-15
RECORD TYPE: User Account Data
====================================
id,name,email,status,created_date
1,Alice Smith,alice@example.com,active,2024-01-01
2,Bob Jones,bob@example.com,inactive,2024-01-02
3,Charlie Brown,charlie@example.com,active,2024-01-03"""


@pytest.fixture
def repeated_headers_csv():
    """CSV with repeated header rows (concatenated extracts)."""
    return """id,name,email,department
1,Alice Smith,alice@example.com,Engineering
2,Bob Jones,bob@example.com,Marketing
id,name,email,department
3,Charlie Brown,charlie@example.com,Sales
4,Diana Prince,diana@example.com,Engineering
id,name,email,department
5,Eve Wilson,eve@example.com,HR"""


@pytest.fixture
def trailer_row_count_csv():
    """CSV with trailer line containing row count."""
    return """id,name,email,score
1,Alice,alice@example.com,95
2,Bob,bob@example.com,87
3,Charlie,charlie@example.com,92
4,Diana,diana@example.com,88
TRAILER:RECORD_COUNT=4"""


@pytest.fixture
def whitespace_and_nulls_csv():
    """CSV with various whitespace issues and null tokens."""
    return """id,name,email,phone,status
1,  Alice Smith  ,alice@example.com,  555-1234  ,active
2,Bob Jones,bob@example.com,NA,-
3,  Charlie  ,,NULL,active
4,Diana,diana@example.com,nan,active"""


@pytest.fixture
def numeric_commas_csv():
    """CSV with numeric values containing commas."""
    return """id,name,salary,revenue,employees
1,Alice,"85,000","1,250,000",5
2,Bob,"120,500","2,750,000",12
3,Charlie,"95,750","950,000",3"""


@pytest.fixture
def utf8_bom_csv():
    """CSV with UTF-8 BOM at the beginning."""
    return """\ufeffid,name,email
1,Alice,alice@example.com
2,Bob,bob@example.com"""


@pytest.fixture
def mixed_issues_csv():
    """CSV combining multiple issues: metadata, BOM, repeated headers, whitespace, nulls."""
    return """\ufeffREPORT: User Export
DATE: 2024-01-15
id,name,email,status
1,  Alice  ,alice@example.com,active
id,name,email,status
2,Bob,NA,-
3,,NULL,active"""


@pytest.fixture
def empty_file_csv():
    """Empty CSV file."""
    return ""


@pytest.fixture
def header_only_csv():
    """CSV with only header, no data rows."""
    return """id,name,email"""


@pytest.fixture
def special_characters_csv():
    """CSV with special characters in data."""
    return '''id,name,description,tags
1,Alice,"Product with ""quotes""","tag1,tag2"
2,Bob,Item with apostrophes,tag3
3,Charlie,Data & symbols!,#special'''


class TestMetadataPreHeader:
    """Test scenarios with metadata lines before the header."""

    def test_skip_metadata_lines(self, metadata_pre_header_csv):
        """Test skipping metadata lines and finding the header."""
        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={
                    "expected_columns": [
                        "id",
                        "name",
                        "email",
                        "status",
                        "created_date",
                    ]
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(metadata_pre_header_csv))
        result = list(rows)

        # Verify rows
        assert len(result) == 3
        assert result[0]["id"] == "1"
        assert result[0]["name"] == "Alice Smith"
        assert result[2]["name"] == "Charlie Brown"

        # Verify report counters
        assert report.metadata_lines_skipped == 5
        assert report.rows_emitted == 3

    def test_metadata_with_max_skip_limit(self, metadata_pre_header_csv):
        """Test max_skip limit when skipping metadata."""
        steps = [
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={
                    "expected_columns": ["id", "name", "email"],
                    "max_skip": 3,  # Only allow 3 lines to be skipped
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(metadata_pre_header_csv))
        _ = list(rows)  # Consume the iterator

        # Should trigger warning and include remaining lines
        assert len(report.warnings) > 0
        assert any("max_skip" in w for w in report.warnings)


class TestRepeatedHeaders:
    """Test scenarios with repeated header rows."""

    def test_drop_repeated_headers(self, repeated_headers_csv):
        """Test dropping repeated header rows from concatenated files."""
        steps = [
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(repeated_headers_csv))
        result = list(rows)

        # Verify all data rows present, headers removed
        assert len(result) == 5
        assert result[0]["id"] == "1"
        assert result[2]["id"] == "3"
        assert result[4]["id"] == "5"

        # Verify report counters
        assert report.duplicate_headers_removed == 2
        assert report.rows_emitted == 5


class TestTrailerRowCount:
    """Test scenarios with trailer lines containing row counts."""

    def test_extract_row_count_from_trailer(self, trailer_row_count_csv):
        """Test extracting row count from trailer line."""
        steps = [
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={
                    "pattern": r"^TRAILER:",
                    "extract_row_count": True,
                    "row_count_regex": r"RECORD_COUNT=(\d+)",
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(trailer_row_count_csv))
        result = list(rows)

        # Verify data rows
        assert len(result) == 4
        assert result[0]["name"] == "Alice"
        assert result[3]["name"] == "Diana"

        # Verify trailer was processed
        assert report.trailer_lines_skipped == 1
        assert report.row_count_trailer_value == 4
        assert report.rows_emitted == 4


class TestWhitespaceAndNulls:
    """Test scenarios with whitespace and null token handling."""

    def test_trim_whitespace_and_normalize_nulls(self, whitespace_and_nulls_csv):
        """Test trimming whitespace and converting null tokens."""
        steps = [
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(whitespace_and_nulls_csv))
        result = list(rows)

        # Verify whitespace trimming
        assert result[0]["name"] == "Alice Smith"
        assert result[0]["phone"] == "555-1234"

        # Verify null conversions
        assert result[1]["phone"] is None  # "NA"
        assert result[1]["status"] is None  # "-"
        assert result[2]["name"] == "Charlie"  # Whitespace trimmed
        assert result[2]["email"] is None  # Empty string converted to None
        assert result[2]["phone"] is None  # "NULL" token converted to None
        assert result[3]["phone"] is None  # "nan"

        assert len(result) == 4
        assert report.rows_emitted == 4


class TestNumericCommas:
    """Test scenarios with numeric values containing commas."""

    def test_numeric_values_with_commas_preserved(self, numeric_commas_csv):
        """Test that numeric values with commas are handled by CSV parser."""
        steps = []
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(numeric_commas_csv))
        result = list(rows)

        # CSV parser handles quoted values with commas
        assert len(result) == 3
        assert result[0]["salary"] == "85,000"
        assert result[0]["revenue"] == "1,250,000"
        assert result[1]["salary"] == "120,500"
        assert result[2]["employees"] == "3"


class TestUTF8BOM:
    """Test scenarios with UTF-8 BOM."""

    def test_remove_utf8_bom(self, utf8_bom_csv):
        """Test removing UTF-8 BOM from the beginning of file."""
        steps = [
            NormalizationStepConfig(
                step="RemoveBOMStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(utf8_bom_csv))
        result = list(rows)

        # Verify data is read correctly
        assert len(result) == 2
        assert result[0]["id"] == "1"
        assert result[0]["name"] == "Alice"

        # Verify BOM warning
        assert any("BOM" in w for w in report.warnings)


class TestMixedIssues:
    """Test comprehensive scenarios combining multiple issues."""

    def test_full_pipeline_with_all_steps(self, mixed_issues_csv):
        """Test full normalization pipeline with multiple steps."""
        steps = [
            NormalizationStepConfig(
                step="RemoveBOMStep",
                config={},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={"expected_columns": ["id", "name", "email", "status"]},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(mixed_issues_csv))
        result = list(rows)

        # Verify final results
        assert len(result) == 3
        assert result[0]["name"] == "Alice"  # Whitespace trimmed
        assert result[1]["email"] is None  # "NA" converted to None
        assert result[1]["status"] is None  # "-" converted to None
        assert result[2]["name"] is None  # Empty converted to None

        # Verify report counters
        assert report.metadata_lines_skipped == 2
        assert report.duplicate_headers_removed == 1
        assert report.rows_emitted == 3
        assert any("BOM" in w for w in report.warnings)


class TestEdgeCases:
    """Test edge cases and error conditions."""

    def test_empty_file(self, empty_file_csv):
        """Test handling of completely empty file."""
        steps = []
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(empty_file_csv))
        result = list(rows)

        assert len(result) == 0
        assert report.rows_emitted == 0

    def test_header_only_file(self, header_only_csv):
        """Test handling of file with only header, no data."""
        steps = []
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(header_only_csv))
        result = list(rows)

        assert len(result) == 0
        assert report.rows_emitted == 0

    def test_special_characters_in_data(self, special_characters_csv):
        """Test handling of special characters in data."""
        steps = []
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(special_characters_csv))
        result = list(rows)

        assert len(result) == 3
        # CSV parser handles quoted strings with special chars
        assert "quotes" in result[0]["description"]
        assert result[1]["description"] == "Item with apostrophes"
        assert result[2]["description"] == "Data & symbols!"


class TestColumnNameNormalization:
    """Test column name normalization scenarios."""

    def test_normalize_column_names(self):
        """Test column name normalization with spaces and special chars."""
        csv_data = """User ID,Full Name,Email Address,Status Flag
1,Alice,alice@example.com,active
2,Bob,bob@example.com,inactive"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert "user_id" in result[0]
        assert "full_name" in result[0]
        assert "email_address" in result[0]
        assert "status_flag" in result[0]

    def test_column_alias_mapping(self):
        """Test column name aliasing."""
        csv_data = """usr_id,usr_nm,usr_email
1,Alice,alice@example.com
2,Bob,bob@example.com"""

        steps = [
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={
                    "alias_map": {
                        "usr_id": "user_id",
                        "usr_nm": "user_name",
                        "usr_email": "email",
                    }
                },
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        assert len(result) == 2
        assert "user_id" in result[0]
        assert "user_name" in result[0]
        assert "email" in result[0]


class TestComplexRealWorldScenario:
    """Test complex real-world scenario with all features."""

    def test_complete_enterprise_extract(self):
        """Test processing a complex enterprise data extract."""
        csv_data = """\ufeffSYSTEM EXTRACT REPORT
Generated: 2024-01-15 10:30:00
Source: HR Database
Classification: Internal Use Only
=====================================
Employee ID,Full Name  ,Department,  Salary  ,Hire Date,Status
1001,  Alice Smith  ,Engineering,"85,000",2023-01-15,active
1002,Bob Jones,Marketing,NA,2023-02-20,-
Employee ID,Full Name  ,Department,  Salary  ,Hire Date,Status
1003,  Charlie Brown  ,Sales,"75,500",2023-03-10,active
1004,,Engineering,NULL,2023-04-05,inactive
TRAILER: Total Records=4 | Extract Complete"""

        steps = [
            NormalizationStepConfig(
                step="RemoveBOMStep",
                config={},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="SkipUntilHeaderStep",
                config={
                    "expected_columns": [
                        "Employee ID",
                        "Full Name",
                        "Department",
                        "Salary",
                    ]
                },
                enabled=True,
            ),
            NormalizationStepConfig(
                step="DropRepeatedHeaderRowsStep",
                config={},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="DropTrailerLinesStep",
                config={
                    "pattern": r"^TRAILER:",
                    "extract_row_count": True,
                    "row_count_regex": r"Total Records=(\d+)",
                },
                enabled=True,
            ),
            NormalizationStepConfig(
                step="ColumnNameNormalizationStep",
                config={},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="TrimWhitespaceStep",
                config={},
                enabled=True,
            ),
            NormalizationStepConfig(
                step="NullTokenNormalizationStep",
                config={},
                enabled=True,
            ),
        ]
        config = NormalizationConfig(strict=False, steps=steps)
        pipeline = NormalizationPipeline(config=config)

        rows, report = pipeline.run(io.StringIO(csv_data))
        result = list(rows)

        # Verify all data rows processed correctly
        assert len(result) == 4
        assert result[0]["employee_id"] == "1001"
        assert result[0]["full_name"] == "Alice Smith"
        assert result[0]["salary"] == "85,000"

        assert result[1]["full_name"] == "Bob Jones"
        assert result[1]["salary"] is None  # NA -> None
        assert result[1]["status"] is None  # - -> None

        assert result[2]["full_name"] == "Charlie Brown"
        assert result[3]["full_name"] is None  # Empty -> None
        assert result[3]["salary"] is None  # NULL -> None

        # Verify report metrics
        assert report.metadata_lines_skipped == 5
        assert report.duplicate_headers_removed == 1
        assert report.trailer_lines_skipped == 1
        assert report.row_count_trailer_value == 4
        assert report.rows_emitted == 4

        # Verify warnings
        assert len(report.warnings) > 0
        assert any("BOM" in w for w in report.warnings)
