"""Tests for DuckDB-based idx index pipeline."""

import gzip
from pathlib import Path
from typing import Any
from unittest.mock import patch

from secbulkdownload import idx

# Canonical column names for SEC index files
CANONICAL_COLUMNS = ["CIK", "Company Name", "Form Type", "Date Filed", "Filename"]


def create_mock_master_gz(
    tmp_path: Path, records: list[dict[str, Any]], encoding: str = "utf-8"
) -> str:
    """Create a mock master.gz SEC index file with given records.

    Args:
        tmp_path: Temporary directory path
        records: List of dicts with keys matching canonical schema

    Returns:
        Path to created gzip file
    """
    # SEC master.idx format has a metadata preamble, a column header row,
    # a dashed separator row, then data rows.
    header_lines = [
        "Description:           Master Index of EDGAR Dissemination Feed",
        "Last Updated:          2024-01-01",
        "Comments:             comments@sec.gov",
        "Anonymous FTP:        ftp://ftp.sec.gov/edgar/",
        "Cloud HTTP:           https://www.sec.gov/Archives/",
        "Anonymous FTP:        ftp://ftp.sec.gov/edgar/full-index/",
        "SFTP:                 sftp://ftp.sec.gov/edgar/full-index/",
        "",  # Empty line 8
        "",  # Empty line 9
    ]

    column_header = "CIK|Company Name|Form Type|Date Filed|Filename"
    separator_line = "-" * 80

    data_lines = []
    for rec in records:
        # Format: CIK|Company Name|Form Type|Date Filed|Filename
        # Don't zfill here - let the CIK be as-is in the file
        cik = str(rec.get("CIK", ""))
        company = rec.get("Company Name", "")
        form = rec.get("Form Type", "")
        date_filed = rec.get("Date Filed", "2024-01-01")
        filename = rec.get("Filename", "")
        data_lines.append(f"{cik}|{company}|{form}|{date_filed}|{filename}")

    content = (
        "\n".join(header_lines)
        + "\n"
        + column_header
        + "\n"
        + separator_line
        + "\n"
        + "\n".join(data_lines)
        + "\n"
    )

    # Tests can write UTF-8 or latin-1 master files to mirror SEC historical quirks.
    gz_path = tmp_path / "master.gz"
    with gzip.open(gz_path, "wt", encoding=encoding) as f:
        f.write(content)

    return str(gz_path)


def create_mock_idxdir(tmp_path: Path, quarters: list[tuple[int, int]]) -> str:
    """Create mock idxfiles directory structure with master.gz files.

    Args:
        tmp_path: Base temporary path
        quarters: List of (year, quarter) tuples

    Returns:
        Path to idxfiles directory
    """
    idxdir = tmp_path / "idxfiles"
    idxdir.mkdir(parents=True, exist_ok=True)

    for year, qtr in quarters:
        qtr_dir = idxdir / str(year) / f"QTR{qtr}"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        # Create mock data for each quarter
        records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": f"{year}-{qtr * 3:02d}-01",
                "Filename": f"edgar/data/0000320193/0000320193-{year % 100:02d}-000001.txt",
            },
            {
                "CIK": "0000789019",
                "Company Name": "MICROSOFT CORP",
                "Form Type": "10-Q",
                "Date Filed": f"{year}-{qtr * 3:02d}-15",
                "Filename": f"edgar/data/0000789019/0000789019-{year % 100:02d}-000002.txt",
            },
            {
                "CIK": "0001067983",
                "Company Name": "BERKSHIRE HATHAWAY INC",
                "Form Type": "13F-HR",
                "Date Filed": f"{year}-{qtr * 3:02d}-20",
                "Filename": f"edgar/data/0001067983/0001067983-{year % 100:02d}-000003.txt",
            },
        ]
        create_mock_master_gz(qtr_dir, records)

    return str(idxdir)


def create_idx_with_mocked_network(datadir: str, **kwargs: Any) -> idx:
    """Create an idx instance with network calls mocked out.

    This prevents the constructor from making network calls during updateidx().
    """
    with patch("secbulkdownload.secmeta.fetch_sec_file"):
        instance = idx(datadir=datadir, **kwargs)
    return instance


class TestCanonicalSchema:
    """Test that index data follows canonical schema."""

    def test_filter_index_handles_utf8_company_names(self, datadir: str) -> None:
        """filter_index should parse UTF-8 company names in live SEC master.gz files."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        records = [
            {
                "CIK": "1040197",
                "Company Name": "GESTÃO CAPITAL LTDA",
                "Form Type": "13F-HR",
                "Date Filed": "2024-02-14",
                "Filename": "edgar/data/1040197/0001040197-24-000001.txt",
            }
        ]
        create_mock_master_gz(qtr_dir, records)
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({"Form Type": ["13F-HR"]})

        assert len(result) == 1
        assert result[0]["Company Name"] == "GESTÃO CAPITAL LTDA"
        assert result[0]["CIK"] == "0001040197"

    def test_filter_index_handles_latin1_company_names(self, datadir: str) -> None:
        """filter_index should fall back for historical latin-1 SEC master.gz rows."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2017,
            start_quarter=3,
            end_year=2017,
            end_quarter=3,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2017" / "QTR3"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        records = [
            {
                "CIK": "1555512",
                "Company Name": "PRAGMA GESTÃO DE PATRIMÔNIO LTDA",
                "Form Type": "13F-HR",
                "Date Filed": "2017-07-11",
                "Filename": "edgar/data/1555512/9999999997-17-007686.txt",
            }
        ]
        create_mock_master_gz(qtr_dir, records, encoding="latin-1")
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({"Form Type": ["13F-HR"]})

        assert len(result) == 1
        assert result[0]["Company Name"] == "PRAGMA GESTÃO DE PATRIMÔNIO LTDA"
        assert result[0]["CIK"] == "0001555512"

    def test_filter_index_handles_live_sec_separator_row(self, datadir: str) -> None:
        """filter_index should parse SEC master.gz files with a dashed separator row."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        records = [
            {
                "CIK": "1040197",
                "Company Name": "SAMPLE MANAGER LLC",
                "Form Type": "13F-HR",
                "Date Filed": "2024-02-14",
                "Filename": "edgar/data/1040197/0001040197-24-000001.txt",
            }
        ]
        create_mock_master_gz(qtr_dir, records)
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({"Form Type": ["13F-HR"]})

        assert len(result) == 1
        assert result[0]["CIK"] == "0001040197"
        assert result[0]["Form Type"] == "13F-HR"

    def test_filter_index_returns_list_of_dicts(self, datadir: str) -> None:
        """filter_index should return a list of dicts."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
            user_agent="Test Co test@test.com",
        )

        # Create mock index file
        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-01",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            }
        ]
        create_mock_master_gz(qtr_dir, records)
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({"Form Type": ["10-K"]})

        assert isinstance(result, list)
        assert len(result) == 1
        assert isinstance(result[0], dict)

    def test_filter_index_has_canonical_columns(self, datadir: str) -> None:
        """filter_index result should have canonical column names as keys."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-01",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            }
        ]
        create_mock_master_gz(qtr_dir, records)
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({"Form Type": ["10-K"]})

        expected_cols = ["CIK", "Company Name", "Form Type", "Date Filed", "Filename"]
        assert list(result[0].keys()) == expected_cols

    def test_filter_index_has_correct_dtypes(self, datadir: str) -> None:
        """filter_index result should have correct data types."""
        import datetime

        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-15",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            }
        ]
        create_mock_master_gz(qtr_dir, records)
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({"Form Type": ["10-K"]})

        # Check types
        assert isinstance(result[0]["CIK"], str)
        assert isinstance(result[0]["Company Name"], str)
        assert isinstance(result[0]["Form Type"], str)
        assert isinstance(result[0]["Date Filed"], datetime.date)
        assert isinstance(result[0]["Filename"], str)

    def test_cik_is_zero_padded_string(self, datadir: str) -> None:
        """CIK column should be zero-padded 10-character string."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        # CIK with fewer than 10 digits
        records = [
            {
                "CIK": "320193",  # Should be zero-padded
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-15",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            }
        ]
        create_mock_master_gz(qtr_dir, records)
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({})

        # CIK should be zero-padded to 10 characters
        assert result[0]["CIK"] == "0000320193"


class TestFiltering:
    """Test filter functionality."""

    def test_filter_by_form_type(self, datadir: str) -> None:
        """Filter by Form Type should return only matching records."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-01",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            },
            {
                "CIK": "0000789019",
                "Company Name": "MICROSOFT CORP",
                "Form Type": "10-Q",
                "Date Filed": "2024-01-02",
                "Filename": "edgar/data/0000789019/0000789019-24-000002.txt",
            },
            {
                "CIK": "0001067983",
                "Company Name": "BERKSHIRE HATHAWAY INC",
                "Form Type": "13F-HR",
                "Date Filed": "2024-01-03",
                "Filename": "edgar/data/0001067983/0001067983-24-000003.txt",
            },
        ]
        create_mock_master_gz(qtr_dir, records)
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({"Form Type": ["10-K"]})

        assert len(result) == 1
        assert result[0]["Form Type"] == "10-K"

    def test_filter_by_multiple_form_types(self, datadir: str) -> None:
        """Filter by multiple Form Types should return all matching records."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-01",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            },
            {
                "CIK": "0000789019",
                "Company Name": "MICROSOFT CORP",
                "Form Type": "10-Q",
                "Date Filed": "2024-01-02",
                "Filename": "edgar/data/0000789019/0000789019-24-000002.txt",
            },
            {
                "CIK": "0001067983",
                "Company Name": "BERKSHIRE HATHAWAY INC",
                "Form Type": "13F-HR",
                "Date Filed": "2024-01-03",
                "Filename": "edgar/data/0001067983/0001067983-24-000003.txt",
            },
        ]
        create_mock_master_gz(qtr_dir, records)
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({"Form Type": ["10-K", "10-Q"]})

        assert len(result) == 2
        assert {r["Form Type"] for r in result} == {"10-K", "10-Q"}

    def test_filter_by_cik(self, datadir: str) -> None:
        """Filter by CIK should return matching records."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-01",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            },
            {
                "CIK": "0000789019",
                "Company Name": "MICROSOFT CORP",
                "Form Type": "10-Q",
                "Date Filed": "2024-01-02",
                "Filename": "edgar/data/0000789019/0000789019-24-000002.txt",
            },
        ]
        create_mock_master_gz(qtr_dir, records)
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({"CIK": ["0000320193"]})

        assert len(result) == 1
        assert result[0]["CIK"] == "0000320193"

    def test_empty_filter_returns_all(self, datadir: str) -> None:
        """Empty filter dict should return all records."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-01",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            },
            {
                "CIK": "0000789019",
                "Company Name": "MICROSOFT CORP",
                "Form Type": "10-Q",
                "Date Filed": "2024-01-02",
                "Filename": "edgar/data/0000789019/0000789019-24-000002.txt",
            },
        ]
        create_mock_master_gz(qtr_dir, records)
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({})

        assert len(result) == 2

    def test_empty_match_returns_empty_list(self, datadir: str) -> None:
        """Filter with no matches should return empty list."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-01",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            }
        ]
        create_mock_master_gz(qtr_dir, records)
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({"Form Type": ["S-1"]})  # No matches

        assert len(result) == 0


class TestOrdering:
    """Test that results maintain expected ordering."""

    def test_results_ordered_by_date_filed(self, datadir: str) -> None:
        """Results should be ordered by Date Filed ascending."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        records = [
            {
                "CIK": "0000789019",
                "Company Name": "MICROSOFT CORP",
                "Form Type": "10-Q",
                "Date Filed": "2024-02-15",
                "Filename": "edgar/data/0000789019/0000789019-24-000002.txt",
            },
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-01",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            },
            {
                "CIK": "0001067983",
                "Company Name": "BERKSHIRE HATHAWAY INC",
                "Form Type": "13F-HR",
                "Date Filed": "2024-03-01",
                "Filename": "edgar/data/0001067983/0001067983-24-000003.txt",
            },
        ]
        create_mock_master_gz(qtr_dir, records)
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({})

        dates = [r["Date Filed"] for r in result]
        assert dates == sorted(dates)


class TestMultiQuarter:
    """Test behavior with multiple quarters."""

    def test_filter_across_multiple_quarters(self, datadir: str) -> None:
        """Filter should combine results from all quarters."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=2,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        create_mock_idxdir(Path(datadir), [(2024, 1), (2024, 2)])

        idx_instance.idxlist = [
            str(idxdir / "2024" / "QTR1" / "master.gz"),
            str(idxdir / "2024" / "QTR2" / "master.gz"),
        ]

        result = idx_instance.filter_index({"Form Type": ["10-K"]})

        # Should have records from both quarters
        assert len(result) >= 2

    def test_quarter_order_preserved(self, datadir: str) -> None:
        """Results should be ordered by quarter then date."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=2,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        create_mock_idxdir(Path(datadir), [(2024, 1), (2024, 2)])

        idx_instance.idxlist = [
            str(idxdir / "2024" / "QTR1" / "master.gz"),
            str(idxdir / "2024" / "QTR2" / "master.gz"),
        ]

        result = idx_instance.filter_index({})

        # All Q1 dates should come before Q2 dates
        dates = [r["Date Filed"] for r in result]
        assert all(d.month <= 3 for d in dates[: len(dates) // 2])


class TestWorkingIdxUpdate:
    """Test that working_idx is properly updated."""

    def test_filter_index_updates_working_idx(self, datadir: str) -> None:
        """filter_index should update the working_idx attribute."""
        idx_instance = create_idx_with_mocked_network(
            datadir=datadir,
            start_year=2024,
            start_quarter=1,
            end_year=2024,
            end_quarter=1,
            user_agent="Test Co test@test.com",
        )

        idxdir = Path(datadir) / "idxfiles"
        idxdir.mkdir(parents=True, exist_ok=True)
        qtr_dir = idxdir / "2024" / "QTR1"
        qtr_dir.mkdir(parents=True, exist_ok=True)

        records = [
            {
                "CIK": "0000320193",
                "Company Name": "APPLE INC",
                "Form Type": "10-K",
                "Date Filed": "2024-01-01",
                "Filename": "edgar/data/0000320193/0000320193-24-000001.txt",
            }
        ]
        create_mock_master_gz(qtr_dir, records)
        idx_instance.idxlist = [str(qtr_dir / "master.gz")]

        result = idx_instance.filter_index({"Form Type": ["10-K"]})

        # working_idx should be the same list
        assert idx_instance.working_idx is not None
        assert isinstance(idx_instance.working_idx, list)
        assert len(idx_instance.working_idx) == len(result)
