import os
from datetime import date, datetime
from typing import TYPE_CHECKING

import pytest
from arro3.core import Array, DataType, Field, Table

from deltalake import DeltaTable, write_deltalake
from deltalake.exceptions import DeltaError

if TYPE_CHECKING:
    import pyarrow as pa


def _normalize_pyarrow_view_types_for_sort(tbl: "pa.Table") -> "pa.Table":
    """Normalize view types so pyarrow Table.sort_by works.

    PyArrow does not currently support sorting RecordBatches that contain
    `binary_view` (and sometimes `string_view`) columns.
    """

    import pyarrow as pa

    # Older PyArrow versions may not expose view types.
    binary_view = getattr(pa, "binary_view", None)
    string_view = getattr(pa, "string_view", None)
    if binary_view is None and string_view is None:
        return tbl

    fields = []
    columns = []
    for field in tbl.schema:
        col = tbl[field.name]
        ty = field.type

        if binary_view is not None and ty == binary_view():
            ty = pa.binary()
            # PyArrow 16.x lacks casts for view types, so rebuild the array.
            col = pa.array(col.to_pylist(), type=ty)
        elif string_view is not None and ty == string_view():
            ty = pa.string()
            col = pa.array(col.to_pylist(), type=ty)

        fields.append(
            pa.field(field.name, ty, nullable=field.nullable, metadata=field.metadata)
        )
        columns.append(col)

    return pa.Table.from_arrays(
        columns, schema=pa.schema(fields, metadata=tbl.schema.metadata)
    )


def test_read_cdf_partitioned_with_predicate():
    dt = DeltaTable("../crates/test/tests/data/cdf-table/")
    data = dt.load_cdf(0, 3, predicate="birthday = '2023-12-25'").read_all()

    values = list(set(data["birthday"].to_pylist()))
    assert len(values) == 1
    assert values[0] == date(2023, 12, 25)


def test_read_cdf_partitioned():
    dt = DeltaTable("../crates/test/tests/data/cdf-table/")
    b = dt.load_cdf(0, 3).read_all()
    assert sorted(b["id"].to_pylist()) == [
        1,
        2,
        2,
        2,
        3,
        3,
        3,
        4,
        4,
        4,
        5,
        5,
        5,
        6,
        6,
        6,
        7,
        7,
        7,
        7,
        8,
        9,
        10,
    ]
    assert sorted(b["name"].to_pylist()) == [
        "Ada",
        "Bob",
        "Bob",
        "Bob",
        "Borb",
        "Carl",
        "Carl",
        "Carl",
        "Claire",
        "Dave",
        "Dave",
        "Dave",
        "Dennis",
        "Dennis",
        "Dennis",
        "Dennis",
        "Emily",
        "Emily",
        "Emily",
        "Kate",
        "Kate",
        "Kate",
        "Steve",
    ]
    assert sorted(b["_change_type"].to_pylist()) == [
        "delete",
        "insert",
        "insert",
        "insert",
        "insert",
        "insert",
        "insert",
        "insert",
        "insert",
        "insert",
        "insert",
        "update_postimage",
        "update_postimage",
        "update_postimage",
        "update_postimage",
        "update_postimage",
        "update_postimage",
        "update_preimage",
        "update_preimage",
        "update_preimage",
        "update_preimage",
        "update_preimage",
        "update_preimage",
    ]
    assert sorted(b["_commit_version"].to_pylist()) == [
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        1,
        1,
        1,
        1,
        1,
        2,
        2,
        2,
        2,
        2,
        2,
        3,
    ]
    assert sorted(b["_commit_timestamp"].to_pylist()) == [
        datetime(2023, 12, 22, 17, 10, 18, 828000),
        datetime(2023, 12, 22, 17, 10, 18, 828000),
        datetime(2023, 12, 22, 17, 10, 18, 828000),
        datetime(2023, 12, 22, 17, 10, 18, 828000),
        datetime(2023, 12, 22, 17, 10, 18, 828000),
        datetime(2023, 12, 22, 17, 10, 18, 828000),
        datetime(2023, 12, 22, 17, 10, 18, 828000),
        datetime(2023, 12, 22, 17, 10, 18, 828000),
        datetime(2023, 12, 22, 17, 10, 18, 828000),
        datetime(2023, 12, 22, 17, 10, 18, 828000),
        datetime(2023, 12, 22, 17, 10, 21, 675000),
        datetime(2023, 12, 22, 17, 10, 21, 675000),
        datetime(2023, 12, 22, 17, 10, 21, 675000),
        datetime(2023, 12, 22, 17, 10, 21, 675000),
        datetime(2023, 12, 22, 17, 10, 21, 675000),
        datetime(2023, 12, 22, 17, 10, 21, 675000),
        datetime(2023, 12, 29, 21, 41, 33, 785000),
        datetime(2023, 12, 29, 21, 41, 33, 785000),
        datetime(2023, 12, 29, 21, 41, 33, 785000),
        datetime(2023, 12, 29, 21, 41, 33, 785000),
        datetime(2023, 12, 29, 21, 41, 33, 785000),
        datetime(2023, 12, 29, 21, 41, 33, 785000),
        datetime(2024, 1, 6, 16, 44, 59, 570000),
    ]
    assert sorted(b["birthday"].to_pylist()) == [
        date(2023, 12, 22),
        date(2023, 12, 22),
        date(2023, 12, 22),
        date(2023, 12, 22),
        date(2023, 12, 23),
        date(2023, 12, 23),
        date(2023, 12, 23),
        date(2023, 12, 23),
        date(2023, 12, 23),
        date(2023, 12, 23),
        date(2023, 12, 24),
        date(2023, 12, 24),
        date(2023, 12, 24),
        date(2023, 12, 24),
        date(2023, 12, 24),
        date(2023, 12, 24),
        date(2023, 12, 25),
        date(2023, 12, 25),
        date(2023, 12, 25),
        date(2023, 12, 29),
        date(2023, 12, 29),
        date(2023, 12, 29),
        date(2023, 12, 29),
    ]


def test_read_cdf_non_partitioned():
    dt = DeltaTable("../crates/test/tests/data/cdf-table-non-partitioned/")
    b = dt.load_cdf(0, 3).read_all()

    assert sorted(b["id"].to_pylist()) == [
        1,
        2,
        2,
        2,
        3,
        3,
        3,
        4,
        4,
        4,
        5,
        5,
        5,
        6,
        6,
        6,
        7,
        7,
        7,
        7,
        8,
        9,
        10,
    ]
    assert sorted(b["name"].to_pylist()) == [
        "Ada",
        "Bob",
        "Bob",
        "Bob",
        "Borb",
        "Carl",
        "Carl",
        "Carl",
        "Claire",
        "Dave",
        "Dave",
        "Dave",
        "Dennis",
        "Dennis",
        "Dennis",
        "Dennis",
        "Emily",
        "Emily",
        "Emily",
        "Kate",
        "Kate",
        "Kate",
        "Steve",
    ]
    assert sorted(b["birthday"].to_pylist()) == [
        date(2024, 4, 14),
        date(2024, 4, 14),
        date(2024, 4, 14),
        date(2024, 4, 14),
        date(2024, 4, 14),
        date(2024, 4, 14),
        date(2024, 4, 14),
        date(2024, 4, 14),
        date(2024, 4, 15),
        date(2024, 4, 15),
        date(2024, 4, 15),
        date(2024, 4, 15),
        date(2024, 4, 15),
        date(2024, 4, 15),
        date(2024, 4, 16),
        date(2024, 4, 16),
        date(2024, 4, 16),
        date(2024, 4, 16),
        date(2024, 4, 16),
        date(2024, 4, 16),
        date(2024, 4, 17),
        date(2024, 4, 17),
        date(2024, 4, 17),
    ]
    assert sorted(b["long_field"].to_pylist()) == [
        1,
        1,
        1,
        1,
        2,
        2,
        2,
        3,
        3,
        3,
        4,
        4,
        4,
        5,
        5,
        5,
        6,
        6,
        6,
        6,
        7,
        8,
        99999999999999999,
    ]
    assert sorted(b["boolean_field"].to_pylist()) == [
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
        True,
    ]
    assert sorted(b["double_field"].to_pylist()) == [
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
        3.14,
    ]
    assert sorted(b["smallint_field"].to_pylist()) == [
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
    ]
    assert sorted(b["_change_type"].to_pylist()) == [
        "delete",
        "insert",
        "insert",
        "insert",
        "insert",
        "insert",
        "insert",
        "insert",
        "insert",
        "insert",
        "insert",
        "update_postimage",
        "update_postimage",
        "update_postimage",
        "update_postimage",
        "update_postimage",
        "update_postimage",
        "update_preimage",
        "update_preimage",
        "update_preimage",
        "update_preimage",
        "update_preimage",
        "update_preimage",
    ]
    assert sorted(b["_commit_version"].to_pylist()) == [
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        1,
        1,
        1,
        1,
        1,
        2,
        2,
        2,
        2,
        2,
        2,
        3,
    ]
    assert sorted(b["_commit_timestamp"].to_pylist()) == [
        datetime(2024, 4, 14, 15, 58, 26, 249000),
        datetime(2024, 4, 14, 15, 58, 26, 249000),
        datetime(2024, 4, 14, 15, 58, 26, 249000),
        datetime(2024, 4, 14, 15, 58, 26, 249000),
        datetime(2024, 4, 14, 15, 58, 26, 249000),
        datetime(2024, 4, 14, 15, 58, 26, 249000),
        datetime(2024, 4, 14, 15, 58, 26, 249000),
        datetime(2024, 4, 14, 15, 58, 26, 249000),
        datetime(2024, 4, 14, 15, 58, 26, 249000),
        datetime(2024, 4, 14, 15, 58, 26, 249000),
        datetime(2024, 4, 14, 15, 58, 29, 393000),
        datetime(2024, 4, 14, 15, 58, 29, 393000),
        datetime(2024, 4, 14, 15, 58, 29, 393000),
        datetime(2024, 4, 14, 15, 58, 29, 393000),
        datetime(2024, 4, 14, 15, 58, 29, 393000),
        datetime(2024, 4, 14, 15, 58, 29, 393000),
        datetime(2024, 4, 14, 15, 58, 31, 257000),
        datetime(2024, 4, 14, 15, 58, 31, 257000),
        datetime(2024, 4, 14, 15, 58, 31, 257000),
        datetime(2024, 4, 14, 15, 58, 31, 257000),
        datetime(2024, 4, 14, 15, 58, 31, 257000),
        datetime(2024, 4, 14, 15, 58, 31, 257000),
        datetime(2024, 4, 14, 15, 58, 32, 495000),
    ]


def test_read_cdf_partitioned_projection():
    dt = DeltaTable("../crates/test/tests/data/cdf-table/")
    columns = ["id", "_change_type", "_commit_version"]
    assert columns == dt.load_cdf(0, 3, columns=columns).schema.names


@pytest.mark.pyarrow
def test_delete_unpartitioned_cdf(tmp_path, sample_data_pyarrow: "pa.Table"):
    import pyarrow as pa
    import pyarrow.compute as pc
    import pyarrow.dataset as ds
    import pyarrow.parquet as pq

    cdc_path = f"{tmp_path}/_change_data"

    write_deltalake(
        tmp_path,
        sample_data_pyarrow,
        mode="append",
        configuration={"delta.enableChangeDataFeed": "true"},
    )
    dt = DeltaTable(tmp_path)
    dt.delete("int64 > 2")

    expected_data = (
        ds.dataset(sample_data_pyarrow)
        .to_table(filter=(pc.field("int64") > 2))
        .append_column(
            field_=pa.field("_change_type", pa.string(), nullable=False),
            column=[["delete"] * 2],
        )
    )
    cdc_data = pq.read_table(cdc_path)

    assert os.path.exists(cdc_path), "_change_data doesn't exist"
    assert cdc_data.to_pydict() == expected_data.to_pydict()


@pytest.mark.pyarrow
def test_delete_partitioned_cdf(tmp_path, sample_data_pyarrow: "pa.Table"):
    import pyarrow as pa
    import pyarrow.compute as pc
    import pyarrow.dataset as ds

    write_deltalake(
        tmp_path,
        sample_data_pyarrow,
        mode="overwrite",
        partition_by=["utf8"],
        configuration={"delta.enableChangeDataFeed": "true"},
    )
    dt = DeltaTable(tmp_path)
    dt.delete("int64 > 2")

    expected_data = (
        ds.dataset(sample_data_pyarrow)
        .to_table(filter=(pc.field("int64") > 2))
        .append_column(
            field_=pa.field("_change_type", pa.string(), nullable=False),
            column=[["delete"] * 2],
        )
        .select(["int64", "_change_type"])
        .sort_by("int64")
    )

    # delta-rs returns Arrow view types (e.g. `string_view`) for zero-copy.
    # PyArrow 16.x lacks compute kernels for view types, and even `Table.filter`
    # can fail because it applies `take` across *all* columns.
    # Convert the only columns we need to kernel-backed types.
    raw_cdc_table = pa.table(dt.load_cdf().read_all())
    change_type_py = raw_cdc_table.column(
        raw_cdc_table.schema.get_field_index("_change_type")
    ).to_pylist()
    cdc_table = pa.table(
        {
            "int64": raw_cdc_table.column(
                raw_cdc_table.schema.get_field_index("int64")
            ),
            "_change_type": pa.array(change_type_py, type=pa.string()),
        }
    )

    delete_mask = pa.array([(v == "delete") for v in change_type_py], type=pa.bool_())
    cdc_data = cdc_table.filter(delete_mask).sort_by("int64")

    assert cdc_data.to_pydict() == expected_data.to_pydict()


@pytest.mark.pyarrow
def test_write_predicate_unpartitioned_cdf(tmp_path, sample_data_pyarrow: "pa.Table"):
    import pyarrow as pa
    import pyarrow.compute as pc
    import pyarrow.dataset as ds
    import pyarrow.parquet as pq

    cdc_path = f"{tmp_path}/_change_data"

    write_deltalake(
        tmp_path,
        sample_data_pyarrow,
        mode="append",
        configuration={"delta.enableChangeDataFeed": "true"},
    )

    dt = DeltaTable(tmp_path)
    write_deltalake(
        dt,
        data=ds.dataset(sample_data_pyarrow).to_table(filter=(pc.field("int64") > 2)),
        mode="overwrite",
        predicate="int64 > 2",
        configuration={"delta.enableChangeDataFeed": "true"},
    )

    expected_data = pa.concat_tables(
        [
            ds.dataset(sample_data_pyarrow)
            .to_table(filter=(pc.field("int64") > 2))
            .append_column(
                field_=pa.field("_change_type", pa.string(), nullable=False),
                column=[["delete"] * 2],
            ),
            ds.dataset(sample_data_pyarrow)
            .to_table(filter=(pc.field("int64") > 2))
            .append_column(
                field_=pa.field("_change_type", pa.string(), nullable=False),
                column=[["insert"] * 2],
            ),
        ]
    )
    cdc_data = pq.read_table(cdc_path)

    assert os.path.exists(cdc_path), "_change_data doesn't exist"
    assert cdc_data.sort_by([("_change_type", "ascending")]) == expected_data.sort_by(
        [("_change_type", "ascending")]
    )
    assert dt.to_pyarrow_table().sort_by([("utf8", "ascending")]) == sample_data_pyarrow


@pytest.mark.pyarrow
def test_write_predicate_partitioned_cdf(tmp_path, sample_data_pyarrow: "pa.Table"):
    import pyarrow as pa
    import pyarrow.compute as pc
    import pyarrow.dataset as ds
    import pyarrow.parquet as pq

    cdc_path = f"{tmp_path}/_change_data"

    write_deltalake(
        tmp_path,
        sample_data_pyarrow,
        mode="overwrite",
        partition_by=["utf8"],
        configuration={"delta.enableChangeDataFeed": "true"},
    )
    dt = DeltaTable(tmp_path)
    write_deltalake(
        dt,
        data=ds.dataset(sample_data_pyarrow).to_table(filter=(pc.field("int64") > 3)),
        mode="overwrite",
        predicate="int64 > 3",
        configuration={"delta.enableChangeDataFeed": "true"},
    )

    expected_data = pa.concat_tables(
        [
            ds.dataset(sample_data_pyarrow)
            .to_table(filter=(pc.field("int64") > 3))
            .append_column(
                field_=pa.field("_change_type", pa.string(), nullable=False),
                column=[["delete"] * 1],
            ),
            ds.dataset(sample_data_pyarrow)
            .to_table(filter=(pc.field("int64") > 3))
            .append_column(
                field_=pa.field("_change_type", pa.string(), nullable=False),
                column=[["insert"] * 1],
            ),
        ]
    )

    table_schema = pa.schema(dt.schema())
    table_schema = table_schema.insert(
        len(table_schema), pa.field("_change_type", pa.string(), nullable=False)
    )
    cdc_data = pq.read_table(cdc_path, schema=table_schema)

    assert os.path.exists(cdc_path), "_change_data doesn't exist"
    assert len(os.listdir(cdc_path)) == 1
    expected_data = expected_data.combine_chunks().sort_by(
        [("_change_type", "ascending")]
    )
    cdc_data = cdc_data.combine_chunks().sort_by([("_change_type", "ascending")])

    assert expected_data == cdc_data

    table = _normalize_pyarrow_view_types_for_sort(dt.to_pyarrow_table())
    assert table.sort_by([("utf8", "ascending")]) == sample_data_pyarrow


@pytest.mark.pyarrow
def test_write_overwrite_unpartitioned_cdf(tmp_path, sample_data_pyarrow: "pa.Table"):
    import pyarrow as pa
    import pyarrow.dataset as ds

    cdc_path = f"{tmp_path}/_change_data"

    write_deltalake(
        tmp_path,
        sample_data_pyarrow,
        mode="append",
        configuration={"delta.enableChangeDataFeed": "true"},
    )

    dt = DeltaTable(tmp_path)
    write_deltalake(
        dt,
        data=ds.dataset(sample_data_pyarrow).to_table(),
        mode="overwrite",
        configuration={"delta.enableChangeDataFeed": "true"},
    )
    sort_values = [("_change_type", "ascending"), ("utf8", "ascending")]
    expected_data = (
        ds.dataset(pa.concat_tables([sample_data_pyarrow] * 3))
        .to_table()
        .append_column(
            field_=pa.field("_change_type", pa.string(), nullable=True),
            column=[["delete"] * 5 + ["insert"] * 10],
        )
    ).sort_by(sort_values)

    assert not os.path.exists(cdc_path), (
        "_change_data shouldn't exist since table was overwritten"
    )

    tbl = dt.load_cdf().read_all()

    select_cols = [
        col
        for col in tbl.column_names
        if col not in ["_commit_version", "_commit_timestamp"]
    ]
    actual = _normalize_pyarrow_view_types_for_sort(pa.table(tbl.select(select_cols)))
    assert actual.sort_by(sort_values) == expected_data

    table = _normalize_pyarrow_view_types_for_sort(dt.to_pyarrow_table())
    assert table.sort_by([("utf8", "ascending")]) == sample_data_pyarrow


@pytest.mark.pyarrow
def test_write_overwrite_partitioned_cdf(tmp_path, sample_data_pyarrow: "pa.Table"):
    import pyarrow as pa
    import pyarrow.compute as pc
    import pyarrow.dataset as ds

    cdc_path = f"{tmp_path}/_change_data"

    write_deltalake(
        tmp_path,
        sample_data_pyarrow,
        mode="append",
        partition_by=["int64"],
        configuration={"delta.enableChangeDataFeed": "true"},
    )

    batch2 = ds.dataset(sample_data_pyarrow).to_table(filter=(pc.field("int64") > 3))

    dt = DeltaTable(tmp_path)
    write_deltalake(
        dt,
        data=batch2,
        mode="overwrite",
        predicate="int64 > 3",
        partition_by=["int64"],
        configuration={"delta.enableChangeDataFeed": "true"},
    )

    table_schema = pa.schema(dt.schema())
    table_schema = table_schema.insert(
        len(table_schema), pa.field("_change_type", pa.string(), nullable=False)
    )

    sort_values = [("_change_type", "ascending"), ("utf8", "ascending")]

    first_batch = sample_data_pyarrow.append_column(
        field_=pa.field("_change_type", pa.string(), nullable=True),
        column=[["insert"] * 5],
    )

    expected_data = pa.concat_tables([batch2] * 2).append_column(
        field_=pa.field("_change_type", pa.string(), nullable=True),
        column=[["delete", "insert"]],
    )

    assert not os.path.exists(cdc_path), (
        "_change_data shouldn't exist since a specific partition was overwritten"
    )

    actual = _normalize_pyarrow_view_types_for_sort(pa.table(dt.load_cdf().read_all()))
    assert actual.drop_columns(["_commit_version", "_commit_timestamp"]).sort_by(
        sort_values
    ).select(expected_data.column_names) == pa.concat_tables(
        [first_batch, expected_data]
    ).sort_by(sort_values)


def test_read_cdf_version_out_of_range():
    dt = DeltaTable("../crates/test/tests/data/cdf-table/")

    with pytest.raises(DeltaError) as e:
        dt.load_cdf(4).read_all()

    assert "invalid table version: 4" in str(e).lower()


def test_read_cdf_version_out_of_range_with_flag():
    dt = DeltaTable("../crates/test/tests/data/cdf-table/")
    b = dt.load_cdf(4, allow_out_of_range=True).read_all()

    assert len(b) == 0


def test_read_timestamp_cdf_out_of_range():
    dt = DeltaTable("../crates/test/tests/data/cdf-table/")
    start = "2033-12-22T17:10:21.675Z"

    with pytest.raises(DeltaError) as e:
        dt.load_cdf(starting_timestamp=start).read_all()

    assert "is greater than latest commit timestamp" in str(e).lower()


def test_read_timestamp_cdf_out_of_range_with_flag():
    dt = DeltaTable("../crates/test/tests/data/cdf-table/")

    start = "2033-12-22T17:10:21.675Z"
    b = dt.load_cdf(starting_timestamp=start, allow_out_of_range=True).read_all()

    assert len(b) == 0


def test_read_cdf_last_version(tmp_path):
    data = Table.from_pydict(
        {"foo": Array([1, 2, 3], type=Field("foo", DataType.int32(), nullable=True))}
    )

    expected = Table.from_pydict(
        {
            "foo": Array([1, 2, 3], type=Field("foo", DataType.int32(), nullable=True)),
            "_change_type": Array(
                ["insert", "insert", "insert"],
                type=Field("_change_type", DataType.string_view(), nullable=True),
            ),
            "_commit_version": Array(
                [0, 0, 0],
                type=Field("_commit_version", DataType.int64(), nullable=True),
            ),
        }
    )

    write_deltalake(
        tmp_path,
        data=data,
        configuration={"delta.enableChangeDataFeed": "true"},
    )

    data = (
        DeltaTable(tmp_path)
        .load_cdf(
            starting_version=0,
            ending_version=0,
            allow_out_of_range=False,
            columns=["foo", "_change_type", "_commit_version"],
        )
        .read_all()
    )

    assert expected == data
