import logging
import os
from typing import Any, Type, Set
from unittest.mock import Mock
import pytest
from coinmetrics.api_client import CoinMetricsClient, requests
from coinmetrics._catalogs import (
    CatalogAssetsData,
    CatalogAssetAlertsData,
    CatalogAssetPairsData,
    CatalogExchangeAssetsData,
    CatalogExchangesData,
    CatalogIndexesData,
    CatalogInstitutionsData,
    CatalogMarketsData,
    CatalogMetricsData,
    CatalogAssetPairCandlesData,
    CatalogMarketTradesData,
)
from coinmetrics._data_collection import DataCollection
from coinmetrics._typing import (
    DataRetrievalFuncType,
    UrlParamTypes,
    Dict,
    DataReturnType
)
import pyarrow as pa
import pyarrow.parquet as pq
cm_api_key_set = os.environ.get("CM_API_KEY") is not None
REASON_TO_SKIP = "Need to set CM_API_KEY as an env var in order to run this test"

print("CM_API_KEY is set - tests will run") if cm_api_key_set else print(
    "CM_API_KEY not set, tests will not run"
)

try:
    import pandas as pd

    DataFrameType: Type[pd.core.frame.DataFrame] = pd.core.frame.DataFrame
except ImportError:
    logging.info("Pandas not imported")

client = CoinMetricsClient(str(os.environ.get("CM_API_KEY")))

catalog_assets_test_data = [
    {
        "asset": "btc",
        "full_name": "Bitcoin",
        "metrics": [
            {
                "metric": "TestMetric",
                "frequencies": [
                    {
                        "frequency": "1b",
                        "min_time": "2009-01-03T18:15:05.000000000Z",
                        "max_time": "2020-06-08T20:22:17.000000000Z",
                        "min_height": "0",
                        "max_height": "10",
                        "min_hash": "0abc",
                        "max_hash": "0xyz",
                    },
                    {
                        "frequency": "1d",
                        "min_time": "2009-01-03T00:00:00.000000000Z",
                        "max_time": "2020-06-07T00:00:00.000000000Z",
                        "community": True,
                    },
                ],
            }
        ],
        "exchanges": ["binance", "coinbase", "kraken"],
        "markets": [
            "binance-btc-usdt-spot",
            "binance-eth-btc-spot",
            "coinbase-btc-usd-spot",
            "coinbase-eth-btc-spot",
            "kraken-btc-usd-spot",
        ],
    }
]

catalog_candles_test_data = [
    {
        "frequencies": [
            {
                "frequency": "1m",
                "max_time": "2022-07-12T17:50:00.000000000Z",
                "min_time": "2020-12-26T01:00:00.000000000Z",
            },
            {
                "frequency": "5m",
                "max_time": "2022-07-12T17:45:00.000000000Z",
                "min_time": "2020-12-26T01:00:00.000000000Z",
            },
            {
                "frequency": "10m",
                "max_time": "2022-07-12T17:40:00.000000000Z",
                "min_time": "2020-12-26T01:00:00.000000000Z",
            },
            {
                "frequency": "15m",
                "max_time": "2022-07-12T17:30:00.000000000Z",
                "min_time": "2020-12-26T01:00:00.000000000Z",
            },
            {
                "frequency": "30m",
                "max_time": "2022-07-12T17:00:00.000000000Z",
                "min_time": "2020-12-26T01:00:00.000000000Z",
            },
            {
                "frequency": "1h",
                "max_time": "2022-07-12T16:00:00.000000000Z",
                "min_time": "2020-12-26T01:00:00.000000000Z",
            },
            {
                "frequency": "4h",
                "max_time": "2022-07-12T12:00:00.000000000Z",
                "min_time": "2020-12-26T00:00:00.000000000Z",
            },
            {
                "frequency": "1d",
                "max_time": "2022-07-11T00:00:00.000000000Z",
                "min_time": "2020-12-26T00:00:00.000000000Z",
            },
        ],
        "pair": "1inch-usd",
    },
    {
        "frequencies": [
            {
                "frequency": "1m",
                "max_time": "2022-07-12T17:50:00.000000000Z",
                "min_time": "2020-10-10T08:13:00.000000000Z",
            },
            {
                "frequency": "5m",
                "max_time": "2022-07-12T17:45:00.000000000Z",
                "min_time": "2020-10-10T08:10:00.000000000Z",
            },
            {
                "frequency": "10m",
                "max_time": "2022-07-12T17:40:00.000000000Z",
                "min_time": "2020-10-10T08:10:00.000000000Z",
            },
            {
                "frequency": "15m",
                "max_time": "2022-07-12T17:30:00.000000000Z",
                "min_time": "2020-10-10T08:00:00.000000000Z",
            },
            {
                "frequency": "30m",
                "max_time": "2022-07-12T17:00:00.000000000Z",
                "min_time": "2020-10-10T08:00:00.000000000Z",
            },
            {
                "frequency": "1h",
                "max_time": "2022-07-12T16:00:00.000000000Z",
                "min_time": "2020-10-10T08:00:00.000000000Z",
            },
            {
                "frequency": "4h",
                "max_time": "2022-07-12T12:00:00.000000000Z",
                "min_time": "2020-10-10T08:00:00.000000000Z",
            },
            {
                "frequency": "1d",
                "max_time": "2022-07-11T00:00:00.000000000Z",
                "min_time": "2020-10-10T00:00:00.000000000Z",
            },
        ],
        "pair": "aave-usd",
    },
    {
        "frequencies": [
            {
                "frequency": "1m",
                "max_time": "2022-07-12T17:50:00.000000000Z",
                "min_time": "2018-02-26T00:00:00.000000000Z",
            },
            {
                "frequency": "5m",
                "max_time": "2022-07-12T17:45:00.000000000Z",
                "min_time": "2018-02-26T00:00:00.000000000Z",
            },
            {
                "frequency": "10m",
                "max_time": "2022-07-12T17:40:00.000000000Z",
                "min_time": "2018-02-26T00:00:00.000000000Z",
            },
            {
                "frequency": "15m",
                "max_time": "2022-07-12T17:30:00.000000000Z",
                "min_time": "2018-02-26T00:00:00.000000000Z",
            },
            {
                "frequency": "30m",
                "max_time": "2022-07-12T17:00:00.000000000Z",
                "min_time": "2018-02-26T00:00:00.000000000Z",
            },
            {
                "frequency": "1h",
                "max_time": "2022-07-12T16:00:00.000000000Z",
                "min_time": "2018-02-26T00:00:00.000000000Z",
            },
            {
                "frequency": "4h",
                "max_time": "2022-07-12T12:00:00.000000000Z",
                "min_time": "2018-02-26T00:00:00.000000000Z",
            },
            {
                "frequency": "1d",
                "max_time": "2022-07-11T00:00:00.000000000Z",
                "min_time": "2018-02-26T00:00:00.000000000Z",
            },
        ],
        "pair": "abt-usd",
    },
]

catalog_metrics_test_data = [
    {
        "metric": "AdrActCnt",
        "full_name": "Addresses, active, count",
        "description": "The sum count of unique addresses that were active in the network (either as a recipient or originator of a ledger change) that interval. All parties in a ledger change action (recipients and originators) are counted. Individual addresses are not double-counted if previously active.",
        "category": "Addresses",
        "subcategory": "Active",
        "unit": "Addresses",
        "data_type": "bigint",
        "frequencies": [
            {"frequency": "1b", "assets": ["btc", "eth"]},
            {"frequency": "1d", "assets": ["ada", "btc", "eth"]},
        ],
    },
    {
        "metric": "FlowInBFXNtv",
        "full_name": "Flow, in, to Bitfinex, native units",
        "description": "The sum in native units sent to Bitfinex that interval.",
        "category": "Exchange",
        "subcategory": "Deposits",
        "unit": "Native units",
        "data_type": "decimal",
        "frequencies": [
            {"frequency": "1b", "assets": ["btc", "eth"]},
            {"frequency": "1d", "assets": ["btc", "eth"]},
        ],
        "reviewable": "True",
    },
]

catalog_markets_test_data = [
    {
        "market": "bitmex-XBTF15-future",
        "min_time": "2014-11-24T13:05:32.850000000Z",
        "max_time": "2015-01-30T12:00:00.000000000Z",
        "trades": {
            "min_time": "2014-11-24T13:05:32.850000000Z",
            "max_time": "2015-01-30T12:00:00.000000000Z",
        },
        "exchange": "bitmex",
        "type": "future",
        "symbol": "XBTF15",
        "base": "btc",
        "quote": "usd",
        "size_asset": "XBT",
        "margin_asset": "USD",
        "contract_size": "1",
        "tick_size": "0.1",
        "listing": "2014-11-24T13:05:32.850000000Z",
        "expiration": "2015-01-30T12:00:00.000000000Z",
    },
    {
        "market": "bitfinex-agi-btc-spot",
        "min_time": "2018-04-07T16:25:55.000000000Z",
        "max_time": "2020-03-25T20:12:09.639000000Z",
        "trades": {
            "min_time": "2018-04-07T16:25:55.000000000Z",
            "max_time": "2020-03-25T20:12:09.639000000Z",
        },
        "exchange": "bitfinex",
        "type": "spot",
        "base": "agi",
        "quote": "btc",
    },
]

catalog_exchanges_test_data = [
    {
        "exchange": "bibox",
        "markets": ["bibox-abt-btc-spot", "bibox-etc-usdt-spot"],
        "min_time": "2019-04-24T11:09:59.000000000Z",
        "max_time": "2019-05-18T16:06:10.927000000Z",
        "metrics": [
            {
                "metric": "volume_reported_spot_usd_1d",
                "frequencies": [
                    {
                        "frequency": "1d",
                        "min_time": "2019-04-30T00:00:00.000000000Z",
                        "max_time": "2019-06-16T00:00:00.000000000Z",
                    }
                ],
            },
            {
                "metric": "volume_reported_spot_usd_1h",
                "frequencies": [
                    {
                        "frequency": "1h",
                        "min_time": "2019-04-29T21:00:00.000000000Z",
                        "max_time": "2019-06-17T15:00:00.000000000Z",
                    }
                ],
            },
        ],
    },
    {
        "exchange": "binance",
        "markets": [
            "binance-BTCUSDT-future",
            "binance-LTCUSDT-future",
            "binance-ada-bnb-spot",
            "binance-btc-usdt-spot",
            "binance-bcpt-btc-spot",
            "binance-bcd-eth-spot",
        ],
        "min_time": "2017-07-14T04:00:00.510000000Z",
        "max_time": "2020-06-08T20:33:28.868000000Z",
        "metrics": [
            {
                "metric": "volume_reported_spot_usd_1d",
                "frequencies": [
                    {
                        "frequency": "1d",
                        "min_time": "2019-04-30T00:00:00.000000000Z",
                        "max_time": "2019-06-16T00:00:00.000000000Z",
                    }
                ],
            },
            {
                "metric": "volume_reported_spot_usd_1h",
                "frequencies": [
                    {
                        "frequency": "1h",
                        "min_time": "2019-04-29T21:00:00.000000000Z",
                        "max_time": "2019-06-17T15:00:00.000000000Z",
                    }
                ],
            },
        ],
    },
]

catalog_exchange_assets_test_data = [
    {
        "exchange_asset": "binance-btc",
        "metrics": [
            {
                "metric": "volume_trusted_spot_usd_1d",
                "frequencies": [
                    {
                        "frequency": "1d",
                        "min_time": "2020-10-16T00:00:00.000000000Z",
                        "max_time": "2021-01-05T00:00:00.000000000Z",
                    }
                ],
            },
            {
                "metric": "volume_trusted_spot_usd_1h",
                "frequencies": [
                    {
                        "frequency": "1h",
                        "min_time": "2020-10-15T03:00:00.000000000Z",
                        "max_time": "2021-01-06T12:00:00.000000000Z",
                    }
                ],
            },
        ],
    },
    {
        "exchange_asset": "coinbase-eth",
        "metrics": [
            {
                "metric": "volume_trusted_spot_usd_1d",
                "frequencies": [
                    {
                        "frequency": "1d",
                        "min_time": "2020-10-11T00:00:00.000000000Z",
                        "max_time": "2021-01-05T00:00:00.000000000Z",
                    }
                ],
            },
            {
                "metric": "volume_trusted_spot_usd_1h",
                "frequencies": [
                    {
                        "frequency": "1h",
                        "min_time": "2020-10-10T19:00:00.000000000Z",
                        "max_time": "2021-01-06T12:00:00.000000000Z",
                    }
                ],
            },
        ],
    },
]

catalog_asset_pairs_test_data = [
    {
        "pair": "aave-bnb",
        "metrics": [
            {
                "metric": "volume_trusted_spot_usd_1d",
                "frequencies": [
                    {
                        "frequency": "1d",
                        "min_time": "2020-10-16T00:00:00.000000000Z",
                        "max_time": "2021-01-05T00:00:00.000000000Z",
                    }
                ],
            },
            {
                "metric": "volume_trusted_spot_usd_1h",
                "frequencies": [
                    {
                        "frequency": "1h",
                        "min_time": "2020-10-15T03:00:00.000000000Z",
                        "max_time": "2021-01-06T12:00:00.000000000Z",
                    }
                ],
            },
        ],
    },
    {
        "pair": "aave-btc",
        "metrics": [
            {
                "metric": "volume_trusted_spot_usd_1d",
                "frequencies": [
                    {
                        "frequency": "1d",
                        "min_time": "2020-10-11T00:00:00.000000000Z",
                        "max_time": "2021-01-05T00:00:00.000000000Z",
                    }
                ],
            },
            {
                "metric": "volume_trusted_spot_usd_1h",
                "frequencies": [
                    {
                        "frequency": "1h",
                        "min_time": "2020-10-10T19:00:00.000000000Z",
                        "max_time": "2021-01-06T12:00:00.000000000Z",
                    }
                ],
            },
        ],
    },
]

catalog_institutions_test_data = [
    {
        "institution": "grayscale",
        "metrics": [
            {
                "metric": "gbtc_total_assets",
                "frequencies": [
                    {
                        "frequency": "1d",
                        "min_time": "2020-10-16T00:00:00.000000000Z",
                        "max_time": "2021-01-05T00:00:00.000000000Z",
                    }
                ],
            },
            {
                "metric": "gbtc_shares_outstanding",
                "frequencies": [
                    {
                        "frequency": "1h",
                        "min_time": "2020-10-15T03:00:00.000000000Z",
                        "max_time": "2021-01-06T12:00:00.000000000Z",
                    }
                ],
            },
        ],
    }
]

catalog_indexes_test_data = [
    {
        "index": "CMBI10",
        "description": "'CMBI10' index.",
        "frequencies": [
            {
                "frequency": "15s",
                "min_time": "2020-06-08T20:12:40.000000000Z",
                "max_time": "2020-06-08T20:29:30.000000000Z",
            }
        ],
    },
    {
        "index": "CMBIBTC",
        "description": "'CMBIBTC' index.",
        "frequencies": [
            {
                "frequency": "15s",
                "min_time": "2010-07-18T20:00:00.000000000Z",
                "max_time": "2020-06-08T20:29:45.000000000Z",
            },
            {
                "frequency": "1d",
                "min_time": "2010-07-19T00:00:00.000000000Z",
                "max_time": "2020-06-08T00:00:00.000000000Z",
            },
            {
                "frequency": "1d-ny-close",
                "min_time": "2010-07-18T20:00:00.000000000Z",
                "max_time": "2020-06-08T20:00:00.000000000Z",
            },
            {
                "frequency": "1d-sg-close",
                "min_time": "2010-07-19T08:00:00.000000000Z",
                "max_time": "2020-06-08T08:00:00.000000000Z",
            },
            {
                "frequency": "1h",
                "min_time": "2010-07-18T20:00:00.000000000Z",
                "max_time": "2020-06-08T20:00:00.000000000Z",
            },
        ],
    },
]

catalog_asset_alerts_test_data = [
    {
        "asset": "btc",
        "name": "block_count_empty_6b_hi",
        "description": "description",
        "threshold": "4",
        "constituents": ["block_count_empty_6b"],
    }
]

catalog_market_trades_data = [
    {
        "market": "binance-aave-btc-spot",
        "min_time": "2020-10-15T03:00:01.286000000Z",
        "max_time": "2022-07-12T21:00:39.933000000Z",
    },
    {
        "market": "binance-algo-usdc-spot",
        "min_time": "2019-06-22T00:00:07.600000000Z",
        "max_time": "2020-01-07T07:06:19.943000000Z",
    },
    {
        "market": "binance-atom-usdc-spot",
        "min_time": "2019-05-07T12:05:42.347000000Z",
        "max_time": "2022-07-12T21:01:25.281000000Z",
    },
]

catalog_market_quotes_data = [
    {
        "market": "coinbase-btc-usd-spot",
        "min_time": "2019-03-25T18:46:25.000000000Z",
        "max_time": "2022-07-12T21:01:40.000000000Z",
    },
    {
        "market": "coinbase-btc-usdt-spot",
        "min_time": "2021-08-20T13:00:00.000000000Z",
        "max_time": "2022-07-12T21:00:00.000000000Z",
    },
    {
        "market": "coinbase-eth-usd-spot",
        "min_time": "2019-03-25T18:46:25.000000000Z",
        "max_time": "2022-07-12T21:01:40.000000000Z",
    },
]

catalog_market_orderbook_data = [
    {
        "market": "binance-aave-btc-spot",
        "min_time": "2021-09-14T16:00:00.000000000Z",
        "max_time": "2022-07-12T20:00:00.000000000Z",
    },
    {
        "market": "binance-atom-usdc-spot",
        "min_time": "2021-09-14T16:00:00.000000000Z",
        "max_time": "2022-07-12T21:00:00.000000000Z",
    },
]


def test_base_url() -> None:
    assert CoinMetricsClient(" ")._api_base_url == "https://api.coinmetrics.io/v4"
    assert (
        CoinMetricsClient()._api_base_url == "https://community-api.coinmetrics.io/v4"
    )
    assert CoinMetricsClient()._api_key_url_str == ""


def test_catalog_assets_dataframe() -> None:
    data = CatalogAssetsData(catalog_assets_test_data)
    df = data.to_dataframe()
    assert len(df) == df.asset.nunique()

    df_markets = data.to_dataframe(secondary_level="markets")
    df_markets["metrics"] = df_markets["metrics"].astype(str)
    df_markets_test = pd.read_csv("test/data/catalog_assets_markets.csv", dtype=str)
    assert (df_markets.values == df_markets_test.values).all()

    df_metrics = data.to_dataframe(secondary_level="metrics")
    df_metrics = df_metrics.fillna("").astype(str)
    df_metrics_test = pd.read_csv(
        "test/data/catalog_assets_metrics.csv", dtype=str
    ).fillna("")
    assert (df_metrics_test.values == df_metrics.values).all()


def test_catalog_metrics_dataframe() -> None:
    data = CatalogMetricsData(catalog_metrics_test_data)
    df = data.to_dataframe().fillna("")
    df_test = pd.read_csv("test/data/catalog_metrics.csv", dtype=str).fillna("")
    assert (df.values == df_test.values).all()


def test_catalog_exchanges_dataframe() -> None:
    data = CatalogExchangesData(catalog_exchanges_test_data)
    df = data.to_dataframe()
    assert len(df) == df.exchange.nunique()
    df_markets = data.to_dataframe(secondary_level="markets")
    df_markets = df_markets.fillna("").astype(str)
    df_markets_test = pd.read_csv(
        "test/data/catalog_exchanges_markets.csv", dtype=str
    ).fillna("")
    assert (df_markets.values == df_markets_test.values).all()

    df_metrics = data.to_dataframe(secondary_level="metrics")
    df_metrics = df_metrics.fillna("").astype(str)
    df_metrics_test = pd.read_csv(
        "test/data/catalog_exchanges_metrics.csv", dtype=str
    ).fillna("")
    assert (df_metrics_test.values == df_metrics.values).all()


def test_catalog_exchange_assets_dataframe() -> None:
    data = CatalogExchangeAssetsData(catalog_exchange_assets_test_data)
    df = data.to_dataframe().fillna("").astype(str)
    df.to_csv("test/data/catalog_exchange_assets.csv", index_label=False)
    df_test = pd.read_csv("test/data/catalog_exchange_assets.csv", dtype=str).fillna("")
    assert (df.values == df_test.values).all()


def test_catalog_indexes_dataframe() -> None:
    data = CatalogIndexesData(catalog_indexes_test_data)
    df = data.to_dataframe().fillna("").astype(str)
    df_test = pd.read_csv("test/data/catalog_indexes.csv", dtype=str).fillna("")
    assert (df.values == df_test.values).all()


def test_catalog_institutions_dataframe() -> None:
    data = CatalogInstitutionsData(catalog_institutions_test_data)
    df = data.to_dataframe().fillna("").astype(str)
    print(os.getcwd())
    df_test = pd.read_csv("test/data/catalog_institutions.csv", dtype=str).fillna("")
    assert (df.values == df_test.values).all()


def test_catalog_asset_pairs_dataframe() -> None:
    data = CatalogAssetPairsData(catalog_asset_pairs_test_data)
    df = data.to_dataframe().fillna("").astype(str)
    df_test = pd.read_csv("test/data/catalog_asset_pairs.csv", dtype=str).fillna("")
    assert (df.values == df_test.values).all()


def test_catalog_asset_pairs_candles_dataframe() -> None:
    data = CatalogAssetPairCandlesData(catalog_candles_test_data)
    df = data.to_dataframe().fillna("").astype(str)
    df_test = pd.read_csv("test/data/catalog_asset_pair_candles.csv", dtype=str).fillna(
        ""
    )
    assert (df.values == df_test.values).all()


def test_catalog_market_trades_dataframe() -> None:
    data = CatalogMarketTradesData(catalog_market_trades_data)
    df = data.to_dataframe().fillna("").astype(str)
    df_test = pd.read_csv("test/data/catalog_market_trades.csv", dtype=str).fillna("")
    assert (df.values == df_test.values).all()


def test_catalog_market_orderbook_dataframe() -> None:
    data = CatalogMarketTradesData(catalog_market_orderbook_data)
    df = data.to_dataframe().fillna("").astype(str)
    df_test = pd.read_csv("test/data/catalog_market_orderbooks.csv", dtype=str).fillna(
        ""
    )
    assert (df.values == df_test.values).all()


def test_catalog_market_quotes() -> None:
    data = CatalogMarketTradesData(catalog_market_quotes_data)
    df = data.to_dataframe().fillna("").astype(str)
    df_test = pd.read_csv("test/data/catalog_market_quotes.csv", dtype=str).fillna("")
    assert (df.values == df_test.values).all()


@pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP)
def test_empty_dataframe() -> None:
    """Test client behavior when query results in no data"""
    # Use a real endpoint with time range that has no data
    empty_response = client.get_asset_metrics(
        assets="btc",
        metrics="ReferenceRateUSD",
        start_time="2009-01-01",  # Before Bitcoin existed
        end_time="2009-01-02",
    )
    assert empty_response.first_page() == []
    df: pd.DataFrame = empty_response.to_dataframe(dataframe_type="pandas")
    assert df.empty


@pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP)
def test_export_to_parquet_asset_metrics() -> None:
    """Test export_to_parquet for asset-metrics endpoint"""
    # Get all schema-defined fields and export to parquet
    asset_metrics_dc = client.get_asset_metrics("btc", ["SplyBMXNtv", "PriceUSD"], limit_per_asset=5)
    schema_fields = set(asset_metrics_dc.get_schema_field_names())
    
    asset_metrics_dc.export_to_parquet("asset_metrics.parquet")
    asset_metrics_pq = pq.read_table("asset_metrics.parquet")
    parquet_columns = set(asset_metrics_pq.column_names)
    
    # Validate that all schema-defined fields are present in Parquet export
    missing_fields = schema_fields - parquet_columns
    assert not missing_fields, \
        f"Missing schema fields in Parquet: {sorted(missing_fields)}"
    extra_fields = parquet_columns - schema_fields
    assert not extra_fields, \
        f"Extra fields in Parquet: {sorted(extra_fields)}"

    # Test parallel export
    client.get_asset_metrics(
        "btc,eth", "ReferenceRateUSD", limit_per_asset=1
    ).parallel().export_to_parquet_files()
    assert os.path.exists("asset-metrics")
    assert set(os.listdir("asset-metrics")).issuperset(['btc.parquet', 'eth.parquet'])

    # Verify that pd.read_parquet and to_dataframe() yield identical schemas
    asset_metrics_df_direct = client.get_asset_metrics("btc", ["PriceUSD"], limit_per_asset=5).to_dataframe()
    client.get_asset_metrics("btc", ["PriceUSD"], limit_per_asset=5).export_to_parquet("asset_metrics.parquet")
    asset_metrics_df_parquet = pd.read_parquet("asset_metrics.parquet")

    # Compare schemas - only check columns that exist in both (all-null columns may be dropped)
    common_columns = set(asset_metrics_df_direct.columns) & set(asset_metrics_df_parquet.columns)
    print(f"Direct to_dataframe() dtypes: {asset_metrics_df_direct.dtypes.to_dict()}")
    print(f"pd.read_parquet() dtypes: {asset_metrics_df_parquet.dtypes.to_dict()}")
    print(f"Common columns: {sorted(common_columns)}")

    # Check that all common columns have equivalent types
    for col in sorted(common_columns):
        direct_dtype = str(asset_metrics_df_direct[col].dtype)
        parquet_dtype = str(asset_metrics_df_parquet[col].dtype)

        # Types should match, with some acceptable variations:
        # - float64 vs Float64 (both are 64-bit floats, Float64 is nullable)
        # - Int64 vs float64 (Int64 is nullable int, but pandas may read as float64 if values are numeric)
        # - object vs string (both store strings, string is pandas extension type)
        # - datetime64[ns, UTC] should match exactly
        if direct_dtype == parquet_dtype:
            continue

        # For datetime, both should be datetime64[ns, UTC]
        # If Parquet inferred datetime but direct is string, that's acceptable (Parquet inference is working)
        if "datetime" in direct_dtype.lower() and "datetime" in parquet_dtype.lower():
            assert "utc" in direct_dtype.lower() and "utc" in parquet_dtype.lower(), \
                f"Column {col}: datetime types should both be UTC. Direct: {direct_dtype}, Parquet: {parquet_dtype}"
            continue
        # If Parquet inferred datetime from string (type inference working), that's acceptable
        if direct_dtype == "string" and "datetime" in parquet_dtype.lower():
            if "utc" in parquet_dtype.lower():
                continue  # Parquet correctly inferred datetime from string

        # For numeric types, both should be float or int (Int64 may be read as float64 by pandas)
        # Note: pandas reads decimal128 as object, so Float64 (direct) vs object (parquet) is acceptable
        if "float" in direct_dtype.lower() and "float" in parquet_dtype.lower():
            continue
        if "float" in direct_dtype.lower() and parquet_dtype == "object":
            # Float64 (direct) vs object (parquet) is acceptable when parquet uses decimal128
            continue
        if "int" in direct_dtype.lower() and "float" in parquet_dtype.lower():
            # Int64 (nullable int) may be read as float64 by pandas if there are nulls
            continue
        if "int" in direct_dtype.lower() and "int" in parquet_dtype.lower():
            continue

        # For string types, both should be string or object
        if direct_dtype == "string" or direct_dtype == "object":
            if parquet_dtype == "string" or parquet_dtype == "object":
                continue

        # If we get here, types don't match and aren't acceptable variations
        assert False, \
            f"Column {col}: types don't match. Direct: {direct_dtype}, Parquet: {parquet_dtype}"

    print("✅ Schema equivalence verified: pd.read_parquet and to_dataframe() yield identical schemas")


@pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP)
def test_export_to_parquet_trades() -> None:
    """Test export_to_parquet for trades endpoint"""
    print("Exporting Trades to Parquet...")
    trades_dc = client.get_market_trades(["binance-btc-usdt-spot", "binance-1000FLOKIUSDT-future", "deribit-ETH-19SEP25-4000-C-option", "uniswap_v3_eth-1-wbtc-weth-spot", "deribit-XRP_USDC-PERPETUAL-future"], limit_per_market=1, paging_from='end')
    schema_fields = set(trades_dc.get_schema_field_names())
    
    trades_dc.export_to_parquet("trades.parquet")
    tb_pq_trades = pq.read_table("trades.parquet")
    parquet_columns = set(tb_pq_trades.column_names)
    
    # Validate that all schema-defined fields are present in Parquet export
    missing_fields = schema_fields - parquet_columns
    assert not missing_fields, \
        f"Missing schema fields in Parquet: {sorted(missing_fields)}"
    extra_fields = parquet_columns - schema_fields
    assert not extra_fields, \
        f"Extra fields in Parquet: {sorted(extra_fields)}"

    # Verify that Parquet files preserve correct types (not all strings)
    trades_table = pq.read_table("trades.parquet")
    time_field = trades_table.schema.field('time')
    price_field = trades_table.schema.field('price')
    assert pa.types.is_timestamp(time_field.type), \
        f"time field should be timestamp, got {time_field.type}"
    # Price should be either decimal128 or float64 (not string)
    assert not pa.types.is_string(price_field.type), \
        f"price field should not be string, got {price_field.type}"
    assert pa.types.is_decimal(price_field.type) or pa.types.is_floating(price_field.type), \
        f"price field should be decimal or float, got {price_field.type}"


@pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP)
def test_export_to_parquet_market_orderbooks() -> None:
    """Test export_to_parquet for market-orderbooks endpoint"""
    print("Exporting Orderbooks to Parquet...")
    orderbooks_dc = client.get_market_orderbooks(["binance-btc-usdt-spot", "binance-1000FLOKIUSDT-future", "deribit-ETH-19SEP25-4000-C-option", "deribit-XRP_USDC-PERPETUAL-future"], limit_per_market=1, paging_from='end')
    schema_fields = set(orderbooks_dc.get_schema_field_names())
    
    orderbooks_dc.export_to_parquet("orderbooks.parquet")
    tb_pq_ob = pq.read_table("orderbooks.parquet")
    parquet_columns = set(tb_pq_ob.column_names)
    
    # Validate that all schema-defined fields are present in Parquet export
    missing_fields = schema_fields - parquet_columns
    assert not missing_fields, \
        f"Missing schema fields in Parquet: {sorted(missing_fields)}"
    extra_fields = parquet_columns - schema_fields
    assert not extra_fields, \
        f"Extra fields in Parquet: {sorted(extra_fields)}"

    # Test parallel export
    orderbooks_parallel_dc = client.get_market_orderbooks(
        ["binance-btc-usdt-spot", "binance-1000FLOKIUSDT-future", "deribit-ETH-19SEP25-4000-C-option", "deribit-XRP_USDC-PERPETUAL-future"],
        limit_per_market=1, paging_from='end'
    )
    schema_fields_parallel = set(orderbooks_parallel_dc.get_schema_field_names())
    
    orderbooks_parallel_dc.parallel().export_to_parquet_files()
    tb_pq_market = pq.read_table('market-orderbooks/binance-btc-usdt-spot.parquet')
    parquet_columns = set(tb_pq_market.column_names)
    
    # Validate parallel export
    missing_fields = schema_fields_parallel - parquet_columns
    assert not missing_fields, \
        f"Missing schema fields in parallel Parquet: {sorted(missing_fields)}"
    extra_fields = parquet_columns - schema_fields_parallel
    assert not extra_fields, \
        f"Extra fields in parallel Parquet: {sorted(extra_fields)}"

    # Verify that Parquet files preserve correct types (not all strings)
    orderbooks_table = pq.read_table("orderbooks.parquet")
    time_field_ob = orderbooks_table.schema.field('time')
    bids_field_ob = orderbooks_table.schema.field('bids')
    assert pa.types.is_timestamp(time_field_ob.type), \
        f"orderbooks time field should be timestamp, got {time_field_ob.type}"
    # bids should be list(struct(price: decimal, size: decimal))
    assert pa.types.is_list(bids_field_ob.type), \
        f"orderbooks bids field should be list, got {bids_field_ob.type}"
    value_type = bids_field_ob.type.value_type
    assert pa.types.is_struct(value_type), \
        f"orderbooks bids value type should be struct, got {value_type}"
    # Check that struct has two decimal fields (price and size)
    struct_fields = value_type
    # Check that price and size are decimal types
    price_field = None
    size_field = None
    for field in struct_fields:
        if field.name == 'price':
            price_field = field
        if field.name == 'size':
            size_field = field
    assert price_field is not None, \
        f"orderbooks bids struct should have a 'price' field, got fields: {[f.name for f in struct_fields]}"
    assert size_field is not None, \
        f"orderbooks bids struct should have a 'size' field, got fields: {[f.name for f in struct_fields]}"
    assert pa.types.is_decimal(price_field.type), \
        f"orderbooks bids price field should be decimal, got {price_field.type}"
    assert pa.types.is_decimal(size_field.type), \
        f"orderbooks bids size field should be decimal, got {size_field.type}"


@pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP)
def test_export_to_parquet_balance_updates() -> None:
    """Test export_to_parquet for balance-updates endpoint"""
    print("Exporting ATLAS data to Parquet...")
    balance_updates_dc = client.get_list_of_balance_updates_v2(asset="btc", accounts="FEES", start_height=1, end_height=10)
    schema_fields = set(balance_updates_dc.get_schema_field_names())

    balance_updates_dc.export_to_parquet("balance_updates.parquet")
    tb_pq_balance_updates = pq.read_table("balance_updates.parquet")
    parquet_columns = set(tb_pq_balance_updates.column_names)
    
    # Validate that all schema-defined fields are present in Parquet export
    missing_fields = schema_fields - parquet_columns
    assert not missing_fields, \
        f"Missing schema fields in Parquet: {sorted(missing_fields)}"
    extra_fields = parquet_columns - schema_fields
    assert not extra_fields, \
        f"Extra fields in Parquet: {sorted(extra_fields)}"

    # Test parallel export
    print("Exporting ATLAS data to Parquet in parallel...")
    balance_updates_parallel_dc = client.get_list_of_balance_updates_v2(asset="btc", accounts=["FEES", "ISSUANCE"], start_height=1, end_height=10)
    schema_fields_parallel = set(balance_updates_parallel_dc.get_schema_field_names())
    
    balance_updates_parallel_dc.parallel().export_to_parquet_files()
    tb_pq_balance_updates_parallel = pq.read_table('./balance-updates/btc.parquet')
    parquet_columns = set(tb_pq_balance_updates_parallel.column_names)
    
    # Validate parallel export
    missing_fields = schema_fields_parallel - parquet_columns
    assert not missing_fields, \
        f"Missing schema fields in parallel Parquet: {sorted(missing_fields)}"
    extra_fields = parquet_columns - schema_fields_parallel
    assert not extra_fields, \
        f"Extra fields in parallel Parquet: {sorted(extra_fields)}"

    # Verify that Parquet files preserve correct types (not all strings)
    balance_table = pq.read_table("balance_updates.parquet")
    # Find a decimal field (e.g., change, previous_balance, new_balance)
    for field_name in ['change', 'previous_balance', 'new_balance']:
        if field_name in balance_table.schema.names:
            field = balance_table.schema.field(field_name)
            assert not pa.types.is_string(field.type), \
                f"{field_name} field should not be string, got {field.type}"
            assert pa.types.is_decimal(field.type) or pa.types.is_floating(field.type), \
                f"{field_name} field should be decimal or float, got {field.type}"
            break


@pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP)
def test_normal_api_call_asset_chains() -> None:
    """
    Tests that when calling get asset chains, the to_dataframe() will contain those with and without reorgs
    """
    asset_chains_fields = set()

    for row in client.get_asset_chains(
            'btc',
            start_time='2022-10-22T05:40:00',  # '2022-10-22T00:00:00',
            end_time='2022-10-22T10:40:00'
        ):
        _ = {asset_chains_fields.add(field) for field in row}  # type: ignore
    df = client.get_asset_chains(
        assets='btc',
        start_time='2022-10-22T05:40:00',  # '2022-10-22T00:00:00',
        end_time='2022-10-22T10:40:00'
    ).to_dataframe(dataframe_type='pandas')

    assert all([field in df.columns for field in asset_chains_fields]), \
        f"Missing fields in df.columns: {set(asset_chains_fields) - set(df.columns)}"

@pytest.mark.skipif(not cm_api_key_set, reason=REASON_TO_SKIP)
def test_get_transaction_tracker_df() -> None:
    tx_tracker_fields = set()

    for row in client.get_transaction_tracker(
            'btc',
            start_time='2023-03-30T00:00:00',
            end_time='2023-03-30T00:05:00',
            page_size=200
        ):
        _ = {tx_tracker_fields.add(field) for field in row}  # type: ignore
    df = client.get_transaction_tracker(
        'btc',
        start_time='2023-03-30T00:00:00',
        end_time='2023-03-30T00:05:00',
        page_size=200
    ).to_dataframe()
    assert all([field in df.columns for field in tx_tracker_fields]), \
        f"Missing fields in df.columns: {set(tx_tracker_fields) - set(df.columns)}"


# ---------------------------------------------------------------------------
# Lightweight tests for blockchain-v2 rebasing-changes endpoint
# These tests verify method signatures, endpoint paths, and parameter wiring
# without making any API calls (DataCollection is constructed lazily).
# ---------------------------------------------------------------------------


def test_get_list_of_rebasing_changes_v2_endpoint() -> None:
    """Verify get_list_of_rebasing_changes_v2 builds the correct endpoint and params."""
    result = client.get_list_of_rebasing_changes_v2(
        asset="steth_eth",
        txids=["tx1", "tx2"],
        block_hashes="abc123",
        page_size=100,
        paging_from="end",
        start_time="2021-01-01",
        end_time="2021-01-02",
        start_height=1000,
        end_height=2000,
        start_chain_sequence_number=500,
        end_chain_sequence_number=600,
        chain="main",
        start_inclusive=True,
        end_inclusive=False,
        timezone="America/Chicago",
        ignore_unsupported_errors=True,
    )
    assert isinstance(result, DataCollection)
    assert result._endpoint == "blockchain-v2/steth_eth/rebasing-changes"
    assert result._url_params["asset"] == "steth_eth"
    assert result._url_params["txids"] == ["tx1", "tx2"]
    assert result._url_params["block_hashes"] == "abc123"
    assert result._url_params["page_size"] == 100
    assert result._url_params["paging_from"] == "end"
    assert result._url_params["start_time"] == "2021-01-01"
    assert result._url_params["end_time"] == "2021-01-02"
    assert result._url_params["start_height"] == 1000
    assert result._url_params["end_height"] == 2000
    assert result._url_params["start_chain_sequence_number"] == 500
    assert result._url_params["end_chain_sequence_number"] == 600
    assert result._url_params["chain"] == "main"
    assert result._url_params["start_inclusive"] is True
    assert result._url_params["end_inclusive"] is False
    assert result._url_params["timezone"] == "America/Chicago"
    assert result._url_params["ignore_unsupported_errors"] is True


def test_get_list_of_rebasing_changes_v2_defaults() -> None:
    """Verify get_list_of_rebasing_changes_v2 defaults are applied correctly."""
    result = client.get_list_of_rebasing_changes_v2(asset="steth_eth")
    assert isinstance(result, DataCollection)
    assert result._endpoint == "blockchain-v2/steth_eth/rebasing-changes"
    assert result._url_params["asset"] == "steth_eth"
    assert result._url_params["txids"] is None
    assert result._url_params["block_hashes"] is None
    assert result._url_params["page_size"] is None
    assert result._url_params["paging_from"] == "start"
    assert result._url_params["start_time"] is None
    assert result._url_params["end_time"] is None
    assert result._url_params["start_height"] is None
    assert result._url_params["end_height"] is None
    assert result._url_params["start_chain_sequence_number"] is None
    assert result._url_params["end_chain_sequence_number"] is None
    assert result._url_params["chain"] is None
    assert result._url_params["start_inclusive"] is None
    assert result._url_params["end_inclusive"] is None
    assert result._url_params["timezone"] is None
    assert result._url_params["ignore_unsupported_errors"] is False
