// FILE-SIZE-OK: Canonical location for all Binance historical data loading (Vision + REST + fromId)
// Issue #92: REST gap backfill + fromId pagination (source-validation skill)
//! Historical data loading utilities
//!
//! Consolidated data loading functionality extracted from examples to eliminate
//! code duplication. Provides unified interface for Binance aggTrades data
//! with support for single-day, multi-day, recent data loading, and intra-day streaming.
//!
//! ## Streaming Architecture (v8.0+)
//!
//! The `IntraDayChunkIterator` enables memory-efficient streaming by yielding
//! trades in configurable hour-based chunks instead of loading entire date ranges.
//!
//! ```rust,ignore
//! use opendeviationbar_providers::binance::{HistoricalDataLoader, IntraDayChunkIterator};
//!
//! let loader = HistoricalDataLoader::new("BTCUSDT");
//! let chunks = IntraDayChunkIterator::new(loader, start_date, end_date, 6); // 6-hour chunks
//!
//! for chunk_result in chunks {
//!     let trades = chunk_result?;
//!     // Process ~400MB of trades at a time instead of entire day
//! }
//! ```

use chrono::{Duration as ChronoDuration, NaiveDate};
use csv::ReaderBuilder;
use reqwest::Client;
use serde::Deserialize;
use std::collections::HashMap;
use std::io::{BufRead, BufReader, Cursor};
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use thiserror::Error;
use tracing::{debug, info, warn};
use zip::ZipArchive;

use opendeviationbar_core::{AggTrade, FixedPoint, normalize_timestamp};

use super::checksum::{self, ChecksumError};
use super::rate_limiter::AdaptiveRateLimiter;
use crate::serde_helpers::deserialize_fixedpoint_from_str;

// =============================================================================
// Symbol → Market registry (compiled from symbols.toml at build time)
// =============================================================================

static SYMBOL_MARKET_MAP: OnceLock<HashMap<String, String>> = OnceLock::new();

fn parse_symbol_markets(toml_str: &str) -> HashMap<String, String> {
    // Minimal TOML parsing: extract [symbols.SYMBOL] market = "..." entries.
    // We avoid a toml dependency by using simple line-based parsing.
    let mut map = HashMap::new();
    let mut current_symbol: Option<String> = None;

    for line in toml_str.lines() {
        let trimmed = line.trim();
        // Match [symbols.BTCUSDT] headers
        if trimmed.starts_with("[symbols.") && trimmed.ends_with(']') {
            let sym = &trimmed[9..trimmed.len() - 1];
            current_symbol = Some(sym.to_string());
        } else if trimmed.starts_with("market") && trimmed.contains('=') {
            if let Some(ref sym) = current_symbol {
                // Extract value from: market = "spot"
                if let Some(val) = trimmed.split('=').nth(1) {
                    let val = val.trim().trim_matches('"');
                    map.insert(sym.clone(), val.to_string());
                }
            }
        } else if trimmed.starts_with('[') {
            // New section that isn't [symbols.*]
            if !trimmed.starts_with("[symbols.") {
                current_symbol = None;
            }
        }
    }
    map
}

/// Look up the market type for a symbol from the compiled-in symbols.toml.
/// Returns None if the symbol is not registered.
pub fn get_symbol_market(symbol: &str) -> Option<&str> {
    let map = SYMBOL_MARKET_MAP.get_or_init(|| {
        let toml_str = include_str!(concat!(env!("OUT_DIR"), "/symbols.toml"));
        parse_symbol_markets(toml_str)
    });
    map.get(symbol).map(|s| s.as_str())
}

/// Resolve market: normalize "futures-um" → "um", "futures-cm" → "cm".
fn normalize_market(market: &str) -> &str {
    match market {
        "futures-um" => "um",
        "futures-cm" => "cm",
        other => other,
    }
}

/// Errors that can occur during historical data loading
#[derive(Debug, Error)]
pub enum HistoricalError {
    /// HTTP request failed
    #[error("HTTP error for {date}: {message}")]
    HttpError { date: String, message: String },

    /// Failed to parse CSV data
    #[error("CSV parse error: {0}")]
    CsvError(#[from] csv::Error),

    /// Failed to read ZIP archive
    #[error("ZIP error: {0}")]
    ZipError(#[from] zip::result::ZipError),

    /// I/O error during file operations
    #[error("IO error: {0}")]
    IoError(#[from] std::io::Error),

    /// Request timeout
    #[error("Request timeout for {date}")]
    Timeout { date: String },

    /// Tokio runtime error
    #[error("Runtime error: {0}")]
    RuntimeError(String),

    /// Checksum verification failed (Issue #43)
    /// This indicates data corruption - the downloaded file does not match
    /// the SHA-256 checksum provided by Binance.
    #[error("Checksum verification failed for {date}: {message}")]
    ChecksumMismatch { date: String, message: String },

    /// REST API error (Issue #92: gap backfill)
    #[error("REST API error: {0}")]
    RestApiError(String),

    /// Rate limited by Binance API (HTTP 429)
    #[error("Rate limited for {symbol}: exceeded {max_retries} retries")]
    RateLimited { symbol: String, max_retries: u32 },
}

#[derive(Debug, Deserialize)]
pub struct CsvAggTrade(
    pub u64,                                             // agg_trade_id
    pub f64,                                             // price
    pub f64,                                             // quantity
    pub u64,                                             // first_trade_id
    pub u64,                                             // last_trade_id
    pub u64,                                             // timestamp
    #[serde(deserialize_with = "python_bool")] pub bool, // is_buyer_maker
);

/// Custom deserializer for Python-style booleans
pub fn python_bool<'de, D>(deserializer: D) -> Result<bool, D::Error>
where
    D: serde::Deserializer<'de>,
{
    let s = String::deserialize(deserializer)?;
    match s.as_str() {
        "True" | "true" => Ok(true),
        "False" | "false" => Ok(false),
        _ => Err(serde::de::Error::custom(format!(
            "Invalid boolean value: {}",
            s
        ))),
    }
}

/// Detect CSV headers
pub fn detect_csv_headers(buffer: &str) -> bool {
    if let Some(first_line) = buffer.lines().next() {
        first_line.contains("agg_trade_id")
            || first_line.contains("price")
            || first_line.contains("quantity")
            || first_line.contains("timestamp")
            || first_line.contains("is_buyer_maker")
    } else {
        false
    }
}

/// Binance REST API `/api/v3/aggTrades` response format (Issue #92)
/// Issue #96: price/quantity deserialize directly to FixedPoint (zero String allocation).
///
/// JSON keys are short single-letter names from the Binance API:
/// `a` = agg_trade_id, `p` = price, `q` = quantity, `f` = first_trade_id,
/// `l` = last_trade_id, `T` = trade_time, `m` = is_buyer_maker
#[derive(Debug, Deserialize)]
pub struct RestAggTrade {
    #[serde(rename = "a")]
    pub agg_trade_id: i64,
    #[serde(rename = "p")]
    #[serde(deserialize_with = "deserialize_fixedpoint_from_str")]
    pub price: FixedPoint,
    #[serde(rename = "q")]
    #[serde(deserialize_with = "deserialize_fixedpoint_from_str")]
    pub quantity: FixedPoint,
    #[serde(rename = "f")]
    pub first_trade_id: i64,
    #[serde(rename = "l")]
    pub last_trade_id: i64,
    #[serde(rename = "T")]
    pub trade_time: i64,
    #[serde(rename = "m")]
    pub is_buyer_maker: bool,
}

impl From<CsvAggTrade> for AggTrade {
    fn from(csv_trade: CsvAggTrade) -> Self {
        AggTrade {
            agg_trade_id: csv_trade.0 as i64,
            price: FixedPoint::from_f64(csv_trade.1),
            volume: FixedPoint::from_f64(csv_trade.2),
            first_trade_id: csv_trade.3 as i64,
            last_trade_id: csv_trade.4 as i64,
            timestamp: csv_trade.5 as i64,
            is_buyer_maker: csv_trade.6,
            is_best_match: None, // Not available in historical CSV data
        }
    }
}

impl From<RestAggTrade> for AggTrade {
    fn from(rest: RestAggTrade) -> Self {
        AggTrade {
            agg_trade_id: rest.agg_trade_id,
            price: rest.price,
            volume: rest.quantity,
            first_trade_id: rest.first_trade_id,
            last_trade_id: rest.last_trade_id,
            timestamp: normalize_timestamp(rest.trade_time as u64),
            is_buyer_maker: rest.is_buyer_maker,
            is_best_match: None,
        }
    }
}

impl CsvAggTrade {
    /// Convert to AggTrade with market-aware timestamp conversion
    pub fn to_agg_trade(&self, _market_type: &str) -> AggTrade {
        // Universal timestamp normalization (market_type no longer needed)
        let normalized_timestamp = normalize_timestamp(self.5);

        AggTrade {
            agg_trade_id: self.0 as i64,
            price: FixedPoint::from_f64(self.1),
            volume: FixedPoint::from_f64(self.2),
            first_trade_id: self.3 as i64,
            last_trade_id: self.4 as i64,
            timestamp: normalized_timestamp,
            is_buyer_maker: self.6,
            is_best_match: None, // Not available in historical CSV data
        }
    }
}

/// Maximum retries for transient errors (429, 5xx, timeout, network)
const TRANSIENT_MAX_RETRIES: u32 = 5;
/// Initial backoff in milliseconds (doubles each retry: 1s → 2s → 4s → 8s → 16s)
const TRANSIENT_INITIAL_BACKOFF_MS: u64 = 1_000;

/// Send an HTTP request with exponential backoff on transient errors.
///
/// Retries up to `TRANSIENT_MAX_RETRIES` times with doubling backoff for:
/// - HTTP 429 (rate limit)
/// - HTTP 5xx (server error)
/// - Network/connection errors
/// - Timeouts (30s per attempt)
///
/// When `rate_limiter` is provided (Issue #162), acquires weight before sending
/// and syncs server-reported usage from response headers.
///
/// Non-transient errors (4xx except 429) are returned immediately.
async fn send_with_rate_limit_backoff(
    request_builder: impl Fn() -> reqwest::RequestBuilder,
    symbol: &str,
    rate_limiter: Option<&AdaptiveRateLimiter>,
    request_weight: u32,
) -> Result<reqwest::Response, HistoricalError> {
    // Issue #162: Proactive rate limiting before sending
    #[allow(clippy::collapsible_if)]
    if let Some(limiter) = rate_limiter {
        if let Err(e) = limiter.acquire(request_weight).await {
            warn!(symbol = %symbol, error = %e, "rate limiter blocked request, proceeding with backoff safety net");
            // Don't fail — fall through to existing 429 backoff as safety net
        }
    }
    let mut backoff_ms = TRANSIENT_INITIAL_BACKOFF_MS;
    let mut last_error: Option<HistoricalError> = None;

    for attempt in 0..=TRANSIENT_MAX_RETRIES {
        // Timeout wrapping the send — retried on timeout
        let timeout_result =
            tokio::time::timeout(Duration::from_secs(30), request_builder().send()).await;

        let send_result = match timeout_result {
            Ok(r) => r,
            Err(_) => {
                // Timeout — transient, retry
                let err = HistoricalError::RestApiError(format!(
                    "Timeout fetching {} (attempt {}/{})",
                    symbol, attempt, TRANSIENT_MAX_RETRIES
                ));
                if attempt == TRANSIENT_MAX_RETRIES {
                    return Err(err);
                }
                warn!(
                    symbol = %symbol,
                    attempt = attempt,
                    backoff_ms = backoff_ms,
                    "Transient error (timeout), backing off"
                );
                last_error = Some(err);
                tokio::time::sleep(Duration::from_millis(backoff_ms)).await;
                backoff_ms = (backoff_ms * 2).min(16_000);
                continue;
            }
        };

        let response = match send_result {
            Ok(r) => r,
            Err(e) => {
                // Network/connection error — transient, retry
                let err = HistoricalError::RestApiError(format!(
                    "Network error for {} (attempt {}/{}): {e}",
                    symbol, attempt, TRANSIENT_MAX_RETRIES
                ));
                if attempt == TRANSIENT_MAX_RETRIES {
                    return Err(err);
                }
                warn!(
                    symbol = %symbol,
                    attempt = attempt,
                    backoff_ms = backoff_ms,
                    error = %e,
                    "Transient error (network), backing off"
                );
                last_error = Some(err);
                tokio::time::sleep(Duration::from_millis(backoff_ms)).await;
                backoff_ms = (backoff_ms * 2).min(16_000);
                continue;
            }
        };

        let status = response.status();

        // 429 rate limit — transient, retry
        if status == reqwest::StatusCode::TOO_MANY_REQUESTS {
            if attempt == TRANSIENT_MAX_RETRIES {
                return Err(HistoricalError::RateLimited {
                    symbol: symbol.to_string(),
                    max_retries: TRANSIENT_MAX_RETRIES,
                });
            }
            warn!(
                symbol = %symbol,
                attempt = attempt,
                backoff_ms = backoff_ms,
                "Transient error (429 rate limit), backing off"
            );
            tokio::time::sleep(Duration::from_millis(backoff_ms)).await;
            backoff_ms = (backoff_ms * 2).min(16_000);
            continue;
        }

        // 5xx server error — transient, retry
        if status.is_server_error() {
            let err = HistoricalError::RestApiError(format!(
                "Server error {} for {} (attempt {}/{})",
                status.as_u16(),
                symbol,
                attempt,
                TRANSIENT_MAX_RETRIES
            ));
            if attempt == TRANSIENT_MAX_RETRIES {
                return Err(err);
            }
            warn!(
                symbol = %symbol,
                attempt = attempt,
                status = status.as_u16(),
                backoff_ms = backoff_ms,
                "Transient error (5xx), backing off"
            );
            last_error = Some(err);
            tokio::time::sleep(Duration::from_millis(backoff_ms)).await;
            backoff_ms = (backoff_ms * 2).min(16_000);
            continue;
        }

        // Issue #162: Sync server-reported weight from response headers
        if let Some(limiter) = rate_limiter {
            limiter.update_from_headers(response.headers());
        }

        // Non-transient response (2xx, 3xx, 4xx except 429) — return as-is
        return Ok(response);
    }

    // All retries exhausted — return last error
    Err(last_error.unwrap_or_else(|| HistoricalError::RateLimited {
        symbol: symbol.to_string(),
        max_retries: TRANSIENT_MAX_RETRIES,
    }))
}

/// Historical data loader for Binance aggTrades
#[derive(Clone)]
pub struct HistoricalDataLoader {
    client: Client,
    symbol: String,
    market_type: String,
    /// Optional adaptive rate limiter (Issue #162).
    /// When set, REST calls acquire weight before sending and sync from response headers.
    rate_limiter: Option<Arc<AdaptiveRateLimiter>>,
}

impl HistoricalDataLoader {
    pub fn new(symbol: &str) -> Self {
        let market = get_symbol_market(symbol)
            .map(normalize_market)
            .unwrap_or("spot");
        Self::new_with_market(symbol, market)
    }

    pub fn new_with_market(symbol: &str, market_type: &str) -> Self {
        Self {
            client: Client::new(),
            symbol: symbol.to_uppercase(),
            market_type: market_type.to_string(),
            rate_limiter: None,
        }
    }

    /// Attach an adaptive rate limiter (Issue #162).
    ///
    /// When set, all REST API calls will:
    /// 1. Call `acquire(weight)` before sending (proactive pacing)
    /// 2. Call `update_from_headers()` after receiving response (server feedback sync)
    ///
    /// The existing 429 backoff in `send_with_rate_limit_backoff()` remains as inner safety net.
    pub fn with_rate_limiter(mut self, limiter: Arc<AdaptiveRateLimiter>) -> Self {
        self.rate_limiter = Some(limiter);
        self
    }

    /// Get market path for URL construction — spot only (2026-03-11 policy).
    fn get_market_path(&self) -> &str {
        // All data sources use Binance spot market exclusively.
        // Vision and REST produce identical day boundaries.
        "spot"
    }

    /// Load single day trades
    pub async fn load_single_day_trades(
        &self,
        date: NaiveDate,
    ) -> Result<Vec<AggTrade>, Box<dyn std::error::Error>> {
        let date_str = date.format("%Y-%m-%d");
        let url = format!(
            "https://data.binance.vision/data/{}/daily/aggTrades/{}/{}-aggTrades-{}.zip",
            self.get_market_path(),
            self.symbol,
            self.symbol,
            date_str
        );

        let response =
            tokio::time::timeout(Duration::from_secs(30), self.client.get(&url).send()).await??;

        if !response.status().is_success() {
            return Err(format!("HTTP {} for {}", response.status(), date_str).into());
        }

        let zip_bytes = response.bytes().await?;
        let csv_filename = format!("{}-aggTrades-{}.csv", self.symbol, date_str);

        // Detect headers by peeking at first line (avoids read_to_string)
        let has_headers = {
            let cursor = Cursor::new(zip_bytes.as_ref());
            let mut archive = ZipArchive::new(cursor)?;
            let csv_file = archive.by_name(&csv_filename)?;
            let mut buf = BufReader::new(csv_file);
            let mut first_line = String::new();
            buf.read_line(&mut first_line)?;
            detect_csv_headers(&first_line)
        };

        // Parse trades using BufReader (no read_to_string allocation)
        let cursor = Cursor::new(zip_bytes.as_ref());
        let mut archive = ZipArchive::new(cursor)?;
        let csv_file = archive.by_name(&csv_filename)?;

        let buf_reader = BufReader::with_capacity(64 * 1024, csv_file);
        let mut reader = ReaderBuilder::new()
            .has_headers(has_headers)
            .from_reader(buf_reader);

        let mut day_trades = Vec::with_capacity(2_000_000);
        for result in reader.deserialize() {
            let csv_trade: CsvAggTrade = result?;
            let agg_trade: AggTrade = csv_trade.to_agg_trade(&self.market_type);
            day_trades.push(agg_trade);
        }

        day_trades.sort_by_key(|trade| trade.timestamp);
        Ok(day_trades)
    }

    /// Load multiple days of historical data
    pub async fn load_historical_range(
        &self,
        days_back: i64,
    ) -> Result<Vec<AggTrade>, Box<dyn std::error::Error>> {
        use chrono::Utc;

        let end_date = Utc::now().date_naive() - chrono::Duration::days(2);
        let start_date = end_date - chrono::Duration::days(days_back - 1);

        let mut all_trades = Vec::with_capacity(days_back as usize * 2_000_000);
        let mut current_date = start_date;

        while current_date <= end_date {
            match self.load_single_day_trades(current_date).await {
                Ok(mut day_trades) => {
                    all_trades.append(&mut day_trades);
                }
                Err(e) => {
                    println!("⚠️  Failed to load {}: {}", current_date, e);
                    return Err(e);
                }
            }
            current_date += chrono::Duration::days(1);
        }

        all_trades.sort_by_key(|trade| trade.timestamp);
        Ok(all_trades)
    }

    /// Try to load recent data (for testing)
    pub async fn load_recent_day(&self) -> Result<Vec<AggTrade>, Box<dyn std::error::Error>> {
        use chrono::Utc;

        for days_back in 1..=7 {
            let test_date = Utc::now().date_naive() - chrono::Duration::days(days_back);

            match self.load_single_day_trades(test_date).await {
                Ok(trades) => return Ok(trades),
                Err(_) => continue,
            }
        }

        Err("No recent data available in the last 7 days".into())
    }

    /// Load single day trades with typed error (for IntraDayChunkIterator)
    ///
    /// # Arguments
    ///
    /// * `date` - The date to load trades for
    /// * `verify_checksum` - If true, verify SHA-256 checksum of downloaded data (Issue #43)
    pub async fn load_single_day_trades_typed(
        &self,
        date: NaiveDate,
    ) -> Result<Vec<AggTrade>, HistoricalError> {
        self.load_single_day_trades_with_checksum(date, true).await
    }

    /// Download a day's ZIP file and detect CSV headers without parsing all trades.
    ///
    /// Returns (zip_bytes, has_csv_headers, date_str, csv_filename).
    /// This is the first half of the download+parse pipeline, allowing callers
    /// to re-parse the ZIP bytes multiple times for different hour ranges without
    /// holding the entire day's trades in memory.
    pub async fn download_day_zip(
        &self,
        date: NaiveDate,
        verify_checksum: bool,
    ) -> Result<(Vec<u8>, bool, String, String), HistoricalError> {
        let date_str = date.format("%Y-%m-%d").to_string();
        let url = format!(
            "https://data.binance.vision/data/{}/daily/aggTrades/{}/{}-aggTrades-{}.zip",
            self.get_market_path(),
            self.symbol,
            self.symbol,
            date_str
        );

        debug!(
            event_type = "download_start",
            symbol = %self.symbol,
            date = %date_str,
            verify_checksum = verify_checksum,
            "Downloading aggTrades ZIP"
        );

        let response = tokio::time::timeout(Duration::from_secs(30), self.client.get(&url).send())
            .await
            .map_err(|_| HistoricalError::Timeout {
                date: date_str.clone(),
            })?
            .map_err(|e| HistoricalError::HttpError {
                date: date_str.clone(),
                message: e.to_string(),
            })?;

        if !response.status().is_success() {
            return Err(HistoricalError::HttpError {
                date: date_str,
                message: format!("HTTP {}", response.status()),
            });
        }

        let raw_bytes = response
            .bytes()
            .await
            .map_err(|e| HistoricalError::HttpError {
                date: date_str.clone(),
                message: e.to_string(),
            })?;

        // Checksum verification (Issue #43) — operates on raw reqwest Bytes
        if verify_checksum {
            match checksum::fetch_and_verify(&self.client, &url, &raw_bytes).await {
                Ok(result) if result.verified => {
                    info!(
                        event_type = "checksum_verified",
                        symbol = %self.symbol,
                        date = %date_str,
                        hash = %result.actual,
                        "Checksum verified successfully"
                    );
                }
                Ok(result) if result.skipped => {
                    warn!(
                        event_type = "checksum_skipped",
                        symbol = %self.symbol,
                        date = %date_str,
                        "Checksum verification skipped (file not available)"
                    );
                }
                Ok(_) => {}
                Err(ChecksumError::Mismatch { expected, actual }) => {
                    return Err(HistoricalError::ChecksumMismatch {
                        date: date_str,
                        message: format!("expected {expected}, got {actual}"),
                    });
                }
                Err(ChecksumError::InvalidFormat(msg)) => {
                    return Err(HistoricalError::ChecksumMismatch {
                        date: date_str,
                        message: format!("invalid checksum format: {msg}"),
                    });
                }
                Err(e) => {
                    warn!(
                        event_type = "checksum_error",
                        symbol = %self.symbol,
                        date = %date_str,
                        error = %e,
                        "Checksum verification error (continuing anyway)"
                    );
                }
            }
        }

        // Convert bytes::Bytes → Vec<u8> (zero-copy when sole owner)
        let zip_bytes: Vec<u8> = raw_bytes.into();
        let csv_filename = format!("{}-aggTrades-{}.csv", self.symbol, date_str);

        // Detect CSV headers by reading just the first line
        let has_headers = {
            let cursor = Cursor::new(zip_bytes.as_slice());
            let mut archive = ZipArchive::new(cursor)?;
            let csv_file = archive.by_name(&csv_filename)?;
            let mut buf = BufReader::new(csv_file);
            let mut first_line = String::new();
            buf.read_line(&mut first_line)?;
            detect_csv_headers(&first_line)
        };

        let zip_size = zip_bytes.len();
        debug!(
            event_type = "zip_downloaded",
            symbol = %self.symbol,
            date = %date_str,
            zip_size_bytes = zip_size,
            has_headers = has_headers,
            "ZIP downloaded, ready for streaming parse"
        );

        Ok((zip_bytes, has_headers, date_str, csv_filename))
    }

    /// Fetch aggregated trades from Binance REST API (Issue #92: gap backfill).
    ///
    /// Paginates through `/api/v3/aggTrades` (max 1000 per request).
    /// Trades are returned sorted by timestamp in ascending order.
    ///
    /// # Arguments
    ///
    /// * `start_ms` - Start time in milliseconds (inclusive)
    /// * `end_ms` - End time in milliseconds (inclusive)
    ///
    /// # Returns
    ///
    /// Vec of `AggTrade` with timestamps in microseconds (opendeviationbar-core convention).
    pub async fn fetch_aggtrades_rest(
        &self,
        start_ms: i64,
        end_ms: i64,
    ) -> Result<Vec<AggTrade>, HistoricalError> {
        let url = "https://api.binance.com/api/v3/aggTrades";
        let mut all_trades: Vec<AggTrade> = Vec::new();
        // None = first page (use startTime); Some(id) = subsequent pages (use fromId)
        let mut next_from_id: Option<i64> = None;

        debug!(
            event_type = "rest_fetch_start",
            symbol = %self.symbol,
            start_ms = start_ms,
            end_ms = end_ms,
            "Fetching aggTrades from REST API"
        );

        loop {
            let client = &self.client;
            let symbol = &self.symbol;
            let end = end_ms;
            let from_id = next_from_id;
            let start = start_ms;

            let response = send_with_rate_limit_backoff(
                || {
                    let req = client.get(url);
                    if let Some(fid) = from_id {
                        // Subsequent pages: fromId-only (zero-gap, trade-ID anchored).
                        // Binance does not allow fromId + endTime together (HTTP 400).
                        // We filter by end_ms post-fetch instead.
                        req.query(&[
                            ("symbol", symbol.as_str()),
                            ("fromId", &fid.to_string()),
                            ("limit", "1000"),
                        ])
                    } else {
                        // First page only: startTime+endTime to establish the entry point.
                        req.query(&[
                            ("symbol", symbol.as_str()),
                            ("startTime", &start.to_string()),
                            ("endTime", &end.to_string()),
                            ("limit", "1000"),
                        ])
                    }
                },
                symbol,
                self.rate_limiter.as_deref(),
                4, // Spot aggTrades endpoint weight (Futures = 20)
            )
            .await?;

            if !response.status().is_success() {
                return Err(HistoricalError::RestApiError(format!(
                    "HTTP {} for {} (from_id={:?})",
                    response.status(),
                    self.symbol,
                    next_from_id,
                )));
            }

            let batch: Vec<RestAggTrade> = response.json().await.map_err(|e| {
                HistoricalError::RestApiError(format!("JSON parse error for {}: {e}", self.symbol))
            })?;

            if batch.is_empty() {
                break;
            }

            let last_trade_time = batch.last().unwrap().trade_time;
            let last_agg_id = batch.last().unwrap().agg_trade_id;
            let batch_len = batch.len();

            // Keep only trades within the requested time range
            all_trades.extend(
                batch
                    .into_iter()
                    .filter(|t| t.trade_time <= end_ms)
                    .map(AggTrade::from),
            );

            // Stop if the last trade in this batch is beyond end_ms
            if last_trade_time > end_ms {
                break;
            }

            // Next page: advance by trade ID (zero-gap — no same-ms boundary drop)
            next_from_id = Some(last_agg_id + 1);

            // If we got fewer than limit, we've exhausted the range
            if batch_len < 1000 {
                break;
            }
        }

        all_trades.sort_by_key(|t| t.timestamp);

        info!(
            event_type = "rest_fetch_complete",
            symbol = %self.symbol,
            trade_count = all_trades.len(),
            range_ms = end_ms - start_ms,
            "Fetched aggTrades from REST API"
        );

        Ok(all_trades)
    }

    /// Fetch aggregated trades by `fromId` (zero-gap pagination).
    ///
    /// Single REST call using `fromId` parameter — caller controls cursor.
    /// This is the **correct** pagination method; `startTime` drops trades
    /// at millisecond boundaries.
    ///
    /// # Arguments
    ///
    /// * `from_id` - Starting agg_trade_id (inclusive)
    /// * `limit` - Maximum trades to return (1-1000)
    ///
    /// # Returns
    ///
    /// Vec of `AggTrade` with timestamps in microseconds.
    pub async fn fetch_aggtrades_by_id(
        &self,
        from_id: i64,
        limit: u16,
    ) -> Result<Vec<AggTrade>, HistoricalError> {
        let url = "https://api.binance.com/api/v3/aggTrades";
        let limit_str = limit.min(1000).to_string();
        let from_id_str = from_id.to_string();
        let client = &self.client;
        let symbol = &self.symbol;

        let response = send_with_rate_limit_backoff(
            || {
                client.get(url).query(&[
                    ("symbol", symbol.as_str()),
                    ("fromId", from_id_str.as_str()),
                    ("limit", limit_str.as_str()),
                ])
            },
            symbol,
            self.rate_limiter.as_deref(),
            4, // Spot aggTrades endpoint weight (Futures = 20)
        )
        .await?;

        if !response.status().is_success() {
            return Err(HistoricalError::RestApiError(format!(
                "HTTP {} for {} fromId={from_id}",
                response.status(),
                self.symbol,
            )));
        }

        let batch: Vec<RestAggTrade> = response.json().await.map_err(|e| {
            HistoricalError::RestApiError(format!("JSON parse error for {}: {e}", self.symbol))
        })?;

        Ok(batch.into_iter().map(AggTrade::from).collect())
    }

    /// Fetch the latest aggregated trade for the symbol.
    ///
    /// Single REST call with `limit=1`, no `fromId` or `startTime`.
    /// Returns the most recent trade as an anchor point for cursor-based pagination.
    pub async fn fetch_latest_aggtrade(&self) -> Result<AggTrade, HistoricalError> {
        let url = "https://api.binance.com/api/v3/aggTrades";
        let client = &self.client;
        let symbol = &self.symbol;

        let response = send_with_rate_limit_backoff(
            || {
                client
                    .get(url)
                    .query(&[("symbol", symbol.as_str()), ("limit", "1")])
            },
            symbol,
            self.rate_limiter.as_deref(),
            1, // minimal weight for limit=1
        )
        .await?;

        if !response.status().is_success() {
            return Err(HistoricalError::RestApiError(format!(
                "HTTP {} for {} (latest)",
                response.status(),
                self.symbol,
            )));
        }

        let batch: Vec<RestAggTrade> = response.json().await.map_err(|e| {
            HistoricalError::RestApiError(format!("JSON parse error for {}: {e}", self.symbol))
        })?;

        batch.into_iter().next().map(AggTrade::from).ok_or_else(|| {
            HistoricalError::RestApiError(format!(
                "No trades returned for {} (latest)",
                self.symbol,
            ))
        })
    }

    /// Load single day trades with optional checksum verification
    ///
    /// # Arguments
    ///
    /// * `date` - The date to load trades for
    /// * `verify_checksum` - If true, verify SHA-256 checksum of downloaded data (Issue #43)
    ///
    /// # Checksum Verification Behavior
    ///
    /// | Scenario | Behavior | Rationale |
    /// |----------|----------|-----------|
    /// | Checksum matches | Continue | Success |
    /// | Checksum mismatch | Hard error | Data corruption detected |
    /// | Checksum file 404 | Soft warning, continue | Old data may not have checksums |
    /// | Network timeout | Soft warning, continue | Network issues shouldn't block |
    pub async fn load_single_day_trades_with_checksum(
        &self,
        date: NaiveDate,
        verify_checksum: bool,
    ) -> Result<Vec<AggTrade>, HistoricalError> {
        let (zip_bytes, has_headers, date_str, csv_filename) =
            self.download_day_zip(date, verify_checksum).await?;

        // Parse all trades from ZIP using BufReader (no read_to_string allocation)
        let cursor = Cursor::new(zip_bytes.as_slice());
        let mut archive = ZipArchive::new(cursor)?;
        let csv_file = archive.by_name(&csv_filename)?;

        let buf_reader = BufReader::with_capacity(64 * 1024, csv_file);
        let mut reader = ReaderBuilder::new()
            .has_headers(has_headers)
            .from_reader(buf_reader);

        let mut day_trades = Vec::with_capacity(2_000_000);
        for result in reader.deserialize() {
            let csv_trade: CsvAggTrade = result?;
            let agg_trade: AggTrade = csv_trade.to_agg_trade(&self.market_type);
            day_trades.push(agg_trade);
        }

        day_trades.sort_by_key(|trade| trade.timestamp);

        info!(
            event_type = "download_complete",
            symbol = %self.symbol,
            date = %date_str,
            trade_count = day_trades.len(),
            "Downloaded and parsed aggTrades"
        );

        Ok(day_trades)
    }
}

/// Default trades per chunk: 500K trades ≈ 30MB of AggTrade structs.
/// For BTCUSDT peak days (~20M trades), this yields ~40 chunks per day.
/// For typical days (~2M trades), this yields ~4 chunks — similar to 6-hour chunking.
const DEFAULT_TRADES_PER_CHUNK: usize = 500_000;

/// Intra-day chunk iterator for memory-efficient streaming.
///
/// Downloads each day's ZIP once, then re-parses the CSV for each chunk
/// using record-count based chunking (by aggregate trade ID position).
/// Only the ZIP bytes (~200MB-1GB compressed) plus the current chunk's
/// trades (~30MB for 500K records) are resident at any time.
///
/// # Why trade-count chunking (not hour-based)
///
/// Aggregate trade IDs are monotonically increasing, so record-count
/// chunking gives predictable, controllable memory usage regardless of
/// how trades are distributed across hours. High-volume periods (flash
/// crashes, liquidation cascades) concentrate millions of trades in
/// minutes — hour-based chunking would create one huge chunk.
///
/// # Memory Efficiency (BTCUSDT high-volume days)
///
/// | Approach              | Peak Memory | Notes                              |
/// |-----------------------|-------------|------------------------------------|
/// | Full-day cache (old)  | ~13-20 GB   | ZIP + String + Vec + clone         |
/// | ZIP-streaming (new)   | ~1-3 GB     | ZIP bytes + one chunk Vec          |
///
/// # Example
///
/// ```rust,ignore
/// use opendeviationbar_providers::binance::{HistoricalDataLoader, IntraDayChunkIterator};
/// use chrono::NaiveDate;
///
/// let loader = HistoricalDataLoader::new("BTCUSDT");
/// let start = NaiveDate::from_ymd_opt(2024, 1, 1).unwrap();
/// let end = NaiveDate::from_ymd_opt(2024, 1, 7).unwrap();
///
/// // Create iterator with default chunk size (500K trades)
/// let chunks = IntraDayChunkIterator::new(loader, start, end, 6);
///
/// for chunk_result in chunks {
///     let trades = chunk_result?;
///     println!("Processing {} trades", trades.len());
/// }
/// ```
pub struct IntraDayChunkIterator {
    /// The underlying data loader
    loader: HistoricalDataLoader,
    /// Current date being processed
    current_date: NaiveDate,
    /// End date (inclusive)
    end_date: NaiveDate,
    /// Maximum trades per chunk (controls memory usage)
    max_trades_per_chunk: usize,
    /// Records already consumed from current day's ZIP
    records_consumed: usize,
    /// Whether current day has been fully consumed
    day_fully_consumed: bool,
    /// Cached ZIP bytes for current day (downloaded once, re-parsed per chunk)
    day_zip_bytes: Option<Vec<u8>>,
    /// Decompressed CSV bytes for current day (decompressed once from ZIP)
    day_csv_bytes: Option<Vec<u8>>,
    /// Whether current day's CSV has headers
    day_has_headers: bool,
    /// CSV filename within the ZIP for current day
    day_csv_filename: String,
    /// Tokio runtime for async operations
    runtime: tokio::runtime::Runtime,
    /// Whether iteration has completed
    exhausted: bool,
    /// Whether to verify checksums (Issue #43)
    verify_checksum: bool,
}

impl IntraDayChunkIterator {
    /// Create a new intra-day chunk iterator.
    ///
    /// The `chunk_hours` parameter controls chunk granularity for backward
    /// compatibility. Internally, it maps to a max-trades-per-chunk value:
    /// - chunk_hours=1 → 100K trades/chunk
    /// - chunk_hours=6 → 500K trades/chunk (default, recommended)
    /// - chunk_hours=12 → 1M trades/chunk
    /// - chunk_hours=24 → 2M trades/chunk (full day, single parse)
    ///
    /// # Panics
    ///
    /// Panics if chunk_hours is 0 or greater than 24.
    pub fn new(
        loader: HistoricalDataLoader,
        start_date: NaiveDate,
        end_date: NaiveDate,
        chunk_hours: u32,
    ) -> Self {
        Self::with_checksum(loader, start_date, end_date, chunk_hours, true)
    }

    /// Create a new intra-day chunk iterator with configurable checksum verification.
    ///
    /// # Panics
    ///
    /// Panics if chunk_hours is 0 or greater than 24.
    pub fn with_checksum(
        loader: HistoricalDataLoader,
        start_date: NaiveDate,
        end_date: NaiveDate,
        chunk_hours: u32,
        verify_checksum: bool,
    ) -> Self {
        assert!(
            chunk_hours > 0 && chunk_hours <= 24,
            "chunk_hours must be 1-24"
        );

        // Map chunk_hours to approximate trades per chunk.
        // Average BTCUSDT day: ~2M trades; peak days: ~20M.
        // chunk_hours=6 → 4 chunks/day → 500K default.
        let max_trades = match chunk_hours {
            1 => 100_000,
            h if h <= 6 => DEFAULT_TRADES_PER_CHUNK,
            h if h <= 12 => 1_000_000,
            _ => 2_000_000,
        };

        let runtime = tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()
            .expect("Failed to create tokio runtime");

        Self {
            loader,
            current_date: start_date,
            end_date,
            max_trades_per_chunk: max_trades,
            records_consumed: 0,
            day_fully_consumed: false,
            day_zip_bytes: None,
            day_csv_bytes: None,
            day_has_headers: false,
            day_csv_filename: String::new(),
            runtime,
            exhausted: false,
            verify_checksum,
        }
    }

    /// Get symbol being processed
    pub fn symbol(&self) -> &str {
        &self.loader.symbol
    }

    /// Get current date being processed
    pub fn current_date(&self) -> NaiveDate {
        self.current_date
    }

    /// Get current chunk position (approximate hour for backward compat)
    pub fn current_hour(&self) -> u32 {
        // Approximate: assume 2M trades/day for hour mapping
        let approx_hour = (self.records_consumed as u64 * 24 / 2_000_000) as u32;
        approx_hour.min(23)
    }

    /// Parse next chunk from cached ZIP bytes.
    ///
    /// Decompresses CSV from ZIP once per day (cached in `day_csv_bytes`),
    /// then skips `records_consumed` records over in-memory bytes.
    /// Eliminates O(N^2) ZIP re-decompression per chunk.
    ///
    /// Returns (chunk, has_more). If has_more is false, the day is fully consumed.
    fn parse_chunk_from_zip(&mut self) -> Result<(Vec<AggTrade>, bool), HistoricalError> {
        // Decompress CSV from ZIP once, cache for subsequent chunks
        if self.day_csv_bytes.is_none() {
            let zip_bytes = self
                .day_zip_bytes
                .as_ref()
                .expect("ZIP bytes must be cached");
            let cursor = Cursor::new(zip_bytes.as_slice());
            let mut archive = ZipArchive::new(cursor)?;
            let mut csv_file = archive.by_name(&self.day_csv_filename)?;
            let mut csv_bytes = Vec::with_capacity(csv_file.size() as usize);
            std::io::Read::read_to_end(&mut csv_file, &mut csv_bytes)?;
            self.day_csv_bytes = Some(csv_bytes);
        }

        let csv_bytes = self.day_csv_bytes.as_ref().unwrap();
        let cursor = Cursor::new(csv_bytes.as_slice());
        let buf_reader = BufReader::with_capacity(64 * 1024, cursor);
        let mut reader = ReaderBuilder::new()
            .has_headers(self.day_has_headers)
            .from_reader(buf_reader);

        let skip = self.records_consumed;
        let take = self.max_trades_per_chunk;
        let mut chunk = Vec::with_capacity(take.min(100_000));
        let mut record_index = 0;
        let mut has_more = false;

        for result in reader.deserialize() {
            let csv_trade: CsvAggTrade = result?;
            record_index += 1;

            // Skip records we've already consumed in previous chunks
            if record_index <= skip {
                continue;
            }

            // Collect up to max_trades_per_chunk
            if chunk.len() < take {
                let trade: AggTrade = csv_trade.to_agg_trade(&self.loader.market_type);
                chunk.push(trade);
            } else {
                // We've filled our chunk and there's at least one more record
                has_more = true;
                break;
            }
        }

        // Defensive sort (no-op for already-sorted data, O(n) verification)
        chunk.sort_by_key(|t| t.timestamp);

        Ok((chunk, has_more))
    }
}

impl Iterator for IntraDayChunkIterator {
    type Item = Result<Vec<AggTrade>, HistoricalError>;

    fn next(&mut self) -> Option<Self::Item> {
        if self.exhausted {
            return None;
        }

        loop {
            // Download day's ZIP if not cached
            if self.day_zip_bytes.is_none() {
                if self.current_date > self.end_date {
                    self.exhausted = true;
                    return None;
                }

                match self.runtime.block_on(
                    self.loader
                        .download_day_zip(self.current_date, self.verify_checksum),
                ) {
                    Ok((zip_bytes, has_headers, _date_str, csv_filename)) => {
                        self.day_zip_bytes = Some(zip_bytes);
                        self.day_has_headers = has_headers;
                        self.day_csv_filename = csv_filename;
                        self.records_consumed = 0;
                        self.day_fully_consumed = false;
                    }
                    Err(e) => {
                        self.current_date += ChronoDuration::days(1);
                        return Some(Err(e));
                    }
                }
            }

            // Parse next chunk from ZIP bytes by record count
            let (chunk, has_more) = match self.parse_chunk_from_zip() {
                Ok(result) => result,
                Err(e) => {
                    self.day_zip_bytes = None;
                    self.day_csv_bytes = None;
                    self.current_date += ChronoDuration::days(1);
                    return Some(Err(e));
                }
            };

            self.records_consumed += chunk.len();

            if !has_more {
                // Day fully consumed — drop ZIP + CSV bytes and move to next day
                self.day_zip_bytes = None;
                self.day_csv_bytes = None;
                self.day_fully_consumed = true;
                self.current_date += ChronoDuration::days(1);
            }

            if !chunk.is_empty() {
                return Some(Ok(chunk));
            }

            // Empty chunk from an exhausted day — loop to try next day
            if self.day_fully_consumed {
                continue;
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_chunk_size_mapping() {
        // Verify chunk_hours → max_trades_per_chunk mapping
        let loader = HistoricalDataLoader::new("BTCUSDT");
        let start = NaiveDate::from_ymd_opt(2024, 1, 1).unwrap();
        let end = NaiveDate::from_ymd_opt(2024, 1, 1).unwrap();

        let iter1 = IntraDayChunkIterator::new(loader.clone(), start, end, 1);
        assert_eq!(iter1.max_trades_per_chunk, 100_000);

        let iter6 = IntraDayChunkIterator::new(loader.clone(), start, end, 6);
        assert_eq!(iter6.max_trades_per_chunk, DEFAULT_TRADES_PER_CHUNK);

        let iter12 = IntraDayChunkIterator::new(loader.clone(), start, end, 12);
        assert_eq!(iter12.max_trades_per_chunk, 1_000_000);

        let iter24 = IntraDayChunkIterator::new(loader.clone(), start, end, 24);
        assert_eq!(iter24.max_trades_per_chunk, 2_000_000);
    }

    #[test]
    fn test_chunk_hours_validation() {
        let loader = HistoricalDataLoader::new("BTCUSDT");
        let start = NaiveDate::from_ymd_opt(2024, 1, 1).unwrap();
        let end = NaiveDate::from_ymd_opt(2024, 1, 1).unwrap();

        // Valid chunk hours
        let _ = IntraDayChunkIterator::new(loader.clone(), start, end, 1);
        let _ = IntraDayChunkIterator::new(loader.clone(), start, end, 6);
        let _ = IntraDayChunkIterator::new(loader.clone(), start, end, 12);
        let _ = IntraDayChunkIterator::new(loader.clone(), start, end, 24);
    }

    #[test]
    #[should_panic(expected = "chunk_hours must be 1-24")]
    fn test_chunk_hours_zero_panics() {
        let loader = HistoricalDataLoader::new("BTCUSDT");
        let start = NaiveDate::from_ymd_opt(2024, 1, 1).unwrap();
        let end = NaiveDate::from_ymd_opt(2024, 1, 1).unwrap();
        let _ = IntraDayChunkIterator::new(loader, start, end, 0);
    }

    #[test]
    #[should_panic(expected = "chunk_hours must be 1-24")]
    fn test_chunk_hours_too_large_panics() {
        let loader = HistoricalDataLoader::new("BTCUSDT");
        let start = NaiveDate::from_ymd_opt(2024, 1, 1).unwrap();
        let end = NaiveDate::from_ymd_opt(2024, 1, 1).unwrap();
        let _ = IntraDayChunkIterator::new(loader, start, end, 25);
    }

    // === Issue #96: detect_csv_headers + CsvAggTrade conversion tests ===

    #[test]
    fn test_detect_csv_headers_with_headers() {
        assert!(detect_csv_headers(
            "agg_trade_id,price,quantity,first_trade_id\n1,50000,1.0,1"
        ));
        assert!(detect_csv_headers("timestamp,is_buyer_maker\n123,true"));
    }

    #[test]
    fn test_detect_csv_headers_without_headers() {
        assert!(!detect_csv_headers(
            "12345,50000.0,1.5,100,102,1640995200000,true,true"
        ));
        assert!(!detect_csv_headers(""));
    }

    #[test]
    fn test_csv_agg_trade_to_agg_trade() {
        let csv = CsvAggTrade(12345, 50000.12345678, 1.5, 100, 102, 1640995200000, true);
        let trade: AggTrade = csv.into();
        assert_eq!(trade.agg_trade_id, 12345);
        assert_eq!(trade.first_trade_id, 100);
        assert_eq!(trade.last_trade_id, 102);
        assert_eq!(trade.timestamp, 1640995200000);
        assert!(trade.is_buyer_maker);
        assert!(trade.price.to_f64() > 49999.0);
        assert!(trade.volume.to_f64() > 1.0);
    }

    #[test]
    fn test_rest_agg_trade_inline_conversion() {
        let rest = RestAggTrade {
            agg_trade_id: 999,
            price: FixedPoint::from_str("50000.0").unwrap(),
            quantity: FixedPoint::from_str("2.5").unwrap(),
            first_trade_id: 10,
            last_trade_id: 12,
            trade_time: 1700000000000,
            is_buyer_maker: false,
        };
        let trade: AggTrade = rest.into();
        assert_eq!(trade.agg_trade_id, 999);
        assert!(!trade.is_buyer_maker);
        assert_eq!(trade.timestamp, 1700000000000 * 1000);
        assert_eq!(trade.price.0, 5000000000000_i64);
        assert_eq!(trade.volume.0, 250000000_i64);
    }

    /// Issue #96: Oracle — JSON round-trip for RestAggTrade deserialization
    #[test]
    fn test_rest_agg_trade_json_roundtrip() {
        let json = r#"[
            {"a":42,"p":"50000.12345678","q":"1.5","f":100,"l":102,"T":1700000000000,"m":true}
        ]"#;
        let batch: Vec<RestAggTrade> = serde_json::from_str(json).unwrap();
        assert_eq!(batch.len(), 1);
        let rest = &batch[0];
        assert_eq!(rest.agg_trade_id, 42);
        assert_eq!(rest.price.0, 5000012345678_i64);
        assert_eq!(rest.quantity.0, 150000000_i64);
        assert!(rest.is_buyer_maker);
        let trade: AggTrade = batch.into_iter().next().unwrap().into();
        assert_eq!(trade.timestamp, 1700000000000 * 1000);
    }

    // === Issue #96: CsvAggTrade.to_agg_trade() + market path tests ===

    #[test]
    fn test_csv_to_agg_trade_normalizes_ms_to_us() {
        let csv = CsvAggTrade(1, 50000.0, 1.0, 1, 1, 1640995200000, false);
        let trade = csv.to_agg_trade("spot");
        // normalize_timestamp converts 13-digit ms → µs
        assert_eq!(trade.timestamp, 1640995200000 * 1000);
        assert_eq!(trade.agg_trade_id, 1);
    }

    #[test]
    fn test_csv_from_vs_to_agg_trade_timestamp_differs() {
        let csv1 = CsvAggTrade(1, 50000.0, 1.0, 1, 1, 1640995200000, false);
        let csv2 = CsvAggTrade(1, 50000.0, 1.0, 1, 1, 1640995200000, false);
        let from_trade: AggTrade = csv1.into();
        let to_trade = csv2.to_agg_trade("spot");
        // From uses raw timestamp; to_agg_trade normalizes ms → µs
        assert_eq!(from_trade.timestamp, 1640995200000);
        assert_eq!(to_trade.timestamp, 1640995200000 * 1000);
        // Other fields identical
        assert_eq!(from_trade.agg_trade_id, to_trade.agg_trade_id);
        assert_eq!(from_trade.price, to_trade.price);
    }

    #[test]
    fn test_get_market_path_always_spot() {
        // 2026-03-11 policy: all data sources use spot exclusively
        assert_eq!(
            HistoricalDataLoader::new("BTCUSDT").get_market_path(),
            "spot"
        );
        assert_eq!(
            HistoricalDataLoader::new_with_market("BTCUSDT", "um").get_market_path(),
            "spot"
        );
        assert_eq!(
            HistoricalDataLoader::new_with_market("BTCUSDT", "cm").get_market_path(),
            "spot"
        );
        assert_eq!(
            HistoricalDataLoader::new_with_market("BTCUSDT", "unknown").get_market_path(),
            "spot"
        );
    }

    // === fromId pagination + 429 backoff tests ===

    #[test]
    fn test_rest_agg_trade_from_impl() {
        let rest = RestAggTrade {
            agg_trade_id: 42,
            price: FixedPoint::from_str("50000.12345678").unwrap(),
            quantity: FixedPoint::from_str("1.5").unwrap(),
            first_trade_id: 100,
            last_trade_id: 102,
            trade_time: 1700000000000,
            is_buyer_maker: true,
        };
        let trade: AggTrade = rest.into();
        assert_eq!(trade.agg_trade_id, 42);
        assert_eq!(trade.first_trade_id, 100);
        assert_eq!(trade.last_trade_id, 102);
        assert!(trade.is_buyer_maker);
        assert_eq!(trade.timestamp, 1700000000000 * 1000);
        assert_eq!(trade.price.0, 5000012345678_i64);
        assert_eq!(trade.volume.0, 150000000_i64);
    }

    #[test]
    fn test_rest_agg_trade_from_consistency_with_csv() {
        // REST and CSV should produce the same normalized timestamp
        let rest = RestAggTrade {
            agg_trade_id: 1,
            price: FixedPoint::from_str("50000.0").unwrap(),
            quantity: FixedPoint::from_str("1.0").unwrap(),
            first_trade_id: 1,
            last_trade_id: 1,
            trade_time: 1640995200000,
            is_buyer_maker: false,
        };
        let csv = CsvAggTrade(1, 50000.0, 1.0, 1, 1, 1640995200000, false);

        let rest_trade: AggTrade = rest.into();
        let csv_trade = csv.to_agg_trade("spot");

        // Both should normalize ms → us identically
        assert_eq!(rest_trade.timestamp, csv_trade.timestamp);
        assert_eq!(rest_trade.agg_trade_id, csv_trade.agg_trade_id);
        assert_eq!(rest_trade.is_buyer_maker, csv_trade.is_buyer_maker);
    }

    #[test]
    fn test_transient_retry_constants() {
        assert_eq!(TRANSIENT_MAX_RETRIES, 5);
        assert_eq!(TRANSIENT_INITIAL_BACKOFF_MS, 1_000);
        // Verify backoff schedule: 1s → 2s → 4s → 8s → 16s (capped)
        let mut backoff = TRANSIENT_INITIAL_BACKOFF_MS;
        let expected = [1_000, 2_000, 4_000, 8_000, 16_000];
        for &exp in &expected {
            assert_eq!(backoff, exp);
            backoff = (backoff * 2).min(16_000);
        }
    }

    #[test]
    fn test_parse_symbol_markets() {
        let toml = r#"
[symbols.BTCUSDT]
enabled = true
market = "spot"

[symbols.BTCUSD_PERP]
enabled = true
market = "futures-cm"
"#;
        let map = parse_symbol_markets(toml);
        assert_eq!(map.get("BTCUSDT").map(|s| s.as_str()), Some("spot"));
        assert_eq!(
            map.get("BTCUSD_PERP").map(|s| s.as_str()),
            Some("futures-cm")
        );
        assert_eq!(map.get("UNKNOWN"), None);
    }

    #[test]
    fn test_get_symbol_market_btcusdt() {
        // BTCUSDT should be in the compiled-in symbols.toml as "spot"
        assert_eq!(get_symbol_market("BTCUSDT"), Some("spot"));
    }

    #[test]
    fn test_get_symbol_market_unknown() {
        assert_eq!(get_symbol_market("UNKNOWNSYMBOL_XYZ"), None);
    }

    #[test]
    fn test_normalize_market() {
        assert_eq!(normalize_market("futures-um"), "um");
        assert_eq!(normalize_market("futures-cm"), "cm");
        assert_eq!(normalize_market("spot"), "spot");
        assert_eq!(normalize_market("um"), "um");
    }
}
