//! Live bar engine for real-time open deviation bar construction (Issue #91)
//! # FILE-SIZE-OK
//!
//! Multiplexes Binance WebSocket streams across symbols and fans out trades
//! to canonical `OpenDeviationBarProcessor` instances (full 58-column microstructure).
//!
//! Architecture:
//! - One tokio task per symbol (independent reconnection via barter-rs pattern)
//! - One `OpenDeviationBarProcessor` per (symbol, threshold) pair
//! - Completed bars emitted to a bounded channel for Python consumption
//! - Graceful shutdown via `CancellationToken`

use opendeviationbar_core::IntraBarConfig;
use opendeviationbar_core::checkpoint::Checkpoint;
use opendeviationbar_core::interbar::InterBarConfig;
use opendeviationbar_core::processor::{OpenDeviationBarProcessor, ProcessingError};
use opendeviationbar_core::{AggTrade, OpenDeviationBar};
use opendeviationbar_providers::binance::{
    AdaptiveRateLimiter, CombinedWebSocketStream, ReconnectionPolicy,
};
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
// Issue #96: Arc<str> for CompletedBar.symbol — cheap clone instead of per-bar String allocation
use tokio::sync::{mpsc, watch};
use tokio::time::Duration;
use tokio_util::sync::CancellationToken;

// Issue #96 Task #9: Ring buffer replaces mpsc channel for memory efficiency
use crate::ring_buffer::ConcurrentRingBuffer;

// Issue #257: Gap detection infrastructure
use crate::gap::{GapEvent, GapFillResult, GapFillSender, TradeIdGapDetector};

/// Microseconds per UTC day — used for midnight boundary detection.
const DAY_US: i64 = 86_400_000_000;

/// A completed bar with metadata identifying its source.
/// Issue #96: `symbol` is `Arc<str>` — created once per symbol task, cheap clone per bar.
#[derive(Debug, Clone)]
pub struct CompletedBar {
    pub symbol: Arc<str>,
    pub threshold_decimal_bps: u32,
    pub bar: OpenDeviationBar,
}

/// Issue #214: A forming (incomplete) bar snapshot for SSE push to consumers.
///
/// Published at 1Hz via `tokio::sync::watch` channels. Each (symbol, threshold)
/// pair has its own watch channel. The daemon thread reads snapshots without
/// blocking the trade-processing hot path.
#[derive(Debug, Clone)]
pub struct FormingBar {
    pub symbol: Arc<str>,
    pub threshold_decimal_bps: u32,
    pub bar: OpenDeviationBar,
    /// Microsecond timestamp of the last trade that updated this forming bar.
    /// Used by the daemon thread to skip broadcasting stale forming bars
    /// (e.g., when markets are closed and no trades arrive).
    pub last_trade_timestamp_us: i64,
}

/// Key type for forming bar watch channels: (symbol, threshold).
type FormingBarKey = (Arc<str>, u32);

/// Map of watch receivers for forming bar snapshots (Issue #214).
/// Extracted from the engine before `start()` (like `take_checkpoint_receiver()`).
pub type FormingBarWatches = HashMap<FormingBarKey, watch::Receiver<Option<FormingBar>>>;

/// Metrics for the live bar engine.
/// Issue #96 Task #6: Added backpressure metrics for monitoring queue behavior
/// Issue #96 Task #12: Expose ring buffer metrics (dropped bars, queue depth)
#[derive(Debug)]
pub struct LiveEngineMetrics {
    pub trades_received: AtomicU64,
    pub bars_emitted: AtomicU64,
    pub reconnections: AtomicU64,
    pub backpressure_events: AtomicU64, // Times ring buffer was full and bar dropped
    pub dropped_bars: AtomicU64,        // Total bars dropped due to full ring buffer
    pub max_queue_depth: AtomicU64,     // Maximum observed queue depth
    pub total_block_time_ms: AtomicU64, // Accumulated time producer waited for queue space
    pub gap_fills: AtomicU64,           // Number of reconnection gap fills performed
    pub gap_trades_recovered: AtomicU64, // Total trades recovered via REST gap fill
}

impl Default for LiveEngineMetrics {
    fn default() -> Self {
        Self {
            trades_received: AtomicU64::new(0),
            bars_emitted: AtomicU64::new(0),
            reconnections: AtomicU64::new(0),
            backpressure_events: AtomicU64::new(0),
            dropped_bars: AtomicU64::new(0),
            max_queue_depth: AtomicU64::new(0),
            total_block_time_ms: AtomicU64::new(0),
            gap_fills: AtomicU64::new(0),
            gap_trades_recovered: AtomicU64::new(0),
        }
    }
}

/// Configuration for the live bar engine.
#[derive(Debug, Clone)]
pub struct LiveEngineConfig {
    /// Symbols to stream (e.g., ["BTCUSDT", "ETHUSDT"])
    pub symbols: Vec<String>,
    /// Thresholds in decimal basis points (e.g., [250, 500, 750, 1000])
    pub thresholds: Vec<u32>,
    /// Whether to compute inter-bar + intra-bar microstructure features
    pub include_microstructure: bool,
    /// Channel capacity for completed bars (backpressure)
    pub bar_channel_capacity: usize,
    /// Reconnection policy for WebSocket connections
    pub reconnection_policy: ReconnectionPolicy,
    /// Initial checkpoints keyed by (symbol, threshold) for resuming incomplete bars
    pub initial_checkpoints: HashMap<(String, u32), Checkpoint>,
    /// Optional rate limiter for REST API gap-fill pacing (Issue #162).
    /// When set, replaces fixed 100ms sleep with budget-aware acquire().
    pub rate_limiter: Option<Arc<AdaptiveRateLimiter>>,
    /// Issue #128: Per-feature computation toggles
    pub compute_tier2: bool,
    pub compute_tier3: bool,
    pub compute_hurst: Option<bool>,
    pub compute_permutation_entropy: Option<bool>,
    /// Per-symbol gap detector seeds from ClickHouse (max trade ID across thresholds).
    /// Propagated from StreamManager's `trade_id_state` before `start()`.
    /// Used to seed `TradeIdGapDetector` so the first WS trade triggers gap
    /// detection even when no checkpoint files exist.
    pub gap_detector_seeds: HashMap<String, i64>,
}

impl LiveEngineConfig {
    /// Get bar channel capacity from environment or use default (10K)
    /// Issue #96 Task #6: OPENDEVIATIONBAR_MAX_PENDING_BARS env var support
    fn get_bar_channel_capacity() -> usize {
        std::env::var("OPENDEVIATIONBAR_MAX_PENDING_BARS")
            .ok()
            .and_then(|v| v.parse::<usize>().ok())
            .unwrap_or(10_000)
    }

    /// Create config with sensible defaults.
    pub fn new(symbols: Vec<String>, thresholds: Vec<u32>) -> Self {
        Self {
            symbols,
            thresholds,
            include_microstructure: true,
            bar_channel_capacity: Self::get_bar_channel_capacity(),
            reconnection_policy: ReconnectionPolicy::default(),
            initial_checkpoints: HashMap::new(),
            compute_tier2: true,
            compute_tier3: false,
            compute_hurst: None,
            compute_permutation_entropy: None,
            rate_limiter: None,
            gap_detector_seeds: HashMap::new(),
        }
    }

    /// Set rate limiter for REST API gap-fill pacing (Issue #162).
    pub fn with_rate_limiter(mut self, limiter: Arc<AdaptiveRateLimiter>) -> Self {
        self.rate_limiter = Some(limiter);
        self
    }

    /// Inject a checkpoint for a specific (symbol, threshold) pair.
    /// Must be called before `LiveBarEngine::start()`.
    pub fn with_checkpoint(
        mut self,
        symbol: String,
        threshold: u32,
        checkpoint: Checkpoint,
    ) -> Self {
        self.initial_checkpoints
            .insert((symbol, threshold), checkpoint);
        self
    }

    /// Set bar channel capacity explicitly (overrides env var)
    pub fn with_bar_channel_capacity(mut self, capacity: usize) -> Self {
        self.bar_channel_capacity = capacity;
        self
    }
}

/// Live bar engine: multiplexes WebSocket streams → canonical processors → completed bars.
///
/// Issue #91: Uses `OpenDeviationBarProcessor` (NOT `ExportOpenDeviationBarProcessor`) for full
/// 3-step feature finalization producing all 58 columns.
pub struct LiveBarEngine {
    config: LiveEngineConfig,
    /// Issue #96 Task #9: Ring buffer replaces mpsc for 10-20% memory savings
    bar_buffer: ConcurrentRingBuffer<CompletedBar>,
    shutdown: CancellationToken,
    metrics: Arc<LiveEngineMetrics>,
    started: bool,
    /// Channel for receiving processor checkpoints on shutdown
    checkpoint_rx: Option<mpsc::Receiver<(String, u32, Checkpoint)>>,
    checkpoint_tx: mpsc::Sender<(String, u32, Checkpoint)>,
    /// Issue #214: Watch senders for forming bar snapshots (one per symbol×threshold).
    /// Passed into symbol tasks on start(). Senders are moved, receivers extracted via
    /// take_forming_bar_watches() before start().
    forming_bar_txs: HashMap<FormingBarKey, watch::Sender<Option<FormingBar>>>,
    /// Issue #214: Watch receivers for forming bar snapshots.
    /// Extracted before start() via take_forming_bar_watches().
    forming_bar_watches: Option<FormingBarWatches>,
    /// Issue #257: Channel for emitting gap events to Python consumers.
    /// Created in `new()`, passed to symbol tasks in `start()`.
    gap_event_tx: mpsc::Sender<GapEvent>,
    /// Issue #257: Receiver half, extracted via `take_gap_event_receiver()`.
    gap_event_rx: Option<mpsc::Receiver<GapEvent>>,
    /// Issue #257: Per-symbol gap-fill command senders for on-demand fill_gap().
    /// Cloneable — Python can call fill_gap(&self) without borrow conflicts.
    gap_fill_senders: HashMap<String, GapFillSender>,
    /// Issue #257: Per-symbol gap-fill receivers, passed to symbol_tasks on start().
    gap_fill_receivers:
        Option<HashMap<String, crate::gap::GapFillReceiver>>,
    /// Issue #286: Pre-spawned WS trade receivers for buffered start.
    /// When `start_ws()` is called, WS connections are spawned and trades
    /// buffer in these channels. `start()` then drains the buffer seamlessly.
    ws_trade_receivers: Option<HashMap<String, mpsc::Receiver<AggTrade>>>,
    /// Issue #286: Pre-created processors from fill_from_rest().
    /// When `fill_from_rest()` runs, processors are created, REST trades are
    /// fed through them, and they're stored here with accumulated state.
    /// `start()` reuses them instead of creating fresh processors.
    pre_created_processors:
        Option<HashMap<String, Vec<(u32, OpenDeviationBarProcessor)>>>,
}

impl LiveBarEngine {
    /// Create a new live bar engine.
    ///
    /// Does NOT start streaming — call `start()` after creation.
    pub fn new(config: LiveEngineConfig) -> Self {
        // Issue #96 Task #9: Initialize ring buffer instead of mpsc channel
        let bar_buffer = ConcurrentRingBuffer::new(config.bar_channel_capacity);
        // Channel for extracting checkpoints on shutdown (one per symbol×threshold)
        let max_checkpoints = config.symbols.len() * config.thresholds.len();
        let (checkpoint_tx, checkpoint_rx) = mpsc::channel(max_checkpoints.max(1));

        // Issue #214: Create watch channels for forming bar snapshots.
        // One (tx, rx) pair per (symbol, threshold). Created here so receivers
        // can be extracted before start() (avoiding borrow conflicts with next_bar).
        let num_pairs = config.symbols.len() * config.thresholds.len();
        let mut forming_bar_txs = HashMap::with_capacity(num_pairs);
        let mut forming_bar_rxs = HashMap::with_capacity(num_pairs);
        for symbol in &config.symbols {
            let sym_arc: Arc<str> = Arc::from(symbol.to_uppercase().as_str());
            for &threshold in &config.thresholds {
                let (tx, rx) = watch::channel(None);
                let key = (sym_arc.clone(), threshold);
                forming_bar_txs.insert(key.clone(), tx);
                forming_bar_rxs.insert(key, rx);
            }
        }

        // Issue #257: Gap event channel (bounded, non-blocking sends from hot path)
        let (gap_event_tx, gap_event_rx) = mpsc::channel(256);

        // Issue #257: Per-symbol gap-fill command channels
        let num_symbols = config.symbols.len();
        let mut gap_fill_senders = HashMap::with_capacity(num_symbols);
        let mut gap_fill_receivers = HashMap::with_capacity(num_symbols);
        for symbol in &config.symbols {
            let sym = symbol.to_uppercase();
            let (tx, rx) = crate::gap::gap_fill_channel(16);
            gap_fill_senders.insert(sym.clone(), tx);
            gap_fill_receivers.insert(sym, rx);
        }

        Self {
            config,
            bar_buffer,
            shutdown: CancellationToken::new(),
            metrics: Arc::new(LiveEngineMetrics::default()),
            started: false,
            checkpoint_rx: Some(checkpoint_rx),
            checkpoint_tx,
            forming_bar_txs,
            forming_bar_watches: Some(forming_bar_rxs),
            gap_event_tx,
            gap_event_rx: Some(gap_event_rx),
            gap_fill_senders,
            gap_fill_receivers: Some(gap_fill_receivers),
            ws_trade_receivers: None,
            pre_created_processors: None,
        }
    }

    /// Inject a checkpoint for a specific (symbol, threshold) pair.
    /// Must be called before `start()`.
    pub fn set_initial_checkpoint(&mut self, symbol: &str, threshold: u32, checkpoint: Checkpoint) {
        self.config
            .initial_checkpoints
            .insert((symbol.to_uppercase(), threshold), checkpoint);
    }

    /// Update gap detector seeds (used by fill_from_rest to determine REST start point).
    /// Called by StreamManager to propagate runtime trade_id_state before fill.
    pub fn set_gap_detector_seeds(&mut self, seeds: HashMap<String, i64>) {
        self.config.gap_detector_seeds = seeds;
    }

    /// Start WebSocket connections ONLY — buffer trades without processing (#286).
    ///
    /// Spawns WS connections immediately so trades start buffering in the
    /// mpsc channels (capacity 50,000 per symbol). Call this BEFORE loading
    /// checkpoints or seeding gap detectors. Then call `start()` to begin
    /// processing the buffered + live trades seamlessly.
    ///
    /// Safe to call multiple times (idempotent) — if WS is already started
    /// or `start()` was called, this is a no-op.
    pub fn start_ws(&mut self) {
        if self.ws_trade_receivers.is_some() || self.started {
            return; // Already started WS or fully started
        }

        let mut receivers = HashMap::new();
        let mut senders = HashMap::new();

        // Create per-symbol channels — same capacity, same Receiver type as before
        for symbol in &self.config.symbols {
            let symbol = symbol.to_uppercase();
            let (trade_tx, trade_rx) = mpsc::channel::<AggTrade>(50_000);
            senders.insert(symbol.clone(), trade_tx);
            receivers.insert(symbol, trade_rx);
        }

        // Single combined WS connection for all symbols (Issue #279)
        let symbols: Vec<String> = self.config.symbols.iter().map(|s| s.to_uppercase()).collect();
        let ws_shutdown = self.shutdown.clone();
        let policy = self.config.reconnection_policy.clone();
        tokio::spawn(async move {
            CombinedWebSocketStream::run_with_reconnect(
                symbols, senders, policy, ws_shutdown,
            )
            .await;
        });

        tracing::info!(
            symbols = self.config.symbols.len(),
            "started WS combined connection (capacity=50000) — trades buffering until start()"
        );
        self.ws_trade_receivers = Some(receivers);
    }

    /// Fill historical gap from REST before starting WS processing (#286).
    ///
    /// Creates processors (same as `start()`), loads checkpoints, feeds REST
    /// trades through them, and stores the processors with accumulated state.
    /// When `start()` is called next, it reuses these processors instead of
    /// creating fresh ones — zero forming-bar loss, zero transition gap.
    ///
    /// Call sequence: `start_ws()` → `fill_from_rest()` → `start()`
    ///
    /// Constraints respected (from 15 failed attempts in git history):
    /// - Runs BEFORE engine is live (not Asclepius — #284)
    /// - Same processors transition to WS (no boundary mismatch — #290)
    /// - No DELETE operations (no clean-slate — #286)
    /// - Per-symbol sequential (~200MB each, no OOM — #285)
    pub async fn fill_from_rest(&mut self) -> Result<u64, ProcessingError> {
        if self.started {
            return Ok(0); // Engine already running, skip
        }

        let fill_start = std::time::Instant::now();

        // Phase 4 (PARA-01): Raise rate limiter ceiling during parallel fill
        if let Some(ref limiter) = self.config.rate_limiter {
            limiter.set_ceiling(4800);
            tracing::info!("fill_from_rest: rate limiter ceiling raised to 4800 for parallel fill");
        }

        // Extract shared state for spawned tasks. initial_checkpoints uses remove()
        // so we take ownership of the whole map and partition per-symbol before spawning.
        let mut initial_checkpoints = std::mem::take(&mut self.config.initial_checkpoints);
        let gap_detector_seeds = self.config.gap_detector_seeds.clone();
        let thresholds = self.config.thresholds.clone();
        let include_microstructure = self.config.include_microstructure;
        let inter_bar_config = InterBarConfig {
            compute_tier2: self.config.compute_tier2,
            compute_tier3: self.config.compute_tier3,
            compute_hurst: self.config.compute_hurst,
            compute_permutation_entropy: self.config.compute_permutation_entropy,
            ..Default::default()
        };
        let intra_hurst = self.config.compute_hurst.unwrap_or(self.config.compute_tier3);
        let intra_pe = self.config.compute_permutation_entropy.unwrap_or(self.config.compute_tier3);
        let intra_bar_config = IntraBarConfig {
            compute_hurst: intra_hurst,
            compute_permutation_entropy: intra_pe,
        };
        let rate_limiter = self.config.rate_limiter.clone();
        let bar_buffer = self.bar_buffer.clone_ref();
        let metrics = Arc::clone(&self.metrics);

        // Phase 4 (PARA-01): Parallel fill with JoinSet + Semaphore(4)
        let semaphore = Arc::new(tokio::sync::Semaphore::new(4));
        let mut join_set = tokio::task::JoinSet::new();

        for symbol in &self.config.symbols {
            let symbol = symbol.to_uppercase();

            // Extract this symbol's checkpoints from the shared map
            let mut symbol_checkpoints: HashMap<(String, u32), opendeviationbar_core::checkpoint::Checkpoint> = HashMap::new();
            for &threshold in &thresholds {
                let key = (symbol.clone(), threshold);
                if let Some(cp) = initial_checkpoints.remove(&key) {
                    symbol_checkpoints.insert(key, cp);
                }
            }

            let ch_seed_tid = gap_detector_seeds.get(&symbol).copied();
            let thresholds = thresholds.clone();
            let inter_bar_config = inter_bar_config.clone();
            let intra_bar_config = intra_bar_config.clone();
            let rate_limiter = rate_limiter.clone();
            let bar_buffer = bar_buffer.clone_ref();
            let metrics = Arc::clone(&metrics);
            let sem = Arc::clone(&semaphore);

            join_set.spawn(async move {
                // Acquire semaphore permit — at most 4 symbols fill concurrently
                let _permit = sem.acquire().await.expect("semaphore closed unexpectedly");

                // Create processors per threshold (same as start())
                let mut processors: Vec<(u32, OpenDeviationBarProcessor)> =
                    Vec::with_capacity(thresholds.len());
                for &threshold in &thresholds {
                    let key = (symbol.clone(), threshold);
                    let p = if let Some(cp) = symbol_checkpoints.remove(&key) {
                        match OpenDeviationBarProcessor::from_checkpoint(cp) {
                            Ok(mut restored) => {
                                if include_microstructure {
                                    restored = restored
                                        .with_inter_bar_config(inter_bar_config.clone())
                                        .with_intra_bar_features()
                                        .with_intra_bar_config(intra_bar_config.clone());
                                }
                                tracing::info!(
                                    %symbol, threshold,
                                    has_incomplete_bar = restored.get_incomplete_bar().is_some(),
                                    "fill_from_rest: processor restored from checkpoint"
                                );
                                restored
                            }
                            Err(e) => {
                                tracing::warn!(
                                    %symbol, threshold, ?e,
                                    "fill_from_rest: checkpoint restore failed, starting fresh"
                                );
                                OpenDeviationBarProcessor::new(threshold)?
                            }
                        }
                    } else {
                        let mut fresh = OpenDeviationBarProcessor::new(threshold)?;
                        if include_microstructure {
                            fresh = fresh
                                .with_inter_bar_config(inter_bar_config.clone())
                                .with_intra_bar_features()
                                .with_intra_bar_config(intra_bar_config.clone());
                        }
                        fresh
                    };
                    processors.push((threshold, p));
                }

                // Determine REST start point: max(checkpoint_last_tid, ch_seed_tid)
                let checkpoint_tid = processors
                    .first()
                    .and_then(|(_, p)| p.last_agg_trade_id());
                let from_tid = match (checkpoint_tid, ch_seed_tid) {
                    (Some(cp), Some(ch)) => Some(cp.max(ch)),
                    (Some(cp), None) => Some(cp),
                    (None, Some(ch)) => Some(ch),
                    (None, None) => None,
                };

                let mut symbol_bars: u64 = 0;
                let mut symbol_trades_total: u64 = 0;

                if let Some(from_id) = from_tid {
                    let symbol_fill_start = std::time::Instant::now();

                    tracing::info!(
                        %symbol, from_id,
                        "fill_from_rest: fetching trades from REST (chunked)"
                    );

                    // Chunked REST pagination: fetch 50K trades at a time to avoid OOM.
                    const CHUNK_SIZE: i64 = 50_000;
                    const MAX_EMPTY_RETRIES: u32 = 3;
                    let mut current_from = from_id + 1;
                    let mut consecutive_errors: u32 = 0;
                    let mut consecutive_empty: u32 = 0;
                    let symbol_arc: Arc<str> = Arc::from(symbol.as_str());
                    let mut last_days: Vec<i64> =
                        processors.iter().map(|_| -1i64).collect();

                    tracing::info!(
                        %symbol, from_id, current_from,
                        "fill_from_rest: starting chunked REST pagination"
                    );

                    loop {
                        let chunk_to = current_from + CHUNK_SIZE;
                        let trades = match opendeviationbar_providers::binance::parallel_fetch::fetch_aggtrades_parallel(
                            &symbol,
                            current_from,
                            chunk_to,
                            rate_limiter.clone(),
                            None,
                        )
                        .await
                        {
                            Ok(t) => {
                                consecutive_errors = 0;
                                t
                            }
                            Err(e) => {
                                consecutive_errors += 1;
                                tracing::warn!(
                                    %symbol, ?e, chunk_from = current_from,
                                    chunk_to, consecutive_errors,
                                    symbol_trades_total,
                                    "fill_from_rest: REST chunk failed"
                                );
                                if consecutive_errors >= 3 {
                                    tracing::error!(
                                        %symbol, consecutive_errors, symbol_trades_total,
                                        chunk_from = current_from,
                                        "fill_from_rest: ABORTING after 3 consecutive errors — \
                                         gap from tid {} will remain until kintsugi repairs",
                                        current_from
                                    );
                                    break;
                                }
                                // Skip this chunk range, try next
                                current_from = chunk_to;
                                continue;
                            }
                        };

                        if trades.is_empty() {
                            consecutive_empty += 1;
                            if consecutive_empty >= MAX_EMPTY_RETRIES {
                                tracing::info!(
                                    %symbol, symbol_trades_total, current_from,
                                    "fill_from_rest: caught up (3 consecutive empty chunks)"
                                );
                                break;
                            }
                            tracing::debug!(
                                %symbol, current_from, chunk_to, consecutive_empty,
                                "fill_from_rest: empty chunk, skipping ahead"
                            );
                            current_from = chunk_to;
                            continue;
                        }
                        consecutive_empty = 0;

                        let chunk_len = trades.len();
                        let last_tid = trades[chunk_len - 1].agg_trade_id;

                        // Process this chunk through all threshold processors
                        for trade in &trades {
                            for (idx, (threshold, processor)) in
                                processors.iter_mut().enumerate()
                            {
                                if let Some(orphan) = maybe_reset_at_midnight(
                                    processor,
                                    trade.timestamp,
                                    &mut last_days[idx],
                                ) {
                                    metrics
                                        .bars_emitted
                                        .fetch_add(1, Ordering::Relaxed);
                                    let completed = CompletedBar {
                                        symbol: symbol_arc.clone(),
                                        threshold_decimal_bps: *threshold,
                                        bar: orphan,
                                    };
                                    let _ = bar_buffer.push(completed);
                                    symbol_bars += 1;
                                }

                                match processor.process_single_trade(trade) {
                                    Ok(Some(bar)) => {
                                        metrics
                                            .bars_emitted
                                            .fetch_add(1, Ordering::Relaxed);
                                        let completed = CompletedBar {
                                            symbol: symbol_arc.clone(),
                                            threshold_decimal_bps: *threshold,
                                            bar,
                                        };
                                        let _ = bar_buffer.push(completed);
                                        symbol_bars += 1;
                                    }
                                    Ok(None) => {}
                                    Err(e) => {
                                        tracing::warn!(
                                            %symbol, threshold = *threshold, ?e,
                                            "fill_from_rest: trade processing error"
                                        );
                                    }
                                }
                            }
                        }

                        symbol_trades_total += chunk_len as u64;
                        current_from = last_tid + 1;

                        tracing::debug!(
                            %symbol, chunk_len, last_tid, symbol_trades_total,
                            "fill_from_rest: chunk processed"
                        );
                    }

                    if symbol_trades_total > 0 {
                        tracing::info!(
                            %symbol,
                            duration_ms = symbol_fill_start.elapsed().as_millis() as u64,
                            trade_count = symbol_trades_total,
                            bars_produced = symbol_bars,
                            last_tid = current_from - 1,
                            "fill_from_rest.symbol_summary"
                        );
                        metrics
                            .gap_trades_recovered
                            .fetch_add(symbol_trades_total, Ordering::Relaxed);
                    } else {
                        tracing::info!(
                            %symbol,
                            duration_ms = symbol_fill_start.elapsed().as_millis() as u64,
                            trade_count = 0u64,
                            bars_produced = 0u64,
                            "fill_from_rest.symbol_summary"
                        );
                    }
                } else {
                    tracing::info!(%symbol, "fill_from_rest: no CH seed — skipping (first-ever start)");
                }

                Ok::<_, ProcessingError>((symbol, processors, symbol_bars, symbol_trades_total))
            });
        }

        // Collect results from all parallel tasks
        let mut total_bars = 0u64;
        let mut all_processors: HashMap<String, Vec<(u32, OpenDeviationBarProcessor)>> =
            HashMap::new();

        while let Some(result) = join_set.join_next().await {
            match result {
                Ok(Ok((symbol, processors, bars, _trades))) => {
                    total_bars += bars;
                    all_processors.insert(symbol, processors);
                }
                Ok(Err(e)) => {
                    tracing::error!(?e, "fill_from_rest: symbol task failed with processing error");
                }
                Err(e) => {
                    tracing::error!(?e, "fill_from_rest: symbol task panicked");
                }
            }
        }

        // Phase 4 (PARA-03): Lower rate limiter ceiling after fill completes
        if let Some(ref limiter) = self.config.rate_limiter {
            limiter.set_ceiling(3000);
            tracing::info!("fill_from_rest: rate limiter ceiling lowered to 3000 after fill");
        }

        let total_duration_ms = fill_start.elapsed().as_millis() as u64;
        tracing::info!(
            total_bars,
            total_duration_ms,
            symbols = all_processors.len(),
            "fill_from_rest: complete — processors preserved for start()"
        );
        self.pre_created_processors = Some(all_processors);
        Ok(total_bars)
    }

    /// Start all WebSocket connections and processing loops.
    ///
    /// If `start_ws()` was called first, reuses the already-buffered WS
    /// channels (zero-gap transition). Otherwise, creates WS connections
    /// inline (legacy behavior).
    ///
    /// Spawns one tokio task per symbol. Each task:
    /// 1. Connects via `run_with_reconnect()` (independent backoff per symbol)
    /// 2. Fans trades out to N `OpenDeviationBarProcessor` instances (one per threshold)
    /// 3. Sends completed bars to the shared output channel
    pub fn start(&mut self) -> Result<(), ProcessingError> {
        if self.started {
            return Ok(());
        }

        for symbol in &self.config.symbols {
            let symbol = symbol.to_uppercase();
            let thresholds = self.config.thresholds.clone();
            let include_microstructure = self.config.include_microstructure;
            // Issue #128: Build tier-aware InterBarConfig + IntraBarConfig
            let inter_bar_config = InterBarConfig {
                compute_tier2: self.config.compute_tier2,
                compute_tier3: self.config.compute_tier3,
                compute_hurst: self.config.compute_hurst,
                compute_permutation_entropy: self.config.compute_permutation_entropy,
                ..Default::default()
            };
            let intra_hurst = self
                .config
                .compute_hurst
                .unwrap_or(self.config.compute_tier3);
            let intra_pe = self
                .config
                .compute_permutation_entropy
                .unwrap_or(self.config.compute_tier3);
            let intra_bar_config = IntraBarConfig {
                compute_hurst: intra_hurst,
                compute_permutation_entropy: intra_pe,
            };
            // Issue #96 Task #9: Clone ring buffer reference for symbol task
            let bar_buffer = self.bar_buffer.clone_ref();
            let shutdown = self.shutdown.clone();
            let policy = self.config.reconnection_policy.clone();
            let metrics = Arc::clone(&self.metrics);
            let checkpoint_tx = self.checkpoint_tx.clone();

            // Issue #286: Reuse pre-created processors from fill_from_rest() if available.
            // These processors already have accumulated state from REST trades —
            // the SAME processors transition seamlessly to WS mode.
            let processors: Vec<(u32, OpenDeviationBarProcessor)> = if let Some(pre) =
                self.pre_created_processors.as_mut().and_then(|m| m.remove(&symbol))
            {
                tracing::info!(
                    %symbol,
                    thresholds = pre.len(),
                    "reusing pre-created processors from fill_from_rest()"
                );
                pre
            } else {
                // Legacy path: create fresh processors (no fill_from_rest was called)
                let mut procs: Vec<(u32, OpenDeviationBarProcessor)> =
                    Vec::with_capacity(thresholds.len());
                for &threshold in &thresholds {
                    // Try to restore from checkpoint, fall back to fresh processor
                    let key = (symbol.clone(), threshold);
                    let p = if let Some(cp) = self.config.initial_checkpoints.remove(&key) {
                        match OpenDeviationBarProcessor::from_checkpoint(cp) {
                            Ok(mut restored) => {
                                // Re-enable microstructure features after checkpoint restore
                                if include_microstructure {
                                    restored = restored
                                        .with_inter_bar_config(inter_bar_config.clone())
                                        .with_intra_bar_features()
                                        .with_intra_bar_config(intra_bar_config.clone());
                                }
                                tracing::info!(
                                    %symbol, threshold,
                                    has_incomplete_bar = restored.get_incomplete_bar().is_some(),
                                    "processor restored from checkpoint"
                                );
                                restored
                            }
                            Err(e) => {
                                tracing::warn!(
                                    %symbol, threshold, ?e,
                                    "checkpoint restore failed, starting fresh"
                                );
                                let mut fresh = OpenDeviationBarProcessor::new(threshold)?;
                                if include_microstructure {
                                    fresh = fresh
                                        .with_inter_bar_config(inter_bar_config.clone())
                                        .with_intra_bar_features()
                                        .with_intra_bar_config(intra_bar_config.clone());
                                }
                                fresh
                            }
                        }
                    } else {
                        let mut fresh = OpenDeviationBarProcessor::new(threshold)?;
                        if include_microstructure {
                            fresh = fresh
                                .with_inter_bar_config(inter_bar_config.clone())
                                .with_intra_bar_features()
                                .with_intra_bar_config(intra_bar_config.clone());
                        }
                        fresh
                    };
                    procs.push((threshold, p));
                }
                procs
            };

            // Issue #214: Extract forming bar senders for this symbol's thresholds
            let symbol_arc: Arc<str> = Arc::from(symbol.as_str());
            let mut forming_txs: Vec<(u32, watch::Sender<Option<FormingBar>>)> = Vec::new();
            for &threshold in &self.config.thresholds {
                let key = (symbol_arc.clone(), threshold);
                if let Some(tx) = self.forming_bar_txs.remove(&key) {
                    forming_txs.push((threshold, tx));
                }
            }

            // Spawn per-symbol task
            let rate_limiter = self.config.rate_limiter.clone();
            let gap_event_tx = self.gap_event_tx.clone();
            // Issue #257: Extract per-symbol gap-fill receiver for command channel
            let gap_fill_rx = self
                .gap_fill_receivers
                .as_mut()
                .and_then(|m| m.remove(&symbol));
            // ClickHouse seed for gap detector (max trade ID across thresholds)
            let ch_seed_tid = self.config.gap_detector_seeds.get(&symbol).copied();
            // Issue #286: Reuse pre-buffered WS receiver if start_ws() was called
            let pre_buffered_rx = self
                .ws_trade_receivers
                .as_mut()
                .and_then(|m| m.remove(&symbol));
            tokio::spawn(symbol_task(
                symbol,
                processors,
                bar_buffer,
                checkpoint_tx,
                policy,
                shutdown,
                metrics,
                rate_limiter,
                forming_txs,
                gap_event_tx,
                gap_fill_rx,
                ch_seed_tid,
                pre_buffered_rx,
            ));
        }

        self.started = true;
        tracing::info!(
            symbols = ?self.config.symbols,
            thresholds = ?self.config.thresholds,
            microstructure = self.config.include_microstructure,
            "live bar engine started"
        );
        Ok(())
    }

    /// Receive next completed bar. Returns `None` on timeout or shutdown.
    /// Issue #96 Task #9: Poll ring buffer with timeout, replacing async channel
    /// Memory v2: Exponential backoff 100μs→10ms reduces idle CPU ~100x
    pub async fn next_bar(&mut self, timeout: Duration) -> Option<CompletedBar> {
        let start = tokio::time::Instant::now();
        let mut poll_interval = Duration::from_micros(100);
        let max_interval = Duration::from_millis(10);

        loop {
            // Try to pop from ring buffer (non-blocking)
            if let Some(bar) = self.bar_buffer.pop() {
                return Some(bar);
            }

            // Check timeout
            if start.elapsed() >= timeout {
                return None;
            }

            // Check shutdown
            if self.shutdown.is_cancelled() {
                return None;
            }

            // Sleep with exponential backoff (resets on successful pop above)
            tokio::time::sleep(poll_interval).await;
            poll_interval = (poll_interval * 2).min(max_interval);
        }
    }

    /// Drain all completed bars from the ring buffer synchronously.
    ///
    /// Returns bars in FIFO order. Purely synchronous — no async, no polling.
    /// Used by sync_flush to bypass the async next_bar() polling loop which
    /// can fail when called via block_on() before engine.start().
    pub fn drain_bars(&self) -> Vec<CompletedBar> {
        self.bar_buffer.drain_all()
    }

    /// Graceful shutdown — cancels all WebSocket tasks.
    pub fn stop(&self) {
        tracing::info!("live bar engine stopping");
        self.shutdown.cancel();
    }

    /// Get engine metrics snapshot.
    pub fn metrics(&self) -> &LiveEngineMetrics {
        &self.metrics
    }

    /// Whether the engine has been started.
    pub fn is_started(&self) -> bool {
        self.started
    }

    /// Collect checkpoints from all processors after shutdown.
    ///
    /// Call after `stop()`. Each symbol task sends its processor checkpoints
    /// through the channel before exiting. Returns a map of `"SYMBOL:THRESHOLD"` → `Checkpoint`.
    pub async fn collect_checkpoints(&mut self, timeout: Duration) -> HashMap<String, Checkpoint> {
        let mut result = HashMap::new();
        let rx = match self.checkpoint_rx.take() {
            Some(rx) => rx,
            None => return result,
        };

        // Drain all available checkpoints within timeout
        let deadline = tokio::time::Instant::now() + timeout;
        let mut rx = rx;
        loop {
            tokio::select! {
                item = rx.recv() => {
                    match item {
                        Some((symbol, threshold, cp)) => {
                            let key = format!("{symbol}:{threshold}");
                            result.insert(key, cp);
                        }
                        None => break, // Channel closed, all senders dropped
                    }
                }
                () = tokio::time::sleep_until(deadline) => {
                    tracing::warn!(
                        collected = result.len(),
                        "checkpoint collection timed out"
                    );
                    break;
                }
            }
        }

        tracing::info!(count = result.len(), "checkpoints collected");
        result
    }

    /// Get the shutdown token (for external coordination).
    pub fn shutdown_token(&self) -> CancellationToken {
        self.shutdown.clone()
    }

    /// Take the checkpoint receiver for external collection (Issue #178).
    ///
    /// Used by OdbEngine to extract the receiver before the engine is moved
    /// into a tokio task. The caller is responsible for draining the receiver
    /// after shutdown.
    pub fn take_checkpoint_receiver(
        &mut self,
    ) -> Option<mpsc::Receiver<(String, u32, Checkpoint)>> {
        self.checkpoint_rx.take()
    }

    /// Take the forming bar watch receivers (Issue #214).
    ///
    /// Must be called BEFORE `start()`. Returns watch receivers keyed by
    /// (symbol, threshold). The caller reads these at 1Hz to get the latest
    /// forming bar snapshot for each pair — zero contention with the trade
    /// processing hot path.
    ///
    /// Follows the same pattern as `take_checkpoint_receiver()` (Issue #178).
    pub fn take_forming_bar_watches(&mut self) -> Option<FormingBarWatches> {
        self.forming_bar_watches.take()
    }

    /// Get a shared reference to the metrics (Issue #178).
    pub fn shared_metrics(&self) -> Arc<LiveEngineMetrics> {
        Arc::clone(&self.metrics)
    }

    /// Take the gap event receiver (Issue #257).
    ///
    /// Returns the receiver half of the gap event channel. Gap events are
    /// emitted by `TradeIdGapDetector` in each symbol task when trade-ID
    /// discontinuities are detected. The caller (StreamManager/Python)
    /// polls this to react to gaps.
    ///
    /// Can only be called once — subsequent calls return None.
    pub fn take_gap_event_receiver(&mut self) -> Option<mpsc::Receiver<GapEvent>> {
        self.gap_event_rx.take()
    }

    /// Get a clone of the per-symbol gap-fill senders (Issue #257).
    ///
    /// Returns a map of symbol → `GapFillSender`. Each sender can be used
    /// with `&self` (no borrow conflict) to submit on-demand gap-fill
    /// commands that are processed inline by the symbol's `tokio::select!` loop.
    pub fn gap_fill_senders(&self) -> HashMap<String, GapFillSender> {
        self.gap_fill_senders.clone()
    }
}

impl LiveEngineMetrics {
    /// Snapshot of current metrics.
    pub fn snapshot(&self) -> LiveEngineMetricsSnapshot {
        LiveEngineMetricsSnapshot {
            trades_received: self.trades_received.load(Ordering::Relaxed),
            bars_emitted: self.bars_emitted.load(Ordering::Relaxed),
            reconnections: self.reconnections.load(Ordering::Relaxed),
            dropped_bars: self.dropped_bars.load(Ordering::Relaxed),
            max_queue_depth: self.max_queue_depth.load(Ordering::Relaxed),
            backpressure_events: self.backpressure_events.load(Ordering::Relaxed),
            gap_fills: self.gap_fills.load(Ordering::Relaxed),
            gap_trades_recovered: self.gap_trades_recovered.load(Ordering::Relaxed),
        }
    }

    /// Get ring buffer queue depth estimate (from snapshot timing).
    /// Note: This is approximate due to concurrent updates.
    pub fn estimate_queue_depth(&self) -> u64 {
        let emitted = self.bars_emitted.load(Ordering::Relaxed);
        let dropped = self.dropped_bars.load(Ordering::Relaxed);
        let received = self.trades_received.load(Ordering::Relaxed);
        // Rough estimate: bars emitted + dropped vs trades received
        // This is not exact but gives a sense of queue pressure
        if emitted + dropped > received {
            0
        } else {
            received - emitted - dropped
        }
    }
}

/// Immutable metrics snapshot for reporting.
#[derive(Debug, Clone)]
pub struct LiveEngineMetricsSnapshot {
    pub trades_received: u64,
    pub bars_emitted: u64,
    pub reconnections: u64,
    pub dropped_bars: u64,
    pub max_queue_depth: u64,
    pub backpressure_events: u64,
    pub gap_fills: u64,
    pub gap_trades_recovered: u64,
}

// (#286) No MAX_GAP_FILL_TRADES limit — Puck fills any gap regardless of size.
// The rate limiter paces REST calls. Clean-slate startup may create gaps
// spanning hours (midnight to now) that must be filled unconditionally.

/// Maximum number of trades to drain from the channel in a single burst-atomic frame.
/// Prevents unbounded draining if the channel is continuously fed.
/// 1024 is generous — typical same-ms bursts are 5-50 trades.
const MAX_BURST_DRAIN: usize = 1024;

/// Process a single trade through all threshold processors, handling midnight resets,
/// bar emission to ring buffer, forming bar updates, and metrics.
///
/// Extracted from the symbol_task loop to enable burst-atomic frame processing (Issue #273).
#[allow(clippy::too_many_arguments)]
fn process_trade_through_processors(
    trade: &AggTrade,
    processors: &mut [(u32, OpenDeviationBarProcessor)],
    last_days: &mut [i64],
    bar_buffer: &ConcurrentRingBuffer<CompletedBar>,
    metrics: &LiveEngineMetrics,
    symbol_arc: &Arc<str>,
    symbol: &str,
    forming_tx_map: &HashMap<u32, watch::Sender<Option<FormingBar>>>,
    committed_floors: &mut HashMap<u32, i64>,
) {
    for (idx, (threshold, processor)) in processors.iter_mut().enumerate() {
        // Midnight reset: if trade crosses day boundary, emit orphan and reset
        if let Some(orphan) = maybe_reset_at_midnight(processor, trade.timestamp, &mut last_days[idx]) {
            // Rust-side dedup: skip orphan bars within already-committed range
            if let Some(&floor) = committed_floors.get(threshold)
                && orphan.last_agg_trade_id > 0
                && orphan.last_agg_trade_id <= floor
            {
                tracing::debug!(%symbol, threshold, last_tid = orphan.last_agg_trade_id, floor, "rust-dedup: skipping orphan (within committed range)");
                continue;
            }
            metrics.bars_emitted.fetch_add(1, Ordering::Relaxed);
            let completed = CompletedBar {
                symbol: symbol_arc.clone(),
                threshold_decimal_bps: *threshold,
                bar: orphan,
            };
            // Update floor after emission
            committed_floors.insert(*threshold, completed.bar.last_agg_trade_id);
            let was_added = bar_buffer.push(completed);
            if !was_added {
                metrics.dropped_bars.fetch_add(1, Ordering::Relaxed);
                metrics.backpressure_events.fetch_add(1, Ordering::Relaxed);
                tracing::warn!(%symbol, threshold, "ring buffer full, old bar dropped (midnight orphan)");
            }
        }

        match processor.process_single_trade(trade) {
            Ok(Some(bar)) => {
                // Rust-side dedup: skip bars within already-committed range
                if let Some(&floor) = committed_floors.get(threshold)
                    && bar.last_agg_trade_id > 0
                    && bar.last_agg_trade_id <= floor
                {
                    tracing::debug!(%symbol, threshold, last_tid = bar.last_agg_trade_id, floor, "rust-dedup: skipping bar (within committed range)");
                    // Still clear forming bar state
                    if let Some(tx) = forming_tx_map.get(threshold) {
                        let _ = tx.send(None);
                    }
                    continue;
                }

                // Issue #214: Clear forming bar BEFORE emitting completed bar
                if let Some(tx) = forming_tx_map.get(threshold) {
                    let _ = tx.send(None);
                }

                metrics.bars_emitted.fetch_add(1, Ordering::Relaxed);
                let completed = CompletedBar {
                    symbol: symbol_arc.clone(),
                    threshold_decimal_bps: *threshold,
                    bar,
                };
                // Update floor after emission
                committed_floors.insert(*threshold, completed.bar.last_agg_trade_id);
                let was_added = bar_buffer.push(completed);
                if !was_added {
                    metrics.dropped_bars.fetch_add(1, Ordering::Relaxed);
                    metrics.backpressure_events.fetch_add(1, Ordering::Relaxed);
                    tracing::warn!(%symbol, threshold, "ring buffer full, old bar dropped");
                }
                let current_depth = bar_buffer.len() as u64;
                let _ = metrics.max_queue_depth.fetch_max(current_depth, Ordering::Relaxed);
            }
            Ok(None) => {
                // Trade absorbed, no bar completed yet.
                // Issue #214: Update forming bar snapshot for consumers.
                if let Some(tx) = forming_tx_map.get(threshold)
                    && let Some(incomplete) = processor.get_incomplete_bar()
                {
                    let _ = tx.send(Some(FormingBar {
                        symbol: symbol_arc.clone(),
                        threshold_decimal_bps: *threshold,
                        bar: incomplete,
                        last_trade_timestamp_us: trade.timestamp,
                    }));
                }
            }
            Err(e) => {
                tracing::warn!(%symbol, threshold = *threshold, ?e, "trade processing error");
            }
        }
    }
}


/// Check if a trade crosses a UTC midnight boundary relative to the processor's last trade,
/// and if so, reset the processor at ouroboros. Returns the orphan bar if one was in progress.
///
/// This prevents cross-midnight bars from being produced. The orphan bar (if any) is emitted
/// as a completed bar with `is_orphan=true`. The trade that triggered the reset is NOT consumed —
/// it will be processed normally after this function returns.
fn maybe_reset_at_midnight(
    processor: &mut OpenDeviationBarProcessor,
    trade_timestamp_us: i64,
    last_day: &mut i64,
) -> Option<OpenDeviationBar> {
    let trade_day = trade_timestamp_us / DAY_US;
    if *last_day >= 0 && trade_day != *last_day {
        *last_day = trade_day;
        // Reset processor — any in-progress bar becomes an orphan
        processor.reset_at_ouroboros()
    } else {
        *last_day = trade_day;
        None
    }
}

/// Puck: inline gap fill on trade-ID discontinuity.
///
/// When a gap is detected between the last processed trade and an incoming
/// WebSocket trade, fetches the missing trades using Binance REST API's
/// parallel `fromId` fetch and processes them through all threshold processors.
/// Sub-3s recovery for typical reconnection gaps.
///
/// Returns the number of trades recovered (0 on error or skip — non-fatal).
#[allow(clippy::too_many_arguments)]
async fn puck_fill(
    symbol: &str,
    last_known_id: i64,
    first_ws_id: i64,
    processors: &mut [(u32, OpenDeviationBarProcessor)],
    bar_buffer: &ConcurrentRingBuffer<CompletedBar>,
    metrics: &LiveEngineMetrics,
    shutdown: &CancellationToken,
    rate_limiter: Option<Arc<AdaptiveRateLimiter>>,
    forming_tx_map: &HashMap<u32, watch::Sender<Option<FormingBar>>>, // Issue #214
    event_type: &str, // "junction_fill", "normal_gap_fill", or "on_demand_gap_fill"
) -> u64 {
    let gap_size = first_ws_id - last_known_id - 1;

    tracing::info!(
        %symbol, gap_size, last_known_id, first_ws_id, event_type,
        "puck: filling gap via parallel REST API"
    );

    // Issue #257: Use parallel fetch for sub-3s gap recovery
    let from_id = last_known_id + 1;
    let trades = match opendeviationbar_providers::binance::parallel_fetch::fetch_aggtrades_parallel(
        symbol,
        from_id,
        first_ws_id,
        rate_limiter,
        None, // Use default concurrency (5)
    )
    .await
    {
        Ok(t) => t,
        Err(e) => {
            tracing::warn!(
                %symbol, ?e,
                "parallel REST gap-fill failed, aborting (backfill service will rectify)"
            );
            return 0;
        }
    };

    // Process fetched trades sequentially through all threshold processors
    // (CRITICAL: processor is order-sensitive, trades must be in agg_trade_id order)
    let mut trades_recovered = 0u64;
    let symbol_arc: Arc<str> = Arc::from(symbol);

    // Track current UTC day per processor for midnight reset
    let mut last_days: Vec<i64> = processors.iter().map(|_| -1i64).collect();

    for trade in &trades {
        // Check shutdown periodically
        if shutdown.is_cancelled() {
            tracing::info!(%symbol, trades_recovered, "gap fill interrupted by shutdown");
            break;
        }

        // Fan out to all threshold processors
        for (idx, (threshold, processor)) in processors.iter_mut().enumerate() {
            // Midnight reset: if trade crosses day boundary, emit orphan and reset
            if let Some(orphan) = maybe_reset_at_midnight(processor, trade.timestamp, &mut last_days[idx]) {
                metrics.bars_emitted.fetch_add(1, Ordering::Relaxed);
                let completed = CompletedBar {
                    symbol: symbol_arc.clone(),
                    threshold_decimal_bps: *threshold,
                    bar: orphan,
                };
                if !bar_buffer.push(completed) {
                    metrics.dropped_bars.fetch_add(1, Ordering::Relaxed);
                    metrics.backpressure_events.fetch_add(1, Ordering::Relaxed);
                }
            }

            match processor.process_single_trade(trade) {
                Ok(Some(bar)) => {
                    // Issue #214: Clear forming bar BEFORE emitting completed bar
                    if let Some(tx) = forming_tx_map.get(threshold) {
                        let _ = tx.send(None);
                    }
                    metrics.bars_emitted.fetch_add(1, Ordering::Relaxed);
                    let completed = CompletedBar {
                        symbol: symbol_arc.clone(),
                        threshold_decimal_bps: *threshold,
                        bar,
                    };
                    if !bar_buffer.push(completed) {
                        metrics.dropped_bars.fetch_add(1, Ordering::Relaxed);
                        metrics.backpressure_events.fetch_add(1, Ordering::Relaxed);
                    }
                }
                Ok(None) => {
                    // Issue #214: Update forming bar snapshot during gap fill
                    if let Some(tx) = forming_tx_map.get(threshold)
                        && let Some(incomplete) = processor.get_incomplete_bar()
                    {
                        let _ = tx.send(Some(FormingBar {
                            symbol: symbol_arc.clone(),
                            threshold_decimal_bps: *threshold,
                            bar: incomplete,
                            last_trade_timestamp_us: trade.timestamp,
                        }));
                    }
                }
                Err(e) => {
                    tracing::warn!(
                        %symbol, threshold = *threshold, ?e,
                        "gap-fill trade processing error"
                    );
                }
            }
        }
        trades_recovered += 1;
    }

    tracing::info!(
        %symbol, trades_recovered, gap_size,
        "reconnection gap fill complete"
    );
    trades_recovered
}

/// Per-symbol WebSocket task with independent reconnection.
/// Issue #91: Each symbol gets its own backoff state (barter-rs pattern).
///
/// TODO(#279): Migrate from per-symbol WebSocket connections to combined streams.
/// Empirical testing (2026-03-16) showed 2x9 hybrid (2 combined connections,
/// 9 symbols each) delivers 4x throughput with zero errors vs 18 individual
/// connections which suffer recv timeouts on low-volume symbols.
#[allow(clippy::too_many_arguments)]
async fn symbol_task(
    symbol: String,
    mut processors: Vec<(u32, OpenDeviationBarProcessor)>,
    bar_buffer: ConcurrentRingBuffer<CompletedBar>, // Issue #96 Task #9: Ring buffer replaces mpsc
    checkpoint_tx: mpsc::Sender<(String, u32, Checkpoint)>,
    _policy: ReconnectionPolicy,
    shutdown: CancellationToken,
    metrics: Arc<LiveEngineMetrics>,
    rate_limiter: Option<Arc<AdaptiveRateLimiter>>,
    forming_txs: Vec<(u32, watch::Sender<Option<FormingBar>>)>, // Issue #214
    gap_event_tx: mpsc::Sender<GapEvent>, // Issue #257
    mut gap_fill_rx: Option<crate::gap::GapFillReceiver>, // Issue #257: on-demand fill commands
    ch_seed_tid: Option<i64>, // ClickHouse seed for gap detector (from StreamManager)
    pre_buffered_rx: Option<mpsc::Receiver<AggTrade>>, // Issue #286: pre-spawned WS receiver
) {
    // Issue #96: Create Arc<str> once per symbol task — cheap clone for each CompletedBar
    let symbol_arc: Arc<str> = Arc::from(symbol.as_str());

    // Issue #214: Index forming bar senders by threshold for O(1) lookup
    let forming_tx_map: HashMap<u32, watch::Sender<Option<FormingBar>>> =
        forming_txs.into_iter().collect();

    // Issue #286: Reuse pre-buffered WS receiver if available (from start_ws()),
    // otherwise create a new WS connection inline (legacy path).
    let mut trade_rx = if let Some(rx) = pre_buffered_rx {
        tracing::info!(%symbol, "using pre-buffered WS receiver (zero-gap start)");
        rx
    } else {
        // Combined stream requires start_ws() to be called first.
        // Per-symbol fallback removed — all symbols use combined WS (COMB-04).
        tracing::error!(%symbol, "combined stream requires pre-buffered receiver — call start_ws() before start()");
        return;
    };

    // Issue #257: Create gap detector for this symbol
    let mut gap_detector = TradeIdGapDetector::new(
        symbol_arc.clone(),
        Some(gap_event_tx),
    );
    // Seed gap detector: max(checkpoint_tid, ch_seed_tid).
    // Checkpoint seed comes from processor state (if restored from checkpoint file).
    // CH seed comes from StreamManager's trade_id_state (seeded from ClickHouse).
    // Using max ensures the detector knows the highest trade ID from ANY source.
    let checkpoint_tid = processors.first().and_then(|(_, p)| p.last_agg_trade_id());
    let effective_seed = match (checkpoint_tid, ch_seed_tid) {
        (Some(cp), Some(ch)) => Some(cp.max(ch)),
        (Some(cp), None) => Some(cp),
        (None, Some(ch)) => Some(ch),
        (None, None) => None,
    };
    if let Some(seed) = effective_seed {
        gap_detector.seed(seed);
        tracing::info!(
            %symbol,
            seed,
            checkpoint_tid = checkpoint_tid.unwrap_or(-1),
            ch_seed_tid = ch_seed_tid.unwrap_or(-1),
            "gap detector seeded"
        );
    }

    tracing::info!(%symbol, thresholds = ?processors.iter().map(|(t, _)| *t).collect::<Vec<_>>(), "symbol task started");

    // Rust-side dedup: per-threshold committed floor.
    // Bars with last_agg_trade_id <= floor are already in ClickHouse — skip emission.
    // Floor is initialized from max(processor.last_completed_bar_tid, ch_seed_tid) per threshold.
    // v1.4: Uses last_completed_bar_tid (not last_agg_trade_id) to avoid suppressing
    // junction bars whose trade IDs overlap with the forming bar tail.
    // After each bar emission, floor is updated to the emitted bar's last_agg_trade_id.
    let mut committed_floors: HashMap<u32, i64> = HashMap::new();
    for (threshold, processor) in processors.iter() {
        let proc_tid = processor.last_completed_bar_tid().unwrap_or(-1);
        let ch_tid = ch_seed_tid.unwrap_or(-1);
        let floor = proc_tid.max(ch_tid);
        if floor > 0 {
            committed_floors.insert(*threshold, floor);
            tracing::info!(%symbol, threshold, floor, completed_tid = proc_tid, ch_tid, "rust-dedup: floor initialized (v1.4: completed bar tid)");
        }
    }

    // Track current UTC day per processor for midnight reset (#264)
    let mut last_days: Vec<i64> = processors.iter().map(|_| -1i64).collect();

    // --- JUNCTION FILL: Bridge REST→WS gap (Issue #XXX) ---
    // Peek first WS trade to determine if a junction gap exists.
    // This runs ONCE at startup, before the main select loop.

    // Drain all immediately-available WS trades to measure buffer depth
    let first_ws_trade = match trade_rx.recv().await {
        Some(t) => t,
        None => {
            tracing::warn!(%symbol, "WS channel closed before first trade, exiting symbol_task");
            return;
        }
    };
    let mut ws_buffer: Vec<AggTrade> = Vec::new();
    while let Ok(t) = trade_rx.try_recv() {
        ws_buffer.push(t);
    }
    let ws_buffer_depth = ws_buffer.len();

    // JUNC-01/JUNC-02: Fill junction gap using the SAME processors (preserves forming bar)
    let last_rest_tid = processors.first().and_then(|(_, p)| p.last_agg_trade_id());
    if let Some(rest_tid) = last_rest_tid {
        let junction_gap = first_ws_trade.agg_trade_id - rest_tid - 1;
        if junction_gap > 0 {
            // Compute forming bar trade count across all processors
            let forming_bar_trades = processors.first()
                .and_then(|(_, p)| p.get_incomplete_bar())
                .map(|b| b.individual_trade_count)
                .unwrap_or(0);

            tracing::info!(
                %symbol,
                junction_gap_tid_delta = junction_gap,
                forming_bar_trades,
                ws_buffer_depth,
                rest_last_tid = rest_tid,
                ws_first_tid = first_ws_trade.agg_trade_id,
                "junction.telemetry"
            );

            let _recovered = puck_fill(
                &symbol,
                rest_tid,
                first_ws_trade.agg_trade_id,
                &mut processors,
                &bar_buffer,
                &metrics,
                &shutdown,
                rate_limiter.clone(),
                &forming_tx_map,
                "junction_fill",
            )
            .await;
        } else {
            let forming_bar_trades = processors.first()
                .and_then(|(_, p)| p.get_incomplete_bar())
                .map(|b| b.individual_trade_count)
                .unwrap_or(0);
            tracing::info!(
                %symbol,
                junction_gap_tid_delta = 0i64,
                forming_bar_trades,
                ws_buffer_depth,
                "junction.telemetry"
            );
        }
    } else {
        tracing::info!(
            %symbol,
            junction_gap_tid_delta = 0i64,
            forming_bar_trades = 0u64,
            ws_buffer_depth,
            "junction.telemetry"
        );
    }

    // JUNC-03: Re-seed gap detector past junction — Puck won't fire on this range
    gap_detector.seed(first_ws_trade.agg_trade_id);

    // JUNC-04: Monotonicity guard — drop WS trade if already processed during junction fill
    let current_last_tid = processors.first().and_then(|(_, p)| p.last_agg_trade_id()).unwrap_or(-1);
    if first_ws_trade.agg_trade_id > current_last_tid {
        process_trade_through_processors(
            &first_ws_trade,
            &mut processors,
            &mut last_days,
            &bar_buffer,
            &metrics,
            &symbol_arc,
            &symbol,
            &forming_tx_map,
            &mut committed_floors,
        );
        gap_detector.update_last_tid(first_ws_trade.agg_trade_id);
    } else {
        tracing::debug!(
            %symbol,
            ws_tid = first_ws_trade.agg_trade_id,
            current_last_tid,
            "dropping overlapping WS trade at junction"
        );
    }
    // --- END JUNCTION FILL ---

    // Process buffered WS trades that arrived during junction fill
    for trade in &ws_buffer {
        if let Some((last_id, gap_size)) = gap_detector.check(trade.agg_trade_id) {
            let recovered = puck_fill(
                &symbol, last_id, trade.agg_trade_id,
                &mut processors, &bar_buffer, &metrics, &shutdown,
                rate_limiter.clone(), &forming_tx_map,
                "junction_fill",
            ).await;
            if recovered > 0 {
                gap_detector.mark_filled(last_id, gap_size);
            }
        }
        process_trade_through_processors(
            trade, &mut processors, &mut last_days,
            &bar_buffer, &metrics, &symbol_arc, &symbol, &forming_tx_map,
            &mut committed_floors,
        );
        gap_detector.update_last_tid(trade.agg_trade_id);
    }
    if !ws_buffer.is_empty() {
        tracing::debug!(%symbol, drained = ws_buffer.len(), "junction: processed buffered WS trades");
    }

    // Issue #96 RUST-03: Reuse burst frame allocation across select loop iterations.
    // Previously allocated Vec::with_capacity(32) per iteration — now cleared and reused.
    let mut frame: Vec<AggTrade> = Vec::with_capacity(32);

    // Process trades → bars
    loop {
        tokio::select! {
            // Issue #257: On-demand gap-fill commands from Python fill_gap()
            Some(cmd) = async {
                match gap_fill_rx.as_mut() {
                    Some(rx) => rx.recv().await,
                    None => std::future::pending().await,
                }
            } => {
                let start_time = std::time::Instant::now();
                tracing::info!(
                    %symbol, from_tid = cmd.from_tid, to_tid = cmd.to_tid,
                    "processing on-demand gap-fill command"
                );

                let _gap_size = cmd.to_tid - cmd.from_tid;
                let recovered = puck_fill(
                    &symbol,
                    cmd.from_tid - 1, // last_known_id (fill expects exclusive start)
                    cmd.to_tid,
                    &mut processors,
                    &bar_buffer,
                    &metrics,
                    &shutdown,
                    rate_limiter.clone(),
                    &forming_tx_map,
                    "on_demand_gap_fill",
                )
                .await;
                // Advance gap detector so it won't re-detect this range (#290)
                if recovered > 0 {
                    gap_detector.update_last_tid(cmd.to_tid);
                }
                let (trades_fetched, bars_produced, partial) =
                    (recovered as usize, 0usize, false);

                let duration_ms = start_time.elapsed().as_millis() as u64;
                let result = GapFillResult {
                    trades_fetched,
                    bars_produced,
                    duration_ms,
                    partial,
                };
                // Send result back — ignore error if caller dropped
                let _ = cmd.response_tx.send(result);
            }
            trade = trade_rx.recv() => {
                match trade {
                    Some(first_trade) => {
                        // Issue #273: Burst-atomic frame processing.
                        //
                        // Collect the triggering trade plus all immediately-buffered
                        // trades from the channel into a single "frame". This ensures
                        // that when a same-millisecond burst of trades arrives (e.g.,
                        // 19 trades at ts=1773485713955), ALL of them are processed
                        // through the threshold processors atomically — no select-loop
                        // yield or gap-fill interruption can split the burst.
                        //
                        // Without this, the select loop processes one trade per
                        // iteration. If a bar closes on trade N and trades N+1..N+k
                        // share the same millisecond, the loop could yield between
                        // them (e.g., to handle a gap-fill command), leaving N+1..N+k
                        // unprocessed until the next select iteration. In edge cases,
                        // those trades may never arrive (WS delivery gap at TCP level)
                        // and the gap detector can't help because they were "expected
                        // to be contiguous" from a timestamp perspective.
                        //
                        // By draining all buffered trades into a frame BEFORE any
                        // processing, we guarantee atomic handling of bursts.
                        frame.clear();
                        frame.push(first_trade);
                        for _ in 0..MAX_BURST_DRAIN {
                            match trade_rx.try_recv() {
                                Ok(t) => frame.push(t),
                                Err(_) => break, // Channel empty or closed
                            }
                        }

                        let frame_size = frame.len();
                        metrics.trades_received.fetch_add(frame_size as u64, Ordering::Relaxed);

                        if frame_size > 1 {
                            tracing::debug!(
                                %symbol, frame_size,
                                first_tid = frame[0].agg_trade_id,
                                last_tid = frame[frame_size - 1].agg_trade_id,
                                "burst-atomic frame: processing {} trades atomically",
                                frame_size
                            );
                        }

                        // Process each trade in the frame sequentially
                        for trade in &frame {
                            // Issue #257: Gap detection via TradeIdGapDetector
                            if let Some((last_id, gap_size)) = gap_detector.check(trade.agg_trade_id) {
                                metrics.reconnections.fetch_add(1, Ordering::Relaxed);
                                metrics.gap_fills.fetch_add(1, Ordering::Relaxed);
                                let recovered = puck_fill(
                                    &symbol,
                                    last_id,
                                    trade.agg_trade_id,
                                    &mut processors,
                                    &bar_buffer,
                                    &metrics,
                                    &shutdown,
                                    rate_limiter.clone(),
                                    &forming_tx_map,
                                    "normal_gap_fill",
                                )
                                .await;
                                metrics
                                    .gap_trades_recovered
                                    .fetch_add(recovered, Ordering::Relaxed);

                                // Mark gap as filled inline if trades were recovered
                                if recovered > 0 {
                                    gap_detector.mark_filled(last_id, gap_size);
                                }
                            }

                            // Issue #273: Process trade through all threshold processors
                            process_trade_through_processors(
                                trade,
                                &mut processors,
                                &mut last_days,
                                &bar_buffer,
                                &metrics,
                                &symbol_arc,
                                &symbol,
                                &forming_tx_map,
                                &mut committed_floors,
                            );

                            // Issue #257: Update gap detector with this trade's ID
                            gap_detector.update_last_tid(trade.agg_trade_id);
                        }
                    }
                    None => {
                        // WebSocket channel closed (terminal error or shutdown)
                        tracing::info!(%symbol, "trade channel closed");
                        break;
                    }
                }
            }
            () = shutdown.cancelled() => {
                tracing::info!(%symbol, "shutdown requested");
                break;
            }
        }
    }

    // Extract checkpoints from all processors before task exits
    for (threshold, processor) in &processors {
        let cp = processor.create_checkpoint(&symbol);
        tracing::info!(
            %symbol, threshold,
            has_incomplete_bar = cp.has_incomplete_bar(),
            "checkpoint extracted on shutdown"
        );
        if checkpoint_tx
            .send((symbol.clone(), *threshold, cp))
            .await
            .is_err()
        {
            tracing::warn!(%symbol, threshold, "checkpoint channel closed");
        }
    }

    tracing::info!(%symbol, "symbol task ended");
}

#[cfg(test)]
mod tests {
    use super::*;
    use opendeviationbar_core::FixedPoint;

    fn make_trade(id: i64, price: f64, timestamp_ms: u64, is_buyer_maker: bool) -> AggTrade {
        let price_str = format!("{price:.8}");
        AggTrade {
            agg_trade_id: id,
            price: FixedPoint::from_str(&price_str).unwrap(),
            volume: FixedPoint::from_str("1.00000000").unwrap(),
            first_trade_id: id,
            last_trade_id: id,
            timestamp: opendeviationbar_core::normalize_timestamp(timestamp_ms),
            is_buyer_maker,
            is_best_match: None,
        }
    }

    #[test]
    fn test_live_engine_config_defaults() {
        let config = LiveEngineConfig::new(
            vec!["BTCUSDT".into(), "ETHUSDT".into()],
            vec![250, 500, 1000],
        );
        assert_eq!(config.symbols.len(), 2);
        assert_eq!(config.thresholds.len(), 3);
        assert!(config.include_microstructure);
        // Default from get_bar_channel_capacity() when OPENDEVIATIONBAR_MAX_PENDING_BARS is unset
        assert_eq!(config.bar_channel_capacity, 10_000);
        // Issue #128: Verify tier config defaults
        assert!(config.compute_tier2);
        assert!(!config.compute_tier3);
        assert_eq!(config.compute_hurst, None);
        assert_eq!(config.compute_permutation_entropy, None);
    }

    #[test]
    fn test_live_engine_config_rate_limiter() {
        let config = LiveEngineConfig::new(vec!["BTCUSDT".into()], vec![250]);
        assert!(config.rate_limiter.is_none());

        let limiter = Arc::new(AdaptiveRateLimiter::binance_default());
        let config = config.with_rate_limiter(limiter);
        assert!(config.rate_limiter.is_some());
        assert_eq!(config.rate_limiter.as_ref().unwrap().remaining(), 4800);
    }

    #[test]
    fn test_completed_bar_metadata() {
        let trade = make_trade(1, 50000.0, 1700000000000, false);
        let bar = OpenDeviationBar::new(&trade);
        let completed = CompletedBar {
            symbol: Arc::from("BTCUSDT"),
            threshold_decimal_bps: 250,
            bar,
        };
        assert_eq!(&*completed.symbol, "BTCUSDT");
        assert_eq!(completed.threshold_decimal_bps, 250);
    }

    #[test]
    fn test_live_engine_creation() {
        let config = LiveEngineConfig::new(vec!["BTCUSDT".into()], vec![250]);
        let engine = LiveBarEngine::new(config);
        assert!(!engine.is_started());
    }

    #[test]
    fn test_metrics_snapshot() {
        let metrics = LiveEngineMetrics::default();
        metrics.trades_received.store(100, Ordering::Relaxed);
        metrics.bars_emitted.store(5, Ordering::Relaxed);
        metrics.reconnections.store(2, Ordering::Relaxed);
        metrics.dropped_bars.store(1, Ordering::Relaxed);
        metrics.max_queue_depth.store(50, Ordering::Relaxed);
        metrics.backpressure_events.store(3, Ordering::Relaxed);
        metrics.gap_fills.store(2, Ordering::Relaxed);
        metrics.gap_trades_recovered.store(150, Ordering::Relaxed);

        let snap = metrics.snapshot();
        assert_eq!(snap.trades_received, 100);
        assert_eq!(snap.bars_emitted, 5);
        assert_eq!(snap.reconnections, 2);
        assert_eq!(snap.dropped_bars, 1);
        assert_eq!(snap.max_queue_depth, 50);
        assert_eq!(snap.backpressure_events, 3);
        assert_eq!(snap.gap_fills, 2);
        assert_eq!(snap.gap_trades_recovered, 150);
    }

    #[test]
    fn test_ring_buffer_metrics() {
        // Issue #96 Task #12: Verify ring buffer metrics tracking
        let metrics = LiveEngineMetrics::default();

        // Initially all zeros
        assert_eq!(metrics.dropped_bars.load(Ordering::Relaxed), 0);
        assert_eq!(metrics.backpressure_events.load(Ordering::Relaxed), 0);
        assert_eq!(metrics.max_queue_depth.load(Ordering::Relaxed), 0);

        // Simulate ring buffer drops
        metrics.dropped_bars.fetch_add(5, Ordering::Relaxed);
        metrics.backpressure_events.fetch_add(5, Ordering::Relaxed);
        let _ = metrics.max_queue_depth.fetch_max(100, Ordering::Relaxed);

        assert_eq!(metrics.dropped_bars.load(Ordering::Relaxed), 5);
        assert_eq!(metrics.backpressure_events.load(Ordering::Relaxed), 5);
        assert_eq!(metrics.max_queue_depth.load(Ordering::Relaxed), 100);

        // Verify snapshot captures all metrics
        let snap = metrics.snapshot();
        assert_eq!(snap.dropped_bars, 5);
        assert_eq!(snap.backpressure_events, 5);
        assert_eq!(snap.max_queue_depth, 100);
    }

    #[tokio::test]
    async fn test_engine_start_and_stop() {
        let config = LiveEngineConfig::new(vec!["BTCUSDT".into()], vec![250]);
        let mut engine = LiveBarEngine::new(config);

        // Start should succeed (spawns tasks but WS won't connect in test)
        assert!(engine.start().is_ok());
        assert!(engine.is_started());

        // Double-start is idempotent
        assert!(engine.start().is_ok());

        // Stop
        engine.stop();

        // next_bar should return None after stop
        let bar = engine.next_bar(Duration::from_millis(50)).await;
        assert!(bar.is_none());
    }

    #[tokio::test]
    async fn test_processor_fan_out() {
        // Verify that trades fan out to all threshold processors correctly
        let thresholds = vec![250u32, 500, 750];
        let mut processors: Vec<(u32, OpenDeviationBarProcessor)> = thresholds
            .iter()
            .map(|&t| (t, OpenDeviationBarProcessor::new(t).unwrap()))
            .collect();

        // Process a series of trades — each processor should maintain independent state
        let base_price = 50000.0;
        for i in 0..10 {
            let trade = make_trade(
                i,
                base_price + (i as f64) * 0.01,
                1700000000000 + (i as u64) * 1000,
                i % 2 == 0,
            );
            for (_, processor) in &mut processors {
                let _ = processor.process_single_trade(&trade);
            }
        }

        // All processors should have processed trades independently
        for (threshold, processor) in &processors {
            let incomplete = processor.get_incomplete_bar();
            assert!(
                incomplete.is_some(),
                "threshold {threshold} should have an incomplete bar"
            );
        }
    }

    #[tokio::test]
    async fn test_microstructure_processor_creation() {
        // Verify processors can be created with full microstructure features
        let threshold = 250u32;
        let p = OpenDeviationBarProcessor::new(threshold)
            .unwrap()
            .with_inter_bar_config(InterBarConfig::default())
            .with_intra_bar_features();

        assert!(p.inter_bar_enabled());
        assert!(p.intra_bar_enabled());
    }

    #[test]
    fn test_ring_buffer_drop_tracking() {
        // Issue #96 Task #12: Verify ring buffer correctly tracks dropped bars
        use crate::ring_buffer::ConcurrentRingBuffer;

        let ring_buf = ConcurrentRingBuffer::new(3);
        let metrics = LiveEngineMetrics::default();

        // Fill the buffer
        let bar1 = CompletedBar {
            symbol: Arc::from("BTC"),
            threshold_decimal_bps: 250,
            bar: OpenDeviationBar::new(&make_trade(1, 50000.0, 1700000000000, false)),
        };
        let bar2 = CompletedBar {
            symbol: Arc::from("BTC"),
            threshold_decimal_bps: 250,
            bar: OpenDeviationBar::new(&make_trade(2, 50001.0, 1700000000001, false)),
        };
        let bar3 = CompletedBar {
            symbol: Arc::from("BTC"),
            threshold_decimal_bps: 250,
            bar: OpenDeviationBar::new(&make_trade(3, 50002.0, 1700000000002, false)),
        };
        let bar4 = CompletedBar {
            symbol: Arc::from("BTC"),
            threshold_decimal_bps: 250,
            bar: OpenDeviationBar::new(&make_trade(4, 50003.0, 1700000000003, false)),
        };

        assert!(ring_buf.push(bar1));
        assert!(ring_buf.push(bar2));
        assert!(ring_buf.push(bar3));

        // Buffer is full (capacity=3), so next push should fail and drop oldest
        assert!(!ring_buf.push(bar4));

        // Simulate metrics tracking (as symbol_task would do)
        metrics.dropped_bars.fetch_add(1, Ordering::Relaxed);
        metrics.backpressure_events.fetch_add(1, Ordering::Relaxed);
        let current_depth = ring_buf.len() as u64;
        let _ = metrics
            .max_queue_depth
            .fetch_max(current_depth, Ordering::Relaxed);

        assert_eq!(metrics.dropped_bars.load(Ordering::Relaxed), 1);
        assert_eq!(metrics.backpressure_events.load(Ordering::Relaxed), 1);
        assert_eq!(metrics.max_queue_depth.load(Ordering::Relaxed), 3);

        // Verify we can still retrieve bars
        assert_eq!(ring_buf.pop().is_some(), true); // bar2
        assert_eq!(ring_buf.pop().is_some(), true); // bar3
        assert_eq!(ring_buf.pop().is_some(), true); // bar4
        assert_eq!(ring_buf.pop().is_none(), true); // empty
    }

    #[test]
    fn test_live_engine_metrics_estimate_queue_depth() {
        // Issue #96 Task #12: Verify queue depth estimation
        let metrics = LiveEngineMetrics::default();

        metrics.trades_received.store(1000, Ordering::Relaxed);
        metrics.bars_emitted.store(500, Ordering::Relaxed);
        metrics.dropped_bars.store(0, Ordering::Relaxed);

        let estimated = metrics.estimate_queue_depth();
        assert_eq!(estimated, 500); // 1000 - 500 - 0
    }

    #[test]
    fn test_forming_bar_watch_channels_created() {
        // Issue #214: Verify watch channels are created for each (symbol, threshold) pair
        let config = LiveEngineConfig::new(
            vec!["BTCUSDT".into(), "ETHUSDT".into()],
            vec![250, 500],
        );
        let mut engine = LiveBarEngine::new(config);

        // Should have 2 symbols × 2 thresholds = 4 watch channels
        let watches = engine.take_forming_bar_watches();
        assert!(watches.is_some());
        let watches = watches.unwrap();
        assert_eq!(watches.len(), 4);

        // Verify all expected keys exist
        let btc: Arc<str> = Arc::from("BTCUSDT");
        let eth: Arc<str> = Arc::from("ETHUSDT");
        assert!(watches.contains_key(&(btc.clone(), 250)));
        assert!(watches.contains_key(&(btc, 500)));
        assert!(watches.contains_key(&(eth.clone(), 250)));
        assert!(watches.contains_key(&(eth, 500)));

        // Second call returns None (already taken)
        assert!(engine.take_forming_bar_watches().is_none());
    }

    #[test]
    fn test_forming_bar_watch_send_receive() {
        // Issue #214: Verify watch channel send/receive semantics
        let (tx, rx) = watch::channel::<Option<FormingBar>>(None);

        // Initially None
        assert!(rx.borrow().is_none());

        // Send a forming bar
        let trade = make_trade(1, 50000.0, 1700000000000, false);
        let bar = OpenDeviationBar::new(&trade);
        let forming = FormingBar {
            symbol: Arc::from("BTCUSDT"),
            threshold_decimal_bps: 250,
            bar: bar.clone(),
            last_trade_timestamp_us: 1700000000000000,
        };
        tx.send(Some(forming)).unwrap();

        // Receiver sees the latest value
        let snapshot = rx.borrow();
        assert!(snapshot.is_some());
        let fb = snapshot.as_ref().unwrap();
        assert_eq!(&*fb.symbol, "BTCUSDT");
        assert_eq!(fb.threshold_decimal_bps, 250);
        assert_eq!(fb.last_trade_timestamp_us, 1700000000000000);
        drop(snapshot);

        // Clear on bar completion (send None)
        tx.send(None).unwrap();
        assert!(rx.borrow().is_none());
    }

    #[test]
    fn test_process_trade_through_processors_basic() {
        // Issue #273: Verify extracted process_trade_through_processors works correctly
        let thresholds = vec![250u32, 500];
        let mut processors: Vec<(u32, OpenDeviationBarProcessor)> = thresholds
            .iter()
            .map(|&t| (t, OpenDeviationBarProcessor::new(t).unwrap()))
            .collect();
        let mut last_days: Vec<i64> = processors.iter().map(|_| -1i64).collect();
        let bar_buffer = ConcurrentRingBuffer::new(100);
        let metrics = LiveEngineMetrics::default();
        let symbol_arc: Arc<str> = Arc::from("BTCUSDT");
        let forming_tx_map: HashMap<u32, watch::Sender<Option<FormingBar>>> = HashMap::new();
        let mut committed_floors: HashMap<u32, i64> = HashMap::new();

        // Process a trade — should be absorbed (no bar completed yet)
        let trade = make_trade(1, 50000.0, 1700000000000, false);
        process_trade_through_processors(
            &trade,
            &mut processors,
            &mut last_days,
            &bar_buffer,
            &metrics,
            &symbol_arc,
            "BTCUSDT",
            &forming_tx_map,
            &mut committed_floors,
        );

        // No bars should be emitted yet (single trade can't breach threshold)
        assert!(bar_buffer.pop().is_none());
        // Both processors should have an incomplete bar
        for (_, processor) in &processors {
            assert!(processor.get_incomplete_bar().is_some());
        }
    }

    #[test]
    fn test_burst_atomic_frame_same_ms_trades() {
        // Issue #273: Verify that a burst of same-ms trades is processed
        // atomically through all processors. Simulates the scenario where
        // 19 trades arrive with the same millisecond timestamp.
        let thresholds = vec![250u32];
        let mut processors: Vec<(u32, OpenDeviationBarProcessor)> = thresholds
            .iter()
            .map(|&t| (t, OpenDeviationBarProcessor::new(t).unwrap()))
            .collect();
        let mut last_days: Vec<i64> = processors.iter().map(|_| -1i64).collect();
        let bar_buffer = ConcurrentRingBuffer::new(100);
        let metrics = LiveEngineMetrics::default();
        let symbol_arc: Arc<str> = Arc::from("BTCUSDT");
        let forming_tx_map: HashMap<u32, watch::Sender<Option<FormingBar>>> = HashMap::new();
        let mut committed_floors: HashMap<u32, i64> = HashMap::new();

        let base_price = 50000.0;
        let same_ms = 1700000000000u64; // All trades at same millisecond

        // Create a burst of 20 trades at the same millisecond
        let mut frame: Vec<AggTrade> = Vec::with_capacity(20);
        for i in 0..20 {
            // Vary price slightly to stay within threshold (not breach)
            let price = base_price + (i as f64) * 0.01;
            frame.push(make_trade(
                100 + i,
                price,
                same_ms,
                i % 2 == 0,
            ));
        }

        // Process all trades in the frame atomically
        for trade in &frame {
            process_trade_through_processors(
                trade,
                &mut processors,
                &mut last_days,
                &bar_buffer,
                &metrics,
                &symbol_arc,
                "BTCUSDT",
                &forming_tx_map,
                &mut committed_floors,
            );
        }

        // All 20 trades should have been processed
        // The processor should have an incomplete bar containing all trades
        let (_, processor) = &processors[0];
        let incomplete = processor.get_incomplete_bar().unwrap();
        assert_eq!(incomplete.first_agg_trade_id, 100);
        assert_eq!(incomplete.last_agg_trade_id, 119);
    }

    #[test]
    fn test_max_burst_drain_constant() {
        // Issue #273: Verify the burst drain limit is sensible
        assert_eq!(MAX_BURST_DRAIN, 1024);
        // Typical same-ms bursts are 5-50 trades. 1024 provides generous headroom
        // without risking unbounded draining.
    }

    #[test]
    fn test_junction_fill_zero_stathera_gaps() {
        // Simulate REST→WS junction scenario
        let (gap_tx, mut gap_rx) = mpsc::channel::<crate::gap::GapEvent>(16);
        let mut gap_detector = crate::gap::TradeIdGapDetector::new(
            Arc::from("TESTUSDT"),
            Some(gap_tx),
        );

        // Simulate fill_from_rest: process trades 100-104
        let mut processor = OpenDeviationBarProcessor::new(250).unwrap();
        for i in 0..5i64 {
            let trade = make_trade(
                100 + i,
                50000.0 + (i as f64) * 0.01,
                1700000000000 + (i as u64) * 1000,
                i % 2 == 0,
            );
            let _ = processor.process_single_trade(&trade);
        }
        assert_eq!(processor.last_agg_trade_id(), Some(104));
        assert!(
            processor.get_incomplete_bar().is_some(),
            "forming bar must exist after REST fill"
        );

        // WS first trade arrives at ID 110 (junction gap: 105-109)
        let first_ws_tid = 110i64;

        // JUNC-03: Re-seed gap detector past junction
        gap_detector.seed(first_ws_tid);

        // Subsequent WS trades should NOT trigger gap detection
        // check(111) -> None because 111 <= 110+1
        assert!(
            gap_detector.check(111).is_none(),
            "no gap after junction re-seed"
        );
        // Must update_last_tid before next check (check doesn't update state)
        gap_detector.update_last_tid(111);
        // check(112) -> None because 112 <= 111+1
        assert!(
            gap_detector.check(112).is_none(),
            "consecutive trade also no gap"
        );

        // No gap events should have been emitted
        assert!(
            gap_rx.try_recv().is_err(),
            "no gap events after junction re-seed"
        );

        // Processor forming bar is still intact (JUNC-01)
        assert!(
            processor.get_incomplete_bar().is_some(),
            "forming bar preserved across junction"
        );
    }

    #[test]
    fn test_junction_zero_gap_no_fill_needed() {
        // Zero-gap junction: first WS trade is exactly last_rest_tid + 1
        let (gap_tx, mut gap_rx) = mpsc::channel::<crate::gap::GapEvent>(16);
        let mut gap_detector = crate::gap::TradeIdGapDetector::new(
            Arc::from("TESTUSDT"),
            Some(gap_tx),
        );

        // REST fill: trades 100-104
        let mut processor = OpenDeviationBarProcessor::new(250).unwrap();
        for i in 0..5i64 {
            let trade = make_trade(
                100 + i,
                50000.0 + (i as f64) * 0.01,
                1700000000000 + (i as u64) * 1000,
                i % 2 == 0,
            );
            let _ = processor.process_single_trade(&trade);
        }
        assert_eq!(processor.last_agg_trade_id(), Some(104));

        // First WS trade at ID 105 — zero gap
        let first_ws_tid = 105i64;
        let last_rest_tid = processor.last_agg_trade_id().unwrap();
        let junction_gap = first_ws_tid - last_rest_tid - 1;
        assert_eq!(
            junction_gap, 0,
            "no junction gap when WS continues exactly from REST"
        );

        // JUNC-03: Still re-seed gap detector
        gap_detector.seed(first_ws_tid);

        // Next WS trade should not trigger gap
        assert!(
            gap_detector.check(106).is_none(),
            "no gap for consecutive trade after zero-gap junction"
        );

        // No gap events emitted
        assert!(
            gap_rx.try_recv().is_err(),
            "no gap events for zero-gap junction"
        );

        // Forming bar intact
        assert!(
            processor.get_incomplete_bar().is_some(),
            "forming bar preserved in zero-gap junction"
        );
    }

    #[test]
    fn test_junction_monotonicity_guard() {
        // JUNC-04: Overlapping WS trades must be silently dropped
        let mut processor = OpenDeviationBarProcessor::new(250).unwrap();

        // Simulate REST + junction fill: trades 100-109
        for i in 0..10i64 {
            let trade = make_trade(
                100 + i,
                50000.0 + (i as f64) * 0.01,
                1700000000000 + (i as u64) * 1000,
                i % 2 == 0,
            );
            let _ = processor.process_single_trade(&trade);
        }
        assert_eq!(processor.last_agg_trade_id(), Some(109));

        // Overlapping WS trades (IDs 107-109) should be dropped
        let last_tid = processor.last_agg_trade_id().unwrap();
        for overlap_id in [107i64, 108, 109] {
            assert!(
                overlap_id <= last_tid,
                "trade {overlap_id} should be dropped (monotonicity guard)"
            );
        }

        // Non-overlapping trade (ID 110) should be processed
        assert!(110 > last_tid, "trade 110 should pass monotonicity guard");
        let trade_110 = make_trade(110, 50000.10, 1700000010000, true);
        let _ = processor.process_single_trade(&trade_110);
        assert_eq!(processor.last_agg_trade_id(), Some(110));
    }

    #[test]
    fn test_junction_forming_bar_trade_count_preserved() {
        // JUNC-01: Forming bar must carry trades across REST→junction→WS
        let mut processor = OpenDeviationBarProcessor::new(250).unwrap();

        // Phase 1: REST fill — 20 trades (no breach at 250 dbps = 2.5%)
        for i in 0..20i64 {
            let trade = make_trade(
                i + 1,
                50000.0 + (i as f64) * 0.001, // tiny increments, no breach
                1700000000000 + (i as u64) * 1000,
                i % 2 == 0,
            );
            let result = processor.process_single_trade(&trade).unwrap();
            assert!(result.is_none(), "should not breach at tiny price changes");
        }
        let bar_after_rest = processor
            .get_incomplete_bar()
            .expect("forming bar after REST");
        assert!(
            bar_after_rest.individual_trade_count >= 20,
            "forming bar has REST trades"
        );

        // Phase 2: Junction fill — 5 more trades through SAME processor
        for i in 20..25i64 {
            let trade = make_trade(
                i + 1,
                50000.0 + (i as f64) * 0.001,
                1700000000000 + (i as u64) * 1000,
                i % 2 == 0,
            );
            let result = processor.process_single_trade(&trade).unwrap();
            assert!(result.is_none(), "still no breach during junction fill");
        }
        let bar_after_junction = processor
            .get_incomplete_bar()
            .expect("forming bar after junction");
        assert!(
            bar_after_junction.individual_trade_count >= 25,
            "forming bar accumulated junction trades: got {}",
            bar_after_junction.individual_trade_count
        );

        // Phase 3: First WS trade — processed through SAME processor
        let ws_trade = make_trade(26, 50000.025, 1700000025000, true);
        let result = processor.process_single_trade(&ws_trade).unwrap();
        assert!(result.is_none(), "still no breach");
        let bar_after_ws = processor
            .get_incomplete_bar()
            .expect("forming bar after WS start");
        assert!(
            bar_after_ws.individual_trade_count >= 26,
            "forming bar spans REST+junction+WS: got {}",
            bar_after_ws.individual_trade_count
        );
    }
}
