-- ClickHouse schema for opendeviationbar cache
-- Stores computed open deviation bars (Tier 2)
--
-- Note: Raw tick data (Tier 1) is stored locally using Parquet files.
-- See opendeviationbar.storage.TickStorage for tick data caching.
--
-- Usage:
--   CREATE DATABASE IF NOT EXISTS opendeviationbar_cache;
--   Then run this file or use OpenDeviationBarCache._ensure_schema()

-- ============================================================================
-- Migration for v5.0.0 (from v4.x)
-- ============================================================================
-- Run this ONCE if upgrading from opendeviationbar-py v4.x with existing cache:
--
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars
--     RENAME COLUMN threshold_bps TO threshold_decimal_bps;
--
-- Note: New installations do not need this migration.

-- ============================================================================
-- Migration for v7.2.0 (Issue #32: rename aggregation_efficiency)
-- ============================================================================
-- Run this ONCE if upgrading from opendeviationbar-py v7.1.x with existing cache:
--
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars
--     RENAME COLUMN aggregation_efficiency TO aggregation_density;
--
-- Note: New installations do not need this migration.

-- ============================================================================
-- Legacy migrations (completed, retained for reference)
-- ============================================================================
-- v12.x: Changed DEFAULT to 'day', all legacy modes removed

-- ============================================================================
-- Migration for v12.x (Issue #8: Exchange sessions integration)
-- ============================================================================
-- Run this ONCE if upgrading from opendeviationbar-py v11.x with existing cache:
--
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars
--     ADD COLUMN exchange_session_sydney UInt8 DEFAULT 0,
--     ADD COLUMN exchange_session_tokyo UInt8 DEFAULT 0,
--     ADD COLUMN exchange_session_london UInt8 DEFAULT 0,
--     ADD COLUMN exchange_session_newyork UInt8 DEFAULT 0;
--
-- Note: New installations do not need this migration.

-- ============================================================================
-- Migration for v13.x (Issue #59: Inter-bar features from lookback window)
-- ============================================================================
-- Run this ONCE if upgrading from opendeviationbar-py v12.x with existing cache:
--
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars
--     -- Tier 1: Core features (7 features)
--     ADD COLUMN lookback_trade_count Nullable(UInt32) DEFAULT NULL,
--     ADD COLUMN lookback_ofi Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN lookback_duration_us Nullable(Int64) DEFAULT NULL,
--     ADD COLUMN lookback_intensity Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN lookback_vwap_raw Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN lookback_vwap_position Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN lookback_count_imbalance Nullable(Float64) DEFAULT NULL,
--     -- Tier 2: Statistical features (5 features)
--     ADD COLUMN lookback_kyle_lambda Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN lookback_burstiness Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN lookback_volume_skew Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN lookback_volume_kurt Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN lookback_price_range Nullable(Float64) DEFAULT NULL,
--     -- Tier 3: Advanced features (4 features)
--     ADD COLUMN lookback_kaufman_er Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN lookback_garman_klass_vol Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN lookback_hurst Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN lookback_permutation_entropy Nullable(Float64) DEFAULT NULL;
--
-- Note: New installations do not need this migration.

-- ============================================================================
-- Migration for v12.4.x (Issue #72: Aggregate trade ID range for data integrity)
-- ============================================================================
-- Run this ONCE if upgrading from opendeviationbar-py v12.3.x with existing cache:
--
-- -- open_deviation_bars table
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars
--     ADD COLUMN first_agg_trade_id Int64 DEFAULT 0,
--     ADD COLUMN last_agg_trade_id Int64 DEFAULT 0;
--
-- -- population_checkpoints table (Full Audit Trail)
-- ALTER TABLE opendeviationbar_cache.population_checkpoints
--     ADD COLUMN first_agg_trade_id_in_bar Int64 DEFAULT 0,
--     ADD COLUMN last_agg_trade_id_in_bar Int64 DEFAULT 0;
--
-- Note: New installations do not need this migration.

-- ============================================================================
-- Migration for Issue #101 (is_liquidation_cascade flag)
-- ============================================================================
-- Run ONCE on existing installations (bigblack and any other cached hosts):
--
-- Step 1: Add column (DEFAULT expression auto-applies to new inserts)
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars
--     ADD COLUMN IF NOT EXISTS is_liquidation_cascade UInt8 DEFAULT toUInt8(
--         (high - low) / open >= toFloat64(threshold_decimal_bps) / 10000.0
--         AND duration_us > 0
--         AND duration_us <= 100000
--     );
--
-- Step 2: Backfill all existing rows (runs async, check system.mutations for progress)
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars
--     UPDATE is_liquidation_cascade = toUInt8(
--         (high - low) / open >= toFloat64(threshold_decimal_bps) / 10000.0
--         AND duration_us > 0
--         AND duration_us <= 100000
--     ) WHERE 1;
--
-- Note: New installations do not need this migration.

-- ============================================================================
-- Migration for v12.8.x (Issue #78: Intra-bar features)
-- ============================================================================
-- Run this ONCE if upgrading from opendeviationbar-py v12.7.x with existing cache:
--
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars
--     ADD COLUMN IF NOT EXISTS intra_bull_epoch_density Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_bear_epoch_density Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_bull_excess_gain Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_bear_excess_gain Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_bull_cv Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_bear_cv Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_max_drawdown Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_max_runup Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_trade_count Nullable(UInt32) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_ofi Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_duration_us Nullable(Int64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_intensity Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_vwap_position Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_count_imbalance Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_kyle_lambda Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_burstiness Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_volume_skew Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_volume_kurt Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_kaufman_er Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_garman_klass_vol Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_hurst Nullable(Float64) DEFAULT NULL,
--     ADD COLUMN IF NOT EXISTS intra_permutation_entropy Nullable(Float64) DEFAULT NULL;
--
-- Note: New installations do not need this migration.

-- ============================================================================
-- Migration for v12.16.x (Issue #90: INSERT block deduplication)
-- ============================================================================
-- Run this ONCE if upgrading from earlier versions with existing cache:
--
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars
--     MODIFY SETTING non_replicated_deduplication_window = 1000;
--
-- Tracks last 1000 INSERT block hashes. Retry INSERTs with same
-- insert_deduplication_token are silently dropped by ClickHouse.
-- Note: New installations do not need this migration.

-- ============================================================================
-- Migration for Issue #164 (Time-range projection + Delta codecs)
-- ============================================================================
-- Run this ONCE on existing installations:
--
-- Step 1: Add projection for flowsurface time-range queries
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars ADD PROJECTION prj_time_range
--     (SELECT * ORDER BY (symbol, threshold_decimal_bps, close_time_us));
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars MATERIALIZE PROJECTION prj_time_range;
--
-- Step 2: Add Delta+ZSTD codecs for monotonic columns (20-30% better compression)
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars
--     MODIFY COLUMN close_time_us CODEC(Delta, ZSTD),
--     MODIFY COLUMN open_time_us CODEC(Delta, ZSTD),
--     MODIFY COLUMN first_agg_trade_id CODEC(Delta, ZSTD),
--     MODIFY COLUMN last_agg_trade_id CODEC(Delta, ZSTD);
--
-- Note: New installations do not need this migration (schema includes these).

-- ============================================================================
-- Migration for Issue #158 Phase 7 (ORDER BY first_agg_trade_id for native dedup)
-- ============================================================================
-- Run this ONCE via scripts/migrate_order_by_first_agg_trade_id.py
-- OR manually on existing installations:
--
-- 1. Create new table with first_agg_trade_id ORDER BY
-- 2. Migrate rows with valid trade IDs (first_agg_trade_id > 0)
-- 3. Rename tables (atomic swap)
-- 4. Add skip index for close_time_us range queries
--
-- See scripts/migrate_order_by_first_agg_trade_id.py for full migration script.
-- Note: New installations do not need this migration.

-- ============================================================================
-- Migration for v14.x (Microsecond column rename: _ms -> _us)
-- ============================================================================
-- Run this ONCE via scripts/migrate_column_rename_us.py (dry-run by default,
-- --apply to execute) OR manually on existing installations:
--
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars
--     RENAME COLUMN close_time_us TO close_time_us;
-- ALTER TABLE opendeviationbar_cache.open_deviation_bars
--     RENAME COLUMN open_time_us TO open_time_us;
-- ALTER TABLE opendeviationbar_cache.data_freshness
--     RENAME COLUMN last_close_time_us TO last_close_time_us;
--
-- Why: Columns have stored microsecond values since the ms->us migration,
-- but retained their _ms names. This rename aligns names with actual units.
--
-- Safety: ALTER TABLE RENAME COLUMN is metadata-only in ClickHouse --
-- instant, no data rewrite. Safe for 42M+ rows. close_time_us is NOT in
-- ORDER BY (changed to first_agg_trade_id in Issue #158), so RENAME works
-- directly with no table recreation.
--
-- Note: New installations do not need this migration (schema already uses _us).

-- ============================================================================
-- Computed Open Deviation Bars Cache (Tier 2)
-- ============================================================================
-- Stores computed open deviation bars with all parameters as cache key
-- Cache hit requires exact match on: symbol, threshold, time range

CREATE TABLE IF NOT EXISTS opendeviationbar_cache.open_deviation_bars (
    -- Cache key components
    symbol LowCardinality(String),
    threshold_decimal_bps UInt32,

    -- OHLCV data
    -- Bar close time in microseconds since Unix epoch. Use intDiv(close_time_us, 1000000) for epoch seconds.
    close_time_us Int64 CODEC(Delta, ZSTD),
    -- Bar open time in microseconds since Unix epoch. Use intDiv(open_time_us, 1000000) for epoch seconds.
    open_time_us Int64 DEFAULT 0 CODEC(Delta, ZSTD),
    open Float64,
    high Float64,
    low Float64,
    close Float64,
    volume Float64,

    -- Market microstructure (from opendeviationbar-core)
    vwap Float64 DEFAULT 0,
    buy_volume Float64 DEFAULT 0,
    sell_volume Float64 DEFAULT 0,
    individual_trade_count UInt32 DEFAULT 0,
    agg_record_count UInt32 DEFAULT 0,

    -- Microstructure features (Issue #25)
    duration_us Int64 DEFAULT 0,
    ofi Float64 DEFAULT 0,
    vwap_close_deviation Float64 DEFAULT 0,
    price_impact Float64 DEFAULT 0,
    kyle_lambda_proxy Float64 DEFAULT 0,
    trade_intensity Float64 DEFAULT 0,
    volume_per_trade Float64 DEFAULT 0,
    aggression_ratio Float64 DEFAULT 0,
    aggregation_density Float64 DEFAULT 1,
    turnover_imbalance Float64 DEFAULT 0,

    -- Ouroboros (cyclical reset boundaries, v10.x)
    -- Day (UTC midnight reset) or Aion (continuous, crypto-only). Per-symbol SSoT: symbols.toml.
    ouroboros_mode LowCardinality(String) DEFAULT 'day',

    -- Exchange session flags (Issue #8: indicates active traditional market sessions)
    exchange_session_sydney UInt8 DEFAULT 0,
    exchange_session_tokyo UInt8 DEFAULT 0,
    exchange_session_london UInt8 DEFAULT 0,
    exchange_session_newyork UInt8 DEFAULT 0,

    -- Inter-bar features (Issue #59: computed from lookback window BEFORE bar opens)
    -- Tier 1: Core features (7 features)
    lookback_trade_count Nullable(UInt32) DEFAULT NULL,
    lookback_ofi Nullable(Float64) DEFAULT NULL,
    lookback_duration_us Nullable(Int64) DEFAULT NULL,
    lookback_intensity Nullable(Float64) DEFAULT NULL,
    lookback_vwap_raw Nullable(Float64) DEFAULT NULL,
    lookback_vwap_position Nullable(Float64) DEFAULT NULL,
    lookback_count_imbalance Nullable(Float64) DEFAULT NULL,
    -- Tier 2: Statistical features (5 features)
    lookback_kyle_lambda Nullable(Float64) DEFAULT NULL,
    lookback_burstiness Nullable(Float64) DEFAULT NULL,
    lookback_volume_skew Nullable(Float64) DEFAULT NULL,
    lookback_volume_kurt Nullable(Float64) DEFAULT NULL,
    lookback_price_range Nullable(Float64) DEFAULT NULL,
    -- Tier 3: Advanced features (4 features)
    lookback_kaufman_er Nullable(Float64) DEFAULT NULL,
    lookback_garman_klass_vol Nullable(Float64) DEFAULT NULL,
    lookback_hurst Nullable(Float64) DEFAULT NULL,
    lookback_permutation_entropy Nullable(Float64) DEFAULT NULL,

    -- Intra-bar features (Issue #78: computed within bar from trade-level data)
    -- ITH features (8 features)
    intra_bull_epoch_density Nullable(Float64) DEFAULT NULL,
    intra_bear_epoch_density Nullable(Float64) DEFAULT NULL,
    intra_bull_excess_gain Nullable(Float64) DEFAULT NULL,
    intra_bear_excess_gain Nullable(Float64) DEFAULT NULL,
    intra_bull_cv Nullable(Float64) DEFAULT NULL,
    intra_bear_cv Nullable(Float64) DEFAULT NULL,
    intra_max_drawdown Nullable(Float64) DEFAULT NULL,
    intra_max_runup Nullable(Float64) DEFAULT NULL,
    -- Statistical features (12 features)
    intra_trade_count Nullable(UInt32) DEFAULT NULL,
    intra_ofi Nullable(Float64) DEFAULT NULL,
    intra_duration_us Nullable(Int64) DEFAULT NULL,
    intra_intensity Nullable(Float64) DEFAULT NULL,
    intra_vwap_position Nullable(Float64) DEFAULT NULL,
    intra_count_imbalance Nullable(Float64) DEFAULT NULL,
    intra_kyle_lambda Nullable(Float64) DEFAULT NULL,
    intra_burstiness Nullable(Float64) DEFAULT NULL,
    intra_volume_skew Nullable(Float64) DEFAULT NULL,
    intra_volume_kurt Nullable(Float64) DEFAULT NULL,
    intra_kaufman_er Nullable(Float64) DEFAULT NULL,
    intra_garman_klass_vol Nullable(Float64) DEFAULT NULL,
    -- Complexity features (2 features)
    intra_hurst Nullable(Float64) DEFAULT NULL,
    intra_permutation_entropy Nullable(Float64) DEFAULT NULL,

    -- Trade ID range (Issue #72: data integrity verification)
    first_agg_trade_id Int64 DEFAULT 0 CODEC(Delta, ZSTD),
    last_agg_trade_id Int64 DEFAULT 0 CODEC(Delta, ZSTD),

    -- Bar flags (Issue #101: derived metadata, always returned)
    -- is_liquidation_cascade: True when a Binance matching-engine atomic batch
    -- collapses many fills into one bar (range >= 10x threshold, duration <= 100ms).
    -- Forensic basis: 2025-10-10 event — 24,433 individual trades all share
    -- microsecond 1760131449755735. No Binance data product can split further.
    -- Historical census (BTCUSDT@250): 427 extreme bars (0.057% of 753,113 total).
    -- 76% concentrated on 2020-03-12 (Black Thursday) and 2021-05-19 (China ban).
    is_liquidation_cascade UInt8 DEFAULT toUInt8(
        (high - low) / open >= toFloat64(threshold_decimal_bps) / 10000.0
        AND duration_us > 0
        AND duration_us <= 100000
    ),

    -- Issue #296: Orphan bar flag (incomplete bar emitted at UTC midnight reset).
    -- Set by processors during Ouroboros reset. Used by midnight preflight
    -- verification (get_yesterday_orphan_tid) to check Stathera continuity.
    is_orphan UInt8 DEFAULT 0,

    -- Cache metadata
    cache_key String,                        -- Hash of full parameters
    opendeviationbar_version String DEFAULT '',      -- Version for invalidation
    source_start_ts Int64 DEFAULT 0,         -- Input data time range
    source_end_ts Int64 DEFAULT 0,
    computed_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC')
)
ENGINE = ReplacingMergeTree(computed_at)
-- Partition by (symbol, threshold) only — 64-72 total partitions.
-- Time-range pruning handled by ORDER BY (first_agg_trade_id is time-monotonic).
-- Removing time from partition key: 192K → 64 partitions (2,600x reduction).
-- Benefits: fast mutations, efficient merges, healthy parts count, optimal FINAL.
-- DELETE for day-recompute: DELETE IN PARTITION (sym, thr) WHERE <trade-ID range>
-- ORDER BY index prunes to matching trade-ID granules — only relevant parts rewritten.
PARTITION BY (symbol, threshold_decimal_bps)
-- Issue #158 Phase 7: ORDER BY first_agg_trade_id for native ReplacingMergeTree dedup.
-- Bars with the same (symbol, threshold, ouroboros, first_agg_trade_id) from different
-- writers are automatically deduplicated on merge (keeping latest computed_at).
-- Previously used close_time_us which couldn't dedup overlapping bars from different
-- processors (they produce different close_time_us for the same trades).
ORDER BY (symbol, threshold_decimal_bps, ouroboros_mode, first_agg_trade_id)
-- NOTE: No market_type column. All current symbols are spot. Before registering
-- futures symbols, add market_type to ORDER BY to prevent first_agg_trade_id
-- collisions between independent market trade-ID sequences. Also update:
-- Redis write lock keys, CacheKey dataclass, all WHERE clauses, Stathera audit.
SETTINGS non_replicated_deduplication_window = 1000,
    deduplicate_merge_projection_mode = 'drop';

-- Issue #164: Projection for flowsurface time-range queries.
-- Primary ORDER BY uses first_agg_trade_id for Stathera dedup, but flowsurface
-- queries by close_time_us. This projection enables index-optimized time-range
-- scans without changing the primary key.
ALTER TABLE opendeviationbar_cache.open_deviation_bars ADD PROJECTION IF NOT EXISTS prj_time_range
    (SELECT * ORDER BY (symbol, threshold_decimal_bps, close_time_us));

-- ============================================================================
-- Data Freshness Tracking (Issue #163: SSE push endpoint)
-- ============================================================================
-- Sidecar updates per bar completion. Flowsurface can fast-poll this table
-- as fallback when SSE is unavailable.

CREATE TABLE IF NOT EXISTS opendeviationbar_cache.data_freshness (
    symbol LowCardinality(String),
    threshold_decimal_bps UInt32,
    last_close_time_us Int64,
    last_bar_count UInt64 DEFAULT 0,
    updated_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC')
)
ENGINE = ReplacingMergeTree(updated_at)
ORDER BY (symbol, threshold_decimal_bps);

-- ============================================================================
-- Materialized Views (Optional - for analytics)
-- ============================================================================

-- View: Daily trade volume by symbol
-- CREATE MATERIALIZED VIEW IF NOT EXISTS opendeviationbar_cache.daily_trade_volume
-- ENGINE = SummingMergeTree()
-- PARTITION BY (symbol)
-- ORDER BY (symbol, date)
-- AS SELECT
--     symbol,
--     toDate(toDateTime(timestamp_us / 1000000)) AS date,
--     count() AS trade_count,
--     sum(quantity) AS total_volume
-- FROM opendeviationbar_cache.raw_trades
-- GROUP BY symbol, date;

-- ============================================================================
-- Population Checkpoints (Issue #69: Cross-machine resume for long backfills)
-- ============================================================================
-- Stores checkpoint state for populate_cache_resumable() to enable:
-- - Cross-machine resume (continue backfill on different machine)
-- - Bar-level resumability (preserves incomplete bar state)
-- - Hybrid storage with local filesystem checkpoints
--
-- Local checkpoints: ~/.cache/opendeviationbar/checkpoints/ (fast, same-machine)
-- ClickHouse checkpoints: This table (cross-machine resume)

CREATE TABLE IF NOT EXISTS opendeviationbar_cache.population_checkpoints (
    -- Key components
    symbol LowCardinality(String),
    threshold_decimal_bps UInt32,
    start_date String,
    end_date String,

    -- Progress tracking
    last_completed_date String,
    last_trade_timestamp_us Int64,
    bars_written UInt64,

    -- Processor state for bar-level resumability (JSON serialized)
    -- Contains incomplete bar state, defer_open flag, etc.
    processor_checkpoint String DEFAULT '',

    -- Aggregate trade ID tracking (Issue #72: Full Audit Trail)
    -- Tracks agg_trade_id range in incomplete bar for resume alignment verification
    first_agg_trade_id_in_bar Int64 DEFAULT 0,
    last_agg_trade_id_in_bar Int64 DEFAULT 0,

    -- Issue #111 (Ariadne): High-water mark for deterministic resume via fromId
    -- Last fully-processed agg_trade_id (all completed bars up to this point)
    last_processed_agg_trade_id Int64 DEFAULT 0,

    -- Metadata
    include_microstructure UInt8 DEFAULT 0,
    -- Day (UTC midnight reset) or Aion (continuous, crypto-only). Per-symbol SSoT: symbols.toml.
    ouroboros_mode LowCardinality(String) DEFAULT 'day',
    created_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'),
    updated_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC')
)
ENGINE = ReplacingMergeTree(updated_at)
ORDER BY (symbol, threshold_decimal_bps, start_date, end_date, ouroboros_mode);

-- and removed. All tables now use DEFAULT 'day'.

-- ============================================================================
-- Backfill Request Queue (Issue #97: On-demand trigger from flowsurface)
-- ============================================================================
-- flowsurface INSERTs a row when it detects stale open deviation bar data.
-- opendeviationbar-py's backfill watcher polls this table and processes requests
-- using existing backfill_gap() from backfill.py.
--
-- Contract: flowsurface writes 'pending', watcher transitions to
-- 'running' → 'completed'|'failed'. TTL auto-cleans after 7 days.

CREATE TABLE IF NOT EXISTS opendeviationbar_cache.backfill_requests (
    request_id UUID DEFAULT generateUUIDv4(),
    symbol LowCardinality(String),
    threshold_decimal_bps UInt32 DEFAULT 0,
    requested_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'),
    status LowCardinality(String) DEFAULT 'pending',
    started_at Nullable(DateTime64(3, 'UTC')) DEFAULT NULL,
    completed_at Nullable(DateTime64(3, 'UTC')) DEFAULT NULL,
    bars_written UInt32 DEFAULT 0,
    gap_seconds Float64 DEFAULT 0,
    error Nullable(String) DEFAULT NULL,
    source LowCardinality(String) DEFAULT 'flowsurface',
    ouroboros_mode LowCardinality(String) DEFAULT 'day'
)
ENGINE = ReplacingMergeTree(requested_at)
ORDER BY (request_id)
TTL requested_at + INTERVAL 7 DAY;

-- ============================================================================
-- Kintsugi Repair Log (Issue #115: Self-Healing Gap Reconciliation)
-- ============================================================================
-- Tracks every gap repair operation for audit and rate-limiting.
-- Kintsugi discovers gaps ("shards") and repairs them using Ariadne (fromId
-- pagination) + Ouroboros (boundary-aware splits). TTL auto-cleans after 90 days.

CREATE TABLE IF NOT EXISTS opendeviationbar_cache.kintsugi_log (
    timestamp DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'),
    symbol LowCardinality(String),
    threshold_decimal_bps UInt32,
    priority LowCardinality(String),          -- legacy: P0/P1/P2 (priority removed in v12.56)
    gap_start_us Int64,
    gap_end_us Int64,
    gap_hours Float64,
    action LowCardinality(String),            -- backfill_gap, populate_cache, skip
    result LowCardinality(String),            -- healed, failed, timeout, skipped
    ariadne_from_id Int64 DEFAULT 0,          -- Ariadne anchor (0 = startTime fallback)
    ouroboros_splits UInt8 DEFAULT 0,          -- Number of Ouroboros boundary splits
    bars_written UInt32 DEFAULT 0,
    duration_seconds Float64 DEFAULT 0,
    error_message String DEFAULT '',
    ouroboros_mode LowCardinality(String) DEFAULT 'day'
)
ENGINE = MergeTree()
ORDER BY (timestamp, symbol, threshold_decimal_bps)
TTL timestamp + INTERVAL 90 DAY;

-- ============================================================================
-- Plugin Migration Registry (Issue #207: Plugin-managed columns)
-- ============================================================================
-- Tracks which FeatureProvider plugins have applied their schema migrations.
-- Plugins call ALTER TABLE ADD COLUMN IF NOT EXISTS on the open_deviation_bars
-- table (via plugins/migration.py). This registry provides observability:
-- query it to see which plugins have run, which columns they added, and when.
--
-- Core columns live in schema.sql above. Plugin columns are NOT listed here —
-- they are managed by each plugin via the FeatureProvider protocol.
-- See: python/opendeviationbar/plugins/CLAUDE.md for the plugin authoring guide.

CREATE TABLE IF NOT EXISTS opendeviationbar_cache.plugin_migrations (
    plugin_name LowCardinality(String),
    plugin_version String,
    columns_added Array(String),
    applied_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC')
)
ENGINE = ReplacingMergeTree(applied_at)
ORDER BY (plugin_name);

-- ============================================================================
-- Indexes (ClickHouse creates automatically based on ORDER BY)
-- ============================================================================
-- No additional indexes needed - ORDER BY creates primary key index
-- ClickHouse uses sparse indexing which is efficient for our access patterns:
-- - Symbol + time range queries
-- - Symbol + threshold lookups
