import logging
import incremental
import tzlocal
from datetime import datetime, timezone
import time
from dateutil import parser
from typing import List, Dict, Union, NoReturn, Any, Tuple
from types import MethodType
from pathlib import Path
import math
import pandas as pd
import numpy as np
import asyncio
from tabulate import tabulate
import inspect

import feedparser # RSS feed parser
from ccxt.base.exchange import Exchange as CcxtExchange
import ccxt
import ccxt.pro as ccxtpro

# https://www.analyticsvidhya.com/blog/2021/06/download-financial-dataset-using-yahoo-finance-in-python-a-complete-guide/
from yahoofinancials import YahooFinancials

# yfinance allows intervals '1m', '5m', '15m', '1h', '1d', '1wk', '1mo'. yahoofinancials not as flexible
import yfinance as yf

from siglab_py.util.retry_util import retry
from siglab_py.exchanges.futubull import Futubull
from siglab_py.exchanges.any_exchange import AnyExchange
from siglab_py.exchanges.deribit import Deribit, DeribitAsync
from siglab_py.exchanges.binance import Binance, BinanceAsync

def instantiate_exchange(
    exchange_name : str,
    api_key : Union[str, None] = None,
    secret : Union[str, None]  = None,
    passphrase : Union[str, None] = None,
    default_type : Union[str, None] = 'spot',
    default_sub_type : Union[str, None] = None,
    default_max_slippage_bps : int = 100, # This is for market orders
    rate_limit_ms : float = 100
) -> Union[AnyExchange, None]:
    exchange_name = exchange_name.lower().strip()

    # Look at ccxt exchange.describe. under 'options' \ 'defaultType' (and 'defaultSubType') for what markets the exchange support.
    # https://docs.ccxt.com/en/latest/manual.html#instantiation
    exchange_params : Dict[str, Any]= {
                        'apiKey' : api_key,
                        'secret' : secret,
                        'enableRateLimit'  : True,
                        'rateLimit' : rate_limit_ms,
                        'options' : {
                            'defaultType' : default_type,
                            'defaultSlippage' : default_max_slippage_bps
                        }
                    }
    
    if default_sub_type:
        exchange_params['defaultSubType'] = default_sub_type

    if api_key:
        exchange_params['apiKey'] = api_key
    if secret:
        exchange_params['secret'] = secret
    if passphrase:
        exchange_params['passphrase'] = passphrase

    if exchange_name=='binance':
        exchange = Binance(exchange_params)  # type: ignore
    elif exchange_name=='bybit':
        exchange = ccxt.bybit(exchange_params) # type: ignore
    elif exchange_name=='okx':
        exchange = ccxt.okx(exchange_params) # type: ignore
    elif exchange_name=='deribit':
        exchange = Deribit(exchange_params)  # type: ignore
    elif exchange_name=='hyperliquid':
        exchange = ccxt.hyperliquid(
            {
                "walletAddress" : api_key, # type: ignore
                "privateKey" : secret, 
                'enableRateLimit'  : True,
                'rateLimit' : rate_limit_ms
            }
        )
        def patch_create_order_response(
                self,            
                average_price : float,
                create_order_response : Dict,
                order_type : str = "market"
            ):
                if order_type=='market':
                    create_order_response['type'] = order_type # Hyperliquid tag market orders as limit orders (but with very wide limit prices)
                    if (
                        'average' not in create_order_response 
                        or ('average' in create_order_response and not create_order_response['average'])
                        or ('average' in create_order_response and create_order_response['average']==0)
                    ):
                        create_order_response['average'] = average_price # For market orders, they tag 'average' null

        exchange.patch_create_order_response = MethodType(
            patch_create_order_response,
            exchange
        )  
    else:
        raise ValueError(f"Unsupported exchange {exchange_name}.")

    exchange.load_markets() # type: ignore

    if not hasattr(exchange, "patch_create_order_response"):
        def default_patch_create_order_response(
            self,
            average_price : float,
            create_order_response : Dict,
            order_type : str = "market"
        ):
            pass

        exchange.patch_create_order_response = MethodType(
            default_patch_create_order_response,
            exchange
        )

    return exchange # type: ignore

async def async_instantiate_exchange(
    gateway_id : str,
    api_key : str,
    secret : str,
    passphrase : str,
    default_type : Union[str, None] = 'spot',
    default_sub_type : Union[str, None] = None,
    default_max_slippage_bps : int = 100, # This is for market orders
    rate_limit_ms : float = 100,
    verbose : bool = False
) -> Union[AnyExchange, None]:
    exchange : Union[AnyExchange, None] = None
    exchange_name : str = gateway_id.split('_')[0]
    exchange_name =exchange_name.lower().strip()

    # Look at ccxt exchange.describe. under 'options' \ 'defaultType' (and 'defaultSubType') for what markets the exchange support.
    # https://docs.ccxt.com/en/latest/manual.html#instantiation
    exchange_params : Dict[str, Any]= {
                        'apiKey' : api_key,
                        'secret' : secret,
                        'enableRateLimit'  : True,
                        'rateLimit' : rate_limit_ms,
                        'options' : {
                            'defaultType' : default_type,
                            'defaultSlippage' : default_max_slippage_bps
                        },
                        'verbose': verbose
                    }
    
    if default_sub_type:
        exchange_params['defaultSubType'] = default_sub_type

    if exchange_name=='binance':
        # spot, future, margin, delivery, option
        # https://github.com/ccxt/ccxt/blob/master/python/ccxt/binance.py#L1298
        exchange = BinanceAsync(exchange_params)  # type: ignore
    elif exchange_name=='bybit':
        # spot, linear, inverse, futures
        # https://github.com/ccxt/ccxt/blob/master/python/ccxt/bybit.py#L1041
        exchange = ccxtpro.bybit(exchange_params) # type: ignore
    elif exchange_name=='okx':
        # 'funding', spot, margin, future, swap, option
        # https://github.com/ccxt/ccxt/blob/master/python/ccxt/okx.py#L1144
        exchange_params['password'] = passphrase
        exchange = ccxtpro.okx(exchange_params) # type: ignore
    elif exchange_name=='deribit':
        # spot, swap, future
        # https://github.com/ccxt/ccxt/blob/master/python/ccxt/deribit.py#L360
        exchange = DeribitAsync(exchange_params)  # type: ignore
    elif exchange_name=='kraken':
        exchange = ccxtpro.kraken(exchange_params) # type: ignore
    elif exchange_name=='hyperliquid':
        '''
        https://app.hyperliquid.xyz/API
        
        defaultType from ccxt: swap
            https://github.com/ccxt/ccxt/blob/master/python/ccxt/hyperliquid.py#L225
        
        How to integrate? You can skip first 6 min: https://www.youtube.com/watch?v=UuBr331wxr4&t=363s

        Example, 
            API credentials created under "\ More \ API":
                    Ledger Arbitrum Wallet Address: 0xAAAAA <-- This is your Ledger Arbitrum wallet address with which you connect to Hyperliquid. 
                    API Wallet Address 0xBBBBB <-- Generated
                    privateKey 0xCCCCC

        Basic connection via CCXT:
            import asyncio
            import ccxt.pro as ccxtpro

            async def main():
                rate_limit_ms = 100
                exchange_params = {
                    "walletAddress" : "0xAAAAA", # Ledger Arbitrum Wallet Address here! Not the generated address.
                    "privateKey" : "0xCCCCC"
                }
                exchange = ccxtpro.hyperliquid(exchange_params) 
                balances = await exchange.fetch_balance()
                print(balances)

            asyncio.run(main())
        '''
        exchange = ccxtpro.hyperliquid(
            {
                "walletAddress" : api_key,
                "privateKey" : secret,
                'enableRateLimit'  : True,
                'rateLimit' : rate_limit_ms,
                'verbose': verbose
            }  # type: ignore
        )
        def patch_create_order_response(
                self,
                average_price : float,
                create_order_response : Dict,
                order_type : str = "market"
            ):
                if order_type=='market':
                    create_order_response['type'] = order_type # Hyperliquid tag market orders as limit orders (but with very wide limit prices)
                    if order_type=='market':
                        create_order_response['type'] = order_type # Hyperliquid tag market orders as limit orders (but with very wide limit prices)
                        if (
                            'average' not in create_order_response 
                            or ('average' in create_order_response and not create_order_response['average'])
                            or ('average' in create_order_response and create_order_response['average']==0)
                        ):
                            create_order_response['average'] = average_price # For market orders, they tag 'average' null

        exchange.patch_create_order_response = MethodType(
            patch_create_order_response,
            exchange
        )
    else:
        raise ValueError(f"Unsupported exchange {exchange_name}, check gateway_id {gateway_id}.")

    await exchange.load_markets() # type: ignore

    if not hasattr(exchange, "patch_create_order_response"):
        def default_patch_create_order_response(
            self,
            average_price : float,
            create_order_response : Dict,
            order_type : str = "market"
        ):
            pass

        exchange.patch_create_order_response = MethodType(
            default_patch_create_order_response,
            exchange
        )

    '''
    Is this necessary? The added trouble is for example bybit.authenticate requires arg 'url'. binance doesn't. And fetch_balance already test credentials.

    try:
        await exchange.authenticate() # type: ignore
    except Exception as swallow_this_error:
        pass
    '''

    return exchange

def timestamp_to_datetime_cols(pd_candles : pd.DataFrame):
    def _fix_timestamp_ms(x):
        if isinstance(x, pd.Timestamp):
            return int(x.value // 10**6)
        elif isinstance(x, np.datetime64):
            return int(x.astype('int64') // 10**6)
        elif isinstance(x, (int, float)):
            x = int(x)
            if len(str(abs(x))) == 13:
                return x
            else:
                return int(x * 1000)
        else:
            raise ValueError(f"Unsupported type {type(x)} for timestamp conversion")
    pd_candles['timestamp_ms'] = pd_candles['timestamp_ms'].apply(_fix_timestamp_ms)
    pd_candles['datetime'] = pd_candles['timestamp_ms'].apply(lambda x: datetime.fromtimestamp(int(x/1000)))
    pd_candles['datetime'] = pd.to_datetime(pd_candles['datetime'])
    pd_candles['datetime'] = pd_candles['datetime'].dt.tz_localize(None)  # type: ignore
    pd_candles['datetime_utc'] = pd_candles['timestamp_ms'].apply(
        lambda x: datetime.fromtimestamp(int(x.timestamp()) if isinstance(x, pd.Timestamp) else int(x / 1000), tz=timezone.utc)
    )
    
    # This is to make it easy to do grouping with Excel pivot table
    pd_candles['year'] = pd_candles['datetime'].dt.year  # type: ignore
    pd_candles['month'] = pd_candles['datetime'].dt.month  # type: ignore
    pd_candles['day'] = pd_candles['datetime'].dt.day  # type: ignore
    pd_candles['hour'] = pd_candles['datetime'].dt.hour  # type: ignore
    pd_candles['minute'] = pd_candles['datetime'].dt.minute  # type: ignore
    pd_candles['dayofweek'] = pd_candles['datetime'].dt.dayofweek  # type: ignore dayofweek: Monday is 0 and Sunday is 6

    pd_candles['week_of_month'] = pd_candles['timestamp_ms'].apply(
        lambda x: timestamp_to_week_of_month(int(x/1000))
    )

    pd_candles['apac_trading_hr'] = pd_candles['timestamp_ms'].apply(
        lambda x: "APAC" in timestamp_to_active_trading_regions(int(x/1000))
    )
    pd_candles['emea_trading_hr'] = pd_candles['timestamp_ms'].apply(
        lambda x: "EMEA" in timestamp_to_active_trading_regions(int(x/1000))
    )
    pd_candles['amer_trading_hr'] = pd_candles['timestamp_ms'].apply(
        lambda x: "AMER" in timestamp_to_active_trading_regions(int(x/1000))
    )

    pd_candles['timestamp_ms_gap'] = pd_candles['timestamp_ms'] - pd_candles['timestamp_ms'].shift(1)
    
    # Depending on asset, minutes bar may have gaps
    timestamp_ms_gap_median = pd_candles['timestamp_ms_gap'].median()
    NUM_MS_IN_1HR = 60*60*1000
    if timestamp_ms_gap_median>=NUM_MS_IN_1HR:
        num_rows_with_expected_gap = pd_candles[~pd_candles.timestamp_ms_gap.isna()][pd_candles.timestamp_ms_gap==timestamp_ms_gap_median].shape[0]
        assert(num_rows_with_expected_gap/pd_candles.shape[0]>0.9)
    pd_candles.drop(columns=['timestamp_ms_gap'], inplace=True)

'''
APAC (Asia-Pacific) Trading Hours
    UTC 21:00 - 09:00 (approximate range)
    Major financial centers: Tokyo, Hong Kong, Singapore, Sydney

EMEA (Europe, Middle East, Africa) Trading Hours
    UTC 07:00 - 16:00 (approximate range)
    Major financial centers: London, Frankfurt, Paris, Zurich, Dubai

US Trading Hours
    UTC 13:00 - 22:00 (approximate range)
    Major financial centers: New York, Chicago
    Key markets: NYSE, NASDAQ

utcnow and utcfromtimestamp been deprecated in Python 3.12 
https://www.pythonmorsels.com/converting-to-utc-time/

Example, UTC 23:00 is 3rd hour in APAC trading session
    utc_hour = 23
    i = get_regions_trading_utc_hours()['APAC'].index(utc_hour)
    assert(i==2)
'''
def get_regions_trading_utc_hours():
    return {
        'APAC' : [21,22,23,0,1,2,3,4,5,6,7,8,9],
        'EMEA' : [7,8,9,10,11,12,13,14,15,16],
        'AMER' : [13,14,15,16,17,18,19,20,21,22]
    }
			
def timestamp_to_active_trading_regions(
        timestamp_ms : int
) -> List[str]:
    active_trading_regions : List[str] = []

    dt_utc = datetime.fromtimestamp(int(timestamp_ms / 1000), tz=timezone.utc)
    utc_hour = dt_utc.hour
    if utc_hour in get_regions_trading_utc_hours()['APAC']:
        active_trading_regions.append("APAC") 

    if utc_hour in get_regions_trading_utc_hours()['EMEA']:
        active_trading_regions.append("EMEA")

    if utc_hour in get_regions_trading_utc_hours()['AMER']:
        active_trading_regions.append("AMER")

    return active_trading_regions

def timestamp_to_week_of_month(timestamp_ms: int) -> int:
    """
    Returns:
        int: Week of the month (0 = first week, 1 = second week, etc.).
    """
    dt = datetime.fromtimestamp(timestamp_ms / 1000)
    day_of_month = dt.day
    week_of_month = (day_of_month - 1) // 7
    return week_of_month

def fix_column_types(pd_candles : pd.DataFrame):
    pd_candles['open'] = pd_candles['open'].astype(float)
    pd_candles['high'] = pd_candles['high'].astype(float)
    pd_candles['low'] = pd_candles['low'].astype(float)
    pd_candles['close'] = pd_candles['close'].astype(float)
    pd_candles['volume'] = pd_candles['volume'].astype(float)

    timestamp_to_datetime_cols(pd_candles)

    '''
    The 'Unnamed: 0', 'Unnamed : 1'... etc columns often appears in a DataFrame when it is saved to a file (e.g., CSV or Excel) and later loaded. 
    This usually happens if the DataFrame's index was saved along with the data, and then pandas automatically treats it as a column during the file loading process.
    We want to drop them as it'd mess up idmin, idmax calls, which will take values from 'Unnamed' instead of actual pandas index.
    '''
    pd_candles.drop(pd_candles.columns[pd_candles.columns.str.contains('unnamed',case = False)],axis = 1, inplace = True)
    pd_candles.reset_index(drop=True, inplace=True)
    pd_candles.sort_values("datetime", inplace=True)

def interval_to_ms(interval : str) -> int:
    interval_ms : int = 0
    if interval=="d":
        interval_ms = 24*60*60*1000
    elif interval=="h":
        interval_ms = 60*60*1000
    elif interval=="m":
        interval_ms = 60*1000

    return interval_ms

'''
https://polygon.io/docs/stocks
'''
class PolygonMarketDataProvider:
    pass

class NASDAQExchange:
    def __init__(self, data_dir : Union[str, None]) -> None:
        if data_dir:
            self.data_dir = data_dir
        else:
            self.data_dir = Path(__file__).resolve().parents[2] / "data/nasdaq" 

    def fetch_ohlcv(
        self,
        symbol : str,
        since : int,
        timeframe : str,
        limit : int = 1
    ) -> List:
        pd_candles = self.fetch_candles(
            symbols=[symbol],
            start_ts=int(since/1000),
            end_ts=None,
            candle_size=timeframe
        )[symbol]
        if pd_candles is not None:
            return pd_candles.values.tolist()
        else:
            return []
    
    def fetch_candles(
        self,
        start_ts,
        end_ts,
        symbols,
        candle_size
    ) -> Dict[str, Union[pd.DataFrame, None]]:
        exchange_candles : Dict[str, Union[pd.DataFrame, None]] = {}

        start_date = datetime.fromtimestamp(start_ts)
        end_date = datetime.fromtimestamp(end_ts) if end_ts else None
        start_date_str = start_date.strftime('%Y-%m-%d')
        end_date_str = end_date.strftime('%Y-%m-%d') if end_date else None
        local_tz = datetime.now().astimezone().tzinfo

        for symbol in symbols:
            # CSV from NASDAQ: https://www.nasdaq.com/market-activity/quotes/historical
            pd_daily_candles = pd.read_csv(f"{self.data_dir}\\NASDAQ_hist_{symbol.replace('^','')}.csv")
            pd_daily_candles.rename(columns={'Date' : 'datetime', 'Open': 'open', 'High': 'high', 'Low': 'low', 'Close/Last' : 'close', 'Adj Close' : 'adj_close', 'Volume' : 'volume' }, inplace=True)
            pd_daily_candles['open'] = pd_daily_candles['open'].astype(str).str.replace('$','')
            pd_daily_candles['high'] = pd_daily_candles['high'].astype(str).str.replace('$','')
            pd_daily_candles['low'] = pd_daily_candles['low'].astype(str).str.replace('$','')
            pd_daily_candles['close'] = pd_daily_candles['close'].astype(str).str.replace('$','')
            pd_daily_candles['datetime']= pd.to_datetime(pd_daily_candles['datetime'])
            pd_daily_candles['timestamp_ms'] = pd_daily_candles.datetime.values.astype(np.int64) // 10 ** 6  # type: ignore
            pd_daily_candles['symbol'] = symbol
            pd_daily_candles['exchange'] = 'nasdaq'
            fix_column_types(pd_daily_candles)

            if candle_size=="1h":
                # Fill forward (i.e. you dont actually have hourly candles)
                start = pd_daily_candles["datetime"].min().normalize()
                end = pd_daily_candles["datetime"].max().normalize() + pd.Timedelta(days=1)
                hourly_index = pd.date_range(start=start, end=end, freq="h") # FutureWarning: 'H' is deprecated and will be removed in a future version, please use 'h' instead.
                pd_hourly_candles = pd.DataFrame({"datetime": hourly_index})
                pd_hourly_candles = pd.merge_asof(
                    pd_hourly_candles.sort_values("datetime"),
                    pd_daily_candles.sort_values("datetime"),
                    on="datetime",
                    direction="backward"
                )

                # When you fill foward, a few candles before start date can have null values (open, high, low, close, volume ...)
                first_candle_dt = pd_hourly_candles[(~pd_hourly_candles.close.isna())  & (pd_hourly_candles['datetime'].dt.time == pd.Timestamp('00:00:00').time())].iloc[0]['datetime']  # type: ignore
                pd_hourly_candles = pd_hourly_candles[pd_hourly_candles.datetime>=first_candle_dt]
                exchange_candles[symbol] = pd_hourly_candles

            elif candle_size=="1d":
                exchange_candles[symbol] = pd_daily_candles

        return exchange_candles

class YahooExchange:
    def fetch_ohlcv(
        self,
        symbol : str,
        since : int,
        timeframe : str,
        limit : int = 1
    ) -> List:
        pd_candles = self.fetch_candles(
            symbols=[symbol],
            start_ts=int(since/1000),
            end_ts=None,
            candle_size=timeframe
        )[symbol]
        if pd_candles is not None:
            return pd_candles.values.tolist()
        else:
            return []

    def fetch_candles(
        self,
        start_ts,
        end_ts,
        symbols,
        candle_size
    ) -> Dict[str, Union[pd.DataFrame, None]]:
        exchange_candles : Dict[str, Union[pd.DataFrame, None]] = {}

        start_date = datetime.fromtimestamp(start_ts)
        end_date = datetime.fromtimestamp(end_ts)
        start_date_str = start_date.strftime('%Y-%m-%d')
        end_date_str = end_date.strftime('%Y-%m-%d')
        local_tz = datetime.now().astimezone().tzinfo

        for symbol in symbols:
            # From yf, "DateTime" in UTC
            # The requested range must be within the last 730 days. Otherwise API will return empty DataFrame.
            pd_candles = yf.download(tickers=symbol, start=start_date_str, end=end_date_str, interval=candle_size)
            pd_candles.reset_index(inplace=True) # type: ignore
            pd_candles.rename(columns={ 'Date' : 'datetime', 'Datetime' : 'datetime', 'Open': 'open', 'High': 'high', 'Low': 'low', 'Close' : 'close', 'Adj Close' : 'adj_close', 'Volume' : 'volume' }, inplace=True) # type: ignore
            pd_candles['datetime'] = pd.to_datetime(pd_candles['datetime']) # type: ignore
            if pd_candles['datetime'].dt.tz is None: # type: ignore
                pd_candles['datetime'] = pd.to_datetime(pd_candles['datetime']).dt.tz_localize('UTC') # type: ignore
            pd_candles['datetime'] = pd_candles['datetime'].dt.tz_convert(local_tz) # type: ignore
            pd_candles['datetime'] = pd_candles['datetime'].dt.tz_localize(None)  # type: ignore
            pd_candles['timestamp_ms'] = pd_candles.datetime.values.astype(np.int64) // 10**6 # type: ignore
            pd_candles = pd_candles.sort_values(by=['timestamp_ms'], ascending=[True]) # type: ignore
            
            fix_column_types(pd_candles)
            pd_candles['symbol'] = symbol
            pd_candles['exchange'] = 'yahoo'
            exchange_candles[symbol] = pd_candles

        return exchange_candles

def aggregate_candles(
    interval : str,
    pd_candles : pd.DataFrame
) -> pd.DataFrame:
    if interval[-1]=='m':
        # 'm' for pandas means months!
        interval = interval.replace('m','min')
    pd_candles.set_index('datetime', inplace=True)
    pd_candles_aggregated = pd_candles.resample(interval).agg({
        'exchange' : 'first',
        'symbol' : 'first',
        'timestamp_ms' : 'first',
        
        'open': 'first',
        'high': 'max',
        'low': 'min',
        'close': 'last',
        'volume': 'sum',

        'datetime_utc' : 'first',
        'year' : 'first',
        'month' : 'first',
        'day' : 'first',
        'hour' : 'first',
        'minute' : 'first',
        'dayofweek' : 'first',
        'week_of_month' : 'first',

        'apac_trading_hr' : 'first',
        'emea_trading_hr' : 'first',
        'amer_trading_hr' : 'first',

        'pct_chg_on_close' : 'sum',

    })
    pd_candles.reset_index(inplace=True)
    pd_candles_aggregated.reset_index(inplace=True)
    return pd_candles_aggregated
    
def fetch_historical_price(
    exchange,
    normalized_symbol : str,
    timestamp_ms : int,
    ref_timeframe : str = '1m'
):
    one_candle = fetch_ohlcv_one_candle(exchange=exchange, normalized_symbol=normalized_symbol, timestamp_ms=timestamp_ms, ref_timeframe=ref_timeframe)
    reference_price = abs(one_candle['close'] + one_candle['open'])/2 if one_candle else None
    return reference_price

def fetch_ohlcv_one_candle(
    exchange,
    normalized_symbol : str,
    timestamp_ms : int,
    ref_timeframe : str = '1m'
):
    candles = exchange.fetch_ohlcv(symbol=normalized_symbol, since=int(timestamp_ms), timeframe=ref_timeframe, limit=1)
    one_candle = {
            'timestamp_ms' : candles[0][0],
            'open' : candles[0][1],
            'high' : candles[0][2],
            'low' : candles[0][3],
            'close' : candles[0][4],
            'volume' : candles[0][5] 
        } if candles and len(candles)>0 else None
    
    return one_candle
    
def fetch_candles(
    start_ts, # in sec
    end_ts, # in sec
    exchange,
    normalized_symbols,
    candle_size,

    logger = None,

    num_candles_limit : int = 100,

    ticker_change_map : List[Dict[str, Union[str, int]]] = [],

    cache_dir : Union[str, None] = None,

    list_ts_field : Union[str, None] = None,

    validation_max_gaps : int = 10,
    validation_max_end_date_intervals : int = 1
) -> Dict[str, Union[pd.DataFrame, None]]:
    exchange_candles = { '' : None }
    num_intervals = int(candle_size.replace(candle_size[-1],''))

    if end_ts>datetime.now().timestamp():
        end_ts = int(datetime.now().timestamp())

    if type(exchange) is YahooExchange:
        exchange_candles = exchange.fetch_candles(
                            start_ts=start_ts,
                            end_ts=end_ts,
                            symbols=normalized_symbols,
                            candle_size=candle_size
                        )
    elif type(exchange) is NASDAQExchange:
        exchange_candles = exchange.fetch_candles(
                            start_ts=start_ts,
                            end_ts=end_ts,
                            symbols=normalized_symbols,
                            candle_size=candle_size
                        )
    elif type(exchange) is Futubull:
        exchange_candles = exchange.fetch_candles(
                            start_ts=start_ts,
                            end_ts=end_ts,
                            symbols=normalized_symbols,
                            candle_size=candle_size
                        )
        for symbol in exchange_candles:
            pd_candles = exchange_candles[symbol]
            if not pd_candles is None:
                fix_column_types(pd_candles) # You don't want to do this from Futubull as you'd need import Futubull from there: Circular references
                
    elif issubclass(exchange.__class__, CcxtExchange):
        exchange_candles = _fetch_candles_ccxt(
            start_ts=start_ts,
            end_ts=end_ts,
            exchange=exchange,
            normalized_symbols=normalized_symbols,
            candle_size=candle_size,
            num_candles_limit=num_candles_limit,
            ticker_change_map=ticker_change_map,
            logger=logger
        )
    if num_intervals!=1:
        for symbol in exchange_candles:
            if not exchange_candles[symbol] is None:
                exchange_candles[symbol] = aggregate_candles(candle_size, exchange_candles[symbol]) #  type: ignore

    # For invalid rows missing timestamps, o/h/l/c/v, fill forward close, set volume to zero.
    for symbol in exchange_candles:
        pd_candles = exchange_candles[symbol]
        
        if pd_candles is not None:
            mask_invalid_candles = pd_candles["timestamp_ms"].isna()
            if mask_invalid_candles.any():
                pd_invalid_candles = pd_candles[mask_invalid_candles]

                if logger is not None:
                    logger.warning(f"Dropping {pd_invalid_candles.shape[0]}/{pd_candles.shape[0]} rows from {symbol} candles (null timestamp_ms)") # type: ignore
                    logger.warning(f"{tabulate(pd_invalid_candles, headers='keys', tablefmt='psql')}") # type: ignore
                    
                def _to_timestamp_ms(dt):
                    if pd.isna(dt):
                        return pd.NA
                    if isinstance(dt, str):
                        dt = pd.to_datetime(dt)
                    return int(dt.timestamp() * 1000)
                
                pd_candles.loc[mask_invalid_candles, "timestamp_ms"] = pd_candles.loc[mask_invalid_candles, "datetime"].apply(_to_timestamp_ms)

                pd_candles["close"] = pd_candles["close"].ffill()
                pd_candles.loc[mask_invalid_candles, ["open", "high", "low"]] = pd_candles.loc[
                                                                                    mask_invalid_candles, ["close"]
                                                                                ]
                pd_candles.loc[mask_invalid_candles, "volume"] = 0.0

    return exchange_candles # type: ignore

'''
Find listing date https://gist.github.com/mr-easy/5185b1dcdd5f9f908ff196446f092e9b

Usage:
    listing_ts = find_start_time(exchange, 'HYPE/USDT:USDT', int(datetime(2024,1,1).timestamp()*1000), int(datetime(2025,5,1).timestamp()*1000), '1h')

Caveats: 
1) If listing date lies outside [start_time, end_time], this function will stackoverflow, 
2) Even if not, it's still very time consuming.

Alternative: market['created']
'''
def search_listing_ts(exchange, symbol, start_time, end_time, timeframe):
    mid_time = (start_time + end_time)//2
    if(mid_time == start_time): return mid_time+1
    ohlcv = exchange.fetch_ohlcv(symbol, timeframe, mid_time, limit=1)
    time.sleep(1)
    if(len(ohlcv) == 0):
        return search_listing_ts(exchange, symbol, mid_time, end_time, timeframe)
    else:
        return search_listing_ts(exchange, symbol, start_time, mid_time, timeframe)
    
def _fetch_candles_ccxt(
    start_ts : int,
    end_ts : int,
    exchange,
    normalized_symbols : List[str],
    candle_size : str,
    num_candles_limit : int = 100,
    ticker_change_map : List[Dict[str, Union[str, int]]] = [],
    logger = None
) -> Dict[str, Union[pd.DataFrame, None]]:
    rsp = {}

    exchange.load_markets()
    
    num_tickers = len(normalized_symbols)
    i = 0
    for ticker in normalized_symbols:
        old_ticker = get_old_ticker(ticker, ticker_change_map)
        ticker_change_mapping = get_ticker_map(ticker, ticker_change_map)

        @retry(num_attempts=3, pause_between_retries_ms=1000, logger=logger)
        def _fetch_ohlcv(exchange, symbol, timeframe, since, limit, params) -> Union[List, NoReturn]:
                one_timeframe = f"1{timeframe[-1]}"
                candles = exchange.fetch_ohlcv(symbol=symbol, timeframe=one_timeframe, since=since, limit=limit, params=params)
                if candles and len(candles)>0:
                    candles.sort(key=lambda x : x[0], reverse=False)

                return candles
            
        def _calc_increment(candle_size):
            increment = 1
            num_intervals = int(candle_size.replace(candle_size[-1],''))
            interval_type = candle_size[-1]
            if interval_type == "m":
                increment = 60
            elif interval_type == "h":
                increment = 60*60
            elif interval_type == "d":
                increment = 60*60*24
            else:
                raise ValueError(f"Invalid candle_size {candle_size}")
            return num_intervals * increment
        
        if logger:
            logger.info(f"{i}/{num_tickers} Fetching {candle_size} candles for {ticker}.")

        '''
        It uses a while loop to implement a sliding window to download candles between start_ts and end_ts. 
        However, start_ts for example can be 1 Jan 2021 for a given ticker. 
        But if that ticker listing date is 1 Jan 2025, this while loop would waste a lot of time loop between 1 Jan 2021 thru 31 Dec 2024, slowly incrementing this_cutoff += _calc_increment(candle_size).
        A more efficient way is to find listing date. Start looping from there.
        '''
        market = exchange.markets[ticker] if ticker in exchange.markets else None
        if not market:
            market = exchange.markets[old_ticker] if old_ticker else None
            if not market:
                raise ValueError(f"market {ticker} not support by exchange {exchange.name}!")

        this_ticker_start_ts = start_ts
        if market['created']:
            this_ticker_start_ts = max(this_ticker_start_ts, int(market['created']/1000))

        all_candles = []
        params = {}
        this_cutoff = this_ticker_start_ts
        while this_cutoff<end_ts:
            _ticker = ticker
            if ticker_change_mapping:
                ticker_change_cutoff_sec = int(ticker_change_mapping['cutoff_ms']) / 1000
                if this_cutoff<ticker_change_cutoff_sec:
                    _ticker = old_ticker
            candles = _fetch_ohlcv(exchange=exchange, symbol=_ticker, timeframe=candle_size, since=int(this_cutoff * 1000), limit=num_candles_limit, params=params)
            if candles and len(candles)>0:
                all_candles = all_candles + [[ int(x[0]), float(x[1]), float(x[2]), float(x[3]), float(x[4]), float(x[5]) ] for x in candles if x[1] and x[2] and x[3] and x[4] and x[5] ]

                record_ts = max([int(record[0]) for record in candles])
                record_ts_str : str = str(record_ts)
                if len(record_ts_str)==13:
                    record_ts = int(int(record_ts_str)/1000) # Convert from milli-seconds to seconds
                
                this_cutoff = record_ts  + _calc_increment(candle_size)
            else:
                this_cutoff += _calc_increment(candle_size)

        columns = ['exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume']
        pd_all_candles = pd.DataFrame([ [ exchange.name, ticker, x[0], x[1], x[2], x[3], x[4], x[5] ] for x in all_candles], columns=columns)
        fix_column_types(pd_all_candles)
        pd_all_candles['pct_chg_on_close'] = pd_all_candles['close'].pct_change()

        rsp[ticker] = pd_all_candles

        i+=1

    return rsp

def fetch_deribit_btc_option_expiries(
    market: str = 'BTC'
) -> Dict[
    str, Union[
        Dict[str, float],
        Dict[str, Dict[str, Union[str, float]]]
    ]
]:
    exchange = ccxt.deribit()
    instruments = exchange.public_get_get_instruments({
        'currency': market,
        'kind': 'option',
        # 'expired': 'true'
    })['result']
    
    index_price = exchange.public_get_get_index_price({
        'index_name': f"{market.lower()}_usd"
    })['result']['index_price']
    index_price = float(index_price)
    
    expiry_data : Dict[str, float] = {}
    expiry_data_breakdown_by_strike : Dict[str, Dict] = {}
    for instrument in instruments:
        expiry_timestamp = int(instrument["expiration_timestamp"]) / 1000
        expiry_date = datetime.utcfromtimestamp(expiry_timestamp)

        strike = float(instrument['strike'])

        option_type = instrument['instrument_name'].split('-')[-1]  # Last part is 'C' or 'P'
        is_call = option_type == 'C'
    
        ticker = exchange.public_get_ticker({
            'instrument_name': instrument['instrument_name']
        })['result']
        
        open_interest = ticker.get("open_interest", 0)  # Open interest in BTC
        open_interest = float(open_interest)
        notional_value : float = open_interest * index_price  # Convert to USD
        
        expiry_str : str = expiry_date.strftime("%Y-%m-%d")
        if expiry_str not in expiry_data:
            expiry_data[expiry_str] = 0
        expiry_data[expiry_str] += notional_value

        if f"{expiry_str}-{strike}" not in expiry_data_breakdown_by_strike:
            expiry_data_breakdown_by_strike[f"{expiry_str}-{strike}"] = {
                'expiry' : expiry_str,
                'strike' : strike,
                'option_type': 'call' if is_call else 'put',
                'notional_value' : notional_value
            }
        else:
            expiry_data_breakdown_by_strike[f"{expiry_str}-{strike}"]['notional_value'] += notional_value
    
    sorted_expiry_data = sorted(expiry_data.items())

    return {
        'index_price' : index_price,
        'by_expiry' : sorted_expiry_data, # type: ignore Otherwise, Error: Type "dict[str, list[tuple[str, float]] | dict[str, Dict[Unknown, Unknown]]]" is not assignable to return type "Dict[str, Dict[str, float] | Dict[str, Dict[str, str | float]]]"
        'by_expiry_and_strike' : expiry_data_breakdown_by_strike
    }

def fetch_funding_rate(
    exchange,
    normalized_symbols : List[str],
    start_ts : int,
    end_ts : int,
    limit : int = 100
) -> Dict[str, pd.DataFrame]:
    results : Dict = {}
    markets = exchange.load_markets()

    funding_rate_annualized_buckets = {
        '< -15' : (float('-inf'), -15),
        '-15 - -10' : (-15, -10),
        '-10 - -8' : (-10, -8),
        '-8 - -5' : (-8, 5),
        '-5 - 0' : (-5, 0),
        '0 - 5' : (0, 5),
        '5 - 8' : (5, 8), 
        '8 - 10' : (8, 10),
        '10  - 15' : (10, 15),
        '> 15' : (15, float('inf'))
    }
    for ticker in normalized_symbols:
        market = exchange.markets[ticker] if ticker in markets else None
        this_ticker_start_ts = start_ts
        if market and market['created']:
            this_ticker_start_ts = max(this_ticker_start_ts, int(market['created']/1000))
        all_funding = []
        params = {}
        this_cutoff = this_ticker_start_ts
        while this_cutoff < end_ts:
            @retry(num_attempts=3, pause_between_retries_ms=1000, logger=None)
            def _fetch_funding_rate_history(exchange, symbol, since, limit, params):
                return exchange.fetchFundingRateHistory(symbol=symbol, since=since, limit=limit, params=params)
            funding = _fetch_funding_rate_history(exchange=exchange, symbol=ticker, since=int(this_cutoff * 1000), limit=limit, params=params)
            if funding and len(funding)>0:
                all_funding = all_funding + funding
                record_ts = max([int(entry['timestamp']) for entry in funding])
                record_ts_str : str = str(record_ts)
                if len(record_ts_str)==13:
                    record_ts = int(record_ts / 1000)
                this_cutoff = record_ts + 1
            else:
                break
        funding_history = [{
            'datetime_utc': datetime.fromtimestamp(int(entry['timestamp']/1000), tz=timezone.utc),
            'timestamp_ms': entry['timestamp'],
            'funding_rate_interval': round(entry['fundingRate'] * 100, 2),
            'funding_rate_annualized': round(entry['fundingRate'] * 100 * 3 * 365, 2),
        } for entry in all_funding]
        pd_funding_history = pd.DataFrame(funding_history)
        pd_funding_history['funding_rate_annualized_bucket'] = pd_funding_history['funding_rate_annualized'].apply(lambda x: next((k for k, (lo, hi) in funding_rate_annualized_buckets.items() if lo <= x < hi), None))
        pd_funding_history['datetime_utc'] = pd_funding_history['datetime_utc'].dt.tz_convert(None)
        results[ticker] = pd_funding_history
    return results

def build_pair_candles(
    pd_candles1 : pd.DataFrame,
    pd_candles2 : pd.DataFrame,
    left_columns_postfix : str = "_1",
    right_columns_postfix : str = "_2"
) -> pd.DataFrame:
    min_timestamp_ms1 = int(pd_candles1.iloc[0]['timestamp_ms'])
    max_timestamp_ms1 = int(pd_candles1.iloc[-1]['timestamp_ms'])
    min_timestamp_ms2 = int(pd_candles2.iloc[0]['timestamp_ms'])
    max_timestamp_ms2 = int(pd_candles2.iloc[-1]['timestamp_ms'])

    pd_candles1 = pd_candles1[(pd_candles1.timestamp_ms>=min_timestamp_ms2) & (pd_candles1.timestamp_ms<=max_timestamp_ms2) & (~pd_candles1.timestamp_ms.isna()) ]
    pd_candles2 = pd_candles2[(pd_candles2.timestamp_ms>=min_timestamp_ms1) & (pd_candles2.timestamp_ms<=max_timestamp_ms1) & (~pd_candles2.timestamp_ms.isna())]
    assert(pd_candles1.shape[0]==pd_candles2.shape[0])

    pd_candles1['timestamp_ms_gap'] = pd_candles1['timestamp_ms'] - pd_candles1['timestamp_ms'].shift(1)
    timestamp_ms_gap = pd_candles1.iloc[-1]['timestamp_ms_gap']
    
    assert(pd_candles1[~pd_candles1.timestamp_ms_gap.isna()][pd_candles1.timestamp_ms_gap!=timestamp_ms_gap].shape[0]==0)
    pd_candles1.drop(columns=['timestamp_ms_gap'], inplace=True)

    pd_candles2['timestamp_ms_gap'] = pd_candles2['timestamp_ms'] - pd_candles2['timestamp_ms'].shift(1)
    timestamp_ms_gap = pd_candles2.iloc[-1]['timestamp_ms_gap']
    assert(pd_candles2[~pd_candles2.timestamp_ms_gap.isna()][pd_candles2.timestamp_ms_gap!=timestamp_ms_gap].shape[0]==0)
    pd_candles2.drop(columns=['timestamp_ms_gap'], inplace=True)

    min_timestamp_ms1 = int(pd_candles1.iloc[0]['timestamp_ms'])
    max_timestamp_ms1 = int(pd_candles1.iloc[-1]['timestamp_ms'])
    min_timestamp_ms2 = int(pd_candles2.iloc[0]['timestamp_ms'])
    max_timestamp_ms2 = int(pd_candles2.iloc[-1]['timestamp_ms'])
    assert(min_timestamp_ms1==min_timestamp_ms2)
    assert(max_timestamp_ms1==max_timestamp_ms2)
    assert(pd_candles1.shape[0]==pd_candles2.shape[0])

    if len([ col for col in pd_candles1.columns if col[-2:]==left_columns_postfix ]) == 0:
        pd_candles1.columns = [str(col) + left_columns_postfix for col in pd_candles1.columns]

    if len([ col for col in pd_candles2.columns if col[-2:]==right_columns_postfix ]) == 0:
        pd_candles2.columns = [str(col) + right_columns_postfix for col in pd_candles2.columns]

    pd_candles1.reset_index(drop=True, inplace=True)
    pd_candles2.reset_index(drop=True, inplace=True)
    pd_candles = pd.concat([pd_candles1, pd_candles2], axis=1)
    pd_candles['timestamp_ms_gap'] = pd_candles[f'timestamp_ms{left_columns_postfix}'] - pd_candles[f'timestamp_ms{right_columns_postfix}']
    assert(pd_candles[pd_candles.timestamp_ms_gap!=0].shape[0]==0)

    pd_candles.drop(pd_candles.columns[pd_candles.columns.str.contains('unnamed',case = False)],axis = 1, inplace = True)

    return pd_candles

def get_old_ticker(
        ticker : str,
        ticker_change_map : List[Dict[str, Union[str, int]]]
    ) -> Union[str, None]:
        if not ticker_change_map:
            return None

        mapping = get_ticker_map(ticker, ticker_change_map)
        if mapping:
            return str(mapping['old_ticker'])
        
        return None

def get_ticker_map(
    ticker : str,
    ticker_change_map : List[Dict[str, Union[str, int]]]
) -> Union[None, Dict[str, Union[str, int]]]:
    if not ticker_change_map:
        return None

    for mapping in ticker_change_map:
        new_ticker = mapping['new_ticker']
        if new_ticker==ticker:
            return mapping
    
    return None

def fetch_headlines_from_rss(
    rss_feeds : Dict[str, str],
    pd_old_headlines : pd.DataFrame, # For purpose of de-duplication
    top_lines : int = 20 # For each feed, we'd only go thru top number of lines
) -> List[Dict[str, Union[str, datetime, int]]]:
    logger: logging.Logger = logging.getLogger()

    new_headlines : List[Dict[str, Union[str, datetime, int]]] = []
    for source, feed_url in rss_feeds.items():
        try:
            feed = feedparser.parse(feed_url)
            logger.info(f"{source}: {len(feed.entries)} headlines found, we're going thru top {top_lines}")

            for entry in feed.entries[:top_lines]:
                try:
                    published_dt = None
                    published = entry.get('published')
                    if published:
                        published_dt = parser.parse(entry.get('published'))
                except Exception as dateparse_err:
                    published_dt = None
                    logger.info(f"{source} {entry.title}: Date parse error {dateparse_err}")

                new_fetch_row = {
                    'source': source,
                    'title': entry.title,
                    'published_utc_dt': published_dt,
                    'published_local_dt': published_dt.astimezone() if published_dt else None,
                    'published_timestamp_ms': int(published_dt.timestamp() * 1000) if published_dt else None,
                    'created_timestamp_ms' : int(datetime.now().timestamp() * 1000),
                    'url': entry.link,
                }
                if (
                        not ((pd_old_headlines['source'] == new_fetch_row['source']) & 
                        (pd_old_headlines['title'] == new_fetch_row['title'])).any()
                ):
                    # Try not append duplicates
                    new_headlines.append(new_fetch_row)
            
        except Exception as e:
            logger.info(f"{source}: feedparser error {e}")
        
    return new_headlines
