#!/usr/bin/env python3
import argparse
import os
import sys
import re
import curses
import textwrap
from collections import defaultdict, deque
from datetime import datetime, date, timedelta
from enum import Enum
from pathlib import Path
from typing import Any, Deque, Dict, List, Optional, Tuple

import pandas
import npyscreen
import pytz

import onetick.py as otp
from onetick.py.db._inspection import DB as InspectionDB


# do not wait after pressing Esc (we don't have escape sequences)
os.environ.setdefault('ESCDELAY', '50')


DESCRIPTION = \
"""
OnetickView is a terminal UI for browsing onetick databases.

It is based on onetick-py library and can work in two modes: localy-installed mode and WebAPI mode.
The mode depends on the set environment variables and installed onetick python api libraries.
See onetick-py documentation for details: https://docs.pip.distribution.sol.onetick.com/static/installation/webapi.html.

Usage (WebAPI mode):

    # set env variables:
    OTP_HTTP_ADDRESS='https://rest.cloud.onetick.com'
    OTP_ACCESS_TOKEN_URL='https://cloud-auth.parent.onetick.com/realms/OMD/protocol/openid-connect/token'
    OTP_CLIENT_ID='?????????????'
    OTP_CLIENT_SECRET='?????????'

    # (get remote databases from WebAPI server)
    onetick view

Usage (locally-installed mode):

    # set env variables:
    MAIN_ONE_TICK_DIR="/opt/one_market_data/one_tick"

    onetick view                                           # (scan current working dir)
    onetick view /tmp                                      # (i.e. for tests with --keep-generated=always)
    onetick view --remote-ts remote.example.com:40001      # (connect to remote server)
    onetick view /tmp --remote-ts remote.example.com:40001 # (both scan specific directory and connect to remote server)

Global settings:
    - press H to show help
    - press F to filter database/symbols list or set tick type
    - press W to add Where EP to filter ticks
    - press Z to change timezone (affects data and locations scan)
    - press D to enable dark mode

Program windows:

1. Databases window: does recursive search in subdirectories and shows available databases
    - press C to show record count (may be slow - scans all DBs)
    - press Q to quit

2. Symbol window: shows available symbols & tick types
    - press C to show record count
    - press + or - to change scroll speed (1, 10, 100 lines)
    - press Esc or Backspace to go back

3. Data window: shows symbol data
    - Use arrows to navigate dataframe
    - press Space to zoom in top left data cell (i.e. view long alert description string)
    - press T to transpose table (switch axis)
    - press + or - to change scroll speed (1, 10, 100 lines)
    - press C to dump dataframe to CSV file
    - press Esc or Backspace to go back

"""


class MyTextfield(npyscreen.Textfield):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.color = 'LABEL'


class Checkbox(npyscreen.Checkbox):
    False_box = 'off'
    True_box = ' ON'

    def set_up_handlers(self):
        super().set_up_handlers()
        self.handlers.update({
            curses.ascii.NL: self.h_exit_down,  # exit down on Enter
            curses.ascii.CR: self.h_exit_down,
        })

    def _create_label_area(self, screen):
        l_a_width = self.width - 5
        if l_a_width < 1:
            raise ValueError("Width of checkbox + label must be at least 6")
        self.label_area = MyTextfield(
            screen, rely=self.rely, relx=self.relx + 5,
            width=self.width - 5, value=self.name, important=True
        )


class OTVSettings:
    # timezone
    _tz_list = [
        'GMT', 'America/Chicago', 'America/New_York', 'Europe/London', 'Europe/Moscow',
        *pytz.all_timezones
    ]
    _selected_tz_idx = None
    # tick type and db name filter
    follow_links = False
    tick_type = ''
    db_str_filter = ''
    db_hide_backups = True
    db_regexp_match = False
    symbol_str_filter = ''
    # where clause
    data_where_clause = ''
    discard_on_match = False

    def __init__(self, timezone_name: Optional[str]):
        timezone_name = timezone_name or 'GMT'  # GMT is default
        try:
            self._selected_tz_idx = self._tz_list.index(timezone_name)
        except ValueError:
            raise ValueError(f"Timezone '{timezone_name}' is not supported")

    @property
    def timezone(self):
        return self._tz_list[self._selected_tz_idx]

    def show_tz_selector(self):
        my_form = npyscreen.Form(name="Settings")
        tz_selector: npyscreen.TitleSelectOne = my_form.add(
            npyscreen.TitleSelectOne,
            value=[self._selected_tz_idx],
            name="Pick Timezone",
            values=self._tz_list,
            scroll_exit=True
        )

        my_form.edit()
        self._selected_tz_idx = tz_selector.value[0]

    def show_where_selector(self):
        my_form = npyscreen.Form(name="Ticks Where Clause")
        data_where_edit = my_form.add(npyscreen.TitleText, name="Where", value=self.data_where_clause)
        discard_on_match = my_form.add(Checkbox, name="Discard On Match", value=self.discard_on_match)
        my_form.edit()
        self.data_where_clause = data_where_edit.value
        self.discard_on_match = discard_on_match.value

    def show_db_filter_selector(self):
        my_form = npyscreen.Form(name="Archive Lookup Filters")
        tick_type_edit = my_form.add(npyscreen.TitleText, name="Tick Type", value=self.tick_type)
        db_filter_edit = my_form.add(npyscreen.TitleText, name="DBs Filter", value=self.db_str_filter)
        symbol_filter_edit = my_form.add(npyscreen.TitleText, name="Symbol Filter", value=self.symbol_str_filter)
        db_regexp_match_edit = my_form.add(Checkbox, name="Regexp Match Filters", value=self.db_regexp_match)
        follow_links_edit = my_form.add(Checkbox, name="Follow Symlinks", value=self.follow_links)
        db_hide_backups_edit = my_form.add(Checkbox, name="Hide Backups DB", value=self.db_hide_backups)
        my_form.edit()
        self.tick_type = tick_type_edit.value
        self.db_str_filter = db_filter_edit.value
        self.db_regexp_match = db_regexp_match_edit.value
        self.follow_links = follow_links_edit.value
        self.db_hide_backups = db_hide_backups_edit.value
        self.symbol_str_filter = symbol_filter_edit.value

    def filter_strings(self, string_list: List[str], filter_str: str, use_regex=False, exact_match=False):
        """
        modes:
         - substring match (default)
         - exact match
         - regex
        """
        if not filter_str:
            return string_list

        if exact_match:  # i.e for tick type
            return [s for s in string_list if filter_str == s]

        if use_regex:
            p = re.compile(filter_str)
            return [s for s in string_list if p.match(s)]

        # substring match
        return [s for s in string_list if filter_str in s]

    def show_help(self):
        my_form = npyscreen.Form(name="Help")
        my_form.add(npyscreen.Pager, values=DESCRIPTION.split('\n'))
        my_form.edit()


class DateData:
    def __init__(self):
        self.stats: Optional[Dict[Tuple[str, str], Any]] = None
        # tick_type -> list_of_symbols
        self.tts_symbols: Optional[Dict[str, List[str]]] = None
        # (tick_type, symbol) -> data
        self.symbols_data: Dict[Tuple[str, str], Any] = defaultdict(None)


class DatabaseLocation:
    def __init__(self, name: str, path: Optional[str] = None, storage_type: str = 'local'):
        if storage_type not in ['local', 'remote']:
            raise ValueError(
                f'Incorrect value for `storage_type` passed: `{storage_type}`. `local` or `remote` expected'
            )

        self.name = name
        self.path = path
        self.storage_type = storage_type
        self.dates: List[date] = []
        self.date_data: Dict[date, DateData] = defaultdict(DateData)

    def __str__(self):
        if self.storage_type == 'remote':
            return f"{self.name} (remote)"

        return self.name

    def get_date_str(self, dt_str: str):
        db_str = str(self)
        return f"{db_str} @ {dt_str}"

    def __repr__(self):
        return f'{self.__class__.__name__}(name={self.name}, path={self.path}, storage_type={self.storage_type}, dates={self.dates})'


class MyThemeNpyscreen(npyscreen.ThemeManager):
    default_colors = {
        'DEFAULT': 'BLACK_YELLOW',
        'FORMDEFAULT': 'BLACK_WHITE',
        'NO_EDIT': 'BLUE_WHITE',
        'STANDOUT': 'CYAN_WHITE',
        'CURSOR': 'BLACK_WHITE',
        'CURSOR_INVERSE': 'WHITE_BLACK',
        'LABEL': 'BLACK_WHITE',
        'LABELBOLD': 'BLACK_YELLOW',
        'CONTROL': 'RED_WHITE',
        'WARNING': 'RED_WHITE',
        'CRITICAL': 'BLACK_RED',
        'GOOD': 'GREEN_BLACK',
        'GOODHL': 'GREEN_WHITE',
        'VERYGOOD': 'WHITE_GREEN',
        'CAUTION': 'YELLOW_WHITE',
        'CAUTIONHL': 'BLACK_YELLOW',
    }


class OnetickBrowserScreens(Enum):
    DATABASES_LOADER = 'DatabasesLoader'
    REMOTE_DATES_LOADER = 'RemoteDatesLoader'
    LOCATION_SELECTOR = 'Databases'
    DATE_SELECTOR = 'Dates'
    TICK_TYPE_SELECTOR = 'Tick type'
    SYMBOL_LIST = 'Symbols'
    TICKS_DATA = 'Data'
    ERROR_SCREEN = 'Error'


class OnetickBrowser:

    def __init__(self, start_path: str, connector: 'OnetickDataConnector', settings):
        self.start_path = start_path
        self.connector: 'OnetickDataConnector' = connector
        self.path_idx = 0  # selected path index
        self.date_idx = 0 # selected date index
        self.symbol_idx = 0  # selected symbol index
        self.settings: OTVSettings = settings
        # data viewer position
        self.data_offset_x = 0
        self.data_offset_y = 0
        self.locator_cursor = 0
        self.date_cursor = 0
        self.symbol_cursor = 0
        self.current_screen = OnetickBrowserScreens.DATABASES_LOADER
        # tick type selector
        self.tick_type_idx = 0
        self.tick_type_cursor = 0
        # config
        self.show_count = False
        self.transpose = False
        self.max_columns = 30  # to fill the screen width
        self.max_colwidth = 50  # maximum width of a single column
        self.scroll_speed = 1
        self.symbols_scroll_speed = 1
        self.show_details = False
        self.use_fallback_colors = False
        self._current_db = None
        # other
        self.lock_controls = False

        self._init_cache()
        self._reset_error_data()

    def _init_cache(self):
        self._cached_locations = []

    def _reset_error_data(self):
        self._error_screen_message = None
        self._error_screen_exception = None
        self._error_screen_return = None
        self._error_screen_shown = False

    def get_locations_cached(self) -> List[DatabaseLocation]:
        if self._cached_locations:
            return self._cached_locations

        dbs_to_dates = self.connector.get_databases()
        self._cached_locations = dbs_to_dates

        return dbs_to_dates

    def get_selected_location(self) -> DatabaseLocation:
        locations = self.get_locations_cached()
        try:
            return locations[self.path_idx]
        except IndexError:
            raise RuntimeError(f"Can't get location for selected database with path index {self.path_idx}")

    def get_selected_location_date(self) -> date:
        location = self.get_selected_location()
        try:
            return location.dates[self.date_idx]
        except IndexError:
            raise RuntimeError(f"Can't get location date for selected database {repr(location)} with date index {self.date_idx}")

    def get_selected_tick_type(self) -> str:
        tick_types = self.get_tick_types_cached()
        try:
            return tick_types[self.tick_type_idx]
        except IndexError:
            raise RuntimeError(f"Can't get tick type with date index {self.tick_type_idx}")

    @staticmethod
    def dt_to_str(dt_obj: date):
        return dt_obj.strftime('%Y%m%d')

    def get_selected_database_path(self):
        database = self.get_selected_location()
        if database.storage_type == 'local':
            return database.path
        else:
            return f'{database.name} (remote)'

    def get_tts_symbols_cached(self) -> Dict[str, List[str]]:
        database = self.get_selected_location()
        db_dt = self.get_selected_location_date()
        if database is None or db_dt is None:
            return []

        if database.date_data[db_dt].tts_symbols is not None:
            return database.date_data[db_dt].tts_symbols

        tts_symbols = self.connector.get_tts_symbols(database, db_dt)
        database.date_data[db_dt].tts_symbols = tts_symbols
        return tts_symbols

    def get_db_symbols_cached_selected_tick_type(self) -> List[str]:
        tts_symbols = self.get_tts_symbols_cached()
        current_tt = self.get_selected_tick_type()
        return tts_symbols[current_tt]

    def get_tick_types_cached(self) -> List[str]:
        tts_symbols = self.get_tts_symbols_cached()
        return list(tts_symbols)

    @staticmethod
    def invalidate_tick_data_cache(locations: List[DatabaseLocation]):
        for db in locations:
            for dt_i in db.date_data.keys():
                db.date_data[dt_i].symbols_data.clear()

    @staticmethod
    def _scroll_window(scroll_rows: int,
                       idx: int, data_len: int,
                       cursor: int, display_height: int) -> Tuple[int, int]:
        """
        Scroll ``scroll_rows`` through some list and window that shows that list.
        If the value is positive scrolls forward, else backward.

        ``idx`` is the current position in the list and ``data_len`` is the total number of values in the list.
        ``cursor`` is the current position in the window with height ``display_height``.

        The list will be cycled, scrolling forward after reaching the end of the list will return rows from the beggining of the list
        and vice versa.

        Returns updated ``idx`` and ``cursor``.
        """
        if scroll_rows == 0:
            return idx, cursor

        out_of_bounds = False
        if idx + scroll_rows < 0 or idx + scroll_rows >= data_len:
            out_of_bounds = True

        # scrolling pages
        if abs(scroll_rows) >= display_height:
            if data_len >= display_height:
                new_idx = (idx + scroll_rows) % data_len
                # by default, cursor stays in the same place (feels like only the window moves)
                if not out_of_bounds:
                    new_cursor = cursor
                # but if we are cycling through list, let the cursor "jump"
                else:
                    new_cursor = 0 if scroll_rows > 0 else display_height - 1
            # if there is less than one display of data, usual cycling doesn't make sense
            # so we are moving cursor between first and last row
            else:
                if scroll_rows > 0:
                    if idx == data_len - 1:
                        new_cursor = new_idx = 0
                    else:
                        new_cursor = new_idx = data_len - 1
                if scroll_rows < 0:
                    if idx == 0:
                        new_cursor = new_idx = data_len - 1
                    else:
                        new_cursor = new_idx = 0

            return new_idx, new_cursor

        # scrolling rows
        new_idx = (idx + scroll_rows) % data_len
        new_cursor = cursor + scroll_rows

        if new_cursor < 0 or new_cursor >= display_height:
            # if we cycled through list, let the cursor "jump" to the bottom or to the top
            if out_of_bounds:
                new_cursor = new_cursor % min(display_height, data_len)
            # otherwise keep it at the bottom or at the top
            elif new_cursor < 0:
                new_cursor = 0
            elif new_cursor >= display_height:
                new_cursor = display_height - 1

        return new_idx, new_cursor

    def handle_location_selector_keys(self, key_pressed, locations_data, display_height):
        if not locations_data:
            self.lock_controls = True

        if self.lock_controls:
            return

        if key_pressed in [curses.KEY_UP, 339]:  # up, pg_up
            scroll_rows = 1 if key_pressed == curses.KEY_UP else display_height
            self.path_idx, self.locator_cursor = self._scroll_window(-scroll_rows,
                                                                     self.path_idx, len(locations_data),
                                                                     self.locator_cursor, display_height)
        elif key_pressed in [curses.KEY_DOWN, 338]:  # down, pg_dn
            scroll_rows = 1 if key_pressed == curses.KEY_DOWN else display_height
            self.path_idx, self.locator_cursor = self._scroll_window(scroll_rows,
                                                                     self.path_idx, len(locations_data),
                                                                     self.locator_cursor, display_height)
        elif key_pressed == 10:  # enter
            previous_db = self._current_db
            selected_db = self.get_selected_location()

            if not selected_db:
                return

            if previous_db and previous_db is not selected_db:
                # clear all tick data cache for database if we switch to another database
                self.invalidate_tick_data_cache([previous_db])

            self._current_db = selected_db
            self.date_cursor = 0
            self.date_idx = 0

            if selected_db.storage_type == 'remote':
                self.current_screen = OnetickBrowserScreens.REMOTE_DATES_LOADER
            else:
                self.current_screen = OnetickBrowserScreens.DATE_SELECTOR

    def handle_date_selector_keys(self, key_pressed, date_data, display_height):
        if not date_data and not self.lock_controls:
            self.lock_controls = True

        if key_pressed in [27, 263, 8]:  # esc, backspace, backspace_win
            self.current_screen = OnetickBrowserScreens.LOCATION_SELECTOR
            self.lock_controls = False
        elif self.lock_controls:
            return
        elif key_pressed in [curses.KEY_UP, 339]:  # up, pg_up
            scroll_rows = 1 if key_pressed == curses.KEY_UP else display_height
            self.date_idx, self.date_cursor = self._scroll_window(-scroll_rows,
                                                                  self.date_idx, len(date_data),
                                                                  self.date_cursor, display_height)
        elif key_pressed in [curses.KEY_DOWN, 338]:  # down, pg_dn
            scroll_rows = 1 if key_pressed == curses.KEY_DOWN else display_height
            self.date_idx, self.date_cursor = self._scroll_window(scroll_rows,
                                                                  self.date_idx, len(date_data),
                                                                  self.date_cursor, display_height)
        elif key_pressed == 10:  # enter
            self.tick_type_idx = 0
            self.tick_type_cursor = 0
            self.current_screen = OnetickBrowserScreens.TICK_TYPE_SELECTOR

    def handle_tick_type_selector_keys(self, key_pressed, tick_types, display_height):
        if not tick_types and not self.lock_controls:
            self.lock_controls = True

        if key_pressed in [27, 263, 8]:  # esc, backspace, backspace_win
            self.current_screen = OnetickBrowserScreens.DATE_SELECTOR
            self.lock_controls = False
        elif self.lock_controls:
            return
        elif key_pressed in [curses.KEY_UP, 339]:  # up, pg_up
            scroll_rows = 1 if key_pressed == curses.KEY_UP else display_height
            self.tick_type_idx, self.tick_type_cursor = self._scroll_window(
                -scroll_rows, self.tick_type_idx, len(tick_types), self.tick_type_cursor, display_height
            )
        elif key_pressed in [curses.KEY_DOWN, 338]:  # down, pg_dn
            scroll_rows = 1 if key_pressed == curses.KEY_DOWN else display_height
            self.tick_type_idx, self.tick_type_cursor = self._scroll_window(
                scroll_rows, self.tick_type_idx, len(tick_types), self.tick_type_cursor, display_height
            )
        elif key_pressed == 10:  # enter
            self.symbol_idx = 0
            self.symbol_cursor = 0
            self.current_screen = OnetickBrowserScreens.SYMBOL_LIST

    def handle_symbol_list_keys(self, key_pressed, db_symbols, display_height):
        if not db_symbols and not self.lock_controls:
            self.lock_controls = True

        if key_pressed in [27, 263, 8]:  # esc, backspace, backspace_win
            self.current_screen = OnetickBrowserScreens.TICK_TYPE_SELECTOR
            self.lock_controls = False
        elif self.lock_controls:
            return
        elif key_pressed in [curses.KEY_UP, 339]:  # up, pg_up
            scroll_rows = self.symbols_scroll_speed if key_pressed == curses.KEY_UP else display_height
            self.symbol_idx, self.symbol_cursor = self._scroll_window(-scroll_rows,
                                                                      self.symbol_idx, len(db_symbols),
                                                                      self.symbol_cursor, display_height)
        elif key_pressed in [curses.KEY_DOWN, 338]:  # down, pg_dn
            scroll_rows = self.symbols_scroll_speed if key_pressed == curses.KEY_DOWN else display_height
            self.symbol_idx, self.symbol_cursor = self._scroll_window(scroll_rows,
                                                                      self.symbol_idx, len(db_symbols),
                                                                      self.symbol_cursor, display_height)
        elif key_pressed in [10, 102, 13]:  # enter, enter_win, enter_win
            self.data_offset_x = 0
            self.data_offset_y = 0
            self.current_screen = OnetickBrowserScreens.TICKS_DATA
        elif key_pressed in [ord('+'), ord('=')]:
            self.symbols_scroll_speed = self.symbols_scroll_speed * 10
        elif key_pressed in [ord('-'), ord('_')]:
            self.symbols_scroll_speed = self.symbols_scroll_speed // 10
            self.symbols_scroll_speed = max(self.symbols_scroll_speed, 1)

    def handle_data_viewer_keys(self, key_pressed, symbol_data, display_height):
        data_max_y, data_max_x = symbol_data.shape
        if key_pressed in [curses.KEY_UP, 339]:  # up, pg_up
            scroll_rows = self.scroll_speed if key_pressed == curses.KEY_UP else display_height
            self.data_offset_y = max(self.data_offset_y - scroll_rows, 0)
        elif key_pressed in [curses.KEY_DOWN, 338]:  # down, pg_dn
            scroll_rows = self.scroll_speed if key_pressed == curses.KEY_DOWN else display_height
            self.data_offset_y = min(self.data_offset_y + scroll_rows, data_max_y - 1)
        elif key_pressed == curses.KEY_LEFT:
            self.data_offset_x = max(self.data_offset_x - self.scroll_speed, 0)
        elif key_pressed == curses.KEY_RIGHT:
            self.data_offset_x = min(self.data_offset_x + self.scroll_speed, data_max_x - 1)
        elif key_pressed in [27, 263, 8]:  # esc, backspace, backspace_win
            if self.show_details:
                self.show_details = False
            else:
                self.current_screen = OnetickBrowserScreens.SYMBOL_LIST
        elif key_pressed == ord(' '):
            self.show_details = not self.show_details
        elif key_pressed in [ord('+'), ord('=')]:
            self.scroll_speed = self.scroll_speed * 10
        elif key_pressed in [ord('-'), ord('_')]:
            self.scroll_speed = self.scroll_speed // 10
            self.scroll_speed = max(self.scroll_speed, 1)

    def handle_config_keys(self, key_pressed):
        if key_pressed == ord('c'):
            if self.current_screen == OnetickBrowserScreens.TICKS_DATA:
                self.dump_csv()
            else:
                self.show_count = not self.show_count
        elif key_pressed == ord('t'):
            self.transpose = not self.transpose
            self.data_offset_x, self.data_offset_y = self.data_offset_y, self.data_offset_x
        elif key_pressed == ord('d'):
            self.use_fallback_colors = not self.use_fallback_colors
            self._init_colors()
        elif key_pressed == ord('f'):  # filters for DBs
            self.settings.show_db_filter_selector()
            self._init_cache()
            self._init_colors()
        elif key_pressed == ord('w'):  # where clause for data
            self.settings.show_where_selector()
            self.invalidate_tick_data_cache(self.get_locations_cached())
            self._init_colors()
        elif key_pressed == ord('z'):  # timezone
            # timezone
            self.settings.show_tz_selector()
            self._init_colors()
            self._init_cache()
        elif key_pressed == ord('h'):  # help
            # timezone
            self.settings.show_help()
            self._init_colors()

    def _init_colors(self):
        if self.use_fallback_colors:
            self._init_fallback_color_scheme()
        else:
            self._init_default_color_scheme()

    def _init_default_color_scheme(self):
        try:
            curses.init_pair(1, curses.COLOR_BLACK, curses.COLOR_WHITE)
            curses.init_pair(2, 8, curses.COLOR_WHITE)  # Status bar
            curses.init_pair(3, 15, 8)  # Selection
            curses.init_pair(4, 15, curses.COLOR_RED)  # One
            curses.init_pair(5, 15, 8)  # Tick
        except Exception as e:
            self._init_fallback_color_scheme()

    @staticmethod
    def _init_fallback_color_scheme():
        curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLACK)  # Background unix
        curses.init_pair(2, curses.COLOR_WHITE, curses.COLOR_BLACK)  # Status bar
        curses.init_pair(3, curses.COLOR_BLACK, curses.COLOR_WHITE)  # Selection
        curses.init_pair(4, curses.COLOR_WHITE, curses.COLOR_RED)  # One
        curses.init_pair(5, curses.COLOR_BLACK, curses.COLOR_WHITE)  # Tick

    def draw_main_display(self, stdscr):
        key_pressed = 0
        stdscr.clear()
        stdscr.refresh()
        curses.start_color()
        npyscreen.setTheme(MyThemeNpyscreen)
        self._init_default_color_scheme()
        stdscr.bkgd(' ', curses.color_pair(1))

        while key_pressed != ord('q'):
            if self.current_screen != OnetickBrowserScreens.ERROR_SCREEN:
                stdscr.clear()

            height, width = stdscr.getmaxyx()
            display_height = height - 1  # minus status bar

            # handle input
            self.handle_config_keys(key_pressed)
            if self.current_screen == OnetickBrowserScreens.ERROR_SCREEN:
                if self._error_screen_shown:
                    _ = stdscr.getch()
                    stdscr.clear()
                    self.current_screen = self._error_screen_return
                    self._reset_error_data()
            elif self.current_screen == OnetickBrowserScreens.LOCATION_SELECTOR:
                locations_data = self.get_locations_cached()
                self.handle_location_selector_keys(key_pressed, locations_data, display_height)
            elif self.current_screen == OnetickBrowserScreens.DATE_SELECTOR:
                database = self.get_selected_location()
                self.handle_date_selector_keys(key_pressed, database.dates, display_height)
            elif self.current_screen == OnetickBrowserScreens.TICK_TYPE_SELECTOR:
                tick_types = self.get_tick_types_cached()
                self.handle_tick_type_selector_keys(key_pressed, tick_types, display_height)
            elif self.current_screen == OnetickBrowserScreens.SYMBOL_LIST:
                db_symbols = self.get_db_symbols_cached_selected_tick_type()
                self.handle_symbol_list_keys(key_pressed, db_symbols, display_height)
            elif self.current_screen == OnetickBrowserScreens.TICKS_DATA:
                symbol_data = self.get_current_symbol_data()
                self.handle_data_viewer_keys(key_pressed, symbol_data, display_height)

            # draw data
            if self.current_screen == OnetickBrowserScreens.DATABASES_LOADER:
                self.draw_databases_loader(stdscr)
                continue
            elif self.current_screen == OnetickBrowserScreens.LOCATION_SELECTOR:
                if self.locator_cursor > self.path_idx:
                    self.locator_cursor = self.path_idx
                locations_data = self.get_locations_cached()
                status_bar = self.draw_location_selector(stdscr, locations_data)
            elif self.current_screen == OnetickBrowserScreens.REMOTE_DATES_LOADER:
                self.draw_remote_date_loader(stdscr)
                key_pressed = 0
                continue
            elif self.current_screen == OnetickBrowserScreens.DATE_SELECTOR:
                if self.date_cursor > self.date_idx:
                    self.date_cursor = self.date_idx

                status_bar = self.draw_date_selector(stdscr)
                if status_bar is None:
                    continue
            elif self.current_screen == OnetickBrowserScreens.TICK_TYPE_SELECTOR:
                if self.tick_type_cursor > self.tick_type_idx:
                    self.tick_type_cursor = self.tick_type_idx

                try:
                    tick_types = self.get_tick_types_cached()
                except Exception as exc:
                    self._save_error_message(
                        'Failed to load tick types', OnetickBrowserScreens.LOCATION_SELECTOR, str(exc),
                    )
                    continue

                status_bar = self.draw_tick_type_selector(stdscr, tick_types)
            elif self.current_screen == OnetickBrowserScreens.SYMBOL_LIST:
                if self.symbol_cursor > self.symbol_idx:
                    self.symbol_cursor = self.symbol_idx

                try:
                    db_symbols = self.get_db_symbols_cached_selected_tick_type()
                except Exception as exc:
                    self._save_error_message(
                        'Failed to load symbols data', OnetickBrowserScreens.LOCATION_SELECTOR, str(exc),
                    )
                    continue

                status_bar = self.draw_symbols_selector(stdscr, db_symbols)
            elif self.current_screen == OnetickBrowserScreens.TICKS_DATA:
                status_bar = self.draw_ticks_data(stdscr)
            elif self.current_screen == OnetickBrowserScreens.ERROR_SCREEN:
                self.draw_error_screen(stdscr)
                continue
            else:
                raise ValueError('Unknown screen ' + str(self.current_screen))

            if self.lock_controls:
                status_bar = '[*] ' + status_bar

            if self.current_screen not in [
                OnetickBrowserScreens.DATABASES_LOADER, OnetickBrowserScreens.REMOTE_DATES_LOADER,
                OnetickBrowserScreens.ERROR_SCREEN,
            ]:
                self.draw_status_bar(stdscr, status_bar)

            stdscr.move(0, width - 1)  # terminal cursor
            stdscr.refresh()  # draw
            key_pressed = stdscr.getch()  # waiting input

    @staticmethod
    def draw_status_bar(stdscr, status_bar: str):
        height, width = stdscr.getmaxyx()
        # add onetick logo
        stdscr.attron(curses.A_BOLD)
        stdscr.attron(curses.color_pair(4))
        stdscr.addstr(height - 1, 0, 'One')
        stdscr.attroff(curses.color_pair(4))
        stdscr.attron(curses.color_pair(5))
        stdscr.addstr(height - 1, 3, 'Tick')
        stdscr.attroff(curses.color_pair(5))
        stdscr.attroff(curses.A_BOLD)
        # display status bar
        stdscr.attron(curses.color_pair(2))
        status_bar = ' ' + status_bar
        max_bar_width = width - 8
        status_bar = status_bar + ' ' * (max_bar_width - len(status_bar))
        stdscr.addstr(height - 1, 7, status_bar[:max_bar_width])
        # stdscr.addstr(height - 1, len(status_bar), " " * (width - len(status_bar) - 1))
        stdscr.attroff(curses.color_pair(2))

    def draw_error_screen(self, stdscr):
        _, width = stdscr.getmaxyx()

        if self._error_screen_message is None:
            self._error_screen_message = 'Unknown error occurred'
        message_lines = textwrap.wrap(self._error_screen_message, width=width)
        message_lines.append("Press any key to continue.")

        if self._error_screen_exception:
            message_lines.extend(["Error details:", ""])
            exception_lines = textwrap.wrap(self._error_screen_exception, width=width)
        else:
            exception_lines = []

        for idx, line in enumerate(message_lines + exception_lines):
            try:
                stdscr.addstr(0 + idx, 0, line)
            except curses.error as exc:
                raise RuntimeError(str(message_lines + exception_lines)) from exc

        stdscr.refresh()
        self._error_screen_shown = True

    def draw_databases_loader(self, stdscr):
        stdscr.addstr(0, 0, "Loading databases...")
        stdscr.refresh()

        try:
            if not self.connector.remote_databases:
                self.connector.preload_databases(self.start_path)
        except Exception as exc:
            self._save_error_message(
                "Failed to connect to remote server, switching to offline mode.",
                OnetickBrowserScreens.LOCATION_SELECTOR, str(exc),
            )
            return
        self.current_screen = OnetickBrowserScreens.LOCATION_SELECTOR

    def draw_remote_date_loader(self, stdscr):
        database = self.get_selected_location()
        stdscr.addstr(0, 0, "Loading dates info...")
        stdscr.refresh()

        try:
            self.connector.preload_remote_database_date(database)
        except Exception as exc:
            self._save_error_message(
                f"Failed to get dates for remote database `{database.name}`.",
                OnetickBrowserScreens.LOCATION_SELECTOR, str(exc),
            )
            return

        self.current_screen = OnetickBrowserScreens.DATE_SELECTOR

    def draw_location_selector(self, stdscr, locations_data: List[DatabaseLocation]):
        height, width = stdscr.getmaxyx()
        lineno = 0
        start_offset = 0
        if len(locations_data) > height:
            start_offset = self.path_idx - self.locator_cursor

        for i, locations in enumerate(locations_data[start_offset:]):
            database = locations
            if lineno == self.locator_cursor:
                stdscr.attron(curses.color_pair(3))
                stdscr.attron(curses.A_BOLD)

            if database.storage_type == 'local':
                db_root = database.path.replace(self.start_path, '.')
            else:
                db_root = ''

            db_str = f"{str(database)} {db_root}"[:width - 1]
            db_str += ' ' * (width - len(db_str) - 1)  # fill right
            stdscr.addstr(lineno, 0, db_str)
            if lineno == self.locator_cursor:
                stdscr.attroff(curses.color_pair(3))
                stdscr.attroff(curses.A_BOLD)
            lineno += 1
            if lineno == height:
                break
        # Status bar
        bar_width = width - 9  # include onetick logo
        status_bar = f"{self.current_screen.value} {self.path_idx + 1}/{len(locations_data)} "
        if locations_data:
            db_path = self.get_selected_database_path()
            path_suffix = db_path[- bar_width + len(status_bar):]
            if path_suffix != db_path:
                status_bar += '...'
            status_bar = status_bar + path_suffix
        else:
            stdscr.addstr(0, 0, 'Databases were not found during '
                                'recursive search starting from '
                                'path: \n' + self.start_path + ' \nPress H to view help.')
        return status_bar[:bar_width]

    def draw_date_selector(self, stdscr):
        height, width = stdscr.getmaxyx()
        lineno = 0
        start_offset = 0

        database = self.get_selected_location()
        if len(database.dates) > height:
            start_offset = self.date_idx - self.date_cursor

        for i, db_dt in enumerate(database.dates[start_offset:]):
            if lineno == self.date_cursor:
                stdscr.attron(curses.color_pair(3))
                stdscr.attron(curses.A_BOLD)

            if self.show_count:
                tts_stats, _, cached = self.connector.get_data_count_cached(database, db_dt)
                if not cached:
                    fetch_str = f'Fetch {i}/{height-1} '
                    fetch_str += '>' * int((width - len(fetch_str) - 2) * (i / (height - 1)))
                    stdscr.addstr(height - 1, 0, fetch_str)
                    stdscr.refresh()  # draw
                if tts_stats is None:
                    tts_stats = '[ error ]'
                else:
                    tts_stats = ', '.join(f"{k}:{v}" for k, v in tts_stats.items())
                    tts_stats = '[' + (tts_stats or 'is empty') + '] '
            else:
                tts_stats = ''

            date_str = f"{self.dt_to_str(db_dt)} {tts_stats}"[:width - 1]
            date_str += ' ' * (width - len(date_str) - 1)  # fill right
            stdscr.addstr(lineno, 0, date_str)
            if lineno == self.date_cursor:
                stdscr.attroff(curses.color_pair(3))
                stdscr.attroff(curses.A_BOLD)
            lineno += 1
            if lineno == height:
                break

        # Status bar
        bar_width = width - 9  # include onetick logo
        status_bar = f"{self.current_screen.value} {self.date_idx + 1}/{len(database.dates)} "
        if database.dates:
            db_path = self.get_selected_database_path()
            path_suffix = db_path[- bar_width + len(status_bar):]
            if path_suffix != db_path:
                status_bar += '...'
            status_bar = status_bar + path_suffix
        else:
            self._save_error_message(
                f'Dates were not found for selected database: `{str(database)}`',
                OnetickBrowserScreens.LOCATION_SELECTOR,
            )
            return None

        return status_bar[:bar_width]

    def draw_tick_type_selector(self, stdscr, tick_types) -> str:
        if not tick_types:
            stdscr.addstr(
                0, 0, f'No data for selected db: {self.get_selected_database_path()} \nPress H to view help.',
            )
            self.tick_type_cursor = 0
            return ''

        database = self.get_selected_location()
        db_dt = self.get_selected_location_date()
        db_readable = database.get_date_str(self.dt_to_str(db_dt))

        height, width = stdscr.getmaxyx()
        lineno = 0
        start_offset = 0
        if len(tick_types) > height:
            start_offset = self.tick_type_idx - self.tick_type_cursor

        for tt in tick_types[start_offset:]:
            if lineno == self.tick_type_cursor:
                stdscr.attron(curses.color_pair(3))
                stdscr.attron(curses.A_BOLD)
            tick_type_str = tt
            tick_type_str = tick_type_str[:width - 1]  # trim right
            tick_type_str += ' ' * (width - len(tick_type_str) - 1)  # fill right
            stdscr.addstr(lineno, 0, tick_type_str)
            if lineno == self.tick_type_cursor:
                stdscr.attroff(curses.color_pair(3))
                stdscr.attroff(curses.A_BOLD)
            lineno += 1
            if lineno == height:
                break
        status_bar = f"{self.current_screen.value} {self.tick_type_idx + 1}/{len(tick_types)} " \
                     f"in {db_readable}. " \
                     f"Press Z to change timezone ({self.settings.timezone}), " \
                     f"press H to see other shortcuts."
        return status_bar

    def draw_symbols_selector(self, stdscr, db_symbols) -> str:
        if not db_symbols:
            stdscr.addstr(
                0, 0, f'No data for selected db: {self.get_selected_database_path()} \nPress H to view help.',
            )
            self.symbol_cursor = 0
            return ''

        database = self.get_selected_location()
        db_dt = self.get_selected_location_date()
        db_readable = database.get_date_str(self.dt_to_str(db_dt))
        tt = self.get_selected_tick_type()

        height, width = stdscr.getmaxyx()
        lineno = 0
        start_offset = 0
        if len(db_symbols) > height:
            start_offset = self.symbol_idx - self.symbol_cursor

        for symbol in db_symbols[start_offset:]:
            if lineno == self.symbol_cursor:
                stdscr.attron(curses.color_pair(3))
                stdscr.attron(curses.A_BOLD)
            if self.show_count:
                _, symbol_stats, _ = self.connector.get_data_count_cached(database, db_dt)
                if symbol_stats is None:
                    symbol_str = "[ error ]"
                else:
                    count = symbol_stats.get((tt, symbol))
                    symbol_str = f"[{tt}:{count}] Symbol {symbol}"
            else:
                symbol_str = f"[{tt}] Symbol {symbol}"
            symbol_str = symbol_str[:width - 1]  # trim right
            symbol_str += ' ' * (width - len(symbol_str) - 1)  # fill right
            stdscr.addstr(lineno, 0, symbol_str)
            if lineno == self.symbol_cursor:
                stdscr.attroff(curses.color_pair(3))
                stdscr.attroff(curses.A_BOLD)
            lineno += 1
            if lineno == height:
                break
        status_bar = f"{self.current_screen.value} {self.symbol_idx + 1}/{len(db_symbols)} " \
                     f"in {db_readable}. " \
                     f"scroll={self.symbols_scroll_speed}. " \
                     f"Press Z to change timezone ({self.settings.timezone}), " \
                     f"press H to see other shortcuts."
        return status_bar

    def get_current_symbol_data(self):
        tt = self.get_selected_tick_type()
        db_symbols = self.get_db_symbols_cached_selected_tick_type()
        if not db_symbols:
            return pandas.DataFrame()
        symbol = db_symbols[self.symbol_idx]

        database = self.get_selected_location()
        db_dt = self.get_selected_location_date()

        cached_data = database.date_data[db_dt].symbols_data.get((tt, symbol))
        if cached_data is not None:
            return cached_data

        symbol_data = self.connector.get_symbol_data(tt, symbol, database, db_dt)
        database.date_data[db_dt].symbols_data[(tt, symbol)] = symbol_data

        return symbol_data

    def draw_ticks_data(self, stdscr):
        height, width = stdscr.getmaxyx()

        symbol_data = self.get_current_symbol_data()
        data_max_y, data_max_x = symbol_data.shape

        db_dt = self.get_selected_location_date()
        db_dt_str = self.dt_to_str(db_dt)

        if symbol_data.empty:
            text_rows = ['No data']
        else:
            if not self.transpose:
                symbol_data = symbol_data[self.data_offset_y: self.data_offset_y + height - 1]
                limit_x = min(symbol_data.shape[1], self.data_offset_x + self.max_columns)
                symbol_data = symbol_data.iloc[:, range(self.data_offset_x, limit_x)]
            else:  # transpose - hack to not transpose all df
                symbol_data = symbol_data[self.data_offset_x: self.data_offset_x + self.max_columns - 1]
                limit_y = min(symbol_data.shape[1], self.data_offset_y + height)
                symbol_data = symbol_data.iloc[:, range(self.data_offset_y, limit_y)]
                symbol_data = symbol_data.transpose()
                data_max_y, data_max_x = data_max_x, data_max_y
            if symbol_data.empty:
                text_rows = [f'X, Y is out of bounds ({data_max_x}, {data_max_y})']
            elif self.show_details:
                # details viewer
                big_str = str(symbol_data.values[0][0])
                start = 0
                chunk = big_str[start: start + width]
                text_rows = [f'Details viewer {type(symbol_data.values[0][0])}. '
                             f'Press Space to go back.']
                while chunk and len(text_rows) < height:
                    text_rows.append(chunk)
                    start += width - 1
                    chunk = big_str[start: start + width]
            else:
                # split dataframe string to rows
                pandas.set_option("display.max_colwidth", self.max_colwidth)
                pandas.set_option("display.max_rows", height)
                text_rows = str(symbol_data)
                text_rows = text_rows.split('\n')

        lineno = 0
        for i, row in enumerate(text_rows[:height - 1]):
            if not i:
                stdscr.attron(curses.color_pair(3))
            stdscr.addstr(lineno, 0, row[:width - 1])
            if not i:
                stdscr.attroff(curses.color_pair(3))
            lineno += 1
        status_bar = f"{self.current_screen.value}" \
                     f"({self.data_offset_y}/{data_max_y},{self.data_offset_x}/{data_max_x}) " \
                     f"scroll={self.scroll_speed}. " \
                     f"Date: {db_dt_str}. " \
                     f"Press Z to change timezone ({self.settings.timezone}), " \
                     "press H to see other shortcuts."
        return status_bar

    def dump_csv(self):
        tt = self.get_selected_tick_type()
        db_symbols = self.get_db_symbols_cached_selected_tick_type()
        symbol = db_symbols[self.symbol_idx]

        database = self.get_selected_location()
        db_dt = self.get_selected_location_date()

        dump_name = f'dump_{database.name}_{self.dt_to_str(db_dt)}_{symbol.split("::")[-1]}_{tt}.csv'
        my_form = npyscreen.Form(name="Dump Data (empty to cancel)")
        csv_path_edit = my_form.add(npyscreen.TitleText, name="CSV Path", value=dump_name)
        my_form.edit()
        if not csv_path_edit.value:
            return
        symbol_data = self.get_current_symbol_data()
        symbol_data.to_csv(csv_path_edit.value)

    def _save_error_message(
        self, message: str, return_screen: OnetickBrowserScreens, error_message: Optional[str] = None,
    ):
        self.current_screen = OnetickBrowserScreens.ERROR_SCREEN
        self._error_screen_message = message
        self._error_screen_return = return_screen
        self._error_screen_exception = error_message
        self._error_screen_shown = False


class OnetickDataConnector:

    def __init__(self, settings, remote_server=None, webapi_mode=False):
        self.settings: OTVSettings = settings
        self.tick_search_max_day_boundary_offset_sec = 86399
        self.day_boundary_tz = os.environ.get('ONETICK_VIEW_DAY_BOUNDARY_TZ', 'GMT')
        self.remote_server = remote_server
        self.local_databases: Dict[str, DatabaseLocation] = {}
        self.remote_databases: Dict[str, List[date]] = {}
        self.webapi_mode = webapi_mode

    @staticmethod
    def _parse_local_db(archive_dir: str):
        archive_dir = archive_dir.rstrip('/')  # fix for cmdline location dir specification
        db_dir = os.path.abspath(os.path.join(archive_dir, '..'))
        _, db_name = os.path.split(db_dir)
        dir_name = os.path.basename(archive_dir)
        db_dt = datetime.strptime(dir_name, '%Y%m%d')
        return db_name, db_dir, db_dt

    def _filter_dbs(self, dbs: Dict[str, DatabaseLocation]):
        filtered_keys = self.settings.filter_strings(
            list(dbs.keys()), self.settings.db_str_filter,
            use_regex=self.settings.db_regexp_match,
        )

        keys_to_delete = set(dbs.keys()).difference(filtered_keys)
        for key in keys_to_delete:
            del dbs[key]

    def _prepare_db_locations(self, dbs: Dict[str, DatabaseLocation]) -> List[DatabaseLocation]:
        self._filter_dbs(dbs)
        for db in dbs.values():
            if db.dates:
                db.dates.sort(reverse=True)

        return sorted(dbs.values(), key=lambda x: x.name)

    def _load_local_databases(self, start_path: str):
        result: Dict[str, DatabaseLocation] = {}
        dir_queue: Deque[Tuple[Path, int]] = deque()
        dir_queue.append((Path(start_path), 0))

        while dir_queue:
            dir_path, depth = dir_queue.popleft()

            if depth > 16 or self.settings.db_hide_backups and dir_path.name == 'backups':
                continue

            try:
                sub_directories: List[Path] = [sub_dir for sub_dir in dir_path.iterdir() if sub_dir.is_dir()]
            except OSError:
                continue
            dates: Dict[Path, date] = {}

            for sub_dir in sub_directories:
                try:
                    dir_dt = datetime.strptime(sub_dir.name, '%Y%m%d').date()
                    dates[sub_dir] = dir_dt
                except Exception:
                    pass

            if len(dates) > 0 and {'meta_data', 'symbols', 'index'} & {
                file.name for file in next(iter(dates.keys())).iterdir() if file.is_file()
            }:
                str_path = str(dir_path)
                result[str_path] = DatabaseLocation(name=dir_path.name, path=str(dir_path.absolute()))
                result[str_path].dates.extend(dates.values())
                continue

            dir_queue.extend([(sub_dir, depth + 1) for sub_dir in sub_directories])

        self.local_databases = result

    def preload_remote_database_date(self, database: DatabaseLocation):
        if database.name not in self.remote_databases:
            return

        if database.dates is not None:
            return

        if self.remote_databases[database.name] is None:
            with otp.Session(override_env=True) as session:
                if not self.webapi_mode:
                    session.use(otp.RemoteTS(self.remote_server))
                self.remote_databases[database.name] = InspectionDB(database.name).dates()

        database.dates = sorted(self.remote_databases[database.name], reverse=True)

    def _load_remote_databases(self):
        if not self.remote_server and not self.webapi_mode:
            self.remote_databases = {}
            return

        with otp.Session(override_env=True) as session:
            if not self.webapi_mode:
                session.use(otp.RemoteTS(self.remote_server))

            result = {db_name: None for db_name in otp.databases()}
            self.remote_databases = result

    def preload_databases(self, start_path: str):
        if start_path:
            self._load_local_databases(start_path)
        self._load_remote_databases()

    def get_databases(self) -> List[DatabaseLocation]:
        local_databases_list = self._prepare_db_locations(self.local_databases)

        remote_databases = {}
        for db_name, dates in self.remote_databases.items():
            remote_databases[db_name] = DatabaseLocation(db_name, storage_type='remote')
            remote_databases[db_name].dates = dates
        remote_databases_list = self._prepare_db_locations(remote_databases)

        return local_databases_list + remote_databases_list

    @staticmethod
    def get_location(archive_dir: str):
        archive_dir = archive_dir.rstrip('/')  # fix for cmdline location dir specification
        db_dir = os.path.abspath(os.path.join(archive_dir, '..'))
        dir_name = os.path.basename(archive_dir)
        date = datetime.strptime(dir_name, '%Y%m%d')
        return db_dir, date

    def setup_db(self, database: DatabaseLocation, session: otp.Session):
        kwargs = {}
        db_properties = {
            'timezone': self.settings.timezone,
            # 'day_boundary_tz': self.day_boundary_tz,
            'day_boundary_tz': self.settings.timezone,
            'tick_search_max_day_boundary_offset_sec': self.tick_search_max_day_boundary_offset_sec,
        }

        if database.storage_type == 'local':
            kwargs['db_locations'] = [{'location': database.path}]

            db = otp.db.DB(
                database.name,
                db_properties=db_properties,
                **kwargs,
            )
            session.use(db)
        elif database.storage_type == 'remote':
            if not self.webapi_mode:
                session.use(otp.RemoteTS(self.remote_server))

    def get_tts_symbols(self, database: DatabaseLocation, db_dt: date) -> Dict:
        tt_symbols = {}

        with otp.Session(override_env=True) as s:
            self.setup_db(database, s)
            db = otp.databases()[database.name]

            tts = db.tick_types(date=db_dt, timezone=self.settings.timezone)
            tts = self.settings.filter_strings(tts, self.settings.tick_type, exact_match=True)
            for tt in tts:
                symbols = db.symbols(tick_type=tt, date=db_dt, timezone=self.settings.timezone)
                symbols = self.settings.filter_strings(symbols, self.settings.symbol_str_filter,
                                                       use_regex=self.settings.db_regexp_match)
                tt_symbols[tt] = symbols

        return tt_symbols

    def get_data_count(self, database: DatabaseLocation, db_dt: date):
        tts_stats = defaultdict(int)
        symbol_stats = defaultdict(int)
        with otp.Session(override_env=True) as s:
            self.setup_db(database, s)
            db = otp.databases()[database.name]

            try:
                tts = db.tick_types(date=db_dt, timezone=self.settings.timezone)
            except:
                return None, None

            tts = self.settings.filter_strings(tts, self.settings.tick_type, exact_match=True)

            for tt in tts:
                symbols = db.symbols(tick_type=tt, date=db_dt, timezone=self.settings.timezone)
                symbols = self.settings.filter_strings(symbols, self.settings.symbol_str_filter,
                                                       use_regex=self.settings.db_regexp_match)
                for symbol in symbols:
                    res = otp.DataSource(db=db, tick_type=tt, symbol=symbol,
                                         start=db_dt, end=db_dt + timedelta(days=1),
                                         schema_policy='manual')
                    res = res.agg({'num_ticks': otp.agg.count()})
                    num_ticks = otp.run(res, timezone=self.settings.timezone)['num_ticks'][0]
                    tts_stats[tt] += num_ticks
                    symbol_stats[(tt, symbol)] += num_ticks
        return dict(tts_stats), dict(symbol_stats)

    def get_data_count_cached(self, database: DatabaseLocation, db_dt: date) -> Tuple[dict, dict, bool]:
        if database.date_data[db_dt].stats is not None:
            tts_stats, symbol_stats = database.date_data[db_dt].stats
            return tts_stats, symbol_stats, True

        data = self.get_data_count(database, db_dt)
        database.date_data[db_dt].stats = data
        return data[0], data[1], False

    def get_symbol_data(self, tt: str, symbol: str, database: DatabaseLocation, db_dt: date):
        with otp.Session(override_env=True) as s:
            self.setup_db(database, s)
            res = otp.DataSource(db=database.name, tick_type=tt, symbol=symbol,
                                 start=db_dt, end=db_dt + timedelta(days=1),
                                 schema_policy='manual')
            if self.settings.data_where_clause:
                condition = otp.raw(self.settings.data_where_clause, dtype=bool)
                res = res.where(condition, discard_on_match=self.settings.discard_on_match)
            res = otp.run(res, timezone=self.settings.timezone)
            return res


def parser_impl(parser):
    parser.add_argument("path", nargs='?', type=str, help="path with OneTick databases", default='')
    parser.add_argument("--remote-ts", type=str, help="address of remote TickServer", default=None)
    parser.add_argument("--tz", type=str, help="the timezone to run the queries and return the data timestamps", default=None)
    parser.description = DESCRIPTION
    parser.formatter_class = argparse.RawTextHelpFormatter
    return parser


def run(args):
    try:
        run_impl(args)
    except KeyboardInterrupt:
        sys.exit(130)

def run_impl(args):
    start_path = args.path
    remote_server = args.remote_ts

    # onetick-py automatically detects which mode it is running
    # depending on env variables and available installed OneTick python packages
    if otp.__webapi__:
        webapi_mode = True
        if start_path or remote_server:
            raise ValueError('onetick-py is running in WebAPI mode.\n'
                             'In this case all databases are loaded from the WebAPI server '
                             'and parameters "path" and "--remote-ts" must not be set.')
    else:
        webapi_mode = False
        if not remote_server:
            # in local mode use current directory if not specified
            start_path = start_path or os.path.abspath(os.getcwd())

    settings = OTVSettings(timezone_name=args.tz)
    connector = OnetickDataConnector(settings=settings, remote_server=remote_server, webapi_mode=webapi_mode)
    otb = OnetickBrowser(start_path, connector, settings=settings)
    curses.wrapper(otb.draw_main_display)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser_impl(parser)
    run(parser.parse_args())
