'''Cross process log sink

The multiprocess executor processses execution steps in child processes and recieves back the system
events. Dagstermill notebooks are executed in a subprocess (opened by jupyter_client.manager.KernelManager).

Given these setups, we need a way to ship logs from the subprocess back to the parent process
(executing the solid compute logic), but since we don't manage the subprocess directly in some cases, we can't use a
Queue/QueueHandler-based scheme. (Socket-based schemes might work, but getting these to play nicely with
Docker and the wide range of possible user networking setups sounds like a headache.)

There are a variety of persistent file-based queue projects in the Python ecosystem, but they are
written for multithreaded cases, not multiprocessing cases; Celery is too heavy for our use
case; and obviously we want to avoid implementing our own locking logic (perhaps on top of
filelock).

Instead we use a sqlite database as a queue. The elements of this setup are exhibited cleanly
in dagster_tests/core/test_xproc_log_sink.py

1. Parent process opens a NamedTemporaryFile for use as a sqlite db.
2. Parent process initializes the db table by calling init_db. We rely on a
   sqlite autoincrement primary key to preserve the order of log messages.
3. Parent process starts a watcher thread that spins up a JsonSqlite3LogWatcher, which polls the
   sqlite db table for new logs until its is_done flag (a threading.Event) is set. New log messages
   are rehydrated using logging.makeLogRecord and handed directly to the logging.Handlers configured
   on the pipeline DagsterLogManager.
4. Child process is started to execute.
5. Child process creates a DagsterLogManager with a single-handler (JsonSqlite3Handler) logger.
   This handler writes the internal dict representation of logging.LogRecords generated by the
   dagster logging machinery to the sqlite table as json.
6. Child process returns. Parent proess sets is_done flag.
7. Parent process waits for watcher process to complete logging.

'''
import copy
import json
import logging
import sqlite3
import sys
import threading
import time

from dagster import check, seven
from dagster.core.log_manager import DagsterLogManager

CREATE_LOG_TABLE_STATEMENT = '''create table if not exists logs (
    timestamp integer primary key asc,
    json_log text
)
'''

INSERT_LOG_RECORD_STATEMENT = '''insert into logs (json_log) values (
    '{json_log}'
)
'''

RETRIEVE_LOG_RECORDS_STATEMENT = '''select * from logs
    where timestamp >= {timestamp}
    order by timestamp asc
'''

if sys.version_info < (3,):  # pragma: nocover
    EVENT_TYPE = threading._Event  # pylint: disable=no-member, protected-access
else:
    EVENT_TYPE = threading.Event


def init_db(sqlite_db_path):
    with sqlite3.connect(sqlite_db_path) as con:
        con.execute(CREATE_LOG_TABLE_STATEMENT)
    con.close()


class JsonSqlite3Handler(logging.Handler):
    def __init__(self, sqlite_db_path, log_msg_only=False):
        check.str_param(sqlite_db_path, 'sqlite_db_path')

        self.sqlite_db_path = sqlite_db_path
        self.log_msg_only = log_msg_only

        super(JsonSqlite3Handler, self).__init__()

    def connect(self):
        return sqlite3.connect(self.sqlite_db_path)

    def emit(self, record):
        try:
            log_dict = copy.copy(record.__dict__)

            if self.log_msg_only and log_dict.get('dagster_meta', {}).get('dagster_event'):
                return

            # while it may seem reasonable hold this connection open, that caused a
            # difficult to debug race-condition-esque problem on py2 where the connect call
            # would sporadically lock up despite setting the timeout argument.
            with self.connect() as con:
                con.execute(INSERT_LOG_RECORD_STATEMENT.format(json_log=seven.json.dumps(log_dict)))
            con.close()

        except Exception as e:  # pylint: disable=W0703
            logging.critical('Error during logging!')
            logging.exception(str(e))


class JsonSqlite3LogWatcher(object):
    def __init__(self, sqlite_db_path, log_manager, is_done):
        check.str_param(sqlite_db_path, 'sqlite_db_path')
        check.inst_param(log_manager, 'log_manager', DagsterLogManager)
        check.inst_param(is_done, 'is_done', EVENT_TYPE)

        self.sqlite_db_path = sqlite_db_path
        self.next_timestamp = 0
        self.log_manager = log_manager
        self.is_done = is_done

    def connect(self):
        return sqlite3.connect(self.sqlite_db_path)

    def watch(self):
        last_pass = False
        while True:
            with self.connect() as conn:
                res = (
                    conn.cursor()
                    .execute(RETRIEVE_LOG_RECORDS_STATEMENT.format(timestamp=self.next_timestamp))
                    .fetchall()
                )
                if res:
                    self.next_timestamp = res[-1][0] + 1
                    json_records = [r[1] for r in res]
                    for json_record in json_records:
                        record = logging.makeLogRecord(json.loads(json_record))

                        for logger in self.log_manager.loggers:
                            for handler in logger.handlers:
                                # Because we're rehydrating the LogMessage, rather than passing
                                # through Logger._log again (which would obscure the original metadata)
                                # we need to filter for log level here
                                if handler.level <= record.levelno:
                                    handler.handle(record)
            conn.close()
            time.sleep(0.5)  # 500 ms
            if last_pass:
                break
            if self.is_done.is_set():
                last_pass = True


def construct_sqlite_logger(sqlite_db_path, log_msg_only=False):
    logger = logging.Logger('xproc_sqlite')
    logger.addHandler(JsonSqlite3Handler(sqlite_db_path, log_msg_only))
    logger.setLevel(10)
    return logger
