'''Logging scheme for dagstermill.

Dagstermill notebooks are executed in a subprocess (opened by jupyter_client.manager.KernelManager).
We need to ship logs from the subprocess back to the parent process (executing the solid compute
logic), but since we don't manage the subprocess directly we can't use a Queue/QueueHandler-based
scheme. (Socket-based schemes might work, but getting these to play nicely with Docker and the wide
range of possible user networking setups sounds like a headache.)

There are a variety of persistent file-based queue projects in the Python ecosystem, but they are
written for multithreaded cases, not multiprocessing cases; Celery is too heavy for our use
case; and obviously we want to avoid implementing our own locking logic (perhaps on top of
filelock).

Instead we use a sqlite database as a queue. The elements of this setup are exhibited cleanly
in dagstermill_tests/test_logger.py:

1. Dagstermill solid compute logic running in parent process opens a NamedTemporaryFile for use as
   a sqlite db.
2. Parent process initializes the db table by calling dagstermill.logger.init_db. We rely on a
   sqlite autoincrement primary key to preserve the order of log messages.
3. Parent process starts a watcher thread that spins up a JsonSqlite3LogWatcher, which polls the
   sqlite db table for new logs until its is_done flag (a threading.Event) is set. New log messages
   are rehydrated using logging.makeLogRecord and handed directly to the logging.Handlers configured
   on the pipeline DagsterLogManager.
4. Child process is started by papermill/nbconvert machinery to execute notebook.
5. Dagstermill notebook manager logic running in child process creates a notebook-specific
   DagsterLogManager with a single-handler (dagstermill.logger.JsonSqlite3Handler) logger. This
   handler writes the internal dict representation of logging.LogRecords generated by the dagster
   logging machinery to the sqlite table as json.
6. Child process returns. Parent proess sets is_done flag.
7. Parent process waits for watcher process to complete logging.

'''
import copy
import json
import logging
import sqlite3
import sys
import threading
import time

from dagster import check, seven
from dagster.core.log_manager import DagsterLogManager
from dagster.utils.log import construct_single_handler_logger


CREATE_LOG_TABLE_STATEMENT = '''create table if not exists logs (
    timestamp integer primary key asc,
    json_log text
)
'''

INSERT_LOG_RECORD_STATEMENT = '''insert into logs (json_log) values (
    '{json_log}'
)
'''

RETRIEVE_LOG_RECORDS_STATEMENT = '''select * from logs
    where timestamp >= {timestamp}
    order by timestamp asc
'''

if sys.version_info < (3,):  # pragma: nocover
    EVENT_TYPE = threading._Event  # pylint: disable=no-member, protected-access
else:
    EVENT_TYPE = threading.Event


def init_db(sqlite_db_path):
    conn = sqlite3.connect(sqlite_db_path)
    cursor = conn.cursor()
    cursor.execute(CREATE_LOG_TABLE_STATEMENT)
    conn.commit()


class JsonSqlite3Handler(logging.Handler):
    def __init__(self, sqlite_db_path):
        check.str_param(sqlite_db_path, 'sqlite_db_path')

        self.sqlite_db_path = sqlite_db_path
        self.conn = sqlite3.connect(self.sqlite_db_path)
        self.cursor = self.conn.cursor()

        init_db(sqlite_db_path)

        super(JsonSqlite3Handler, self).__init__()

    def emit(self, record):
        try:
            log_dict = copy.copy(record.__dict__)
            self.cursor.execute(
                INSERT_LOG_RECORD_STATEMENT.format(json_log=seven.json.dumps(log_dict))
            )
            self.conn.commit()

        except Exception as e:  # pylint: disable=W0703
            logging.critical('Error during logging!')
            logging.exception(str(e))


class JsonSqlite3LogWatcher(object):
    def __init__(self, sqlite_db_path, log_manager, is_done):
        check.str_param(sqlite_db_path, 'sqlite_db_path')
        check.inst_param(log_manager, 'log_manager', DagsterLogManager)
        check.inst_param(is_done, 'is_done', EVENT_TYPE)

        self.sqlite_db_path = sqlite_db_path
        self.conn = sqlite3.connect(self.sqlite_db_path)
        self.cursor = self.conn.cursor()
        self.next_timestamp = 0
        self.log_manager = log_manager
        self.is_done = is_done

    def watch(self):
        last_pass = False
        while True:
            res = self.cursor.execute(
                RETRIEVE_LOG_RECORDS_STATEMENT.format(timestamp=self.next_timestamp)
            ).fetchall()
            if res:
                self.next_timestamp = res[-1][0] + 1
                json_records = [r[1] for r in res]
                for json_record in json_records:
                    record = logging.makeLogRecord(json.loads(json_record))

                    for logger in self.log_manager.loggers:
                        for handler in logger.handlers:
                            # Because we're rehydrating the LogMessage, rather than passing
                            # through Logger._log again (which would obscure the original metadata)
                            # we need to filter for log level here
                            if handler.level <= record.levelno:
                                handler.handle(record)

            time.sleep(1)
            if last_pass:
                break
            if self.is_done.is_set():
                last_pass = True


def construct_sqlite_logger(sqlite_db_path):
    return construct_single_handler_logger(
        'dagstermill', 'debug', JsonSqlite3Handler(sqlite_db_path)
    )
