# -*- coding:utf-8 -*-
"""
@Author       : xupingmao
@email        : 578749341@qq.com
@Date         : 2022-02-12 18:13:41
@LastEditors  : xupingmao
@LastEditTime : 2024-07-14 00:20:28
@FilePath     : /xnote/handlers/system/system_sync/node_follower.py
@Description  : 从节点管理
"""

import time
import logging
import typing
import xutils

from xnote.core import xconfig, xtables

from xutils import Storage
from xutils import textutil, cacheutil
from xutils import dbutil, six
from xnote.service import DatabaseLockService

from xutils.db.binlog import BinLog, FileLog, BinLogOpType, BinLogRecord
from .node_base import NodeManagerBase
from .node_base import convert_follower_dict_to_list
from .system_sync_proxy import HttpClient
from .system_sync_proxy import empty_http_client
from .models import FileIndexInfo, LeaderStat, FollowerInfo
from xutils.mem_util import log_mem_info_deco
from .dao import ClusterConfigDao
from .system_sync_indexer import count_fs_index

def filter_result(result, offset):
    data = []
    for item in result.data:
        if item.get("_id", "") == offset:
            logging.debug("跳过offset:%s", offset)
            continue
        data.append(item)

    result.data = data
    return result

class Follower(NodeManagerBase):

    # PING的时间间隔，单位是秒
    # Leader侧的失效时间是1小时
    PING_INTERVAL = 600

    def __init__(self):
        self.ping_error = ""
        self.ping_result = None # type: LeaderStat|None
        self.follower_list = [] # type: list[FollowerInfo]
        self.admin_token = ""
        self.access_token = ""
        self.last_ping_time = -1
        self.fs_index_count = -1
        self.fs_max_index = -1
        # 同步完成的时间
        self.fs_sync_done_time = -1
        self._debug = False
        self.http_client = self.get_client()
        self.file_syncer = FileSyncer(self.http_client)
        self.db_syncer = DBSyncer(file_syncer = self.file_syncer)

    def create_http_client(self):
        leader_host = self.get_leader_url()
        leader_token = self.get_leader_token()
        return HttpClient(leader_host, leader_token, self.admin_token)

    def get_client(self):
        return self.create_http_client()

    def get_node_id(self):
        return xconfig.WebConfig.cluster_node_id

    def get_leader_node_id(self):
        if self.ping_result != None:
            return self.ping_result.node_id
        return "<unknown>"

    def get_current_port(self):
        return xconfig.WebConfig.port
    
    def is_token_active(self):
        now = time.time()
        is_active = (now - self.last_ping_time) < self.PING_INTERVAL
        return self.access_token != "" and is_active

    def ping_leader(self, force=True):
        if self.is_token_active() and not force:
            return self.ping_result
        
        return self.do_ping_leader()

    def do_ping_leader(self):
        port = self.get_current_port()

        fs_sync_offset = self.get_fs_sync_last_id()

        leader_host = self.get_leader_url()
        if leader_host != "":
            client = self.get_client()
            cluster_node_id = self.get_node_id()
            result_obj = client.get_stat(port = port, fs_sync_offset = fs_sync_offset, cluster_node_id = cluster_node_id)
            self.update_ping_result(result_obj)
            return result_obj

        return None

    def update_ping_result(self, result: typing.Optional[LeaderStat]):
        if result is None:
            logging.error("PING主节点:返回None")
            return

        if result.code != "success":
            self.ping_error = result.message
            logging.error("PING主节点失败:%s", self.ping_error)
            return

        logging.debug("PING主节点成功")

        self.ping_error = ""
        self.ping_result = result
        follower_dict = result.follower_dict
        self.follower_list = convert_follower_dict_to_list(follower_dict)
        self.last_ping_time = time.time()
        self.fs_max_index = result.fs_max_index
        self.access_token = result.access_token

        if len(self.follower_list) > 0:
            item = self.follower_list[0]
            self.admin_token = item.admin_token
            self.fs_index_count = item.fs_index_count
            self.http_client.admin_token = self.admin_token

    def get_follower_list(self):
        return self.follower_list

    def get_leader_url(self):
        return ClusterConfigDao.get_leader_host()
    
    def get_leader_info(self):
        if self.ping_result != None:
            return self.ping_result.leader
        return None

    def get_ping_error(self):
        return self.ping_error

    def need_sync(self):
        if self.fs_sync_done_time < 0:
            return True

        last_sync = time.time() - self.fs_sync_done_time
        return last_sync >= self.PING_INTERVAL

    def get_fs_sync_last_id(self):
        return ClusterConfigDao.get_fs_sync_last_id()
        
    def sync_file_step(self):
        client = self.get_client()    
        last_id = self.get_fs_sync_last_id()
        
        logging.debug("fs_sync_last_id=%s", last_id)
        result = client.list_files(last_id)
        logging.debug("list files result=(%s), last_id=(%s)", result, last_id)

        if result is None:
            logging.error("返回结果为空")
            return []

        data = result.get("data", [])
        assert isinstance(data, list)
        
        if len(data) == 0:
            logging.debug("返回文件列表为空")
            self.fs_sync_done_time = time.time()
            return data
        
        for item in data:
            item = FileIndexInfo(**item)
            last_id = max(last_id, item.id)

        # logging.debug("result:%s", result)
        client.download_files(result)

        # offset可能不变
        logging.debug("result.sync_offset:%s", result.sync_offset)
        logging.debug("last_id = %s", last_id)

        ClusterConfigDao.put_fs_sync_last_id(last_id)
        return data

    def sync_files_from_leader(self):
        result = self.ping_leader()
        if result == None:
            logging.error("ping_leader结果为空")
            return

        if not self.need_sync():
            # logging.debug("没到SYNC时间")
            return

        has_next = True
        loop_count = 0
        while has_next and loop_count < 100:
            result = self.sync_file_step()
            has_next = len(result) > 0
            loop_count += 1

        return result

    def get_sync_process(self):
        if self.fs_index_count < 0:
            return "-1"

        count = self.count_sync_done()
        if count == 0:
            return "0%"
        return "%.2f%%" % (count / self.fs_index_count * 100.0)

    def sync_for_home_page(self):
        return self.ping_leader(force=False) != None

    def get_fs_index_count(self):
        return self.fs_index_count

    def count_sync_done(self):
        return count_fs_index()

    def reset_sync(self):
        ClusterConfigDao.reset_sync()
        self.fs_sync_done_time = -1

    def reset_fs_offset(self):
        ClusterConfigDao.put_fs_sync_last_id(0)

    def sync_db_from_leader(self):
        with DatabaseLockService.lock(lock_key="sync_db_from_leader", timeout_seconds=60):
            return self.sync_db_from_leader_no_lock()

    def sync_db_from_leader_no_lock(self):
        leader_host = self.get_leader_url()
        if leader_host == "":
            logging.debug("leader_host为空")
            raise Exception("leader_host为空")

        ping_result = self.ping_leader()
        if ping_result == None:
            logging.debug("ping_leader结果为空")
            raise Exception("ping_leader结果为空")

        leader_token = self.get_leader_token()
        if leader_token == "":
            logging.debug("leader_token为空")
            raise Exception("leader_token为空")
        
        self.db_syncer.sync_db(self.get_client())
    
    def is_at_full_sync(self):
        return self.db_syncer.get_db_sync_state() == "full"


class FileSyncer:
    """文件同步器"""
    def __init__(self, http_client = empty_http_client):
        self.http_client = http_client

    def handle_file_binlog(self, key, value):
        self.http_client.handle_token()
        if value == None:
            logging.warning("value is None, key=%s", key)
            return
        self.sync_file(value)
    
    def sync_file(self, item):
        new_item = FileIndexInfo(**item)
        self.http_client.download_file(new_item)

empty_file_syncer = FileSyncer()

class DBSyncer:

    MAX_LOOPS = 1000 # 最大循环次数
    FULL_SYNC_MAX_LOOPS = 10000 # 全量同步最大循环次数

    def __init__(self, *, debug = True, file_syncer: FileSyncer):
        self._binlog = BinLog.get_instance()
        self.debug = debug
        self.file_syncer = file_syncer
    
    def get_table_by_key(self, key: str):
        table_name = key.split(":")[0]
        if dbutil.TableInfo.is_registered(table_name):
            return dbutil.get_table(table_name)
        return None
    
    def get_table_name_by_key(self, key: str):
        return key.split(":")[0]
    
    def get_binlog_last_seq(self):
        return ClusterConfigDao.get_binlog_last_seq()

    def put_binlog_last_seq(self, last_seq: int):
        return ClusterConfigDao.put_binlog_last_seq(last_seq)

    def get_db_sync_state(self):
        return ClusterConfigDao.get_db_sync_state()

    def put_db_sync_state(self, state):
        ClusterConfigDao.put_db_sync_state(state)
    
    def get_db_last_key(self):
        # type: () -> str
        # 全量同步使用，按照key进行遍历
        return ClusterConfigDao.get_db_last_key()

    def put_db_last_key(self, last_key):
        ClusterConfigDao.put_db_last_key(last_key)

    def get_table_v2_or_None(self, table_name):
        try:
            return dbutil.get_table_v2(table_name)
        except:
            return None
    
    def put_and_log(self, key, value):
        assert key != None
        assert value != None
        table_name = self.get_table_name_by_key(key)
        table_v2 = self.get_table_v2_or_None(table_name)
        if table_v2 != None:
            table_v2.put_by_key(key, value, fix_index=True)
            return
        
        table_info = dbutil.get_table_info(table_name)
        if table_info == None:
            logging.warning("table not exists: %s", table_name)
            return
        
        if not table_info.need_sync:
            logging.info("no need sync, key=%s", key)
            return
        
        if table_info.type == "hash":
            dbutil.put(key, value)
        else:
            table = dbutil.get_table(table_name, skip_user_check = True)
            table.update_by_key(key, value)
    
    def delete_and_log(self, key:str):
        assert key != None
        table_name = self.get_table_name_by_key(key)
        table_v2 = self.get_table_v2_or_None(table_name)
        if table_v2 != None:
            table_v2.delete_by_key(key)
            return
        
        table = self.get_table_by_key(key)
        if table != None:
            table.delete_by_key(key)
    
    @log_mem_info_deco("follower.sync_db")
    def sync_db(self, proxy: HttpClient):
        proxy.handle_token()
        sync_state = self.get_db_sync_state()
        if sync_state == "binlog":
            # 增量同步
            self.sync_by_binlog(proxy)
        else:
            # 全量同步
            self.sync_db_full(proxy)

    def sync_db_full(self, proxy: HttpClient):
        from xnote_handlers.system.backup import DBImporter
        backup_info = proxy.get_backup_info()
        if backup_info is None:
            return
        proxy.download_file(backup_info.backup_file_info, force_download=True)
        filepath = backup_info.backup_file_info.realpath
        DBImporter().import_db(filepath)
        self.put_db_sync_state("binlog")
        self.put_binlog_last_seq(backup_info.backup_binlog_id)


    def sync_db_full_step(self, proxy, last_key: str):
        # type: (HttpClient, str) -> int
        result, data = proxy.list_db(last_key)

        if self.debug:
            logging.debug("-------------\nresp:%s\n\n", result)

        code = result.code
        count = 0
        assert code == "success"
        assert data != None, "data不能为空"

        if last_key == "":
            # last_key为空说明开始全量同步
            # 这里需要保存一下位点，后面增量同步从这里开始
            binlog_last_seq = data.binlog_last_seq
            assert isinstance(binlog_last_seq, int)
            self.put_binlog_last_seq(binlog_last_seq)

        new_last_key = last_key
        for row in data.rows:
            key = row.record_key
            value = row.value_obj
            assert key != None
            assert value != None
            if key == last_key:
                continue
            
            self.put_and_log(key, value)
            new_last_key = key
            count += 1

        if count == 0:
            self.put_db_sync_state("binlog")
        else:
            self.put_db_last_key(new_last_key)
        
        return count

    @log_mem_info_deco("sync_by_binlog")
    def sync_by_binlog(self, proxy: HttpClient): # type: (HttpClient) -> object
        has_next = True
        loops = 0
        last_seq = ""

        while has_next:
            loops += 1
            if loops > self.MAX_LOOPS:
                logging.error("too deep loops, last_seq=%s", last_seq)
                raise Exception("too deep loops")

            last_seq = self.get_binlog_last_seq()
            result_obj, data_list = proxy.list_binlog(last_seq)
            
            if self.debug:
                logging.debug("list binlog result=%s" % result_obj)

            code = result_obj.code
            has_next = (data_list != None and len(data_list) > 1)
            logging.info("code=%s, has_next=%s", code, has_next)

            if code == "success":
                self.sync_by_binlog_step(data_list)
            elif code == "sync_broken":
                logging.error("同步binlog异常, 重新全量同步...")
                self.put_binlog_last_seq(0)
                self.put_db_sync_state("full")
                self.put_db_last_key("")
                self.sync_db_full(proxy)
                return "sync_by_full"
            else:
                logging.error("code=%s, message=%s", code, result_obj.message)
                raise Exception("未知的code:%s" % code)
        
        return None

    @log_mem_info_deco("sync_by_binlog_step")
    def sync_by_binlog_step(self, data_list: typing.List[BinLogRecord]):
        last_seq = self.get_binlog_last_seq()
        max_seq = last_seq
        for data in data_list:
            seq = data.binlog_id
            if seq == last_seq:
                continue

            optype = data.op_type
            key = data.record_key
            value = data.value_obj

            if optype in (BinLogOpType.put, BinLogOpType.delete):
                self.handle_kv_binlog(data)
            elif optype == "file_upload":
                self.file_syncer.handle_file_binlog(key, value)
            elif optype == "file_rename":
                # TODO 重命名需要考虑删除原来的文件
                self.file_syncer.handle_file_binlog(key, value)
            elif optype == "file_delete":
                # TODO 考虑移动到一个删除文件夹下面
                logging.info("【不处理】删除文件操作: %s", data)
            elif optype in (BinLogOpType.sql_upsert, BinLogOpType.sql_delete):
                self.handle_sql_binlog(data)
            else:
                logging.error("未知的optype:%s", optype)

            max_seq = max(max_seq, seq)
        if max_seq != last_seq:
            self.put_binlog_last_seq(max_seq)
        else:
            logging.info("db已经保持同步")
    
    def handle_kv_binlog(self, data: BinLogRecord):
        key = data.record_key
        value = data.value_obj
        assert key != None
        if value == None or data.op_type == BinLogOpType.delete:
            self.delete_and_log(key)
        else:
            self.put_and_log(key, value)

    def handle_sql_binlog(self, data: BinLogRecord):
        optype = data.op_type
        table_name = data.table_name
        value = data.value_obj
        pk_value = data.record_key

        if table_name == None:
            return
        
        if not xtables.is_table_exists(table_name):
            logging.info("表不存在, table_name=%s", table_name)
            return
        
        table = xtables.get_table_by_name(table_name)
        table.enable_binlog = False
        
        pk_name = table.table_info.pk_name
        where = {pk_name:pk_value}
        if optype == BinLogOpType.sql_delete:
            table.delete(where=where)

        if optype == BinLogOpType.sql_upsert:
            assert isinstance(value, dict)
            value = table.filter_record(value)
            # 使用replace解决冲突的问题
            try:
                table.replace(**value)
            except Exception as err:
                xutils.print_exc()
                print(f"error value={value}")
                self.ignore_or_raise(err)
                
    def ignore_or_raise(self, err: Exception):
        err_msg = str(err)
        print(f"err_msg={err_msg}")
        err_msg = err_msg.lower()
        if err_msg.startswith("(1292, \"incorrect datetime value"):
            # mysql: datetime异常,无法执行成功
            return
        if err_msg.startswith("(1062, \"duplicate entry"):
            # mysql: 主键冲突
            return
        if err_msg.startswith("(1366, \"incorrect integer value:"):
            # mysql: 类型错误
            return
        if err_msg.startswith("unique constraint failed:"):
            # sqlite: 主键冲突
            return
        raise err
