import os
import sys
import json
import shutil
import codecs
import textwrap
import numpy as np
import traceback
import socket
import threading
import re
import atexit
import queue

from time import time as tm
from time import strftime, localtime, strptime, mktime
from collections import OrderedDict, deque
from datetime import datetime as dt
from datetime import timedelta, timezone, tzinfo
from dateutil import tz
from pathlib import Path

from .tzlocal import get_localzone_name

SDK_HOME = ".ratio1"
SDK_HOME_PREV = [
  ".ratio1"
]

from .._ver import __VER__

_HTML_START = "<HEAD><meta http-equiv='refresh' content='5' ></HEAD><BODY><pre>"
_HTML_END = "</pre></BODY>"

COLORS = {
  'n': "\x1b[1;37m", # normal white
  'd': "\033[90m", # dark gray
  # 'd': "\x1b[0;37m", # grey white
  'r': "\x1b[1;31m",
  'g': "\x1b[1;32m",
  'y': "\x1b[1;33m",
  # 'b': "\x1b[1;34m",
  'b': "\x1b[1;36m", # changed to bright cyan
  'm': "\x1b[1;35m",
  'c': "\x1b[1;36m",  # bright cyan
  'v': "\x1b[0;36m",  # dim cyan  
  'a': "\x1b[41m",
  'e': "\x1b[41m",
  'w': "\x1b[1;31m", # warning == red

  '__end__': "\x1b[0m",
}

_LOGGER_LOCK_ID = '_logger_print_lock' 


class LockResource():
  def __init__(self, owner, resource, condition):
    self.__owner = owner
    self.__resource = resource
    self.__condition = condition
    
  def __enter__(self):
    if self.__condition:
      self.__owner.lock_resource(self.__resource)
    return self
  
  def __exit__(self, type, value, traceback):
    if self.__condition:
      self.__owner.unlock_resource(self.__resource)
    return


class BaseLogger(object):

  def __init__(self, lib_name="",
              lib_ver="",
              config_file="",
              config_data={},
              base_folder=None,
              app_folder=None,
              show_time=True,
              config_file_encoding=None,
              no_folders_no_save=False,
              max_lines=None,
              HTML=False,
              DEBUG=True,
              data_config_subfolder=None,
              check_additional_configs=False,
              append_spaces=True,    
              silent=False,          
              default_color='n',
              ):
    """
    Build and initialize the logger runtime.

    This constructor wires together console formatting, filesystem paths,
    lock primitives, and the asynchronous persistence components used by the
    optimized logging pipeline.

    Parameters
    ----------
    lib_name : str, optional
      Logical logger/library name used in prefixes and output filenames.
    lib_ver : str, optional
      Version string displayed in startup logs.
    config_file : str, optional
      Path to a JSON configuration file. When empty, defaults are used.
    config_data : dict, optional
      In-memory configuration overrides merged on top of file config.
    base_folder : str or None, optional
      Root folder for data/log/output directories.
    app_folder : str or None, optional
      Application subfolder under ``base_folder``.
    show_time : bool, optional
      Whether to prepend timestamps to console/log messages.
    config_file_encoding : str or None, optional
      Explicit text encoding for ``config_file`` loading.
    no_folders_no_save : bool, optional
      Disable persistence and folder setup when ``True``.
    max_lines : int or None, optional
      Maximum in-memory lines before log splitting is triggered.
    HTML : bool, optional
      Enable HTML log output mode.
    DEBUG : bool, optional
      Toggle debug behavior for selected logger features.
    data_config_subfolder : str or None, optional
      Optional subfolder used when loading additional config payloads.
    check_additional_configs : bool, optional
      Enable automatic discovery of additional config files.
    append_spaces : bool, optional
      Pad console messages for aligned terminal output.
    silent : bool, optional
      Suppress console output while still collecting log messages.
    default_color : str, optional
      Default terminal color key.

    Notes
    -----
    The optimization path initializes:

    - buffered save policy controls (`_save_idle_seconds`, `_save_buffer_len`)
    - append-only write offsets (`_save_file_offsets`)
    - bounded writer queue/thread state and telemetry counters
    - optional repeat-suppression state for rate control
    """

    super(BaseLogger, self).__init__()
    if os.name == 'nt':
      os.system('color')
    self.__lib__ = lib_name
    self.__folder_setup_done = False
    self.append_spaces = append_spaces
    self.show_time = show_time
    self.no_folders_no_save = no_folders_no_save
    self.max_lines = max_lines
    self.HTML = HTML
    self.silent = silent
    self.DEBUG = DEBUG
    self.log_suffix = lib_name
    self.default_color = default_color
    self.__first_print = False
    
    self._lock_table = OrderedDict({
      _LOGGER_LOCK_ID: threading.Lock(),
      })
    self._lock_table_identities = {}
    
    self._lock_table_mutex = threading.Lock()
    
    if base_folder is None:
      base_folder = self.get_user_folder(as_str=True)
      
    if app_folder is None:
      app_folder = "_local_cache"

    self._base_folder = base_folder
    self._app_folder = app_folder
    self._normalize_path_sep()

    self.check_additional_configs = check_additional_configs
    self.data_config_subfolder = data_config_subfolder

    self.__version__ = __VER__
    self.version = self.__version__
    self.file_prefix = None
    self.refresh_file_prefix()

    self.last_time = tm()
    self.start_timestamp = tm()
    self.utc_offset = self.get_utc_offset()
    
    try:
      self.timezone = get_localzone_name()
    except Exception as exc:
      self.timezone = str(exc)
      
    self.app_log = list()
    self.err_log = list()
    # Flush policy defaults: save on idle gaps, otherwise batch by line count.
    self._save_idle_seconds = 1.0
    self._save_buffer_len = 100
    self._error_flush_immediate = True

    # Track producer-side enqueue offsets so disk writes can be append-only.
    self._last_enqueued_app_len = 0
    self._last_enqueued_err_len = 0
    self._save_file_offsets = {}

    # Dedicated writer queue/thread keeps file I/O off the hot producer path.
    self._writer_queue_maxsize = 4096
    self._writer_queue = queue.Queue(maxsize=self._writer_queue_maxsize)
    self._writer_stop_event = threading.Event()
    self._writer_thread = None

    # Internal telemetry for queue pressure, drops, and writer latency.
    self._writer_stats_lock = threading.Lock()
    self._writer_dropped_lines = 0
    self._writer_queue_high_watermark = 0
    self._writer_batches_written = 0
    self._writer_lines_written = 0
    self._writer_latency_ms = deque(maxlen=5000)

    # Optional repeat suppression state; disabled by default.
    self._rate_limit_enabled = False
    self._rate_limit_window_seconds = 1.0
    self._rate_limit_max_repeats = 0
    self._rate_limit_state = {}
    self._rate_limit_suppressed_messages = 0
    self.split_part = 1
    self.split_err_part = 1
    self.config_data = None
    self.__init_config_data = config_data if config_data is not None else {}
    self.MACHINE_NAME = None
    self.COMPUTER_NAME = None
    self.processor_platform = None
    self.processor_cores = None
    self.python_version = sys.version.split(' ')[0]
    self.python_major = int(self.python_version.split('.')[0])
    self.python_minor = int(self.python_version.split('.')[1])
    if self.python_major < 3:
      msg = "ERROR: Python 2 or lower detected. Run will fail!"
      print(msg)
      raise ValueError(msg)
      
    _ = self.get_machine_name()
    
    
    # START: bundling -- se also properties
    if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
      if not self.silent:
        print('  Running in a PyInstaller bundle')
      self.__is_bundled = True
      self.__bundle_path = sys._MEIPASS
    else:
      if not self.silent:
        print('  Running in a normal Python process')
      self.__is_bundled = False
      self.__bundle_path = None
    # END: bundling -- se also properties
    
    self.analyze_processor_platform()
    
    self._save_enabled = False
    if not self.no_folders_no_save:
      try:
        self._configure_data_and_dirs(config_file, config_file_encoding)        
        self._save_enabled = True
      except:
        self.P("Failed to configure data and dirs. No log persistance possible.\n{}".format(
          traceback.format_exc()
          ), color='r'
        )
        self.no_folders_no_save = True
      #endtry config data and dirs
    #endif 
    
    self._generate_log_path()
    self._generate_error_log_path()
    self._check_additional_configs()
    self._init_log_writer()
    
    self.git_branch = self.get_active_git_branch()
    self.conda_env = self.get_conda_env()

    if lib_ver == "":
      lib_ver = __VER__
    ver = "v{}".format(lib_ver) if lib_ver != "" else ""
    self.P(
      "R1SDK {} initialized on [{}][{}].".format(
        ver, self.MACHINE_NAME, self.get_processor_platform(),
      ),
      color='green',
      boxed=True,
    )
    self.P("  Timezone: {}.".format(self.timezone),color='green')


    if self.DEBUG:
      self.P('  DEBUG is enabled in Logger', color='g')
    else:
      self.P('  WARNING: Debug is NOT enabled in Logger, some functionalities are DISABLED', color='y')

    return
  
  @staticmethod
  def get_user_folder(as_str=False, include_sdk_home=True):
    """
    Returns the user folder.
    """    
    path =  Path.home() / SDK_HOME if include_sdk_home else Path.home()
    if as_str:
      return str(path)
    return path
  
  def get_unique_id(self, size=8):
    """
    efficient and low-colision function for small unique id generation
    """
    import string
    import random
    alphabet = string.ascii_lowercase + string.digits
    uid = ''.join(random.choices(alphabet, k=size))
    return uid
  
  @property
  def is_bundled(self):
    return self.__is_bundled
  
  @property
  def bundle_path(self):
    return self.__bundle_path
    
  
  def is_running(self, verbose=True):
    return self.same_script_already_running(verbose=verbose)
  
  @staticmethod
  def version_to_int(version):
    comps = version.split('.')
    val = 0
    power = 3
    for i, comp in enumerate(comps):
      v = int(comp)
      v = v * 100**power
      power -= 1
      val += v
    return val
    
  @staticmethod
  def get_packages(as_text=False, indent=0, as_dict=False, mandatory={}):
    """
    Will return the currently installed (and visible) packages

    Parameters
    ----------
    as_text : bool, optional
      If true return as text. The default is False.
      
    indent : int, optional
      If return text then return it with indent. The default is 0.
      
    mandatory : dict, optional
      Will raise error if any packages from the dict of key:ver are missing. The default is {}.
      
    as_dict: bool, optional
      Return as package_name:ver dict the result. Default False

    Returns
    -------
    packs : list/str/dict
      the list of packages as list of str, a full text to show or a dict of name:ver.

    """
    try:
      import pkg_resources
      raw_packs = [x for x in pkg_resources.working_set]
      lst_pack_ver = [(x.key, x.version) for x in raw_packs]
    except ModuleNotFoundError:
      from importlib import metadata as importlib_metadata
      import re

      def normalize_package_name(name):
        if not isinstance(name, str):
          return ''
        return re.sub(r"[-_.]+", "-", name).lower()

      lst_pack_ver = []
      for dist in importlib_metadata.distributions():
        dist_name = None
        try:
          dist_name = dist.metadata.get('Name')
        except Exception:
          dist_name = None
        dist_name = normalize_package_name(dist_name or getattr(dist, 'name', None) or '')
        dist_version = getattr(dist, 'version', None)
        if dist_name and isinstance(dist_version, str):
          lst_pack_ver.append((dist_name, dist_version))

      # Keep the latest discovered version for duplicate normalized names.
      lst_pack_ver = sorted(set(lst_pack_ver), key=lambda x: x[0])

    def ver_to_int(version, package=None):
      comps = version.split('.')
      val = 0
      power = 3
      try:
        for i, comp in enumerate(comps):
          v = int(comp)
          v = v * 100**power
          power -= 1
          val += v
      except:
        BaseLogger.print_color("Failed to convert version '{}' to int for package `{}`, version so far: {}".format(version, package, val), color='r')
      return val    
    
    lst_pack_ver = sorted(lst_pack_ver, key=lambda x:x[0])
    maxlen = max([len(x[0]) for x in lst_pack_ver]) + 1 if len(lst_pack_ver) > 0 else 1
    dct_packs = OrderedDict(lst_pack_ver)
    
    if len(mandatory) > 0:      
      for mandatory_pack in mandatory:
        if mandatory_pack not in dct_packs:
          msg = "Mandatory package `{}:{}` is missing. Please check your deployment!".format(
            mandatory_pack, mandatory[mandatory_pack])
          BaseLogger.print_color(msg, color='r')          
          raise ValueError(msg)
        mandatory_ver = ver_to_int(mandatory[mandatory_pack])
        package_ver = ver_to_int(dct_packs[mandatory_pack], package=mandatory_pack)
        if  mandatory_ver > package_ver:
          msg = "Mandatory installed package `{}:{}` ({}) below required version `{}` ({}). Please check your deployment!".format(
            mandatory_pack, dct_packs[mandatory_pack], package_ver, mandatory[mandatory_pack], mandatory_ver)
          BaseLogger.print_color(msg, color='r')
          raise ValueError(msg)        
    #endif check for packages and versions
    
    if as_dict:
      result = dct_packs
    else:
      result = []
      for x in lst_pack_ver:
        result.append("{}{}".format(x[0] + ' ' * (maxlen - len(x[0])), x[1] + ' ' * (14 - len(x[1]))))
        if x[0] in mandatory:
          result[-1] = result[-1] + ' ==> OK ({} > {})'.format(x[1], mandatory[x[0]])
      if as_text:
        fmt = "\n{}".format(' ' * indent)
        result = ' ' * indent + fmt.join(result)
    return result  
    
  
  def same_script_already_running(self, verbose=True):
    import psutil
    CMD = 'python'
    script_file = sys.argv[0]
    if script_file == '':
      self.P("Cannot get script file name", color='r')
      return False
    for q in psutil.process_iter():
      if q.name().startswith(CMD):
        if (
            len(q.cmdline())>1 and 
            script_file in q.cmdline()[1] and 
            q.pid != os.getpid()
            ):
          if verbose:
            self.P("Python '{}' process is already running".format(script_file), color='m')
          return True
    return False
  
  @staticmethod
  def replace_secrets(dct_config, pattern='$EE_', allow_missing=True, missing_default=None):
    """
    TODO: maybe move to Jinja2 style     
    "{{ some_variable | default('my_default') }}"
    
    
    """
    matches = []
    missing = []
    stack = [dct_config]
  
    while stack:
      current = stack.pop()
      if isinstance(current, dict):
        for key, value in current.items():
          if isinstance(value, str) and value.startswith(pattern):
            matches.append(value)
            env_var_name = value[1:] 
            if env_var_name not in os.environ:
              missing.append(env_var_name)
              if allow_missing:
                current[key] = missing_default
            else:
              current[key] = os.environ[env_var_name]
          elif isinstance(value, (dict, list)):
            stack.append(value)
      elif isinstance(current, list):
        for item in current:
          if isinstance(item, (dict, list)):
            stack.append(item)
    if len(missing) > 0:
      msg = 'Required environment configuration for keys {} was not found in current envirnoment. Please setup your docker or bare-metal config to provide this missing key(s)'.format(
        ",".join(['"' + x + '"' for x in missing])
      )
      if allow_missing:
        BaseLogger.print_color(msg, color='r')
      else:
        raise ValueError(msg)
    return matches  
  
  
  
  def lock_process(self, str_lock_name, nt_file_lock=False):
    if os.name == 'nt':
      # windows
      if nt_file_lock:
        # naive lock ...
        self.P("Attempting to create file lock '{}'".format(str_lock_name), color='m')
        fn = str_lock_name + '.lock'
        if os.path.isfile(fn):
          self.P("Another Windows process has already acquired file lock '{}'".format(str_lock_name), color='r')
          return None
        else:
          str_stamp = self.time_to_str()
          with open(fn, "wt") as fh:
            fh.write("LOCKED AT {}".format(str_stamp))
          self.P("Current Windows process has acquired file lock '{}'".format(fn))
          return fn
      else:
        # nice lock but not always working (if not superuser...)
        from win32event import CreateMutex
        from win32api import GetLastError
        from winerror import ERROR_ALREADY_EXISTS
        str_lock_name = "Global\\" + str_lock_name.replace("\\","")
        self.P("Attempting to create lock on current Windows process for id '{}'".format(str_lock_name), color='m')
        
        try:
          mutex_handle = CreateMutex(None, 1, str_lock_name)
          err = GetLastError()
        except:
          self.P("Exception in process locking id '{}'".format(str_lock_name), color='r')
          err = ERROR_ALREADY_EXISTS
          
        if err == ERROR_ALREADY_EXISTS:
          # maybe show some text
          self.P("Another Windows process has already acquired id '{}'".format(str_lock_name), color='r')
          return None
        else:
          # maybe show some text
          self.P("Current Windows process has acquired id '{}':{} ({})".format(
            str_lock_name, mutex_handle, err), color='g')
          return mutex_handle    
    else:
      import platform
      str_platform = platform.system()
      if str_platform.lower() == 'darwin':
        # macos
        self.P("Running on MacOS. Skipping mutex and checking if script is running", color='m')
        if self.same_script_already_running():
          return None        
        return -1
      else:         
        import socket
        self.P("Attempting to create lock on current Linux process for id '{}'".format(str_lock_name), color='m')
        _lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
        try:
          _lock_socket.bind('\0' + str_lock_name)
          # maybe show some text
          self.P("Current Linux process has acquired id '{}': {}".format(
            str_lock_name, _lock_socket), color='g')
          return _lock_socket
        except Exception as err:
          # maybe show some text
          self.P("Another Linux process has already acquired id '{}'. Error: {}".format(
            str_lock_name, err), color='r')
          return None
      # end if platform
    # end if not windows
    return
  
  def maybe_unlock_windows_file_lock(self, str_lock_name):
    if os.name == 'nt' and str_lock_name is not None and isinstance(str_lock_name, str):
      self.P("Attempting to unlock windows lock file...")
      if os.path.isfile(str_lock_name):
        os.remove(str_lock_name)
        self.P("Released windows file lock {}".format(str_lock_name))
      else:
        self.P("Unknown file lock '{}'".format(str_lock_name))
    return
  
  def analyze_processor_platform(self):
    import platform
    import subprocess
    import re
    import multiprocessing
    str_system = platform.system()

    cores = multiprocessing.cpu_count()
    self.processor_cores = cores

    if str_system == "Windows":
      self.processor_platform = platform.processor()
      
    elif str_system == "Darwin":
      os.environ['PATH'] = os.environ['PATH'] + os.pathsep + '/usr/sbin'
      command ="sysctl -n machdep.cpu.brand_string"
      self.processor_platform = subprocess.check_output(command, shell=True).strip().decode('utf-8')
      
    elif str_system == "Linux":
      command = "cat /proc/cpuinfo"
      all_info = subprocess.check_output(command, shell=True).decode().strip()
      proc_platform = None
      for line in all_info.split("\n"):
        if "model name" in line:
          proc_platform = re.sub( ".*model name.*:", "", line,1)    
          break
      if proc_platform is None:
        proc_platform = "Unknown"
        try:
          lscpu_out = subprocess.check_output("lscpu", shell=True).decode().strip()
          match_vendor = re.search(r"Vendor ID:\s+(.*)", lscpu_out)
          if match_vendor:
            proc_platform = match_vendor.group(1)
        except:
          pass          
        proc_platform = f"{proc_platform} {cores} cores"
      #endif no model name
      self.processor_platform = proc_platform
    return
  
  def get_processor_platform(self):
    return self.processor_platform
    

  def get_processor_cores(self):
    return self.processor_cores


  def managed_lock_resource(self, str_res, condition=True):
    """
    Managed lock resource. Will lock and unlock resource automatically. 
    To be used in a with statement.
    The condition parameter allows users to disable the lock if desired.

    Parameters
    ----------
    str_res : str
      The resource to lock.
    condition : bool, optional
      If False the lock will not be acquired. The default is True.

    Returns
    -------
    LockResource
      The lock resource object.

    Example
    -------
    ```
    with self.managed_lock_resource('my_resource'):
      # do something
    ```

    ```
    # will control if the following operation is locked or not based on this flag
    locking = False
    with self.managed_lock_resource('my_resource', condition=locking):
      # do something
    ```
    """
    return LockResource(self, str_res, condition)

  def lock_resource(self, str_res):
    """
    Possible critical failure:
    
    1. base plugin runs try stuff etc
    2. plugin runs lock
    3. threading.Lock() fails
    4. base plugin runs except
    5. except locks in log (no output) due to _lock_table_mutex.acquire(blocking=True)
    6. any thread running lock_reource will hang with NO LOG OUTPUT
    """
    result = None
    self._lock_table_mutex.acquire(blocking=True)
    try:
      if str_res not in self._lock_table:
        self._lock_table[str_res] = threading.Lock()
    except:
      print("**************************************************************\nPANIC: Failed to create lock for resource '{}'\n**************************************************************".format(str_res))
    finally:
      self._lock_table_mutex.release()

    if str_res in self._lock_table:      
      self._lock_table[str_res].acquire(blocking=True)
      self._lock_table_identities[str_res] = threading.get_ident()
      result = self._lock_table[str_res]
      
    return result
  
  def unlock_resource(self, str_res):
    if str_res in self._lock_table:
      self._lock_table_identities[str_res] = None
      self._lock_table[str_res].release()
    return
  
  def managed_lock_logger(self):
    return self.managed_lock_resource(_LOGGER_LOCK_ID)
  
  def lock_logger(self):
    self.lock_resource(_LOGGER_LOCK_ID)
    return
  
  def unlock_logger(self):
    self.unlock_resource(_LOGGER_LOCK_ID)
    
  def get_file_path(self, fn, folder, subfolder_path=None, force=False):
    lfld = self.get_target_folder(target=folder)
    if lfld is None:
      datafile = fn
    else:
      datafolder = lfld
    if subfolder_path is not None:
      datafolder = os.path.join(datafolder, subfolder_path.lstrip('/'))
    datafile = os.path.join(datafolder, fn)
    if os.path.isfile(datafile) or force:
      return datafile
    return 
    

  @property
  def session_id(self):
    return self.file_prefix
  
  
  def cleanup_timestamped_files(self, folder, keep=5):
    if os.path.isdir(folder):
      files = [x for x in os.listdir(folder) if len(x) > 6 and x[:6].isnumeric()]
      if len(files) > keep:
        files = sorted(files)
        n_delete = len(files) - keep
        to_delete = files[:n_delete]
        self.P("Cleaning {} timestamped files between {} and {}, preserving {} in '{}'...".format(
          n_delete, to_delete[0], to_delete[-1], keep, folder), color='y'
        )
        for fn in to_delete:
          os.remove(os.path.join(folder, fn))
          print(".", flush=True, end='')
        print("Done.\r", flush=True, end='')
      #endif delete extra files
    #endif folder valir
    return


  def maybe_cleanup_timestamped_files(self, folder, keep=4):
    return self.cleanup_timestamped_files(folder=folder, keep=keep)
  
  
  def cleanup_logs(self, archive_older_than_days=2, MAX_ARCHIVE_SIZE=5*1024**2):
    if self.no_folders_no_save:
      return
    self.P("Cleaning logs older than {} days...".format(archive_older_than_days), color='y')
    str_old_date = (dt.today() - timedelta(days=archive_older_than_days)).strftime('%Y%m%d')
    int_old_date = int(str_old_date)
    logs = os.listdir(self._logs_dir)
    archive_list = []
    show_list = []
    base_fn = "_logs_archive"
    zip_fn = os.path.join(self._logs_dir, base_fn + '.zip')
    if os.path.isfile(zip_fn):
      stats = os.stat(zip_fn)
      if stats.st_size > MAX_ARCHIVE_SIZE:
        self.P("  Current archive larger than {:.1f} MB. Renaming and marking for manual deletion".format(
          MAX_ARCHIVE_SIZE / 1024**2), color='y'
        )
        new_fn = os.path.join(self._logs_dir, base_fn + '_' + self.file_prefix.split('_')[0] + '.zip')
        if os.path.isfile(new_fn):
          self.P("  Something is strange, file already exists. Deleting...")
          try:
            os.unlink(new_fn)
          except:
            self.P("    Failed to remove {}:\n{}".format(new_fn, traceback.format_exc()), color='r')
        try:
          os.rename(zip_fn, new_fn)
        except:
          self.P("  Archiving logs file failed:\n{}".format(
            traceback.format_exc()
          ), color='r')
      #endif file to large
    #endif already exists
    for fn in logs:
      if fn[-4:] == '.txt':
        str_date = fn[:8]
        int_date = None
        if len(str_date) == 8:
          try:
            int_date = int(str_date)
          except:
            pass
        if int_date is not None and int_date < int_old_date:
          full_fn = os.path.join(self._logs_dir, fn)
          archive_list.append(full_fn)
          show_list.append(fn)
        
    if len(archive_list) > 0:
      self.P("  Archiving {} logs...".format(len(archive_list)), color='y')
      self.add_files_to_zip(zip_fn, archive_list)
      for full_fn in archive_list:
        os.remove(full_fn)
    else:
      self.P("  Nothing to clean.")
    return
                    


  def _logger(self, logstr, show=None, noprefix=False, show_time=False, color=None):
    """
    Process one log event on the producer path.

    The method serializes producers through the logger lock, appends the
    in-memory message, and decides whether pending messages should be flushed
    to disk through the asynchronous writer queue.

    Parameters
    ----------
    logstr : str
      Message payload to log.
    show : bool or None, optional
      Console display flag. ``None`` delegates to default logger behavior.
    noprefix : bool, optional
      Skip automatic prefix/timestamp formatting.
    show_time : bool, optional
      Append call elapsed time marker to the message.
    color : str or None, optional
      Console color key override.

    Returns
    -------
    float
      Elapsed seconds since the previous log call, used by idle-flush logic.

    Notes
    -----
    - Error messages can force immediate enqueue/flush.
    - Disk I/O is intentionally delegated to `_log_writer_loop`.
    """
    with self.managed_lock_logger():
      self.start_timer('_logger', section='LOGGER_internal')

      elapsed = tm() - self.last_time
      # Fast drop path for optional repeat suppression.
      if not self._should_emit_log(logstr):
        self.last_time = tm()
        self.end_timer('_logger', section='LOGGER_internal')
        return elapsed

      self.start_timer('_logger_add_log', section='LOGGER_internal')
      # Compare error-list length to detect error-line insertion.
      err_len_before = len(self.err_log)
      self._add_log(
        logstr, show=show,
        noprefix=noprefix,
        show_time=show_time,
        color=color
      )
      err_len_after = len(self.err_log)
      self.end_timer('_logger_add_log', section='LOGGER_internal')

      self.start_timer('_logger_save_log', section='LOGGER_internal')
      if self._save_enabled:
        # Errors may bypass buffering policy to reduce persistence lag.
        immediate_error_flush = self._error_flush_immediate and err_len_after > err_len_before
        if immediate_error_flush or self._should_save_logs(elapsed):
          self._flush_pending_logs_async(force=immediate_error_flush)
      self.end_timer('_logger_save_log', section='LOGGER_internal')
      
      self.last_time = tm()
      self._check_log_size()
      self._sync_enqueued_log_lengths()

      self.end_timer('_logger', section='LOGGER_internal')
    # endwith lock
    return elapsed

  def configure_flush_policy(self, idle_seconds=None, save_buffer_len=None, error_immediate=None):
    """
    Configure persistence trigger policy for normal/error logs.

    Parameters
    ----------
    idle_seconds : float or None, optional
      Flush when time since previous call is above this threshold.
    save_buffer_len : int or None, optional
      Flush when unsaved lines reach this count.
    error_immediate : bool or None, optional
      Force immediate flush attempts for new error lines.

    Returns
    -------
    None
      Updates in-place logger policy fields.
    """
    if idle_seconds is not None:
      self._save_idle_seconds = max(float(idle_seconds), 0.0)
    if save_buffer_len is not None:
      self._save_buffer_len = max(int(save_buffer_len), 1)
    if error_immediate is not None:
      self._error_flush_immediate = bool(error_immediate)
    return

  def configure_rate_control(self, enabled=None, window_seconds=None, max_repeats=None):
    """
    Configure optional repeated-message suppression.

    Parameters
    ----------
    enabled : bool or None, optional
      Toggle rate control.
    window_seconds : float or None, optional
      Sliding window size for duplicate detection.
    max_repeats : int or None, optional
      Maximum identical messages emitted inside one window.

    Returns
    -------
    None
      Updates in-place rate control fields.
    """
    if enabled is not None:
      self._rate_limit_enabled = bool(enabled)
    if window_seconds is not None:
      self._rate_limit_window_seconds = max(float(window_seconds), 0.01)
    if max_repeats is not None:
      self._rate_limit_max_repeats = max(int(max_repeats), 0)
    return

  def get_log_writer_telemetry(self):
    """
    Return internal metrics for the asynchronous writer.

    Returns
    -------
    dict
      Telemetry snapshot containing queue occupancy, drop counters, writer
      throughput counters, and p50/p95 writer latency in milliseconds.

    Notes
    -----
    Latency quantiles are computed over a bounded rolling history.
    """
    with self._writer_stats_lock:
      latencies = list(self._writer_latency_ms)
      dropped_lines = self._writer_dropped_lines
      queue_high_watermark = self._writer_queue_high_watermark
      write_batches = self._writer_batches_written
      write_lines = self._writer_lines_written

    # Quantiles are computed outside the lock to minimize telemetry contention.
    latencies_sorted = sorted(latencies)
    if latencies_sorted:
      idx50 = int((len(latencies_sorted) - 1) * 0.50)
      idx95 = int((len(latencies_sorted) - 1) * 0.95)
      p50_ms = latencies_sorted[idx50]
      p95_ms = latencies_sorted[idx95]
    else:
      p50_ms = 0.0
      p95_ms = 0.0
    return {
      "queue_maxsize": self._writer_queue_maxsize,
      "queue_depth": self._writer_queue.qsize(),
      "queue_high_watermark": queue_high_watermark,
      "dropped_lines": dropped_lines,
      "write_batches": write_batches,
      "write_lines": write_lines,
      "write_latency_p50_ms": p50_ms,
      "write_latency_p95_ms": p95_ms,
      "rate_limited_messages": self._rate_limit_suppressed_messages,
    }

  def _init_log_writer(self):
    """
    Start the background writer thread if persistence is available.

    Returns
    -------
    None
      Initializes writer lifecycle state and registers shutdown hook.
    """
    if self.no_folders_no_save or self._save_enabled is False:
      return
    if self._writer_thread is not None:
      return

    # The writer is daemonized and drained via explicit shutdown hook.
    self._writer_stop_event.clear()
    self._writer_thread = threading.Thread(
      target=self._log_writer_loop,
      name=f"{self.__lib__}-log-writer",
      daemon=True,
    )
    self._writer_thread.start()
    atexit.register(self._shutdown_log_writer)
    return

  def _shutdown_log_writer(self, timeout=5.0):
    """
    Flush and stop the writer thread.

    Parameters
    ----------
    timeout : float, optional
      Maximum seconds to wait for thread termination.

    Returns
    -------
    None
      Best-effort shutdown; safe to call multiple times.
    """
    if self._writer_thread is None:
      return
    if not self._writer_thread.is_alive():
      return
    try:
      # Force enqueue of all outstanding lines before stop signal is issued.
      with self.managed_lock_logger():
        self._flush_pending_logs_async(force=True)
    except:
      pass
    self._writer_stop_event.set()
    try:
      self._writer_queue.put_nowait(None)
    except queue.Full:
      pass
    self._writer_thread.join(timeout=timeout)
    return

  def _log_writer_loop(self):
    """
    Consume queued save tasks and persist append-only deltas.

    Returns
    -------
    None
      Runs until stop signal is set and queue is drained.
    """
    while True:
      if self._writer_stop_event.is_set() and self._writer_queue.empty():
        break
      try:
        task = self._writer_queue.get(timeout=0.5)
      except queue.Empty:
        continue
      if task is None:
        self._writer_queue.task_done()
        continue

      log_ref, log_file, start_idx, end_idx = task
      if end_idx > start_idx:
        t0 = tm()
        # Save only the delta segment prepared by the producer path.
        self._save_log(
          log=log_ref,
          log_file=log_file,
          start_idx=start_idx,
          end_idx=end_idx,
        )
        duration_ms = (tm() - t0) * 1000.0
        with self._writer_stats_lock:
          # Update writer throughput/latency telemetry atomically.
          self._writer_batches_written += 1
          self._writer_lines_written += (end_idx - start_idx)
          self._writer_latency_ms.append(duration_ms)
      self._writer_queue.task_done()
    return

  def _enqueue_save_task(self, log_ref, log_file, start_idx, end_idx, force=False):
    """
    Enqueue a persistence task for the writer thread.

    Parameters
    ----------
    log_ref : list
      In-memory log list backing the write.
    log_file : str
      Target file path.
    start_idx : int
      Inclusive begin index for delta write.
    end_idx : int
      Exclusive end index for delta write.
    force : bool, optional
      When ``True``, block briefly or fallback to direct write.

    Returns
    -------
    bool
      ``True`` when task is accepted or handled through fallback path.
    """
    if end_idx <= start_idx:
      return True
    task = (log_ref, log_file, start_idx, end_idx)
    put_timeout = 0.2 if force else 0.0
    try:
      if force:
        self._writer_queue.put(task, timeout=put_timeout)
      else:
        self._writer_queue.put_nowait(task)
      with self._writer_stats_lock:
        self._writer_queue_high_watermark = max(
          self._writer_queue_high_watermark, self._writer_queue.qsize()
        )
      return True
    except queue.Full:
      # Force path preserves durability for critical/error flush attempts.
      if force:
        self._save_log(log=log_ref, log_file=log_file, start_idx=start_idx, end_idx=end_idx)
        return True
      # Non-forced path accounts for dropped lines to avoid hidden data loss.
      self._mark_dropped_lines(end_idx - start_idx)
      return True

  def _mark_dropped_lines(self, nr_lines):
    """
    Increase dropped-line telemetry counter.

    Parameters
    ----------
    nr_lines : int
      Number of lines dropped due to queue pressure.

    Returns
    -------
    None
      Counter is updated under telemetry lock.
    """
    if nr_lines <= 0:
      return
    with self._writer_stats_lock:
      self._writer_dropped_lines += nr_lines
    return

  def _sync_enqueued_log_lengths(self):
    """
    Clamp producer enqueue cursors to current in-memory list sizes.

    Returns
    -------
    None
      Keeps cursor state valid after log rotation/truncation events.
    """
    if self._last_enqueued_app_len > len(self.app_log):
      self._last_enqueued_app_len = len(self.app_log)
    if self._last_enqueued_err_len > len(self.err_log):
      self._last_enqueued_err_len = len(self.err_log)
    return

  def _flush_pending_logs_async(self, force=False):
    """
    Enqueue all unsaved deltas for app/error logs.

    Parameters
    ----------
    force : bool, optional
      Propagates urgency to enqueue layer (used for error or shutdown flushes).

    Returns
    -------
    None
      Updates enqueue cursors after queuing deltas.
    """
    self._sync_enqueued_log_lengths()
    app_len = len(self.app_log)
    err_len = len(self.err_log)
    if app_len > self._last_enqueued_app_len:
      # Queue only newly appended normal log lines.
      self._enqueue_save_task(
        log_ref=self.app_log,
        log_file=self.log_file,
        start_idx=self._last_enqueued_app_len,
        end_idx=app_len,
        force=force,
      )
      self._last_enqueued_app_len = app_len
    if err_len > self._last_enqueued_err_len:
      # Error path can be forced independently for lower durability latency.
      self._enqueue_save_task(
        log_ref=self.err_log,
        log_file=self.log_e_file,
        start_idx=self._last_enqueued_err_len,
        end_idx=err_len,
        force=True if force else False,
      )
      self._last_enqueued_err_len = err_len
    return

  def _should_save_logs(self, elapsed_since_last_call):
    """
    Decide whether current call should trigger a flush enqueue.

    Parameters
    ----------
    elapsed_since_last_call : float
      Seconds elapsed since previous producer call.

    Returns
    -------
    bool
      ``True`` when idle or buffer thresholds indicate a flush opportunity.
    """
    self._sync_enqueued_log_lengths()
    if elapsed_since_last_call > self._save_idle_seconds:
      return True
    # Buffer policy: flush once enough unsaved normal/error lines accumulate.
    app_unsaved = len(self.app_log) - self._last_enqueued_app_len
    err_unsaved = len(self.err_log) - self._last_enqueued_err_len
    if app_unsaved >= self._save_buffer_len or err_unsaved >= self._save_buffer_len:
      return True
    return False

  def _should_emit_log(self, logstr):
    """
    Apply optional duplicate-message rate limiting.

    Parameters
    ----------
    logstr : str
      Candidate message before formatting/output.

    Returns
    -------
    bool
      ``True`` when message should be emitted; ``False`` when suppressed.

    Notes
    -----
    Suppression is disabled unless explicitly configured through
    :meth:`configure_rate_control`.
    """
    if not self._rate_limit_enabled or self._rate_limit_max_repeats <= 0:
      return True

    msg_key = logstr if type(logstr) == str else str(logstr)
    now = tm()
    state = self._rate_limit_state.get(msg_key)
    if state is None:
      # First occurrence starts a new counting window.
      self._rate_limit_state[msg_key] = [now, 1]
      return True

    window_start, counter = state
    if now - window_start > self._rate_limit_window_seconds:
      # Window expired: reset counter and allow message.
      self._rate_limit_state[msg_key] = [now, 1]
      return True

    if counter < self._rate_limit_max_repeats:
      # Still below configured repeat budget in current window.
      state[1] = counter + 1
      return True

    # Suppress overflow repeats and periodically prune state growth.
    self._rate_limit_suppressed_messages += 1
    if len(self._rate_limit_state) > 2048:
      self._rate_limit_state = {}
    return False

  def _normalize_path_sep(self):
    if self._base_folder is not None:
      if os.path.sep == '\\':
        self._base_folder = self._base_folder.replace('/', '\\')
      else:
        self._base_folder = self._base_folder.replace('\\', '/')
      #endif
    #endif

    if self._app_folder is not None:
      if os.path.sep == '\\':
        self._app_folder = self._app_folder.replace('/', '\\')
      else:
        self._app_folder = self._app_folder.replace('\\', '/')
      #endif
    #endif

    return

  def print_on_columns(self, *objects, nr_print_columns=4, nr_print_chars=12, header=None, color=None):
    if header:
      self.P(header, color=color)

    print_columns = [[] for _ in range(nr_print_columns)]

    crt_column = 0
    _fmt = "{:>" + str(nr_print_chars) + "}"

    nr_labels_per_column = int(np.ceil(len(objects) / nr_print_columns))
    for i, obj in enumerate(objects):
      if i // nr_labels_per_column != crt_column:
        crt_column += 1

      print_columns[crt_column].append(_fmt.format(obj[:nr_print_chars]))
    # endfor

    for i in range(nr_labels_per_column):
      str_line = ''
      for j in range(nr_print_columns):
        if i >= len(print_columns[j]):
          continue

        str_line += print_columns[j][i] + '    '

      self.P(str_line, noprefix=True, color=color)
    # endfor
    return

  def _add_log(self, logstr, show=None, noprefix=False, show_time=False, color=None):
    if show is None:
      show = not self.silent    
    if type(logstr) != str:
      logstr = str(logstr)
    if logstr == "":
      logstr = " "
    if 'WARNING' in logstr and color is None:
      color = 'warning'
    if 'ERROR' in logstr and color is None:
      color = 'error'
    elapsed = tm() - self.last_time
    nowtime = dt.now()
    prefix = ""
    strnowtime = nowtime.strftime("[{}][%y-%m-%d %H:%M:%S]".format(self.__lib__))
    if self.show_time and (not noprefix):
      prefix = strnowtime
    if logstr[0] == "\n":
      logstr = logstr[1:]
      prefix = "\n" + prefix
    res_log = logstr
    if (len(logstr) == 0 or logstr[0] != '[') and not noprefix:
      prefix = prefix + ' '
    logstr = prefix + logstr
    if show_time:
      logstr += " [{:.2f}s]".format(elapsed)
    print_logstr = logstr
    if color is None:
      color = self.default_color
      if not self.__first_print and show:
        if not self.silent:
          BaseLogger.print_color("<Logging with default color: {}>".format(color), color=color)
        self.__first_print = True
    if show:
      if self.append_spaces:
        spaces = " " * max(60 - len(print_logstr), 0)
      else:
        spaces = ''
      print_logstr = print_logstr + spaces
      #endif use default color
      BaseLogger.print_color(print_logstr, color=color)
    if color.lower()[0] in ['e', 'r']:
      self.err_log.append(logstr)
    self.app_log.append(logstr)
      
    #endif
    return

  def _save_log(
    self,
    log,
    log_file,
    DEBUG_ERRORS=False,
    start_idx=None,
    end_idx=None,
    force_rewrite=False,
  ):
    """
    Persist log lines to disk using append-only deltas or full rewrite mode.

    Parameters
    ----------
    log : list[str]
      In-memory list that stores logger lines.
    log_file : str
      Destination file path.
    DEBUG_ERRORS : bool, optional
      Print save-stage diagnostics on exceptions.
    start_idx : int or None, optional
      Inclusive delta start index. Defaults to internal saved offset.
    end_idx : int or None, optional
      Exclusive delta end index. Defaults to current ``len(log)``.
    force_rewrite : bool, optional
      Rewrite the full file when ``True``.

    Returns
    -------
    None
      Persists data and updates internal per-file save offset.

    Notes
    -----
    - Normal optimized path writes only ``log[start_idx:end_idx]`` in append
      mode.
    - HTML mode always rewrites full file to preserve wrappers.
    """
    if self.no_folders_no_save or self._save_enabled is False:
      return

    if log is None:
      return

    nowtime = dt.now()
    strnowtime = nowtime.strftime("[{}][%Y-%m-%d %H:%M:%S] ".format(self.__lib__))
    stage = 0
    try:
      if self.HTML:
        # HTML wrapper mode requires full rewrite to keep valid document shape.
        force_rewrite = True

      if end_idx is None:
        end_idx = len(log)
      end_idx = min(end_idx, len(log))

      if force_rewrite:
        start_idx = 0
        open_mode = "w"
      else:
        if start_idx is None:
          # Default delta starts from last persisted index for this file.
          start_idx = self._save_file_offsets.get(log_file, 0)
        start_idx = max(start_idx, 0)
        open_mode = "a"

      if end_idx <= start_idx:
        return

      log_output = codecs.open(log_file, open_mode, "utf-8")
      stage += 1
      if self.HTML and force_rewrite:
        log_output.write(_HTML_START)
        stage += 1
        iter_list = reversed(log[start_idx:end_idx])
      else:
        iter_list = log[start_idx:end_idx]

      for log_item in iter_list:
        log_output.write("{}\n".format(log_item))
        stage += 1
      if self.HTML and force_rewrite:
        log_output.write(_HTML_END)
        stage += 1
      log_output.close()
      stage += 1
      # Track last persisted index so next write can continue as delta append.
      self._save_file_offsets[log_file] = end_idx if not force_rewrite else len(log)
    except:
      if DEBUG_ERRORS:
        print(strnowtime + "LogWErr S: {} [{}]".format(stage,
                                                       sys.exc_info()[0]), flush=True)
    return

  def _check_log_size(self):
    """
    Handle log splitting when in-memory line limits are reached.

    Returns
    -------
    None
      Forces pending writes, rotates file paths, and resets cursor state.
    """
    if self.max_lines is None:
      return

    if len(self.app_log) >= self.max_lines:
      # Flush before reset so rotation does not lose pending lines.
      self._add_log("Ending log part {}".format(self.split_part))
      self._flush_pending_logs_async(force=True)
      self.app_log = []
      self._last_enqueued_app_len = 0
      self.split_part += 1
      self._generate_log_path()
      self._add_log("Starting log part {}".format(self.split_part))
      self._flush_pending_logs_async(force=True)
    if len(self.err_log) >= self.max_lines:
      self._add_log("Ending error log part {}".format(self.split_err_part))
      self._flush_pending_logs_async(force=True)
      self.err_log = []
      self._last_enqueued_err_len = 0
      self.split_err_part += 1
      self._generate_error_log_path()
      self._add_log("Starting error log part {}".format(self.split_err_part))
      self._flush_pending_logs_async(force=True)
      return

  def verbose_log(self, str_msg, show_time=False, noprefix=False, color=None, show=None):
    return self._logger(
      str_msg,
      show=show,
      show_time=show_time,
      noprefix=noprefix, color=color
    )

  def P(
    self, 
    str_msg, 
    show_time=False, 
    noprefix=False, 
    color=None, 
    boxed=False, 
    show=None, 
    **kwargs
  ):
    """
    Will print & log a message. If show is None it will be set to not self.silent.
    """
    return self.p(
      str_msg, 
      show_time=show_time, 
      noprefix=noprefix, 
      color=color, 
      boxed=boxed, 
      show=show,
      **kwargs
    )

  def D(self, str_msg, show_time=False, noprefix=False, color=None, **kwargs):
    if False:
      return self.P(str_msg, show_time=show_time, noprefix=noprefix, color=color, **kwargs)

  @staticmethod
  def Pr(str_msg, show_time=False, noprefix=False):
    if type(str_msg) != str:
      str_msg = str(str_msg)
    print("\r" + str_msg, flush=True, end='')

  def __convert_to_box(self, str_msg, box_char='#', indent=None, **kwargs):
    lst_msg_lines = str_msg.split('\n')
    max_line_len = max(map(len, lst_msg_lines))

    center = 4 if max_line_len > 80 else 10
    box_line_len = center + 1 + max_line_len + 1 + center
    default_indent = 4 if box_line_len > 100 else 20
    indent = indent if indent is not None else default_indent
    str_indent = ' ' * indent

    msg = box_char * 3 + 'IMPORTANT' + box_char * 3 + '\n\n'
    msg += str_indent + box_char * box_line_len + '\n'
    msg += str_indent + box_char + (box_line_len - 2) * ' ' + box_char + '\n'

    for line in lst_msg_lines:
      left_box_edge = str_indent + box_char
      right_box_edge = box_char

      left_shift = ' ' * center
      right_shift = ' ' * (center + max_line_len - len(line)) # adjust for different line lengths
      shifted_line = left_shift + line + right_shift
      msg += left_box_edge + shifted_line + right_box_edge + '\n'
    # end for

    msg += str_indent + box_char + (box_line_len - 2) * ' ' + box_char + '\n'
    msg += str_indent + box_char * box_line_len + '\n'

    return msg

  def p(self, str_msg, show_time=False, noprefix=False, color=None, boxed=False, show=None, **kwargs):
    if boxed:
      msg = self.__convert_to_box(str_msg, **kwargs)
      self._logger(msg, show=show, noprefix=noprefix, color=color)
    else:
      return self._logger(
        str_msg,
        show=show,
        show_time=show_time,
        noprefix=noprefix, color=color
      )
      

  def Pmd(self, s=''):
    print_func = None
    try:
      from IPython.display import Markdown, display
      def print_func(s):
        display(Markdown(s))
    except:
      pass
    if type(s) != str:
      s = str(s)

    if print_func is not None:
      self._add_log(
        logstr=s,
        show=False,
        noprefix=False,
        show_time=False,
      )
      print_func(s)
    else:
      self.P(s)
    return

  def Pmdc(self, s=''):
    print_func = None
    try:
      from IPython.display import Markdown, display
      def print_func(s):
        display(Markdown(s))
    except:
      pass
    if type(s) != str:
      s = str(s)

    if print_func is not None:
      self._add_log(
        logstr=s,
        show=False,
        noprefix=False,
        show_time=False,
      )
      print_func('<strong>' + s + '</strong>')
    else:
      self.P(s)
    return

  def print_pad(self, str_msg, str_text, n=3):
    if type(str_msg) != str:
      str_msg = str(str_msg)
    if type(str_text) != str:
      str_text = str(str_text)
    str_final = str_msg + "\n" + textwrap.indent(str_text, n * " ")
    self._logger(str_final, show=None, show_time=False)
    return

  def log(self, str_msg, show=None, show_time=False, color=None):
    return self._logger(str_msg, show=show, show_time=show_time, color=color)

  def _generate_log_path(self):
    """
    Create/refresh the active normal-log filepath metadata.

    Returns
    -------
    None
      Updates `self.log_file`, sidecar pointer metadata, and save offset cache.
    """
    if self.no_folders_no_save:
      return
    part = '{:03d}'.format(self.split_part)
    lp = self.file_prefix
    ls = self.log_suffix
    if self.HTML:
      self.log_file = lp + '_' + ls + '_' + part + '_log_web.html'
    else:
      self.log_file = lp + '_' + ls + '_' + part + '_log.txt'

    self.log_file = os.path.join(self._logs_dir, self.log_file)
    # Reset append offset for newly generated filename.
    self._save_file_offsets[self.log_file] = 0
    path_dict = {}
    path_dict['CURRENT_LOG'] = self.log_file
    file_path = os.path.join(self._logs_dir, self.__lib__ + '.txt')
    with open(file_path, 'w') as fp:
      json.dump(path_dict, fp, sort_keys=True, indent=4)
    self._add_log("{} log changed to {}...".format(file_path, self.log_file))
    return

  def _generate_error_log_path(self):
    """
    Create/refresh the active error-log filepath metadata.

    Returns
    -------
    None
      Updates `self.log_e_file`, sidecar pointer metadata, and save offset cache.
    """
    if self.no_folders_no_save:
      return
    part = '{:03d}'.format(self.split_err_part)
    lp = self.file_prefix
    ls = self.log_suffix
    if self.HTML:
      self.log_e_file = lp + '_' + ls + '_' + part + '_error_log_web.html'
    else:
      self.log_e_file = lp + '_' + ls + '_' + part + '_error_log.txt'

    self.log_e_file = os.path.join(self._logs_dir, self.log_e_file)
    # Reset append offset for newly generated filename.
    self._save_file_offsets[self.log_e_file] = 0
    path_dict = {}
    path_dict['CURRENT_E_LOG'] = self.log_e_file
    file_path = os.path.join(self._logs_dir, self.__lib__ + '.txt')
    with open(file_path, 'w') as fp:
      json.dump(path_dict, fp, sort_keys=True, indent=4)
    self._add_log("{} error log changed to {}...".format(file_path, self.log_e_file))
    return
  
  

  def _get_cloud_base_folder(self, base_folder):
    upper = base_folder.upper()
    google = "GOOGLE" in upper
    dropbox = "DROPBOX" in upper

    if google and not "/DATA/" in upper:
      base_folder = self.get_google_drive()
    if dropbox and not "/DATA/" in upper:
      base_folder = self.get_dropbox_drive()
    return base_folder
  
  def reload_config(self):
    self._configure_data_and_dirs(self.config_file, self.config_file_encoding)
    return

  @staticmethod
  def maybe_migrate_user_folder(verbose=False):
    current_user_folder = BaseLogger.get_user_folder()
    if current_user_folder.is_dir():
      if verbose:
        BaseLogger.print_color(f"{str(current_user_folder)} already exists. Skipping migration.")
      return
    if verbose:
      BaseLogger.print_color(f'User folder {str(current_user_folder)} does not exist. Checking for previous versions...')
    for old_sdk_home in SDK_HOME_PREV:
      old_user_folder = current_user_folder.parent / old_sdk_home
      if old_user_folder.is_dir():
        if verbose:
          BaseLogger.print_color(f"Found previous user folder '{old_user_folder}'. Renaming to '{current_user_folder}'...")
        old_user_folder.rename(current_user_folder)
        return
    if verbose:
      BaseLogger.print_color("No previous user folders found.")
    return

  def _maybe_migrate_folder(self):
    user_base_folder = BaseLogger.get_user_folder(as_str=True, include_sdk_home=True)
    if user_base_folder in self._base_folder:
      BaseLogger.maybe_migrate_user_folder()
    return

  def _configure_data_and_dirs(self, config_file, config_file_encoding=None):
    self._maybe_migrate_folder()
    
    self.config_file_encoding = config_file_encoding
    
    if self.no_folders_no_save:
      return

    if config_file != "":
      if config_file_encoding is None:
        f = open(config_file)
      else:
        f = open(config_file, encoding=config_file_encoding)
      
      try:
        self.config_data = json.load(f, object_pairs_hook=OrderedDict)
      except:
        msg = "Failed to load config file '{}'".format(config_file)
        self.P(msg, color='r', boxed=True)
        self.P("Exception details:\n{}".format(traceback.format_exc()), color='r')
        self.config_data = {}

      if self._base_folder is None and self._app_folder is None:
        assert ("BASE_FOLDER" in self.config_data.keys())
        assert ("APP_FOLDER" in self.config_data.keys())
        self._base_folder = self.config_data["BASE_FOLDER"]
        self._app_folder = self.config_data["APP_FOLDER"]
      #endif no defaults for base/app folders

      if not self.silent:
        BaseLogger.print_color("Loaded config [{}]".format(config_file))
      self.config_file = config_file
    else:

      
      
      self.config_data = {
        'BASE_FOLDER' : self._base_folder,
        'APP_FOLDER' : self._app_folder
      }
      self.config_file = "default_config.txt"
      
      if not self.silent:
        BaseLogger.print_color("No config file provided. Using default config.")
    #endif have or not config file
    
    self.config_data = {
      **self.config_data,
      **self.__init_config_data,
    }
    
    matches = self.replace_secrets(self.config_data, allow_missing=True)
    if not self.silent:    
      if len(matches) > 0:
        BaseLogger.print_color("  Config modified with following env vars: {}".format(matches))
      else:
        BaseLogger.print_color("  No secrets/template found in config")
        
    if not self.__folder_setup_done:
      # if not already done lets setup de folders
      self._base_folder = self.expand_tilda(self._base_folder)
      self._base_folder = self._get_cloud_base_folder(self._base_folder)
      self._root_folder = os.path.abspath(self._base_folder)
      self._base_folder = os.path.join(self._base_folder, self._app_folder)
    
      if not self.silent:
        BaseLogger.print_color("BASE: {}".format(self._base_folder))

      self._normalize_path_sep()

      if not os.path.isdir(self._base_folder):
        BaseLogger.print_color(
          f"WARNING! No application base folder '{self._base_folder}'!",
          color='r'
        )
      #endif

      self._logs_dir = os.path.join(self._base_folder, self.get_logs_dir_name())
      self._outp_dir = os.path.join(self._base_folder, self.get_output_dir_name())
      self._data_dir = os.path.join(self._base_folder, self.get_data_dir_name())
      self._modl_dir = os.path.join(self._base_folder, self.get_models_dir_name())

      self._setup_folders([
        self._outp_dir,
        self._logs_dir,
        self._data_dir,
        self._modl_dir
      ])
      self.__folder_setup_done = True
    else:
      BaseLogger.print_color("Folders already configured.", color='y')
    #endif apply only first time
    return

  @staticmethod
  def get_logs_dir_name():
    return '_logs'

  @staticmethod
  def get_output_dir_name():
    return '_output'

  @staticmethod
  def get_data_dir_name():
    return '_data'

  @staticmethod
  def get_models_dir_name():
    return '_models'
  
  
  @staticmethod
  def get_all_files_in_folder(root_folder=None):
    """
    Walks through all directories and sub-directories of the given root_folder and returns a list of all file paths.

    Parameters
    ----------
    root_folder : str
      The path of the folder you want to walk through. Default `None` will generate 
      all files in current folder

    Returns
    -------
    List[str]
      A list containing the paths of all files in the folder and its subfolders.
    """
    
    if root_folder is None:
      root_folder = os.getcwd()
    
    file_paths = []
    
    for dirpath, dirnames, filenames in os.walk(root_folder):
      for filename in filenames:
        full_path = os.path.join(dirpath, filename)
        file_paths.append(full_path)
    
    return file_paths

  @staticmethod
  def get_all_subfolders(root_folder=None, as_package=False):
    """
    Walks through all directories and sub-directories of the given root_folder and returns a list of all subfolder paths.

    Parameters
    ----------
    root_folder : str
      The path of the folder you want to walk through. Default `None` will generate 
      all subfolders in current folder

    Returns
    -------
    List[str]
      A list containing the paths of all files in the folder and its subfolders.
    """
    
    if root_folder is None:
      root_folder = os.getcwd()
    
    folder_paths = []
    
    for dirpath, dirnames, filenames in os.walk(root_folder):
      for dirname in dirnames:
        full_path = os.path.join(dirpath, dirname)
        if as_package:
          full_path = full_path.replace('/', '.').replace('\\', '.')
        folder_paths.append(full_path)
    
    return folder_paths
  
  
  
  def get_code_base_folder(self):
    if getattr(sys, 'frozen', False):
      # Running in a bundled exe
      folder = sys._MEIPASS
    else:
      # Running in a normal Python environment
      folder = self.root_folder
    return folder  
  
  @property
  def code_base_folder(self):
    return self.get_code_base_folder()

  def _setup_folders(self, folder_list):
    self.folder_list = folder_list
    for folder in folder_list:
      if not os.path.isdir(folder):
        print("Creating folder [{}]".format(folder))
        os.makedirs(folder)
    return

  def update_config(self, dict_newdata=None):
    """
     saves config file with current config_data dictionary
    """
    if dict_newdata is not None:
      for key in dict_newdata:
        self.config_data[key] = dict_newdata[key]
    with open(self.config_file, 'w') as fp:
      json.dump(self.config_data, fp, indent=4)
    self.P("Config file '{}' has been updated.".format(self.config_file))
    return
  
  
  def update_config_values(self, dct_newdata):
    self.P("Selective update of config file '{}' on {}".format(self.config_file, list(dct_newdata.keys())))
    with open(self.config_file, 'r') as fp:
      dct_cache_config = json.load(fp, object_pairs_hook=OrderedDict)
    for k,v in dct_newdata.items():
      old_value = dct_cache_config[k]
      dct_cache_config[k] = v
      self.P("  Modified '{}'='{}' => '{}'='{}'".format(k, old_value, k, v))
    with open(self.config_file, 'w') as fp:
      json.dump(dct_cache_config, fp, indent=4)
    self.P("Config file '{}' has been updated.".format(self.config_file))
    return


  def _check_additional_configs(self):
    additional_configs = []
    
    if not self.check_additional_configs:
      return 

    check_dir = self.get_data_folder()
    if self.data_folder_additional_configs is not None:
      check_dir = self.get_data_subfolder(self.data_folder_additional_configs)
      if check_dir is None:
        self.P("Additional configs folder '{}' not found in '{}'"
               .format(self.data_folder_additional_configs, self.get_data_folder()[-50:]))
        return

    data_files = list(filter(lambda x: os.path.isfile(os.path.join(check_dir, x)), os.listdir(check_dir)))
    data_files = list(filter(lambda x: any(ext in x for ext in ['.txt', 'json']), data_files))

    for f in data_files:
      if any(x in f for x in ['config', 'cfg', 'conf']):
        fn = self.get_data_file(f)
        self.P("Found additional config in '{}'".format(fn))
        dct_config = json.load(open(fn), object_pairs_hook=OrderedDict)
        self.replace_secrets(dct_config, allow_missing=True)
        additional_configs.append(dct_config)

    if len(additional_configs) > 0:
      dct_final = {}
      for d in additional_configs:
        dct_final.update(d)
      for k, v in dct_final.items():
        if k in self.config_data:
          self.P("[WARNING] Overriding key '{}'".format(k))
        self.config_data[k] = v
    return

  def raise_error(self, error_message):
    self.P("ERROR: {}".format(error_message))
    raise ValueError(str(error_message))

  def get_config_value(self, key, default=0):
    if key in self.config_data.keys():
      _val = self.config_data[key]
    else:
      # create key if does not exist
      _val = default
      self.config_data[key] = _val
    return _val

  def clear_folder(self, folder, include_subfolders=False):
    self.P("Clearing {}".format(folder))
    for the_file in os.listdir(folder):
      file_path = os.path.join(folder, the_file)
      try:
        if os.path.isfile(file_path):
          self.P("  Deleting {}".format(file_path[-30:]))
          os.unlink(file_path)
        elif os.path.isdir(file_path) and include_subfolders:
          self.P("  Removing ...{} subfolder".format(file_path[-30:]))
          shutil.rmtree(file_path)
      except Exception as e:
        self.P("{}".format(e))

  def clear_model_folder(self, include_subfolders=False):
    folder = self.get_models_folder()
    self.clear_folder(folder, include_subfolders=include_subfolders)

  def clear_log_folder(self, include_subfolders=False):
    folder = self._logs_dir
    self.clear_folder(folder, include_subfolders=include_subfolders)

  def clear_output_folder(self, include_subfolders=False):
    folder = self.get_output_folder()
    self.clear_folder(folder, include_subfolders=include_subfolders)

  def clear_all_results(self):
    self.P("WARNING: removing all files from models, logs and output!")
    self.clear_log_folder()
    self.clear_model_folder()
    self.clear_output_folder()

  def get_base_folder(self):
    return self._base_folder if hasattr(self, '_base_folder') else ''

  @property
  def base_folder(self):
    return self.get_base_folder()

  @property
  def root_folder(self):
    return self._root_folder

  @property
  def app_folder(self):
    return self._app_folder

  def get_data_folder(self):
    return self._data_dir if hasattr(self, '_data_dir') else ''

  def get_logs_folder(self):
    return self._logs_dir if hasattr(self, '_logs_dir') else ''

  def get_output_folder(self):
    return self._outp_dir if hasattr(self, '_outp_dir') else ''

  def get_models_folder(self):
    return self._modl_dir if hasattr(self, '_modl_dir') else ''

  def get_target_folder(self, target):
    if target is None:
      return

    if target.lower() in ['data', '_data', 'data_dir', 'dat']:
      return self.get_data_folder()

    if target.lower() in ['logs', 'log', 'logs_dir', 'log_dir', '_log', '_logs']:
      return self.get_logs_folder()

    if target.lower() in ['models', 'model', '_models', '_model', 'model_dir', 'models_dir', 'modl']:
      return self.get_models_folder()

    if target.lower() in ['output', '_output', 'output_dir', 'outp', '_outp']:
      return self.get_output_folder()

    self.P("Inner folder of type '{}' not found".format(target))
    return
  
  
  def get_subfolder(self, where, subfolder, force_create=False):
    folder = self.get_target_folder(target=where)
    path = os.path.join(folder, subfolder)
    if force_create:
      os.makedirs(path, exist_ok=True)
    if os.path.isdir(path):
      return path
    return None
    

  def get_data_subfolder(self, _dir, force_create=False):
    return self.get_subfolder(where='data', subfolder=_dir, force_create=force_create)

  def get_models_subfolder(self, _dir, force_create=False):
    return self.get_subfolder(where='models', subfolder=_dir, force_create=force_create)
  
  def get_output_subfolder(self, _dir, force_create=False):
    return self.get_subfolder(where='output', subfolder=_dir, force_create=force_create)

  def get_path_from_node(self, dct):
    if 'PARENT' in dct:
      path = self.get_path_from_node(dct['PARENT'])
      os.path.join(path, dct['PATH'])
      return path
    elif 'USE_DROPBOX' in dct and int(dct['USE_DROPBOX']) == 1:
      return os.path.join(self.get_base_folder(), dct['PATH'])
    else:
      return dct['PATH']

  def get_root_subfolder(self, folder):
    fld = os.path.join(self._root_folder, folder)
    if os.path.isdir(fld):
      return fld
    else:
      return None

  def get_base_subfolder(self, folder):
    fld = os.path.join(self._base_folder, folder)
    if os.path.isdir(fld):
      return fld
    else:
      return None

  def get_root_file(self, str_file):
    fn = os.path.join(self._root_folder, str_file)
    assert os.path.isfile(fn), "File not found: {}".format(fn)
    return fn

  def get_base_file(self, str_file):
    fn = os.path.join(self.get_base_folder(), str_file)
    assert os.path.isfile(fn), "File not found: {}".format(fn)
    return fn

  def get_file_from_folder(self, s_folder, s_file):
    s_fn = os.path.join(self.get_base_folder(), s_folder, s_file)
    if not os.path.isfile(s_fn):
      s_fn = None
    return s_fn

  def get_data_file(self, s_file):
    """
    returns full path of a data file or none is file does not exist
    """
    fpath = os.path.join(self.get_data_folder(), s_file)
    if not os.path.isfile(fpath):
      fpath = None
    return fpath

  def get_model_file(self, s_file):
    """
    returns full path of a data file or none is file does not exist
    """
    fpath = os.path.join(self.get_models_folder(), s_file)
    if not os.path.isfile(fpath):
      fpath = None
    return fpath

  def get_models_file(self, s_file):
    return self.get_model_file(s_file)

  def get_output_file(self, s_file):
    fpath = os.path.join(self.get_output_folder(), s_file)
    if not os.path.isfile(fpath):
      fpath = None
    return fpath

  def check_folder(self, sub_folder, root=None):
    if root is None:
      root = self.get_base_folder()
    sfolder = os.path.join(root, sub_folder)
    if sfolder not in self.folder_list:
      self.folder_list.append(sfolder)

    if not os.path.isdir(sfolder):
      self.verbose_log(" Creating folder [...{}]".format(sfolder[-40:]))
      os.makedirs(sfolder)
    return sfolder

  def check_folder_data(self, sub_folder):
    root = self.get_data_folder()
    return self.check_folder(sub_folder, root)

  def check_folder_models(self, sub_folder):
    root = self.get_models_folder()
    return self.check_folder(sub_folder, root)

  def check_folder_output(self, sub_folder):
    root = self.get_output_folder()
    return self.check_folder(sub_folder, root)

  @staticmethod    
  def is_url_friendly(s):
    """
    Check if a string is URL-friendly.

    Parameters
    ----------
    s : str
      The string to be checked for URL-friendliness.

    Returns
    -------
    bool
      True if the string is URL-friendly, False otherwise.
    """
    # Regular expression for matching only letters, numbers, underscores, and hyphens
    pattern = r'^[a-zA-Z0-9_-]+$'
    return bool(re.match(pattern, s))  

  @staticmethod
  def get_folders(path):
    lst = [os.path.join(path, x) for x in os.listdir(path)]
    return [x for x in lst if os.path.isdir(x)]

  @staticmethod
  def expand_tilda(path):
    if '~' in path:
      path = path.replace('~', os.path.expanduser('~'))
    return path

  def refresh_file_prefix(self):
    self.file_prefix = dt.now().strftime("%Y%m%d_%H%M%S")
    return

  @staticmethod
  def now_str(nice_print=False, short=False):
    if nice_print:
      if short:        
        return dt.now().strftime("%Y-%m-%d %H:%M:%S")
      else:
        return dt.now().strftime("%Y-%m-%d %H:%M:%S.%f")
    else:
      if short:
        return dt.now().strftime("%Y%m%d%H%M%S") 
      else:
        return dt.now().strftime("%Y%m%d%H%M%S%f")
      
  @staticmethod
  def get_utc_offset(as_string=True):

    # Get the local timezone
    local_timezone = tz.tzlocal()
    # Get the current datetime in the local timezone
    now = dt.now(local_timezone)
    # Get the timezone offset in minutes
    offset_minutes = now.utcoffset().total_seconds() / 60
    # Get the timezone offset in hours
    offset_hours = offset_minutes / 60
    if as_string:
      result = "UTC{}{}".format("+" if offset_hours >= 0 else "-", int(offset_hours)) 
    else:
      result = offset_hours
    return result
  
  
  def utc_to_local(self, remote_datetime, remote_utc, fmt='%Y-%m-%d %H:%M:%S', as_string=False):
    """
    Given a "remote" datetime (in datetime or str format) and a string or int denoting an offset 
    will return local datetime as a datetime object.

    Parameters
    ----------
    remote_datetime: datetime or str or tzinfo
      The remote datetime
      
    remote_utc: int or str
      The UTC offset of the remote datetime as int or as string - i.e. "UTC+3" or even "+3"

    Returns
    -------
      datetime      
    """
    if remote_utc is None:
      remote_utc = 'UTC+3'
    try:
      if isinstance(remote_utc, str):
        remote_utc = tz.gettz(remote_utc)
      elif isinstance(remote_utc, int):
        utc_offset = remote_utc
        remote_utc = tz.tzoffset(None, timedelta(hours=utc_offset))
      elif not isinstance(remote_utc, tzinfo):
        raise ValueError("Unknown remote_utc type: {}".format(type(remote_utc)))
    except Exception as exc:
      # if it breaks use local time
      remote_utc = tz.tzlocal()
      self.P("Error in remote_datetime: {} requested remote_utc: {}, Error: {}. Using local timezone {} for given date.".format(
        remote_datetime, remote_utc, exc, remote_utc), color='r'
      )
      

    if isinstance(remote_datetime, str):
      remote_datetime = dt.strptime(remote_datetime, fmt)

    remote_datetime = remote_datetime.replace(tzinfo=remote_utc)
    local_timezone = tz.tzlocal()
    local_datetime = remote_datetime.astimezone(local_timezone)
    local_datetime = local_datetime.replace(tzinfo=None)
    if as_string:
      local_datetime = local_datetime.strftime(fmt)
    return local_datetime
  
  @staticmethod
  def str_to_sec(s):
    res = None
    try:
      import time
      x = time.strptime(s,'%H:%M:%S')
      res = timedelta(hours=x.tm_hour,minutes=x.tm_min,seconds=x.tm_sec).total_seconds()
    except:
      pass
    return res  
  
  
  @staticmethod
  def second_of_minute():
    """
    Returns the current second of the minute.
    
    Returns
    -------
    int
      The current second of the minute (0-59).
    """
    return dt.now().second
  
  
  @staticmethod
  def time_to_str(t=None, fmt='%Y-%m-%d %H:%M:%S'):
    if t is None:
      t = tm()
    return strftime(fmt, localtime(t))
  
  
  def elapsed_to_str(self, elapsed=None, show_days=False):
    """
    Pretty format a number of seconds

    Parameters
    ----------
    elapsed : float, optional
      The amount of time in seconds. The default is None and will use Logger init start as reference.
    show_days : bool, optional
      Show in days, hours, etc instead of HH:MM:SS. The default is False.

    Returns
    -------
    s : str
      Formatted time.

    """
    if elapsed is None:
      elapsed = tm() - self.start_timestamp

    if elapsed < 0:
      sign = -1
      elapsed = abs(elapsed)
    else:
      sign = 1
    
    hours, rem = divmod(elapsed, 3600)
    minutes, seconds = divmod(rem, 60)
    if show_days:
      s = str(timedelta(seconds=int(elapsed)))
    else:
      s = "{:0>2}:{:0>2}:{:0>2}".format(int(hours),int(minutes),int(seconds))
    s = "-" + s if sign == -1 else s
    return s


  @staticmethod
  def str_to_time(s, fmt='%Y-%m-%d %H:%M:%S'):
    return mktime(strptime(s, fmt))
  
  
  @staticmethod
  def str_to_date(s, fmt='%Y-%m-%d %H:%M:%S'):
    return dt.strptime(s, fmt)


  @staticmethod
  def now_str_fmt(fmt=None):
    if fmt is None:
      fmt = '%Y-%m-%d %H:%M:%S.%f'
    return dt.now().strftime(fmt)
  
  @staticmethod
  def str_to_bool(s):
    return str(s).lower() in ['true', 't', 'yes', 'y', '1', 'on']

  def get_error_info(self, return_err_val=False):
    """
    Returns error_type, file, method, line for last error if available

    Parameters
    ----------
    return_err_val: boolean, optional
      Whether the method returns or not the error message (err_val)

    Returns
    -------
    if not return_err_val:
      (tuple) str, str, str, str : errortype, file, method, line
    else:
      (tuple) str, str, str, str, str : errortype, file, method, line, err message
    """
    err_type, err_val, err_trace = sys.exc_info()
    if False:
      # dont try this at home :) if you want to pickle a logger instance after
      # `get_error_info` was triggered, then it won't work because `self._last_extracted_error`
      # contain an object of type `traceback` and tracebacks cannot be pickled
      self._last_extracted_error = err_type, err_val, err_trace
    # endif
    if err_type is not None:
      str_err = err_type.__name__
      stack_summary = traceback.extract_tb(err_trace)
      last_error_frame = stack_summary[-1]
      fn = os.path.splitext(os.path.split(last_error_frame.filename)[-1])[0]
      lineno = last_error_frame.lineno
      func_name = last_error_frame.name
      if not return_err_val:
        return str_err, 'File: ' + fn, 'Func: ' + func_name, 'Line: ' + str(lineno)
      else:
        return str_err, 'File: ' + fn, 'Func: ' + func_name, 'Line: ' + str(lineno), str(err_val)
    else:
      return "", "", "", "", ""

  @staticmethod
  def tqdm_enumerate(_iter):
    from tqdm import tqdm
    i = 0
    for y in tqdm(_iter):
      yield i, y
      i += 1

  @staticmethod
  def set_nice_prints(linewidth=500,
                      precision=3,
                      np_precision=None,
                      df_precision=None,
                      suppress=False):

    if np_precision is None:
      np_precision = precision
    if df_precision is None:
      df_precision = precision
    np.set_printoptions(precision=np_precision)
    np.set_printoptions(floatmode='fixed')
    np.set_printoptions(linewidth=linewidth)
    np.set_printoptions(suppress=suppress)

    try:
      import pandas as pd
      pd.set_option('display.max_rows', 500)
      pd.set_option('display.max_columns', 500)
      pd.set_option('display.width', 1000)
      pd.set_option('display.max_colwidth', 1000)
      _format = '{:.' + str(df_precision) + 'f}'
      pd.set_option('display.float_format', lambda x: _format.format(x))
    except:
      pass

    return

  @staticmethod
  def get_google_drive():
    home_dir = os.path.expanduser("~")
    valid_paths = [
      os.path.join(home_dir, "Google Drive"),
      os.path.join(home_dir, "GoogleDrive"),
      os.path.join(os.path.join(home_dir, "Desktop"), "Google Drive"),
      os.path.join(os.path.join(home_dir, "Desktop"), "GoogleDrive"),
      os.path.join("C:/", "GoogleDrive"),
      os.path.join("C:/", "Google Drive"),
      os.path.join("D:/", "GoogleDrive"),
      os.path.join("D:/", "Google Drive"),
    ]

    drive_path = None
    for path in valid_paths:
      if os.path.isdir(path):
        drive_path = path
        break

    if drive_path is None:
      raise Exception("Couldn't find google drive folder!")

    return drive_path

  @staticmethod
  def get_dropbox_drive():
    # TODO: change this to not be restricted by the folder name
    home_dir = os.path.expanduser("~")
    valid_paths = [
      os.path.join(home_dir, "Lummetry.AI Dropbox/DATA"),
      os.path.join(home_dir, "Lummetry.AIDropbox/DATA"),
      os.path.join(os.path.join(home_dir, "Desktop"), "Lummetry.AI Dropbox/DATA"),
      os.path.join(os.path.join(home_dir, "Desktop"), "Lummetry.AIDropbox/DATA"),
      os.path.join("C:/", "Lummetry.AI Dropbox/DATA"),
      os.path.join("C:/", "Lummetry.AIDropbox/DATA"),
      os.path.join("D:/", "Lummetry.AI Dropbox/DATA"),
      os.path.join("D:/", "Lummetry.AIDropbox/DATA"),
      os.path.join(home_dir, "Dropbox/DATA"),
      os.path.join(os.path.join(home_dir, "Desktop"), "Dropbox/DATA"),
      os.path.join("C:/", "Dropbox/DATA"),
      os.path.join("D:/", "Dropbox/DATA"),
    ]

    drive_path = None
    for path in valid_paths:
      if os.path.isdir(path):
        drive_path = path
        break

    if drive_path is None:
      raise Exception("Couldn't find google drive folder!")

    return drive_path

  @staticmethod
  def get_dropbox_subfolder(sub_folder):
    drop_root = BaseLogger.get_dropbox_drive()
    full = os.path.join(drop_root, sub_folder)
    if os.path.isdir(full):
      return full
    else:
      return None
    
  @staticmethod
  def print_color(s, color=None):
    color = color or 'n'
    color = color.lower()[0]
    color_start = COLORS[color] if color in COLORS else COLORS['n']
    color_end = COLORS['__end__']
    print('\r' + color_start + s + color_end, flush=True)
    return

  @staticmethod
  def import_string(dotted_path):
    """
    Import a dotted module path and return the attribute/class designated by the
    last name in the path. Raise ImportError if the import failed.
    Credits: django 3.1
    """
    from importlib import import_module
    try:
      module_path, class_name = dotted_path.rsplit('.', 1)
    except ValueError as err:
      raise ImportError("%s doesn't look like a module path" % dotted_path) from err

    module = import_module(module_path)

    try:
      return getattr(module, class_name)
    except AttributeError as err:
      raise ImportError(
        'Module "%s" does not define a "%s" attribute/class' % \
        (module_path, class_name)
      ) from err

  def get_machine_name(self):
    """
    if socket.gethostname().find('.')>=0:
        name=socket.gethostname()
    else:
        name=socket.gethostbyaddr(socket.gethostname())[0]
    """

    self.MACHINE_NAME = socket.gethostname()
    self.COMPUTER_NAME = self.MACHINE_NAME
    return self.MACHINE_NAME


  def _link(self, src_path, target_subpath, is_dir, target=None):
    """
    Creates a symbolic link.

    Parameters:
    ----------
    src_path: str, mandatory
      Symlink src full path

    target_subpath: str, mandatory
      Subpath in the target directory of the logger

    is_dir: bool, mandatory
      Whether is directory or file

    target: str, optional
      Target directory of the logger (data, models, output or logs)
      The default is None ('data')
    """
    if target is None:
      target = 'data'

    if not os.path.exists(src_path):
      self.verbose_log("ERROR! Could not create symlink, because '{}' does not exist".format(src_path))
      return

    target_path = self.get_target_folder(target)
    if target_path is None:
      return

    target_path = os.path.join(target_path, target_subpath)
    if os.path.exists(target_path):
      return

    target_path_parent = Path(target_path).parent
    if not os.path.exists(target_path_parent):
      os.makedirs(target_path_parent)

    os.symlink(
      src_path, target_path,
      target_is_directory=is_dir
    )

    return

  def link_file(self, src_path, target_subpath, target=None):
    self._link(src_path, target_subpath, is_dir=False, target=target)
    return

  def link_folder(self, src_path, target_subpath, target=None):
    self._link(src_path, target_subpath, is_dir=True, target=target)
    return

  @property
  def is_main_thread(self):
    return threading.current_thread() is threading.main_thread()
  
  @staticmethod
  def get_os_name():
    import platform
    return platform.platform()
    
  @staticmethod
  def get_conda_env():
    folder = os.environ.get("CONDA_PREFIX", None)
    env = None
    if folder is not None and len(folder) > 0:
      try:
        env = os.path.split(folder)[-1]
      except:
        env = None
    return env

  @staticmethod
  def get_active_git_branch():
    fn = './.git/HEAD'
    if os.path.isfile(fn):
      with open(fn, 'r') as f:
        content = f.readlines()
      for line in content:
        if line.startswith('ref:'):
          return line.partition('refs/heads/')[2].replace('\n','')
    else:
      return None
  
  
  def dict_show(self, d):
    self.dict_pretty_format(d=d, display=True)
    return

  def show_dict(self, d):
    self.dict_pretty_format(d=d, display=True)
    return
  
  def dict_pretty_format(self, d, indent=4, as_str=True, display_callback=None, display=False, limit_str=250): 
    assert isinstance(d, dict), "`d` must be dict"
    if display and display_callback is None:
      display_callback = self.P
    lst_data = []
    def deep_parse(dct, ind=indent):
      for ki, key in enumerate(dct):
        # dct actually can be dict or list
        str_key = str(key) if not isinstance(key, str) else '"{}"'.format(key)
        if isinstance(dct, dict):
          value = dct[key]
          lst_data.append(' ' * ind + str(str_key) + ' : ')
        else:
          value = key
        if isinstance(value, dict):
          if lst_data[-1][-1] in ['[', ',']:
            lst_data.append(' ' * ind + '{')
          else:
            lst_data[-1] = lst_data[-1] + '{'
          deep_parse(value, ind=ind + indent)
          lst_data.append(' ' * ind + '}')
        elif isinstance(value, list) and len(value) > 0 and isinstance(value[0], dict):
          lst_data[-1] = lst_data[-1] + '['      
          deep_parse(value, ind=ind + indent)
          lst_data.append(' ' * ind + ']')
        else:
          str_value = str(value) if not isinstance(value,str) else '"{}"'.format(value)
          if isinstance(value,str) and len(str_value) > limit_str:
            str_value = str_value[:limit_str]
          lst_data[-1] = lst_data[-1] + str_value

        if ki < (len(dct) - 1):
          lst_data[-1] = lst_data[-1] + ','            
      return
    
    deep_parse(dct=d,ind=0)
    
    displaybuff = "{\n"
    for itm in lst_data:
      displaybuff += '  ' + itm + '\n'
    displaybuff += "}"
    
    if display_callback is not None:
      displaybuff = "Dict pretty formatter:\n" + displaybuff
      display_callback(displaybuff)
    if as_str:
      return displaybuff
    else:
      return lst_data
      
  def get_log_files(self):
    return [os.path.join(self._logs_dir, x) for x in os.listdir(self._logs_dir) if '.txt' in x.lower()]
  
  
  def camel_to_snake(self, s):
    import re    
    if s.isupper():
      result = s.lower()
    else:
      s = re.sub(r'(?<!^)(?=[A-Z])', '_', s).lower()
      s = s.replace('__', '_')
      result = s
    return result

  @staticmethod
  def match_template(json_data: dict, template: dict) -> bool:
    """
    Check if all keys (including subkeys) within the template can be found with the same values in the given JSON.

    Parameters
    ----------
    json_data : dict
      The JSON (dict) to check against the template.
    template : dict
      The template JSON (dict) containing the keys and values to match.

    Returns
    -------
    bool
      True if the JSON matches the template, False otherwise.
    """
    # Initialize the stack with the top-level dictionaries from json_data and template
    stack = [(json_data, template)]

    # Process each pair of current data and template dictionaries/lists from the stack
    while stack:
      current_data, current_tmpl = stack.pop()

      # Check if current_tmpl is a dictionary
      if isinstance(current_tmpl, dict):
        for key, value in current_tmpl.items():
          # If the key is not in current_data, return False
          if key not in current_data:
            return False

          # If the value in the template is a dictionary, add the corresponding pair to the stack
          if isinstance(value, dict):
            if not isinstance(current_data[key], dict):
              return False
            stack.append((current_data[key], value))

          # If the value in the template is a list, process each item in the list
          elif isinstance(value, list):
            if not isinstance(current_data[key], list):
              return False

            tmpl_list = value
            data_list = current_data[key]

            # For each item in the template list, ensure there is a matching item in the data list
            for tmpl_item in tmpl_list:
              matched = False
              for data_item in data_list:
                # If both are dictionaries, add them to the stack for further processing
                if isinstance(tmpl_item, dict) and isinstance(data_item, dict):
                  stack.append((data_item, tmpl_item))
                  matched = True
                  break
                # If both are lists, add them to the stack for further processing
                elif isinstance(tmpl_item, list) and isinstance(data_item, list):
                  stack.append((data_item, tmpl_item))
                  matched = True
                  break
                # If they are of the same type and equal, mark as matched
                elif tmpl_item == data_item:
                  matched = True
                  break
              # If no matching item is found, return False
              if not matched:
                return False

          # If the value is not a dictionary or list, directly compare the values
          elif current_data[key] != value:
            return False

      # Check if current_tmpl is a list
      elif isinstance(current_tmpl, list):
        for tmpl_item in current_tmpl:
          matched = False
          for data_item in current_data:
            # If both are dictionaries, add them to the stack for further processing
            if isinstance(tmpl_item, dict) and isinstance(data_item, dict):
              stack.append((data_item, tmpl_item))
              matched = True
              break
            # If both are lists, add them to the stack for further processing
            elif isinstance(tmpl_item, list) and isinstance(data_item, list):
              stack.append((data_item, tmpl_item))
              matched = True
              break
            # If they are of the same type and equal, mark as matched
            elif tmpl_item == data_item:
              matched = True
              break
          # If no matching item is found, return False
          if not matched:
            return False

    # If all checks passed, return True
    return True
