'''opencos.util -- support global logging, argparser for printing (colors)'''

# pylint: disable=too-many-lines

import argparse
import atexit
import datetime
import json
import multiprocessing
import os
import re
import shutil
import subprocess
import sys
import time
import traceback

from enum import Enum
from pathlib import Path, PosixPath
from importlib import import_module
from dotenv import load_dotenv
from supports_color import supportsColor

import opencos
from opencos.files import safe_shutil_which
from opencos.utils import status_constants
from opencos.utils.str_helpers import strip_ansi_color

INITIAL_CWD = os.getcwd()

global_exit_allowed = False # pylint: disable=invalid-name
progname = "UNKNOWN"        # pylint: disable=invalid-name
progname_in_message = True  # pylint: disable=invalid-name
debug_level = 0             # pylint: disable=invalid-name
dot_f_files_expanded = []   # pylint: disable=invalid-name
env_files_loaded = []       # pylint: disable=invalid-name

args = {                    # pylint: disable=invalid-name
    'color' : bool(supportsColor.stdout),
    'emoji' : bool(getattr(supportsColor.stdout, 'level', 0) >= 2),
    'quiet' : False,
    'verbose' : False,
    'debug' : False,
    'fancy' : sys.stdout.isatty(),
    'warnings' : 0,
    'errors' : 0,
    'artifacts-json': True,
}
modified_args = {}

max_error_code = 0  # pylint: disable=invalid-name


class Colors:
    '''Namespace class for color printing help

    Avoid calling these directly, other than perhapas calling info(*txt, color=Colors.red)
    with 'color' set. It is preferred for outside callers to use one of the print_<color>(..)
    functions, or one of info|warning|error|debug
    '''

    off    = "\033[0m"

    # styling
    bold =      "\033[1m"
    faint =     "\033[2m"
    italic =    "\033[3m"
    underline = "\033[4m"
    blink =     "\033[5m"
    negative =  "\033[7m"
    crossed =   "\033[9m"

    # base colors, does not clear styling
    black =  "\033[30m"
    red =    "\033[31m"
    green =  "\033[32m"
    yellow = "\033[33m" # This looks orange/brown, but it's techincally yellow
    blue =   "\033[34m"
    purple = "\033[35m"
    cyan =   "\033[36m"
    white =  "\033[37m"
    light_gray = "\033[37m"
    foreground = "\033[39m"

    # bolds, shortname
    bred =      "\033[1;31m"
    bgreen =    "\033[1;32m"
    byellow =   "\033[1;33m"
    bblue =     "\033[1;34m"
    bpurple =   "\033[1;35m"
    bcyan =     "\033[1;36m"
    bwhite =    "\033[1;37m"
    bforeground = "\033[1;39m"

    # off/normal/clear + shortnames
    off_black =  "\033[0;30m"
    off_red =    "\033[0;31m"
    off_green =  "\033[0;32m"
    off_yellow = "\033[0;33m"
    off_blue =   "\033[0;34m"
    off_purple = "\033[0;35m"
    off_cyan =   "\033[0;36m"
    off_white =  "\033[0;37m"
    off_light_gray = "\033[0;37m"
    off_foreground = "\033[0;39m"

    # off/normal/clear + bold + shortnames
    off_bred =      "\033[0;1;31m"
    off_bgreen =    "\033[0;1;32m"
    off_byellow =   "\033[0;1;33m"
    off_bblue =     "\033[0;1;34m"
    off_bpurple =   "\033[0;1;35m"
    off_bcyan =     "\033[0;1;36m"
    off_bwhite =    "\033[0;1;37m"
    off_bforeground = "\033[0;1;39m"

    info = "\033[0;2;39m" # off + faint + foreground
    binfo = "\033[1;2;39m" # bold + faint + foreground
    iinfo = "\033[2;3;39m" # faint + italic + foreground
    biinfo = "\033[1;2;3;39m" # bold + faint + italic + foreground
    sinfo = "\033[2;39m" # faint + foreground (no OFF, leave prev style)

    err = "\033[0;31m" # off + red
    berr = "\033[1;31m" # bold + red

    warn = "\033[0;33m" # off + yellow
    bwarn = "\033[1;33m" # bold + yellow

    @staticmethod
    def color_text(text: str, color: str) -> str:
        '''Wraps 'text' (str) with color (one of red|green|yellow|foreground) prefix and

        color (normal) suffix. Disables color prefix/suffix wrapping if args['color']=False
        '''
        if args['color']:
            return color + text + "\033[0m" # (normal)
        return text

    def disable(self) -> None:
        '''Clears all color str in class Colors, to prevent print() methods that use
        util.Colors from printing color'''
        for x in dir(self):
            if not callable(getattr(self, x, None)) and isinstance(x, str) and \
               not x.startswith('_'):
                setattr(self, x, '')


def safe_emoji(emoji: str, default: str = '') -> str:
    '''Returns emoji character if args['emoji'] is True'''
    if args['emoji']:
        return emoji
    return default


class ArtifactTypes(Enum):
    '''Types that are allow-listed for artifacts.add* methods. If you don't use one of
    these, you'll get a warning'''
    TEXT = 0
    JSON = 1
    YAML = 2
    WAVEFORM = 3
    DOTF = 4
    TCL = 5
    SHELL = 6
    BITSTREAM = 7
    LICENSE = 8
    HTML = 9
    COVERAGE_DATABASE = 10
    UNKNOWN = 255

class Artifacts:
    '''Class to hold file artifacts, for logs generated by EDA, or other artifcats created
    by a specific tool.

    If enabled, will write out an artifacts.json file upon util.exit()
    '''

    ARTIFACTS_NAME = 'artifacts.json'
    EDA_ERRORS_NAME = 'eda_errors.json'

    data: dict = {}
    unresolved_data: dict = {}
    enabled: bool  = True
    artifacts_json_filepath: str = os.path.join('eda.work', ARTIFACTS_NAME)
    eda_errors_json_filepath: str = os.path.join('eda.work', EDA_ERRORS_NAME)
    tool_errors: dict = {}
    tool_warnings: dict = {}
    job_errors_or_warnings: dict = {}

    def _get_type_str(self, typ: str) -> (bool, str):
        '''Gets the name from ArtifactTypes, and returns tuple of (True, str)

        if the ArtifactTypes exists, else (False, str.upper())'''
        ret = getattr(ArtifactTypes, typ.upper(), '')
        if ret:
            return True, ret.name.lower()
        return False, typ.lower()

    def incr_tool_error(self, tool: str, count: int = 1) -> None:
        '''Increments self.tool_errors[tool]'''
        if tool not in self.tool_errors:
            self.tool_errors[tool] = count
        else:
            self.tool_errors[tool] += count

    def incr_tool_warning(self, tool: str, count: int = 1) -> None:
        '''Increments self.tool_warnings[tool]'''
        if tool not in self.tool_warnings:
            self.tool_warnings[tool] = count
        else:
            self.tool_warnings[tool] += count

    def add(self, name: str, typ: str, description: str = '') -> None:
        '''Adds file to artifacts'''

        if not self.enabled:
            return # artifacts dumping disabled.

        _, shortname = os.path.split(name)
        if shortname in self.data:
            return # artifact already registered

        type_exists, type_str = self._get_type_str(typ)
        if not type_exists:
            warning(f'Unknown artifacts {typ=} add for {name=} {description=}')

        self.data[shortname] = {
            'name': name,
            'type': type_str,
            'size': 0, # will be filled in when JSON written upon exit.
            'description': description,
        }

    def add_extension(
            self, search_paths: list, file_extension: str, typ: str, description: str = ''
    ) -> None:
        '''Adds a placeholder for file extensions to add to artifacts

        For example, when saving a Simulation waves, we may know it's a .fst or .vcd file,
        but we do not know what the testbench SV file has set in $dumpfile(filepath)
        '''
        type_exists, type_str = self._get_type_str(typ)
        if not type_exists:
            warning(f'Unknown artifacts {typ=} add_extension for {search_paths=}',
                    f'{file_extension=} {description=}')
        else:
            info(f'artifacts: add_extension "{file_extension}", search paths: {search_paths}')

        if isinstance(search_paths, (str, PosixPath)):
            sp = [search_paths]
        else:
            sp = search_paths
        self.unresolved_data[file_extension] = {
            'search_paths': sp,
            'type': type_str,
            'description': description
        }

    def set_artifacts_json_dir(self, dirpath: str) -> None:
        '''Sets the artifacts_json_filepath'''
        self.artifacts_json_filepath = os.path.join(dirpath, self.ARTIFACTS_NAME)
        self.eda_errors_json_filepath = os.path.join(dirpath, self.EDA_ERRORS_NAME)

    def reset(self, enable: bool = True) -> None:
        '''Clears internal data (dict). Called by util.process_tokens()'''
        self.data = {}
        self.unresolved_data = {}
        self.tool_errors = {}
        self.tool_warnings = {}
        self.job_errors_or_warnings = {}
        self.enabled = enable or args['artifacts-json']
        self.artifacts_json_filepath = os.path.join('eda.work', self.ARTIFACTS_NAME)
        self.eda_errors_json_filepath = os.path.join('eda.work', self.EDA_ERRORS_NAME)

    def _resolve_unresolved_data(self, ext: str) -> None:
        '''Find files for this extension that were registered using add_extension(...)

        adds to self.data'''
        entry = self.unresolved_data[ext]
        if not entry:
            return

        for search_path in entry['search_paths']:
            p = Path(search_path)
            # Don't add the JSON if it's artifacts.json or eda_errors.json:
            if p.resolve() == Path(self.artifacts_json_filepath).resolve():
                continue
            if p.resolve() == Path(self.eda_errors_json_filepath).resolve():
                continue
            for posix_filename in p.glob(f'*.{ext}'):
                self.add(
                    name=str(posix_filename), typ=entry['type'], description=entry['description']
                )

    def add_eda_errors_json(self) -> None:
        '''Adds eda_errors.json to artifacts'''

        safe_mkdir_for_file(self.eda_errors_json_filepath)

        # Save error/warning information to same directory:
        # Note - the global artifacts.tool_errors and tool_warnings are incremented by
        # Command.update_tool_warn_err_counts_from_log_lines(..)
        error_data = {
            'errors': args['errors'],
            'warnings': args['warnings'],
            'tool_errors': self.tool_errors,
            'tool_warnings': self.tool_warnings
        }
        if self.job_errors_or_warnings:
            error_data['job_errors_or_warnings'] = self.job_errors_or_warnings

        try:
            with open(self.eda_errors_json_filepath, 'w', encoding='utf-8') as f:
                json.dump(error_data, f, indent=4)
        except PermissionError:
            error(f'PermissionError trying to write file: {self.eda_errors_json_filepath}')
        except IOError as e:
            error(f'IOError while trying to write file: {self.eda_errors_json_filepath}, {e}')

        self.add(
            name=self.eda_errors_json_filepath, typ='json',
            description='opencos-eda EDA errors/warnings, and tool reported errors/warnings'
        )


    def write_json(self) -> None: # pylint: disable=too-many-branches
        '''Write out the artifacts.json file, called by util.exit()'''

        if not self.enabled:
            return # artifacts dumping disabled.

        if self.unresolved_data:
            for ext in self.unresolved_data:
                self._resolve_unresolved_data(ext)

        self.add_eda_errors_json()

        if not self.data:
            return # do nothing if we have no artifacts

        safe_mkdir_for_file(self.artifacts_json_filepath)

        # Update all file sizes:
        remove_keys = set()
        for key, entry in self.data.items():
            if entry['type'] == 'license':
                # License entries are metadata, not files - keep them as-is
                entry['size'] = 0
            elif os.path.isfile(entry['name']):
                entry['size'] = os.path.getsize(entry['name'])
            else:
                # file doesn't exist, remove it from artifacts.
                warning(f'Removing {key} ({entry["name"]}) from artifacts (file does not exist)')
                remove_keys.add(key)
        for key in remove_keys:
            del self.data[key]

        try:
            with open(self.artifacts_json_filepath, 'w', encoding='utf-8') as f:
                json.dump(self.data, f, indent=4)
        except PermissionError:
            error(f'PermissionError trying to write file: {self.artifacts_json_filepath}')
        except IOError as e:
            error(f'IOError while trying to write file: {self.artifacts_json_filepath}, {e}')

        info(f'Wrote artifacts JSON: {self.artifacts_json_filepath}')
        for entry in self.data.values():
            if entry["name"] and entry["type"] and \
               (args['errors'] or entry["name"] != self.eda_errors_json_filepath):
                # If the file is eda_errors_json_filepath, only info that line if
                # we had errors (we always save that file though, it's in artifacts)
                info(f'  --> {entry["name"]}  ({entry["type"]})')


artifacts = Artifacts()


class UtilLogger:
    '''Class for the util.global_log'''
    file = None
    filepath = ''
    time_last = 0 #timestamp via time.time()

    # disabled by default, eda.py enables it. Can also be disabled via
    # util's argparser: --no-default-log, --logfile=<name>, or --force-logfile=<name>
    default_log_enabled = False
    default_log_filepath = os.path.join('eda.work', 'eda.log')
    default_log_disable_with_args = ['-h', '--help'] # common argparse help
    enable = True

    def clear(self) -> None:
        '''Resets some internals for logging, but preserve others'''
        self.file = None
        self.filepath = ''
        self.time_last = 0

    def stop(self) -> None:
        '''Closes open log, resets some internals'''
        if self.file:
            self.write_timestamp(f'stop - {self.filepath}')
            info(f"Closing logfile: {self.filepath}")
            self.file.close()
        self.clear()

    def start(self, filename: str, force: bool = False) -> None:
        '''Starts (opens) log'''
        if not self.enable:
            self.file = None
            return
        if not filename:
            error('Trying to start a logfile, but filename is missing')
            return
        if os.path.exists(filename):
            if force:
                debug(f"Overwriting logfile '{filename}', which exists, due to --force-logfile.")
            else:
                error(f"The --logfile path '{filename}' exists.  Use --force-logfile",
                      "(vs --logfile) to override.")
                return
        else:
            safe_mkdir_for_file(filename)
        try:
            self.file = open(filename, 'w', encoding='utf-8') # pylint: disable=consider-using-with
            debug(f"Opened logfile '{filename}' for writing")
            self.filepath = filename
            self.write_timestamp(f'start - {self.filepath}')
            # add to global artifacts:
            artifacts.add(name=filename, typ='text', description='EDA stdout logfile')
        except PermissionError:
            error(f'PermissionError opening log: {filename}')
            self.clear()
        except IOError as e:
            error(f'IOError while opening log: {filename}, {e}')
            self.clear()
        except Exception as e:
            error(f"Error opening '{filename}' for writing, {e}")
            self.clear()

    def write_timestamp(self, text: str = "") -> None:
        '''Writes timestamp to opened log'''
        if not self.file:
            return
        dt = datetime.datetime.now().ctime()
        print(f"INFO: [{progname}] Time: {dt} {text}", file=self.file)
        self.time_last = time.time()

    def write(self, text: str, end: str = '\n') -> None:
        '''Writes text to opened log'''
        if not self.file:
            return

        if ((time.time() - self.time_last) > 10) and \
           any(text.startswith(x) for x in [
               f"DEBUG: [{progname}]",
               f"INFO: [{progname}]",
               f"WARNING: [{progname}]",
               f"ERROR: [{progname}]"]):
            self.write_timestamp()
        print(strip_ansi_color(text), end=end, file=self.file)
        self.file.flush()
        os.fsync(self.file)


global_log = UtilLogger()


def start_log(filename: str, force: bool = False) -> None:
    '''Starts the global_log, if not already started'''
    global_log.start(filename=filename, force=force)

def write_log(text: str, end: str = '\n') -> None:
    '''Writes to the global_log, if started'''
    global_log.write(text=text, end=end)

def stop_log() -> None:
    '''Stops/closed the global_log'''
    global_log.stop()


atexit.register(stop_log)


def get_thread_budget() -> int:
    '''
    Returns an int, a la $(nproc), but takes the sched_getaffinity if available (Linux) to see
    what the OS actually allows this process to use. Otherwise, use cpu_count.
    '''
    try:
        return max(1, len(os.sched_getaffinity(0)))
    except AttributeError:
        return max(1, multiprocessing.cpu_count())
    except Exception:
        return 1


def get_tool_args_threads(total_budget: int, internal_parallelism: int) -> int:
    '''
    Returns an int, used by CommandParallel tools, to avoid --parallel=2 (yes 2x!)
    but then each of those oversubscribes to what the system advertises.

    Minimum return value 1. Example use:
        child_threads = util.get_tool_args_threads(
            self.args.get('threads', 1) / self.args.get('parallel', 1))
        )
        # now call sub eda command with: f'-j {child_threads}'
    '''
    # Ensure at least 1 thread minimum
    return max(1, total_budget // internal_parallelism)


def get_argparse_bool_action_kwargs() -> dict:
    '''Returns args for BooleanOptionalAction kwargs for an ArgumentParser.add_argument

    This is mostly for python compatibility to support --some-enable and --no-some-enable
    '''
    bool_kwargs = {}
    x = getattr(argparse, 'BooleanOptionalAction', None)
    if x is not None:
        bool_kwargs['action'] = x
    else:
        bool_kwargs['action'] = 'store_true'
    return bool_kwargs


def get_argparser() -> argparse.ArgumentParser:
    '''Returns the opencos.util ArgumentParser'''
    parser = argparse.ArgumentParser(
        prog=f'{safe_emoji("🔎 ")}opencos common options', add_help=False, allow_abbrev=False
    )
    # We set allow_abbrev=False so --force-logfile won't try to attempt parsing shorter similarly
    # named args like --force, we want those to go to unparsed list.
    # For bools, support --color and --no-color with this action=argparse.BooleanOptionalAction
    # -- however Python3.8 and older does not support this, so as a workaround, use kwargs for
    #    boolean actions:
    bool_action_kwargs = get_argparse_bool_action_kwargs()

    parser.add_argument('--version', default=False, action='store_true',
                        help=('Shows our version:'
                              f' {opencos.__version__} ({opencos.__pyproject_name__})')
                        )
    parser.add_argument('--color', **bool_action_kwargs, default=bool(supportsColor.stdout),
                        help='Use shell colors for info/warning/error messaging')
    parser.add_argument('--emoji', **bool_action_kwargs, default=args['emoji'],
                        help=f'Support emojis in terminal{safe_emoji(" 💪")}')
    parser.add_argument('--quiet', **bool_action_kwargs, default=args['quiet'],
                        help='Do not display info messaging')
    parser.add_argument('--verbose', **bool_action_kwargs, default=args['verbose'],
                        help='Display additional messaging level 2 or higher')
    parser.add_argument('--fancy', **bool_action_kwargs, default=args['fancy'])
    parser.add_argument('--debug', **bool_action_kwargs, default=args['debug'],
                        help='Display additional debug messaging level 1 or higher')
    parser.add_argument('--debug-level', type=int, default=0,
                        help='Set debug level messaging (default: 0)')
    parser.add_argument('--logfile', type=str, default=None,
                        help=('Write eda messaging to safe logfile that will not be overwritten'
                              ' (default disabled)'))
    parser.add_argument('--force-logfile', type=str, default=None,
                        help='Set to force overwrite the logfile')
    parser.add_argument('--default-log', **bool_action_kwargs,
                        default=global_log.default_log_enabled,
                        help=('Enable/Disable default logging to'
                              f' {global_log.default_log_filepath}. Default logging is disabled'
                              ' if --logfile or --force-logfile is set'))
    parser.add_argument('--no-respawn', action='store_true',
                        help=('Legacy mode (default respawn disabled) for respawning eda.py'
                              ' using $OC_ROOT/bin'))
    parser.add_argument('--artifacts-json', **bool_action_kwargs,
                        help='Store a work-dir/artifacts.json file of known tool output files')
    parser.add_argument('-f', '--input-file', default=[], action='append',
                        help=(
                            'Input .f file to be expanded as eda'
                            ' args/defines/incdirs/files/targets'))
    parser.add_argument('--env-file', default=[], action='append',
                        help=(
                            "dotenv file(s) to pass ENV vars, (default: .env loaded first,"
                            " subsequent files' vars override .env"
                        ))
    return parser


def get_argparser_args_list(parser: object = None) -> list:
    '''Returns list of all args, all items will include the -- prefix (--help, etc)'''
    if not isinstance(parser, argparse.ArgumentParser):
        return []
    return list(vars(parser).get('_option_string_actions', {}).keys())


def get_argparsers_args_list(parsers: list) -> list:
    '''Returns list of all args from list of parsers, all items have -- prefix (--help, etc)'''
    _args_list = []
    for parser in parsers:
        _args_list.extend(get_argparser_args_list(parser))
    return list(dict.fromkeys(_args_list)) # uniqify, preserve order


def get_argparser_short_help(parser: object = None) -> str:
    '''Returns short help for our ArgumentParser'''
    if not parser:
        parser = get_argparser()

    if not args['color']:
        # Since python3.14 doesn't care about our custom color settings,
        # need to remove any ANSI colors from argparse help formatter:
        full_lines = strip_ansi_color(parser.format_help()).split('\n')
    else:
        full_lines = parser.format_help().split('\n')

    lineno = 0
    for lineno, line in enumerate(full_lines):
        # Again, strip any ANSI colors when searching for starting text:
        # - options:
        # - optional arguments:
        if args['color']:
            line = strip_ansi_color(line)
        if any(line.startswith(x) for x in ('options:', 'optional arguments:')):
            break

    # skip the line that says 'options:', replace with the progname:
    return f'{Colors.off_cyan}{parser.prog}:{Colors.off}\n' + '\n'.join(full_lines[lineno + 1:])


def process_token(arg: list) -> bool:
    '''Returns true if we parsed arg (list, because we may pass --arg value)

    This is legacy holdover for oc_cli.py, that would process one token at a time.
    Simply run through our full argparser.
    '''
    _, unparsed = process_tokens(arg)
    if not unparsed: # empty list
        debug(f"Processed token or pair: {arg}")
        return True
    return False


def load_env_file(env_file: str) -> None:
    '''Handles .env file (from util CLI args --env-file)'''
    if os.path.isfile(env_file):
        load_dotenv(env_file, override=True)
        if os.path.abspath(env_file) not in env_files_loaded:
            env_files_loaded.append(os.path.abspath(env_file))
    else:
        warning(f'--env-file {env_file} does not exist and is not loaded.')


def patch_args_for_dir(tokens: list, patch_dir: str, caller_info: str) -> list:
    '''Given list of args, attempt to correct for relative dir'''

    # deal with relative path args or files.
    # Note the dot-f file could have been in a different directory (-f=path/to/my.f)
    # As a workaround to deal with relative paths, attempt to replace relative
    # path args within this dotf contents - so they are relative the dotf dir:
    if not os.path.isdir(patch_dir):
        return tokens

    ret = []
    for word in tokens:
        if word.startswith('-') and '=' in word:
            parts = word.split('=')
            leftarg = parts[0] + '='
            word = '='.join(parts[1:])
        elif word.startswith('+incdir+'):
            # do for +incdir+ too
            leftarg = '+incdir+'
            word = word[len('+incdir+'):]
        else:
            leftarg = ''

        if word and not os.path.isabs(word) and \
           os.path.exists(os.path.join(patch_dir, word)):
            # fix relative path of word, or --arg=word
            word = os.path.abspath(os.path.join(patch_dir, word))
            info(f'Using relative path {patch_dir} for arg/token: {leftarg}{word}',
                 f'{caller_info}')
        ret.append(f'{leftarg}{word}')
    return ret

def read_tokens_from_dot_f(filepath: str, caller_info: str = '', verbose: bool = False) -> list:
    '''Returns list of tokens from a .f file, with ENV vars expanded'''

    # Let's defer 'info' printing out what input files were opened until after
    # args['quiet'] and debug is resolved (which may be in these .f files)
    start_str = f"Opening -f / --input-file '{filepath}' for contents {caller_info}"
    if verbose:
        info(start_str)
    else:
        debug(start_str)
    if not os.path.isfile(filepath):
        error(f'-f (or --input-file): {filepath} does not exist {caller_info}',
              error_code=status_constants.EDA_GENERAL_FILE_NOT_FOUND)
        return []
    if os.path.abspath(filepath) in dot_f_files_expanded:
        error(f'-f (or --input-file): {filepath} has already been expanded',
              f'cannot traverse again (duplicate arg or nested .f files) {caller_info}')
    else:
        dot_f_files_expanded.append(os.path.abspath(filepath))
    tokens = []
    dotf_file_dir, _ = os.path.split(filepath)
    with open(filepath, encoding='utf-8') as f:
        for line in f:
            line = os.path.expandvars(line.strip())
            if not line or line.startswith('#') or line.startswith('//'):
                continue
            words = []
            if line.startswith("'") and line.endswith("'"):
                # treat as single word:
                words = [line]
            else:
                words = line.split()
            tokens.extend(patch_args_for_dir(
                tokens=words, patch_dir=dotf_file_dir, caller_info=f"(from dotf {filepath})"
            ))
    return tokens


def process_debug_args(parsed: argparse.Namespace) -> None:
    '''Sets global debug_level based on parsed args'''
    global debug_level # pylint: disable=global-statement

    if parsed.debug_level:
        set_debug_level(parsed.debug_level)
    elif parsed.debug:
        set_debug_level(1)
    elif parsed.debug is False:
        # not None, explicitly set to False via --no-debug
        debug_level = 0


def process_tokens( # pylint: disable=too-many-branches,too-many-statements
        tokens: list, caller_info: str = ''
) -> (argparse.Namespace, list):
    '''Processes tokens (unparsed args list) on util's ArgumentParser

    Returns tuple of (parsed Namespace, unparsed args list)
    '''
    global debug_level # pylint: disable=global-statement
    debug_level = 0

    # Deal with --version, --debug, --debug-level, --env-file, -f/--input-file(s)
    # tokens first, for now put dot-f file contents in front of of tokens, do this first
    # in a separate custom argparser:
    bool_action_kwargs = get_argparse_bool_action_kwargs()
    parser = argparse.ArgumentParser(
        prog='opencos -f/--input-file', add_help=False, allow_abbrev=False
    )
    parser.add_argument('--version', default=False, action='store_true')
    parser.add_argument('--debug', **bool_action_kwargs,
                        help='Display additional debug messaging level 1 or higher')
    parser.add_argument('--debug-level', type=int, default=0,
                        help='Set debug level messaging (default: 0)')
    parser.add_argument('--env-file', default=[], action='append',
                        help=(
                            "dotenv file(s) to pass ENV vars, (default: .env loaded first,"
                            " subsequent files' vars override .env"
                        ))
    parser.add_argument('-f', '--input-file', default=[], action='append',
                        help=(
                            'Input .f file to be expanded as eda args, defines, incdirs,'
                            ' files, or targets.'
                        ))
    try:
        parsed, unparsed = parser.parse_known_args(tokens + [''])
        tokens2 = list(filter(None, unparsed))
    except argparse.ArgumentError:
        error(f'util -f/--input-file, problem attempting to parse_known_args for: {tokens}')

    if parsed.version:
        # Stop processing, return nothing, caller can print version and exit.
        return parsed, []

    process_debug_args(parsed=parsed)
    debug(f'util.process_tokens: {parsed=} {unparsed=} from: {tokens}')

    if os.path.isfile(str(Path('.env'))):
        parsed.env_file.insert(0, '.env')
    if parsed.env_file:
        for env_file in parsed.env_file:
            load_env_file(env_file)

    if parsed.input_file:
        dotf_tokens = []
        for filepath in parsed.input_file:
            dotf_tokens.extend(read_tokens_from_dot_f(
                filepath=filepath, caller_info=caller_info
            ))

        # put the .f files before the unparsed args.
        tokens2 = dotf_tokens + tokens2

        # recurse until we've resolved nested .f files.
        return process_tokens(tokens=tokens2, caller_info=f'(from {parsed.input_file[-1]})')


    # Continue with all normal parsing beyond --debug and -f/--input-file,
    # Note that we re-parse everything in case there was --debug or --debug-level in
    # the .f file(s), or anything else that this argparser would pick up.
    parser = get_argparser()
    try:
        parsed, unparsed = parser.parse_known_args(tokens2 + [''])
        unparsed = list(filter(None, unparsed))
    except argparse.ArgumentError:
        error(f'util: problem attempting to parse_known_args for tokens={tokens2}')

    process_debug_args(parsed=parsed)

    if parsed.input_file:
        warning(f'python error, nested -f/--input-file(s) {parsed.input_file} should',
                'have already been resolved')

    if not parsed.color:
        Colors.disable(Colors) # strip strings in Colors class

    # clear existing artifacts dicts (mostly for pytests repeatedly calling eda.main),
    # set artifacts.enabled based on args['artifacts-json']
    artifacts.reset(enable=parsed.artifacts_json)

    if parsed.force_logfile:
        start_log(parsed.force_logfile, force=True)
    elif parsed.logfile:
        start_log(parsed.logfile, force=False)
    elif parsed.default_log and \
         not parsed.version and \
         not any(x in unparsed for x in global_log.default_log_disable_with_args) and \
         (parsed.force_logfile is None and parsed.logfile is None):
        # Use a forced logfile in the eda.work/eda.log:
        # avoid this if someone has --help arg not yet parsed.
        # avoid this if someone called --version, b/c that will print and exit.
        start_log(global_log.default_log_filepath, force=True)

    parsed_as_dict = vars(parsed)
    for key, value in parsed_as_dict.items():
        key = key.replace('_', '-')
        if key in args and value is not None:
            if value != args.get(key, None):
                modified_args[key] = True
            args[key] = value # only update with non-None values to our global 'args' dict

    for filepath in env_files_loaded:
        info(f"Loaded --env-file '{os.path.relpath(filepath)}'")
    for filepath in dot_f_files_expanded:
        info(f"Processed -f / --input-file '{os.path.relpath(filepath)}' for contents")

    return parsed, unparsed

# ********************
# fancy support
# In fancy mode, we take the bottom _FANCY_LINES lines of the screen to be written using
# fancy_print, while the lines above that show regular scrolling content (via info, debug, warning,
# error above). User should not use print() when in fancy mode

_FANCY_LINES = []

def fancy_start(fancy_lines: int = 4, min_vanilla_lines: int = 4) -> None:
    '''Starts fancy line support. This is not called by util internally

    It is called by an opencos.eda Command handling class, it can check if util.args['fancy']
    is set
    '''
    _, lines = shutil.get_terminal_size()
    if fancy_lines < 2:
        error("Fancy mode requires at least 2 fancy lines")
    if fancy_lines > (lines - min_vanilla_lines):
        error(f"Fancy mode supports at most {(lines - min_vanilla_lines)} fancy lines, given",
              f"{min_vanilla_lines} non-fancy lines")
    if _FANCY_LINES:
        error("We are already in fancy line mode, cannot call fancy_start() again")
    for _ in range(fancy_lines - 1):
        print() # create the requisite number of blank lines
        _FANCY_LINES.append("")
    print("", end="") # the last line has no "\n" because we don't want ANOTHER blank line below
    _FANCY_LINES.append("")
    # the cursor remains at the leftmost character of the bottom line of the screen

def fancy_stop() -> None:
    '''Stops fancy mode. Intended to be called by an opencos.eda Command handling class'''
    if not _FANCY_LINES:
        # don't do anything if we aren't in fancy mode.
        return

    # user is expected to have painted something into the fancy lines, we can't "pull down"
    # the regular lines above, and we don't want _FANCY_LINES blank or garbage lines either,
    # that's not pretty
    _FANCY_LINES.clear()
    # since cursor is always left at the leftmost character of the bottom line of the screen,
    # which was one of the fancy lines which now has the above-mentioned "something", we want
    # to move one lower
    print()


def fancy_print(text: str, line: int) -> None:
    '''Fancy print, intended to be called by an opencos.eda Command handling class'''

    # strip any newline, we don't want to print that
    text = text.rstrip("\n")
    lines_above = len(_FANCY_LINES) - line - 1
    if lines_above:
        print(
            (f"\033[{lines_above}A" # move cursor up
             f"{text}\033[1G" # desired text, then move cursor to the first character of the line
             f"\033[{lines_above}B" # move the cursor down
             ),
            end="", flush=True
        )
    else:
        print(
            f"{text}\033[1G", # desired text, then move cursor to the first character of the line
            end="", flush=True
        )
    _FANCY_LINES[line] = text

def print_pre() -> None:
    '''called by all util info/warning/debug/error. Handles fancy mode'''
    # stuff we do before printing any line
    if not _FANCY_LINES:
        return

    # Also, note that in fancy mode we don't allow the "above lines" to be partially written, they
    # are assumed to be full lines ending in "\n"
    # As always, we expect the cursor was left in the leftmost character of bottom line of screen
    print(
        (f"\033[{len(_FANCY_LINES)-1}A" # move the cursor up to where the first fancy line is drawn
         f"\033[0K"                     # clear the old fancy line 0
         ),
        end="", flush=True
    )

def print_post(text: str, end: str) -> None:
    '''called by all util info/warning/debug/error. Handles fancy mode'''
    # stuff we do after printing any line
    if _FANCY_LINES:
        # we just printed a line, including a new line, on top of where fancy line 0 used to be,
        # so cursor is now at the start of fancy line 1. move cursor down to the beginning of the
        # final fancy line (i.e. standard fancy cursor resting place)
        for x, line in enumerate(_FANCY_LINES):
            print("\033[0K", end="") # erase the line to the right
            print(line, flush=True,
                  end=('' if x == (len(_FANCY_LINES) - 1) else '\n'))

        print("\033[1G", end="", flush=True)
    if global_log.file:
        write_log(text, end=end)

def print_wrap(text: str, end: str = '\n') -> None:
    '''Same a print_color, but without a color change'''
    print_pre()
    print(text + Colors.off, end=end, flush=True)
    print_post(text, end)

def print_color(text: str, color: str, end: str = '\n') -> None:
    '''Note that color(str) must be one of Colors.[red|green|yellow|normal]'''
    print_pre()
    print(Colors.color_text(text, color), end=end, flush=True)
    print_post(text, end)

def print_foreground_color(text: str, end: str = '\n') -> None:
    '''Print text as foreground color, goes back to normal color'''
    print_pre()
    print(Colors.color_text(text, color=Colors.foreground), end=end, flush=True)
    print_post(text, end)


def set_debug_level(level) -> None:
    '''Sets global debug level, sets args['debug'] and args['verbose']'''
    global debug_level # pylint: disable=global-statement
    debug_level = level
    args['debug'] = level > 0
    args['verbose'] = level > 1
    info(f"Set debug level to {debug_level}")


def debug(
        *text, level: int = 1, start: object = None, end: str = '\n', color=Colors.foreground
) -> None:
    '''Print debug messaging (in foreground color if possible). If args['debug'] is false,
    prints nothing.

    *text: (positional str args) to be printed
    level: (int) debug level to decide if printed or not.
    start: (optional str) prefix to message; if None: chooses default start str
    end: (optional str) suffix to print

    Note these messages append to global logging (but require args['debug'] to be set)
    '''
    if start is None:
        start = "DEBUG: " + (f"[{progname}] " if progname_in_message else "")
    if args['debug'] and \
       (((level==1) and args['verbose']) or (debug_level >= level)):
        print_color(f"{start}{' '.join(list(text))}", color=color, end=end)


def info(*text, start: object = None, end='\n', color=None) -> None:
    '''Print information messaging (in green if possible). If args['quiet'], prints nothing.

    *text: (positional str args) to be printed
    start: (optional str) prefix to message; if None: chooses default start str
    end: (optional str) suffix to print

    Note these messages append to global logging even if args['quiet'] is set
    '''
    if color is None:
        color = Colors.info
    if start is None:
        start = (
            f"{Colors.off}{Colors.green}INFO: " + \
            (f"[{progname}] " if progname_in_message else "")
        )
    if not args['quiet']:
        print_wrap(f"{start}{color}{' '.join(list(text))}", end=end)

def warning(*text, start: object = None, end: str = '\n') -> None:
    '''Print warning messaging (in yellow if possible).

    *text: (positional str args) to be printed
    start: (optional str) prefix to message; if None: chooses default start str
    end: (optional str) suffix to print

    Note these messages append to global logging. Increments global args['warnings'] int.
    '''
    if start is None:
        start = (
            f"{Colors.off}{Colors.bwarn}WARNING: " + \
            (f"[{progname}] " if progname_in_message else "")
        )
    args['warnings'] += 1
    print_wrap(f"{start}{Colors.warn}{' '.join(list(text))}", end=end)


def error(
        *text,
        error_code: int = status_constants.EDA_DEFAULT_ERROR,
        do_exit: bool = True, start: object = None, end: str = '\n'
) -> int:
    '''Print error messaging (in red if possible).

    *text: (positional str args) to be printed
    error_code: (int) shell style return code (non-zero is error, but prefer > 1 b/c those are
                python exceptions)
    do_exit: (bool) if True will call exit based on global_exit_allowed.
    start: (optional str) prefix to message; if None: chooses default start str
    end: (optional str) suffix to print

    Note these messages append to global logging. Increments global args['errors'] int.
    '''
    global max_error_code # pylint: disable=global-statement

    if start is None:
        start = f"{Colors.off}{Colors.berr}ERROR: "
        if progname_in_message:
            start += f"[{progname}] "
        start += safe_emoji("❌ ")
    args['errors'] += 1
    max_error_code = max(max_error_code, error_code)
    print_wrap(f"{start}{Colors.err}{' '.join(list(text))}", end=end)
    if do_exit:
        if args['debug']:
            print(traceback.print_stack())
        # Call our overriden-builtin function for 'exit':
        return exit(error_code) # pylint: disable=consider-using-sys-exit

    if error_code is None:
        return 0
    return abs(int(error_code))


class AlreadyMessage:
    '''
    Namespace, so we only make the same warnings once, aka ToolVerilator may be called > 1 times
    but we only want to see those warnings once, vs N times for every N class handles created.

    Used by warn_once(warn_name, "text")
    '''
    info_names: set = set()
    warn_names: set = set()


def info_once(*text, info_name: str, **kwargs) -> None:
    '''Wrapper for util.info, only runs if info_name hasn't happened before'''
    if info_name not in AlreadyMessage.info_names:
        info(*text, **kwargs)
        AlreadyMessage.info_names.add(info_name)


def warn_once(*text, warn_name: str, **kwargs) -> None:
    '''Wrapper for util.warning, only runs if warn_name hasn't happened before'''
    if warn_name not in AlreadyMessage.warn_names:
        warning(*text, **kwargs)
        AlreadyMessage.warn_names.add(warn_name)


def get_return_code() -> int:
    '''Checks global max_error_code and args['errors']'''
    if args['errors']:
        if max_error_code == 0:
            return 255
        return max_error_code
    return 0


def exit( # pylint: disable=redefined-builtin
        error_code: int = 0, quiet: bool = False
) -> int:
    '''sys.exit(int) wrapper, returns the error_code if global_exit_allowed=False'''

    # Save out artifacts file:
    artifacts.write_json()

    if global_exit_allowed:
        if not quiet:
            info_color = Colors.info
            start = safe_emoji('🔚 ')
            if args['errors']:
                info_color = Colors.red
                start = safe_emoji('❗ ')
            elif args['warnings']:
                info_color = Colors.yellow
            info(
                f"{start}Exiting with {Colors.bold}{args['warnings']} warnings{info_color},",
                f"{Colors.bold}{args['errors']} errors",
                color=info_color
            )
        sys.exit(error_code)

    if error_code is None:
        return 0

    return abs(int(error_code))


def getcwd():
    '''Wrapper for os.getcwd() for current working dir'''
    try:
        return os.getcwd()
    except Exception as e:
        error(f"Unable to getcwd(), did it get deleted from under us? Exception: {e}")
        return None

_OC_ROOT = None       # pylint: disable=invalid-name
_OC_ROOT_SET = False  # pylint: disable=invalid-name

def get_oc_root(error_on_fail: bool = False) -> None:
    '''Returns a str or False for the root directory of *this* repo.

    If environment variable OC_ROOT is set, that is used instead, otherwise attempts to use
    `git rev-parse`
    '''

    global _OC_ROOT      # pylint: disable=global-statement
    global _OC_ROOT_SET  # pylint: disable=global-statement
    # if we've already run through here once, just return the memorized result
    if _OC_ROOT_SET:
        return _OC_ROOT

    # try looking for an env var
    s = os.environ.get('OC_ROOT')
    if s:
        debug(f'get_oc_root() -- returning from env: {s=}')
        _OC_ROOT = s.strip()
    else:
        # try asking GIT
        cp = subprocess.run(
            'git rev-parse --show-toplevel', stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
            shell=True, check=False, universal_newlines=True
        )
        if cp.returncode != 0:
            # TODO(drew): at some point, address the fact that not all repos are oc_root.
            # Is this function asking for the repo we are in?  or a pointer to the oc_root which
            # maybe elsewhere on the system?
            print_didnt_find_it = debug
            if error_on_fail:
                print_didnt_find_it = error
            print_didnt_find_it('Unable to get a OC_ROOT directory using git rev-parse')
        else:
            _OC_ROOT = cp.stdout.strip()
            if sys.platform.startswith('win'):
                _OC_ROOT = _OC_ROOT.replace('/', '\\') # git gives us /, but we need \

    # there is no sense running through this code more than once
    _OC_ROOT_SET = True
    return _OC_ROOT


def safe_cp(source: str, destination: str, create_dirs: bool = False) -> None:
    '''shutil.copy2 wrapper to optionally make the destination directories'''
    try:
        # Infer if destination is a directory
        if destination.endswith('/') or os.path.isdir(destination):
            if not os.path.exists(destination) and create_dirs:
                os.makedirs(destination, exist_ok=True)
            destination = os.path.join(destination, os.path.basename(source))
        else:
            # Ensure parent directory exists if needed
            parent_dir = os.path.dirname(destination)
            if create_dirs and parent_dir and not os.path.exists(parent_dir):
                os.makedirs(parent_dir, exist_ok=True)
        # actually copy the file
        shutil.copy2(source, destination)
        info(f"Copied {source} to {destination}")
    except Exception as e:
        print(f"Error copying file from '{source}' to '{destination}': {e}")


def safe_mkdir(path : str) -> None:
    '''Attempt to make dir at path, and make all subdirs up to that path'''
    if os.path.exists(path):
        return
    left, _ = os.path.split(os.path.relpath(path))
    if left and left not in ['.', '..', os.path.sep]:
        safe_mkdir(left)
    try:
        os.mkdir(path)
    except FileExistsError:
        pass
    except Exception as e1:
        try:
            os.system(f'mkdir -p {path}')
        except Exception as e2:
            error(f'unable to mkdir {path=}, exceptions {e1}, {e2=}')


def safe_mkdirs(base : str, new_dirs : list) -> None:
    '''Create new_dirs at base'''
    for p in new_dirs:
        safe_mkdir( os.path.join(base, p) )


def safe_mkdir_for_file(filepath: str) -> None:
    '''Given a new filepath, create dir for that filepath'''
    left, _ = os.path.split(filepath)
    if left:
        safe_mkdir(left)


def import_module_from_string(full_module_name: str) -> None:
    """
    Imports a pymodule given its full name as a str.

    Args:
        full_module_name: The full name of the pymodule,
                           e.g., "module.submodule".

    Returns:
        The imported module, or None if an error occurs.
    """
    try:
        return import_module(full_module_name)
    except Exception as e:
        error(f"Error importing pymodule {full_module_name=}: {e}")
        return None


def import_class_from_string(full_class_name: str) -> None:
    """
    Imports a class given its full name as a str.

    Args:
        full_class_name: The full name of the class,
                         e.g., "module.submodule.ClassName".

    Returns:
        The imported class, or None if an error occurs.
    """
    try:
        module_path, class_name = full_class_name.rsplit(".", 1)
        module = import_module(module_path)
        return getattr(module, class_name)
    except Exception as e:
        error(f"Error importing class {full_class_name=}: {e}")
        return None


class ShellCommandList(list):
    '''Wrapper around a list, of str that we'll run as a subprocess command

    included member vars for:
      - tee_path, to save a log from this subprocess commands list
      - work_dir - in case we want to run this from non-default location.
    '''
    def __init__(self, obj: object = None, tee_fpath: str = '', work_dir: str = ''):
        super().__init__(obj)
        for k in ('tee_fpath', 'work_dir'):
            setattr(self, k, getattr(obj, k, None))
        if tee_fpath:
            self.tee_fpath = tee_fpath
        if work_dir:
            self.work_dir = work_dir


def write_shell_command_file(
        dirpath : str, filename : str, command_lists : list, line_breaks : bool = False
) -> None:
    ''' Writes new file at {dirpath}/{filename} as a bash shell command, using command_lists
    (list of lists)

    -- dirpath (str)        -- directory where file is written (usually eda.work/{target}_sim
    -- filename (str)       -- filename, for example compile_only.sh
    -- command_lists (list) -- list of (list or ShellCommandList), each item in the list is a
                               list of commands (aka, how subprocess.run(args) uses a list of
                               commands.
    -- line_breaks (bool)   -- Set to True to have 1 word per line in the file followed by a line
                               break. Default False has an entry in command_lists all on a single
                               line.

    Returns None, writes the file and chmod's it to 0x755.
    '''

    # command_lists should be a list-of-lists.
    bash_path = safe_shutil_which('bash')
    assert isinstance(command_lists, list), f'{command_lists=}'
    fullpath = os.path.join(dirpath, filename)
    try:
        with open(fullpath, 'w', encoding='utf-8') as f:
            if not bash_path:
                bash_path = "/bin/bash" # we may not get far, but we'll try
            f.write('#!' + bash_path + '\n\n')
            for obj in command_lists:
                assert isinstance(obj, list), \
                    f'{obj=} (obj must be list/ShellCommandList) {command_lists=}'
                clist = list(obj).copy()
                tee_fpath = getattr(obj, 'tee_fpath', None)
                if tee_fpath:
                    # Note the | tee foo.log will ruin bash error codes, so if we're bash is
                    # available, we'll check that ${PIPESTATUS} is 0 to percolate the
                    # a non-zero on the first command (sim.exe).
                    if safe_shutil_which('bash'):
                        clist.append(f'2>&1 | tee {tee_fpath}' + ' && test ${PIPESTATUS} -eq 0')
                    else:
                        clist.append(f'2>&1 | tee {tee_fpath}')

                if clist:
                    if line_breaks:
                        # line_breaks=True - have 1 word per line, followed by \ and \n
                        sep = " \\" + "\n"
                        f.write(sep.join(clist))
                        f.write(" \n")
                    else:
                        # line_break=False (default) - all words on 1 line.
                        f.write(' '.join(clist))
                        f.write(" \n")
                else:
                    f.write("\n")
            f.write("\n")
            f.close()
            os.chmod(fullpath, 0o755)
    except PermissionError:
        error(f'PermissionError trying to write file: {fullpath}')
    except IOError as e:
        error(f'IOError while trying to write file: {fullpath}, {e}')


def get_inferred_top_module_name(module_guess: str, module_fpath: str) -> str:
    '''Returns the best guess as the 'top' module name name, given a fpath where

    module_fpath = /some/path/to/module_guess.[v|sv]

    Use the module_guess if it exists in the file as: module <module_guess>
    Otherwise use the last observed: module <best_guess>
    Otherwise use blank str
    '''

    best_guess = ''
    if not os.path.isfile(module_fpath):
        return ''
    with open(module_fpath, encoding='utf-8') as f:
        for line in f.readlines():
            line = line.strip()
            if line.startswith('module '):
                parts = line.split()
                module_name = parts[1]
                rstrip_nonword_pattern = r'\W+.*$'
                module_name = re.sub(rstrip_nonword_pattern, '', module_name)
                if bool(re.fullmatch(r'^\w+$', module_name)):
                    if module_name == module_guess:
                        return module_guess
                    if module_name:
                        best_guess = module_name
    if best_guess:
        return best_guess

    return ''
