'''opencos.eda_base holds base classes for opencos.tools and opencos.commands

Note that opencos.Command.process_tokens (on a derived class) is the main entrypoint
from eda.py, it will find a command handler class for a command + tool, and call
its obj.process_tokens(args)
'''

# TODO(drew): Clean up the following pylint
# pylint: disable=too-many-lines

import argparse
import copy
import json
import os
from pathlib import Path
import queue
import re
import shutil
import signal
import subprocess
import sys
import threading
import time

from rich.pretty import pprint

from opencos import eda_config, export_helper, files, seed, util
from opencos.deps.deps_file import DepsFile, deps_data_get_all_targets
from opencos.deps.deps_processor import DepsProcessor
from opencos.eda_tool_helper import get_all_handler_commands
from opencos.files import safe_shutil_which
from opencos.util import Colors, safe_emoji
from opencos.utils import status_constants
from opencos.utils.str_helpers import sprint_time, strip_outer_quotes, string_or_space, \
    indent_wrap_long_text, pretty_list_columns_manual, get_terminal_columns, \
    flatten_dict_to_str, pretty_2dlist_columns, PARAMETER_NAME_RSTR
from opencos.utils import subprocess_helpers
from opencos.utils.subprocess_helpers import subprocess_run_background, IS_WINDOWS


def get_all_commands_help_str(config: dict) -> str:
    '''Returns a str of help based on what commands eda supports, from config'''
    all_commands_help = []
    all_handler_commands = get_all_handler_commands(config=config)
    max_command_str_len = max(len(s) for s in config.get('DEFAULT_HANDLERS_HELP', {}).keys())
    for key, value in config.get('DEFAULT_HANDLERS_HELP', {}).items():
        all_commands_help.append(
            f'    {Colors.bcyan}{key:<{max_command_str_len}}{Colors.off} - {value.strip()}'
        )
        if key in all_handler_commands:
            all_commands_help[-1] += (
                f'\n    {" ":<{max_command_str_len}}   '
                f'--tool: {Colors.cyan}{", ".join(all_handler_commands[key])}{Colors.off}'
            )
    if all_commands_help:
        all_commands_help = [
            f'Where {Colors.byellow}COMMAND{Colors.off} is one of:',
            '',
        ] + all_commands_help
    return '\n'.join(all_commands_help)


def print_base_help() -> None:
    '''Prints help() information from other argparsers we use, without "usage"'''
    # using bare 'print' here, since help was requested, avoids --color and --quiet
    print(util.get_argparser_short_help())
    print(eda_config.get_argparser_short_help())
    print(get_argparser_short_help())


def print_eda_usage_line(no_targets: bool = False, command_name='COMMAND') -> None:
    '''Prints line for eda [options] COMMAND [options] FILES|TARGETS,...'''
    print(f'{safe_emoji("🔦 ")}Usage:')
    print(f'    {Colors.bold}{Colors.yellow}eda_targets PATTERN')
    print(f'    {Colors.bold}{Colors.yellow}eda_deps_schema DEPS_FILE')
    print(f'    {Colors.bold}{Colors.yellow}eda_show_autocomplete{Colors.off}')
    print()
    if not command_name:
        command_name = 'COMMAND'
    if no_targets:
        print(
            (f'    {Colors.bold}{Colors.yellow}eda {Colors.cyan}[options]'
             f' {Colors.yellow}{command_name} {Colors.cyan}[options]{Colors.off}')
        )
    else:
        print(
            (f'    {Colors.bold}{Colors.yellow}eda {Colors.cyan}[options]'
             f' {Colors.yellow}{command_name} {Colors.cyan}[options]'
             f' {Colors.yellow}FILES|TARGETS,...{Colors.off}')
        )
    print()


def get_argparser() -> argparse.ArgumentParser:
    '''Returns the ArgumentParser for general eda CLI'''
    parser = argparse.ArgumentParser(
        prog=f'{safe_emoji("🔎 ")}eda options', add_help=False, allow_abbrev=False
    )
    parser.add_argument('-q', '--quit', action='store_true',
                        help=(
                            'For interactive mode (eda called with no options, command, or'
                            ' targets)'))
    parser.add_argument('--exit', action='store_true', help='same as --quit')
    parser.add_argument('-h', '--help', action='store_true')

    # Note - these top level eda.py args, that aren't from eda_config.py or util.py, need to
    # be fed back into CommandMulti or CommandSweep, since they re-invoke eda.
    parser.add_argument('--tool', type=str, default=None,
                        help='Tool to use for this command, such as: modelsim_ase, verilator,' \
                        + ' modelsim_ase=/path/to/bin/vsim, verilator=/path/to/bin/verilator')
    parser.add_argument('--eda-safe', action='store_true',
                        help=('disable all DEPS file deps shell commands, overrides values from'
                              ' --config-yml'))
    parser.add_argument('--ssh', type=str, default=None, metavar='USER@HOST[:/PATH]',
                        help=(
                            'Experimental. Run on a remote host via SSH. Requires --tool. '
                            'Local tools not necessary. '
                            'Sandbox defaults to ~/eda.work/ on remote; '
                            ':/path overrides (e.g. --ssh=user@hw1:/scratch/me). '
                            'Works with eda multi, may not work on all eda commands'
                        ))
    parser.add_argument('--ssh-setup', type=str, default=None, metavar='CMD',
                        help=(
                            'Experimental. Shell command run on remote before eda (use with '
                            '--ssh). Example: --ssh-setup='
                            '"source /opt/Xilinx/Vivado/2024.2/settings64.sh"'
                        ))

    parser.add_argument('--show-tool-versions', **util.get_argparse_bool_action_kwargs(),
                        default=True,
                        help='Use --no-show-tool-versions to avoid looking up versions'
                        )

    return parser


def get_argparser_short_help() -> str:
    '''Returns str for the eda_base argparser (--help, --tool, etc)'''
    return util.get_argparser_short_help(parser=get_argparser())


def get_argparsers_args_list() -> list:
    '''Returns list of all args that we know about from eda_config, util, eda.

    All items will include the -- prefix (--help, etc)'''
    return util.get_argparsers_args_list(parsers=[
        eda_config.get_argparser(),
        util.get_argparser(),
        get_argparser()
    ])


def get_eda_exec(command: str = '') -> str:
    '''Returns the full path of `eda` executable to be used for a given eda COMMAND'''
    # NOTE(drew): This is kind of flaky. 'eda multi' reinvokes 'eda'. But the executable for 'eda'
    # is one of:
    # 1. executable script 'eda' from uv, or venv pip installed
    # 2. (opencos repo)/bin/eda - a python wrapper to link to (opencos repo)/opencos/eda.py, but
    #    packages cannot be run standalone, they need to be called as: python3 -m opencos.eda,
    #    and do not work with relative paths. This only works if env OC_ROOT is set or can be found.
    # 3. If you ran 'source bin/addpath' then you are always using the local (opencos repo)/bin/eda

    # Prefer the 'eda' script that lives alongside the currently-running Python interpreter.
    # This is the venv/bin/ directory when running inside a venv, so we consistently re-invoke
    # the same installation rather than whatever 'eda' happens to be first in PATH.
    installed_bin_dir = os.path.join(sys.prefix, 'Scripts' if IS_WINDOWS else 'bin')
    installed_eda_exe = os.path.join(installed_bin_dir, 'eda.exe' if IS_WINDOWS else 'eda')
    if os.path.isfile(installed_eda_exe):
        return installed_eda_exe

    eda_path = safe_shutil_which('eda')
    if not eda_path:
        # Can we run from OC_ROOT/bin/eda?
        oc_root = util.get_oc_root()
        if not oc_root:
            util.error(f"Need 'eda' in our path to run 'eda {command}', could not find env",
                       f"OC_ROOT, {eda_path=}, {oc_root=}")
        else:
            bin_eda = os.path.join(oc_root, 'bin', 'eda')
            if not os.path.exists(bin_eda):
                util.error(f"Need 'eda' in our path to run 'eda {command}', cound not find",
                           f"bin/, {eda_path=}, {oc_root=}, {bin_eda=}")
            else:
                util.info(f"'eda' not in path, using {bin_eda=} for 'eda' {command} executable")
                eda_path = os.path.abspath(bin_eda)

    return eda_path


def which_tool(command: str, config: dict) -> str:
    '''Returns which tool (str) will be used for a command, given the command_handlers in

    config dict.'''
    if config is None:
        util.error(f'which_tool({command=}) called w/out config')
    if not command in config.get('command_handler', {}):
        util.error("which_tool called with invalid command?")

    # Note: we could create a throw-away Command object using config, and check its
    # args['tool']:
    #    cmd_obj = config['command_handler'][command](config=config)
    #    return cmd_obj.args.get('tool', None)
    # But that has side effects and prints a lot of garbage, and does a lot
    # of loading and setting values to create that throw away Command object.

    # Instead, we'll directly look up at the class _TOOL w/out creating the obj.
    tool = getattr(config['command_handler'][command], '_TOOL', None)
    return tool


def get_class_tool_name(command_obj: object) -> str:
    '''Attempts to return command_obj._TOOL via command_obj.get_tool_name()'''
    if f := getattr(command_obj, 'get_tool_name', None):
        if callable(f):
            ret = f()
            if ret and isinstance(ret, str):
                return ret
    return ''


class Tool:
    '''opencos.eda_base.Tool is a base class used by opencos.tools.<name>.

    If you do not have a config (dict) for the contructor, please use Tool(config={})'''

    error = util.error # use that module's method
    _TOOL = '' # this is the only place a class._TOOL should be set to non-empty str
    _URL = None
    _EXE = None

    DEFAULT_DOCKER_IMAGE = ''
    docker_arg_set = None
    no_exe_but_has_docker = None

    def __init__(self, config: dict):
        # Because Command child classes (CommandSimVerilator, for example), will
        # inherit both Command and Tool classes, we'd like them to reference
        # a Command object's self.args instead of the class Tool.args. Safely create it
        # if it doesn't exist:
        self._VERSION = None

        self.tool_warning_count = 0
        self.tool_error_count = 0

        if getattr(self, 'args', None) is None:
            self.args = {}
        if getattr(self, 'args_help', None) is None:
            self.args_help = {}
        if getattr(self, 'defines', None) is None:
            self.defines = {}
        if getattr(self, 'config', None) is None:
            self.config = config
        self.args.update({
            'tool':   self._TOOL, # Set for all derived classes.
        })
        self.args_help.update({
            'tool': 'Tool to use for this command, such as: verilator',
        })
        # update self._EXE if config says to:
        self.set_exe(config)
        self.get_versions(quiet=True)

        if getattr(self, 'set_tool_config_from_config', None):
            # Hook for classes like CommandSim that have a Tool derived class attached:
            # have the Tool constructor run set_tool_config_from_config(), otherwise
            # every derived class will have to do this.
            # Allows command handlers to make thier own customizations from --config-yaml=YAML.
            self.set_tool_config_from_config()


    def set_exe(self, config: dict) -> None:
        '''Sets self._EXE based on config'''
        if self._TOOL and self._TOOL in config.get('auto_tools_found', {}):
            # config['auto_tools_found'] has the first exe full path:
            exe = config.get('auto_tools_found', {}).get(self._TOOL)
            if exe and exe != self._EXE:
                # Note that safe_shutil_which() on the exe leaf may not match, this does NOT
                # modify os.environ['PATH'].
                util.debug(f'{self._TOOL} using exe: {exe}')
                self._EXE = exe

    def get_tool_name(self) -> str:
        '''Returns _TOOL'''
        return str(self._TOOL)

    def get_full_tool_and_versions(self) -> str:
        '''Returns tool:version, such as: verilator:5.033'''
        if not self._VERSION:
            self.get_versions()
        return str(self._TOOL) + ':' + str(self._VERSION)

    def get_versions( # pylint: disable=unused-argument
            self, **kwargs
    ) -> str:
        '''Sets and returns self._VERSION. Note that this is overriden by nearly all

        child Tool classes, and should limit info/warning messaging since it is part
        of opencos.eda.init_config(..)
        '''
        self.get_set_default_docker_image()
        return self._VERSION

    def get_version_allow_certain_command_if_tool_not_installed(self) -> str:
        '''Returns blank str or 'UNKNOWN, and sets self._VERSION'''
        if _cmd_name := getattr(self, 'command_name', ''):
            if _cmd_name and _cmd_name in self.config.get(
                    'command_available_with_tool_if_tool_not_installed', []
            ):
                util.info(f'--tool={self._TOOL} is not installed, but can perform command:',
                          f'{_cmd_name}', color=util.Colors.cyan)
                self._VERSION = 'UNKNOWN'
                return self._VERSION

        return ''


    def get_set_default_docker_image(self) -> str:
        '''Attempts to fish out the docker-default-image from config.tools.TOOL'''
        if not self.config or not self._TOOL:
            return self.DEFAULT_DOCKER_IMAGE

        tool_cfg = self.config.get('tools', {}).get(self._TOOL, {})
        if not tool_cfg.get('docker-support', False):
            return self.DEFAULT_DOCKER_IMAGE

        from_config = tool_cfg.get('docker-default-image', '')
        if from_config:
            self.DEFAULT_DOCKER_IMAGE = from_config

        return self.DEFAULT_DOCKER_IMAGE


    def report_tool_warn_error_counts(self, clear_counts: bool = False) -> None:
        '''Reports info line based on self.tool_error_count and self.tool_warning_count.'''
        tool_name = get_class_tool_name(self)
        if not tool_name:
            return

        info_color = Colors.info
        error_color = Colors.binfo
        start = ''
        if self.tool_warning_count:
            info_color = Colors.yellow
        if self.tool_error_count:
            info_color = Colors.yellow
            error_color = Colors.underline + Colors.bred
            start = safe_emoji('🔶 ')
        msg = (
            f"{start}Tool - {Colors.bold}{tool_name}{Colors.off}{info_color}, total counts:",
            f"{Colors.bold}{self.tool_warning_count} tool warnings{Colors.off}{info_color},",
            f"{error_color}{self.tool_error_count} tool errors"
        )
        if self.tool_error_count:
            # Treat tool errors as eda error:
            # No Tool.error method, use util.error(..)
            util.error(*msg, do_exit=False)
        elif self.tool_warning_count:
            # Treat tool warnings as eda warning:
            util.warning(*msg)
        else:
            util.info(*msg)

        if clear_counts:
            self.tool_warning_count = 0
            self.tool_error_count = 0

    def set_tool_defines(self) -> None:
        '''Derived classes may override, sets any additional defines based on tool.

        If overriding, you likely want to call:
            super().set_tool_defines()
        to pick up anything from self.config['tools'] (from the eda_config YAML)
        '''
        if not self._TOOL:
            return

        _tool_config = self.config.get('tools', {}).get(self._TOOL, {})
        self.defines.update(
            _tool_config.get('defines', {})
        )



class Command: # pylint: disable=too-many-public-methods,too-many-instance-attributes
    '''Base class for all: eda COMMAND

    The Command class should be used when you don't require files, otherwise consider
    CommandDesign.
    '''

    command_name: str = ''
    subcommand = None

    def __init__(self, config:dict):
        if getattr(self, 'args', None) is None:
            self.args = {}
        if getattr(self, 'args_help', None) is None:
            self.args_help = {}
        if getattr(self, 'args_kwargs', None) is None:
            self.args_kwargs = {}
        if getattr(self, 'args_args', None) is None:
            self.args_args = {} # support for multiple named --arg that map to same self.args key.
        self.args.update({
            "keep" : False,
            "force" : False,
            "fake" : False,
            "stop-before-compile": False,
            "stop-after-compile": False,
            "stop-after-elaborate": False,
            "lint": False, # Same as stop-after-elaborate
            "eda-dir" : "eda.work", # all eda jobs go in here
            "job-name" : "", # this is used to create a certain dir under "eda_dir"
            "work-dir" : "", # default is usually <eda-dir>/<job-name> or <eda-dir>/<target>.<cmd>
            "sub-work-dir" : "", # this can be used to name the dir built under <eda-dir>
            "work-dir-use-target-dir": False,
            "suffix" : "",
            "design" : "", # not sure how this relates to top
            'export':     False,
            'export-run': False,  # run from the exported location if possible
            'export-json': False, # generates an export.json suitable for a testrunner
            'enable-tags': [],
            'disable-tags': [],
            'test-mode': False,
            'error-unknown-args': True,
            'threads': 1, # want small default value
            'use-max-threads': config.get('use_max_threads', False),
            'subprocess-timeout': 0,
            'subprocess-progress-timeout': 0,
        })
        self.args_help.update({
            'keep': (
                'Determined by eda <tool>, generally prevents jobs from overwriting artifacts'
            ),
            'force': 'Determined by eda <tool>, force override',
            'fake': (
                'Determined by eda <tool>, generally is a dry-run that does not execute tool'
            ),
            'stop-before-compile': (
                'stop this run before any compile steps (if possible for tool) and'
                ' save .sh scripts in eda-dir/'
            ),
            'stop-after-compile': (
                'stop this run after any compile steps (if possible for tool) and'
                ' save .sh scripts in eda-dir/'
            ),
            'stop-after-elaborate': (
                'stop this run after any elaborate steps (if possible for tool) and'
                ' save .sh scripts in eda-dir/'
            ),
            'lint': 'tool dependent, run with linting options',
            "job-name" : 'Optional, used to create a sub directory under base work-dir (eda-dir)',
            "sub-work-dir" : (
                'Optional, similar to job-name, can be used to name the directory created under'
                ' eda-dir'
            ),
            'eda-dir':   'Optional, relative base directory where eda logs are saved',
            'export':      'export results for these targets in eda-dir',
            'export-run':  'export, and run, results for these targets in eda-dir',
            'export-json': 'export, and save a JSON file per target',
            'work-dir':    ('Optional override for working directory, if unset will often use '
                            ' ./eda.work/[TOP|TARGET].COMMAND'),
            "work-dir-use-target-dir": ('Set the work-dir to be the same as the in-place location'
                                       ' where the target (DEPS) exists'),
            "suffix": (
                "Optional, determined by eda COMMAND, used by 'multi' jobs information logging"
            ),
            "design": (
                "Optional, used to override both the work-dir, and if unset the top DEPS target"
            ),
            'enable-tags': (
                'DEPS markup tag names to be force enabled for this'
                ' command (mulitple appends to list).'
            ),
            'disable-tags': (
                'DEPS markup tag names to be disabled (even if they match the criteria) for this'
                ' command (mulitple appends to list). --disable-tags has higher precedence than'
                ' --enable-tags.'
            ),
            'test-mode': ('command and tool dependent, usually stops the command early without'
                          ' executing.'),
            'error-unknown-args': (
                'Enable errors on unknown/unparsable args, or unknown/nonexistent files, or targets'
            ),
            'threads': (
                'Logical CPU core count passed to tools that support it. This arg is entirely'
                ' dependent on --tool, and will be divided if running an eda COMMAND that supports'
                ' --parallel=P. The default value is 1, and is internally limited to the system'
                ' max value as advertised (includes Linux/docker sched_getaffinity for containers'
                f' that are limited). Arg --use-max-threads will set to {util.get_thread_budget()}'
            ),
            'use-max-threads': f'Internally sets --threads={util.get_thread_budget()}',
            'subprocess-timeout': (
                'Overall subprocess timeout value for all subprocesses in seconds, 0 disables'
            ),
            'subprocess-progress-timeout': (
                'Progess subprocess timeout value for all subprocesses in seconds, 0 disables.'
                ' If active, will error if nothing has been written to stdout for this time,'
                ' use with tools like a simulation testbench that displays or shows progress.'
            ),
        })
        self.args_args['threads'] = ['-j'] # support -j | --threads
        self.modified_args = {}
        self.config = copy.deepcopy(config) # avoid external modifications.
        self.target = "" # is set as the 'top' or final target short-name (no path info)
        self.target_path = ""
        self.status = 0
        self.errors_log_f = None
        self.auto_tool_applied = False
        self.tool_changed_respawn = {}
        self.top_details = {}


    def get_info_job_name(self) -> str:
        '''Returns an informational string of the job name, using: command - tool - top'''
        return ' - '.join(
            x for x in (self.command_name, self.args.get('tool', ''),
                        self.args.get('top', '')) if x
        )

    def error(self, *args, **kwargs) -> None:
        '''Returns None, child classes can call self.error(..) instead of util.error,

        which updates their self.status. Also will write out to eda.errors.log if the work-dir
        exists. Please consider using Command.error(..) (or self.error(..)) in place of util.error
        so self.status is updated.
        '''

        if self.args.get('work-dir', ''):
            if not self.errors_log_f:
                try:
                    fullpath = os.path.join(self.args['work-dir'], 'eda.errors.log')
                    self.errors_log_f = open( # pylint: disable=consider-using-with
                        fullpath, 'w', encoding='utf-8'
                    )
                    util.artifacts.add(
                        name=fullpath,
                        typ='text', description='EDA reported errors'
                    )

                except Exception:
                    pass
            if self.errors_log_f:
                print(
                    f'ERROR: [eda] ({self.command_name}) {" ".join(list(args))}',
                    file=self.errors_log_f
                )
        self.status = util.error(*args, **kwargs) # error_code passed and returned via kwargs

    def stop_process_tokens_before_do_it(self) -> bool:
        '''Used by derived classes process_tokens() to know an error was reached
        and to not perform the command (avoid calling do_it())

        Also used to know if a DEPS target requested a --tool=<value> change and that
        we should respawn the job.'''
        util.debug('stop_process_tokens_before_do_it:',
                   f'{self.status=} {self.tool_changed_respawn=} {self.args.get("tool", "")=}')
        if self.tool_changed_respawn or self.status_any_error():
            return True
        if self.config.get('ssh'):
            return True
        return False

    def status_any_error(self, report: bool = True) -> bool:
        '''Used by derived classes process_tokens() to know an error was reached
        and to not perform the command. Necessary for pytests that use eda.main()

        Note we also check any parent Tool class for tool_error_count > 0
        '''
        any_err = False
        if self.status > 0:
            any_err = True
        elif isinstance(self, Tool) and getattr(self, 'tool_error_count', 0):
            util.warning(f'Command status={self.status}, but',
                         f'Tool tool_error_count={getattr(self, "tool_error_count", 0)}')
            any_err = True
        if report and any_err:
            util.error(f"Command '{self.command_name}' has previous errors")
        return any_err

    def report_pass_fail(self) -> None:
        '''Reports an INFO line with pass/fail information'''
        job_name = self.get_info_job_name()
        if not self.status_any_error():
            util.info(f'{safe_emoji("✅ ")}{job_name}: {Colors.bold}No errors observed.')

    def which_tool(self, command:str) -> str:
        '''Returns a str for the tool name used for the requested command'''
        return which_tool(command, config=self.config)

    def safe_which_tool(self, command: str = '') -> str:
        '''Returns a str for the tool name used for the requested command,

        avoids NotImplementedError (for CommandMulti)'''

        if getattr(self, '_TOOL', ''):
            return self._TOOL

        if not command:
            command = getattr(self, 'command_name', '')

        try:
            if getattr(self, 'which_tool', None):
                return self.which_tool(command)
        except NotImplementedError:
            pass

        return which_tool(command, config=self.config)


    def create_work_dir( # pylint: disable=too-many-branches,too-many-statements
            self
    ) -> str:
        '''Creates the working directory and populates self.args['work-dir']

        Generally uses ./ self.args['eda-dir'] / TARGET-NAME.COMMAND /
        however, self.args['job-name'] or ['sub-work-dir'] can override that.

        Additionally, the work-dir is attempted to be deleted if it already exists
        AND it is beneath the caller's original working directory. We also will
        not delete 'work-dir' that is ./ or $PWD.
        '''
        util.debug(f"create_work_dir: {self.args['eda-dir']=} {self.args['work-dir']=}")
        if self.args['work-dir-use-target-dir']:
            if not self.target_path:
                self.target_path = '.'
            util.info("create_work_dir: --work-dir-use-target-dir: using:",
                      f"{os.path.abspath(self.target_path)}",
                      f'target={self.target}')
            self.args['work-dir'] = self.target_path
            self.args['sub-work-dir'] = ''
            return self.args['work-dir']

        if not os.path.exists(self.args['eda-dir']):
            util.safe_mkdir(self.args['eda-dir'])
            util.info(f"create_work_dir: created {self.args['eda-dir']}")

        if self.args['design'] == "":
            if ('top' in self.args) and (self.args['top'] != ""):
                self.args['design'] = self.args['top']
                util.debug(f"create_work_dir: set {self.args['design']=} from",
                           f"{self.args['top']=}, since it was empty")
            else:
                self.args['design'] = "design" # generic, i.e. to create work dir "design_upload"
                util.debug(f"create_work_dir: set {self.args['design']=} to 'design', since it",
                           "was empty and we have no top")

        if self.target == "":
            self.target = self.args['design']
            util.debug(f"create_work_dir: set {self.target=} from design name, since it was empty")

        if self.args['work-dir'] == '':
            if self.args['sub-work-dir'] == '':
                if self.args['job-name'] != '':
                    self.args['sub-work-dir'] = self.args['job-name']
                    util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
                               f"{self.args['job-name']=}, since it was empty")
                else:
                    self.args['sub-work-dir'] = f'{self.target}.{self.command_name}'
                    if getattr(self, 'subcommand', None):
                        self.args['sub-work-dir'] += f'.{self.subcommand}'
                    util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
                               f"{self.target=} and {self.command_name=}, since it was empty and",
                               "we have no job-name")
            self.args['work-dir'] = os.path.join(self.args['eda-dir'], self.args['sub-work-dir'])
            util.debug(f"create_work_dir: set {self.args['work-dir']=}")

        keep_file = os.path.join(self.args['work-dir'], "eda.keep")
        if os.path.exists(self.args['work-dir']):
            if os.path.exists(keep_file) and not self.args['force']:
                self.error(f"Cannot remove old work dir due to '{keep_file}'")
            elif os.path.abspath(self.args['work-dir']) in os.getcwd():
                # This effectively checks if
                # --work-dir=.
                # --work-dir=$PWD
                # --work-dir=/some/path/almost/here
                # Allow it, but preserve the existing directory, we don't want to blow away
                # files up-hier from us.
                # Enables support for --work-dir=.
                util.info(f"Not removing existing work-dir: '{self.args['work-dir']}' is within",
                          f"{os.getcwd()=}")
            elif str(Path(self.args['work-dir'])).startswith(str(Path('/'))):
                # Do not allow other absolute path work dirs if it already exists.
                # This prevents you from --work-dir=~ and eda wipes out your home dir.
                self.error(f'Cannot use work-dir={self.args["work-dir"]} starting with',
                           'absolute path "/"')
            elif str(Path('..')) in str(Path(self.args['work-dir'])):
                # Do not allow other ../ work dirs if it already exists.
                self.error(f'Cannot use work-dir={self.args["work-dir"]} with up-hierarchy'
                           '../ paths')
            else:
                # If we made it this far, on a directory that exists, that appears safe
                # to delete and re-create:
                util.info(f"Removing previous '{self.args['work-dir']}'")
                try:
                    shutil.rmtree(self.args['work-dir'])
                    util.safe_mkdir(self.args['work-dir'])
                    util.debug(f'create_work_dir: created {self.args["work-dir"]}')
                except PermissionError as e:
                    self.error('Could not remove existing dir and create new due to filesystem',
                               f'PermissionError: {self.args["work-dir"]}; exception: {e}')
                except Exception as e:
                    self.error('Could not remove existing dir and create new due to internal',
                               f'Exception: {self.args["work-dir"]}; exception: {e}')
        else:
            util.safe_mkdir(self.args['work-dir'])
            util.debug(f'create_work_dir: created {self.args["work-dir"]}')

        if self.args['keep']:
            open(keep_file, 'w', encoding='utf-8').close() # pylint: disable=consider-using-with
            util.debug(f'create_work_dir: created {keep_file=}')

        # Set the util.artifacts path with our work-dir:
        util.artifacts.set_artifacts_json_dir(self.args['work-dir'])

        # Since work-dir has now safely been created (or error flagged) we can override the
        # inferred 'top' if top was not specified (aka, we guessed based on target name or
        # file/module name)
        self.update_top_if_inferred()

        return self.args['work-dir']


    def update_top_if_inferred(self) -> None:
        '''Child classes can override (CommandDesign does)

        Idea is to overwrite self.args['top'] if it was inferred, but Command.args
        does not initially set self.args['top'], so this method takes no action.
        '''
        return


    def artifacts_add(self, name: str, typ: str, description: str) -> None:
        '''Adds a file to util.artifacts, derived classes may override'''
        util.artifacts.add(name=name, typ=typ, description=description)


    # pylint: disable=unknown-option-value
    # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
    def exec(
            self, work_dir: str, command_list: list, background: bool = False,
             stop_on_error: bool = True, quiet: bool = False, tee_fpath: str = '',
             shell: bool = False, ignore_error: bool = False,
             timeout: int = 0, timeout_str: str = '',
             progress_timeout: int = 0, progress_timeout_str: str = ''
    ) -> (str, str, int):
        '''Runs a command via subprocess and returns a tuple of (stderr, stdout, rc)

        - work_dir: is passed to the subprocess call, so os.chdir(..) is not invoked.
        - command_list: should be a list of individual strings w/out spaces.
        - background: if False, does not print to stdout (but will print to any
                      tee_fpath or the global log
        - shell: arg passed to subprocess, defaults to False.
        '''

        if not tee_fpath and getattr(command_list, 'tee_fpath', None):
            tee_fpath = getattr(command_list, 'tee_fpath', '')
        if not quiet:
            util.info(f"{safe_emoji('⏩ ')}exec: {' '.join(command_list)}",
                      f"(in {work_dir}, {tee_fpath=})")

        if timeout == 0 or (self.args.get('subprocess-timeout', 0) > 0 and \
                            self.args.get('subprocess-timeout', 0) > timeout):
            # If timeout is default (0) then pick up from --subprocess-timeout
            # If timeout is set, then pickup from --subprocess-timeout if larger value
            #  (give user flexibility to set longer timeouts if caller hardcoded)
            timeout = self.args.get('subprocess-timeout', 0)
            timeout_str = f'from --subprocess-timeout={self.args.get("subprocess-timeout", 0)}'

        if progress_timeout == 0 or (self.args.get('subprocess-timeout', 0) > 0 and \
                                     self.args.get('subprocess-timeout', 0) > progress_timeout):
            progress_timeout = self.args.get('subprocess-progress-timeout', 0)
            progress_timeout_str = (
                'from --subprocess-progress-timeout='
                f'{self.args.get("subprocess-progress-timeout", 0)}'
            )

        stdout, stderr, return_code = subprocess_run_background(
            work_dir=work_dir,
            command_list=command_list,
            background=background,
            fake=self.args.get('fake', False),
            tee_fpath=tee_fpath,
            shell=shell,
            timeout=timeout,
            timeout_str=timeout_str,
            progress_timeout=progress_timeout,
            progress_timeout_str=progress_timeout_str
        )

        if not ignore_error and return_code > 0:
            if return_code == 1:
                self.status = status_constants.EDA_EXEC_NONZERO_RETURN_CODE1
            elif return_code == 255:
                self.status = status_constants.EDA_EXEC_NONZERO_RETURN_CODE255
            else:
                self.status = status_constants.EDA_EXEC_NONZERO_RETURN_CODE2

            if stop_on_error:
                self.error(f"exec: returned with error (return code: {return_code})",
                           error_code=self.status)
            elif not quiet:
                util.info(f"{safe_emoji('❌ ')}exec: returned with error (return code:",
                          f"{return_code})")
        else:
            util.debug(f"exec: returned without error (return code: {return_code})")
        return stderr, stdout, return_code

    def set_arg( # pylint: disable=too-many-branches
            self, key, value
    ) -> None:
        '''Sets self.args[key] with value, and self.modified_args[key]=True

        Does type checking, handles list type append
        '''

        # Do some minimal type handling, preserving the type(self.args[key])
        if key not in self.args:
            self.error_ifarg(
                f'set_arg, {key=} not in self.args {value=}',
                f'({self.command_name=}, {self.__class__.__name__=})',
                arg='error-unknown-args',
                error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR
            )

        cur_value = self.args[key]

        if isinstance(cur_value, dict):
            # if dict, update
            self.args[key].update(value)

        elif isinstance(cur_value, list):
            # if list, append (allow duplicates)
            if isinstance(value, list):
                # new value also a list
                for x in value:
                    self.args[key].append(x)
            elif value not in cur_value:
                self.args[key].append(value)

        elif isinstance(cur_value, bool):
            # if bool, then attempt to convert new value string or int --> bool
            if isinstance(value, (bool, int)):
                self.args[key] = bool(value)
            elif isinstance(value, str):
                if value.lower() in ['false', '0']:
                    self.args[key] = False
                else:
                    self.args[key] = True
            else:
                self.error(f'set_arg({key=}, {value=}) bool {type(cur_value)=} {type(value)=}')

        elif isinstance(cur_value, int):
            # if int, attempt to convert string or bool --> int
            if isinstance(value, str):
                try:
                    self.args[key] = int(value, 0)
                except ValueError:
                    self.error(f'set_arg({key=}, {value=}) int {type(cur_value)=} {type(value)=}')
            elif isinstance(value, (bool, int)):
                self.args[key] = int(value)
            else:
                self.error(f'set_arg({key=}, {value=}) int {type(cur_value)=} {type(value)=}')


        else:
            # else overwrite it as-is.
            self.args[key] = value

        self.modified_args[key] = True
        util.debug(f'Set arg["{key}"]="{self.args[key]}"')


    def get_argparser( # pylint: disable=too-many-branches
            self, parser_arg_list=None, support_underscores: bool = True,
    ) -> argparse.ArgumentParser:
        ''' Returns an argparse.ArgumentParser() based on self.args (dict)

        If parser_arg_list is not None, the ArgumentParser() is created using only the keys in
        self.args provided by the list parser_arg_list.

        If support_underscores=False, then only return an ArgumentParser() with --arg-posix-dashes
        '''

        # Preference is --args-with-dashes, which then become parsed.args_with_dashes, b/c
        # parsed.args-with-dashes is not legal python. Some of self.args.keys() still have - or _,
        # so this will handle both.
        # Also, preference is for self.args.keys(), to be str with - dashes
        parser = argparse.ArgumentParser(
            prog=f'{safe_emoji("🔎 ")}eda', add_help=False, allow_abbrev=False
        )
        bool_action_kwargs = util.get_argparse_bool_action_kwargs()

        if not parser_arg_list:
            args = self.args
        else:
            args = {key: self.args[key] for key in parser_arg_list}

        for key,value in args.items():
            if '_' in key and '-' in key:
                assert False, f'{args=} has {key=} with both _ and -, which is not allowed'
            if '_' in key:
                util.warning(f'{key=} has _ chars, prefer -')

            keys = [key] # make a list
            if support_underscores and '_' in key:
                keys.append(key.replace('_', '-')) # switch to POSIX dashes for argparse
            elif support_underscores and '-' in key:
                keys.append(key.replace('-', '_')) # also support --some_arg_with_underscores

            arguments = [] # list supplied to parser.add_argument(..) so one liner supports both.
            for this_key in keys:
                arguments.append(f'--{this_key}')
            for arg in self.args_args.get(this_key, []):
                if arg not in arguments:
                    if arg.startswith('-'):
                        arguments.insert(0, arg)
                    else:
                        arguments.append(f'--{arg}')

            if self.args_help.get(key, ''):
                help_kwargs = {'help': f'{self.args_help[key]} (default={value})'}
            elif value is None:
                help_kwargs = {'help': f'default={value}'}
            else:
                help_kwargs = {'help': f'{type(value).__name__} default={value}'}
            help_kwargs['help'] = help_kwargs['help'].replace('%', '%%')

            # Update with any self.args_kwargs for this key
            if self.args_kwargs.get(key, {}):
                help_kwargs.update(self.args_kwargs.get(key, {}))


            # It's important to set the default=None on these, except for list types where default
            # is []. If the parsed Namespace has values set to None or [], we do not update. This
            # means that as deps are processed that have args set, they cannot override the top
            # level args that were already set, nor be overriden by defaults.
            try:
                if isinstance(value, bool):
                    # For bool, support --key and --no-key with
                    # action=argparse.BooleanOptionalAction. Note, this means you cannot use
                    # --some-bool=True, or --some-bool=False, has to be --some-bool or
                    # --no-some-bool.
                    # Also we cannot have self.args.keys() that start with 'no-', which is
                    # why this entire thing is wrapped with try/except.
                    parser.add_argument(
                        *arguments, default=None, **bool_action_kwargs, **help_kwargs)
                elif isinstance(value, (list, set)):
                    parser.add_argument(*arguments, default=value, action='append', **help_kwargs)
                elif isinstance(value, (int, str)):
                    parser.add_argument(*arguments, default=value, type=type(value), **help_kwargs)
                elif value is None:
                    parser.add_argument(*arguments, default=None, **help_kwargs)
                else:
                    assert False, f'{key=} {value=} how do we do argparse for this type of value?'
            except Exception as e:
                if isinstance(value, bool):
                    self.error(f'Could not add argument: {key=} {value=} {type(value)=}',
                               f'{arguments=} {bool_action_kwargs=} {help_kwargs=} {e=}')
                else:
                    self.error(f'Could not add argument: {key=} {value=} {type(value)=}',
                               f'{arguments=} {help_kwargs=} {e=}')



        return parser


    def run_argparser_on_list(
            self, tokens: list, parser_arg_list=None, apply_parsed_args: bool = True
    ) -> (argparse.Namespace, list):
        ''' Creates an argparse.ArgumentParser() for all the keys in self.args, and attempts to
        parse from the provided list. Parsed args are applied to self.args.

        Returns a tuple of (parsed argparse.Namespace obj, list of unparsed args)

        If parser_arg_list is not None, the ArgumentParser() is created using only the keys in
        self.args provided by the list parser_arg_list.
        If apply_parsed_args=False, the parsed args were not applied.
        '''

        parser = self.get_argparser(parser_arg_list=parser_arg_list)
        try:
            parsed, unparsed = parser.parse_known_args(tokens + [''])
            unparsed = list(filter(None, unparsed))
        except argparse.ArgumentError:
            self.error(f'problem {self.command_name=} attempting to parse_known_args for {tokens=}',
                       error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR)

        parsed_as_dict = vars(parsed)

        args_to_be_applied = {}

        for key, value in parsed_as_dict.items():
            # key currently has _ instead of POSIX dashes, so we convert to dashes, for
            # most self.args, like self.args['build-file'], etc.
            if key not in self.args and '_' in key:
                key = key.replace('_', '-')
            assert key in self.args, f'{key=} not in {self.args=}'

            args_to_be_applied[key] = value

        if apply_parsed_args:
            self.apply_args_from_dict(args_to_be_applied)

        return parsed, unparsed


    def apply_args_from_dict(self, args_to_be_applied: dict) -> list:
        '''Given a dict of key/value for self.args, apply them to self.args'''
        util.debug('apply_args_from_dict() -- called by:',
                   f'{self.command_name=}, {self.__class__.__name__=},',
                   f'{args_to_be_applied=}')
        for key, value in args_to_be_applied.items():

            if value is None:
                continue # don't update a self.args[key] to None
            if isinstance(value, list) and len(value) == 0:
                continue # don't update a self.args[key] that's a list --> to an empty list.
            if not isinstance(value, list) and self.args[key] == value:
                continue # don't update non list if the value already matches.
            if not isinstance(value, list) and self.modified_args.get(key, None):
                # For list types, we append. For all others they overwrite, so if we've already
                # modified the arg once, do not modify it again. Such as, command line set an arg,
                # but then a target tried to set it again; or a target set it, and then a dependent
                # target tried to set it again.
                util.warning(f"Command.run_argparser_on_list ({self.command_name}) -",
                             f"skipping {key=} {value=} b/c arg is already modified",
                             f"with different cur value (cur value={self.args.get(key, None)})")
                continue
            if self.args[key] != value:
                util.debug("Command.run_argparser_on_list - setting set_arg b/c",
                           f" argparse -- {key=} {value=} (cur value={self.args[key]})")
                self.set_arg(key, value) # Note this has special handling for lists already.
                self.modified_args[key] = True


    def process_tokens(self, tokens: list, process_all: bool = True,
                       pwd: str = os.getcwd()) -> list:
        '''Command.process_tokens(..) for all named self.args.keys() returns the
        unparsed tokens list

        Derived classes do not need to return a list of unparsed args, nor
        return self.status
        '''

        _, unparsed = self.run_argparser_on_list(tokens)

        self._apply_deps_file_defaults_if_present()

        if process_all and unparsed:
            self.warning_show_known_args()
            self.error_ifarg(
                f"Didn't understand argument: '{unparsed=}' in",
                f"{self.command_name=} context, {pwd=}",
                arg='error-unknown-args',
                error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR
            )

        if self.args.get('use-max-threads', False):
            self.args['threads'] = util.get_thread_budget()

        return unparsed

    def get_sub_command_from_config(
            self, error_if_no_command: bool = True,
    ) -> str:
        '''If this is a command with a sub-command (multi, sweep), then opencos.eda would

        have already parsed in an place it in config['sub_command']. Get this value and return
        it.
        '''
        ret = self.config.get('sub_command', '')

        if not ret and error_if_no_command:
            util.debug(f'get_sub_command_from_config: see: {self.config=}')
            self.error(f"Looking for a valid eda {self.command_name} COMMAND",
                       "but didn't find one in config",
                       error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR)
        return ret



    def set_tool_config_from_config(self) -> None:
        '''Returns None. Hook for classes like CommandSim that have a Tool derived class attached.

        Have the Tool constructor run set_tool_config_from_config() if it exists, otherwise every
        derived class will have to call this (error-prone if the omit it but needed it).

        Allows command handlers to make thier own customizations from --config-yaml=YAML.
        '''
        return

    def update_tool_config(self) -> None:
        '''Returns None. Hook for classes like CommandSim to make tool specific overrides.'''
        return

    def write_eda_config_and_args(self):
        '''Attempts to write eda_output_config.yml to our work-dir'''
        if not self.args.get('work-dir', None):
            util.warning(f'Ouput work-dir not set, saving ouput eda_config to {os.getcwd()}')
        eda_config.write_eda_config_and_args(
            dirpath=self.args.get('work-dir', os.getcwd()), command_obj_ref=self
        )

    def is_export_enabled(self) -> bool:
        '''Returns True if any self.args['export'] is set in any way (but not set to False
        or empty list)'''
        return any(arg.startswith('export') and v for arg,v in self.args.items())

    def run(self) -> None:
        '''Alias for do_it(self)'''
        self.do_it()

    def do_it(self) -> None:
        '''Main entrypoint of running a command, usually called by process_tokens.

        process_tokens(..) is the starting entrypoint from eda.py'''
        self.write_eda_config_and_args()
        self.error(f"No tool bound to command '{self.command_name}', you",
                   " probably need to setup tool, or use '--tool <name>'",
                   error_code=status_constants.EDA_CONFIG_ERROR)

    def do_export(self, **kwargs) -> None:
        '''Derived classes supporting --export[--run|...] may wish to have this

        as a minimal viable support for --export. Note a derived can do whatever it
        likes and does not have to use **kwargs, these are more for simple uses of:
            super().do_export(deps_file_args=['--compile-args', ...], support_export_run=True)


        kwargs:
          -- deps_file_args -- list of allow-listed args (if arg startswith) that will
                               be included in the exported DEPS.yml file target args.
          -- support_export_run -- (bool), if true, have a bare minimum --export-run
                                   supported.
        '''
        out_dir = os.path.join(self.args['work-dir'], 'export')
        target = self.target
        if not target:
            target = 'test'

        export_obj = export_helper.ExportHelper(
            cmd_design_obj=self,
            eda_command=self.command_name,
            out_dir=out_dir,
            # Note this may not be the correct target for debug infomation,
            # so we'll only have the first one.
            target=target
        )

        # Set things in the exported: DEPS.yml
        tool = self.args.get('tool', None)
        # Certain args are allow-listed here
        deps_file_args = []
        for a in self.get_command_line_args():
            if any(a.startswith(x) for x in kwargs.get('deps_file_args', [])):
                deps_file_args.append(a)

        export_obj.run(
            deps_file_args=deps_file_args,
            export_json_eda_config={
                'tool': tool,
            }
        )

        if self.args['export-run']:
            if not kwargs.get('support_export_run', False):
                util.warning(
                    f'--export-run is not supported by eda command: {self.command_name}'
                )
                return

            # remove the '--export' named args, we don't want those.
            args_no_export = self.get_command_line_args(remove_args_startswith=['export'])
            command_list = ['eda', self.command_name] + args_no_export + [target]
            util.info(f'export-run: from {export_obj.out_dir=}: {command_list=}')
            self.exec(
                work_dir=export_obj.out_dir,
                command_list=command_list,
            )



    def command_safe_set_tool_defines(self) -> None:
        '''Safe wrapper for calling self.set_tool_defines() in case a Tool parent class is

        not yet present. If you have Tool parent, you can instead simply call:
           self.set_tool_defines()
        '''
        set_tool_defines = getattr(self, 'set_tool_defines', None)
        if set_tool_defines and callable(set_tool_defines):
            self.set_tool_defines()


    def help( # pylint: disable=dangerous-default-value,too-many-branches
            self, tokens: list = [], no_targets: bool = False,
            show_all_eda_commands: bool = False
    ) -> None:
        '''Since we don't quite follow standard argparger help()/usage(), we'll format our own

        if self.args_help has additional help information.
        '''

        # Indent long lines (>100) to indent=24 (this is where usual argparse help leaves off)
        def indent_me(text: str, initial_indent: int = 23):
            return indent_wrap_long_text(
                ' ' * initial_indent + text, width=get_terminal_columns(), indent=24
            )

        util.info('Help:', color=Colors.cyan)
        # using bare 'print' here, since help was requested, avoids --color and --quiet
        print()
        print_eda_usage_line(no_targets=no_targets, command_name=self.command_name)

        if show_all_eda_commands:
            print(get_all_commands_help_str(config=self.config))
            print()
            print(indent_wrap_long_text((
                f'{safe_emoji("❕ ")}where {Colors.byellow}FILES|TARGETS{Colors.off}'
                ' is one or more source file or DEPS markup file target,'
                ' such as .v, .sv, .vhd[l], .cpp files, or a target key in a'
                f' {Colors.byellow}DEPS.[yml|yaml|toml|json]{Colors.off}. Note that you can'
                f' prefix source files with {Colors.bcyan}sv@{Colors.off},'
                f' {Colors.bcyan}v@{Colors.off}, {Colors.bcyan}vhdl@{Colors.off},'
                f' or {Colors.bcyan}cpp@{Colors.off}'
                ' to force use that file as systemverilog, verilog, vhdl, or C++, respectively.'
            ), width=get_terminal_columns(), indent=4))
            print()
            print(indent_wrap_long_text((
                f'{safe_emoji("🌐 ")}Remote execution via {Colors.bcyan}--ssh{Colors.off}:'
                f' Add {Colors.bcyan}--ssh=user@host{Colors.off}'
                f' and {Colors.bcyan}--tool=TOOLNAME{Colors.off}'
                ' to run any command on a remote host. Local tools not necessary.'
                ' Assumes POSIX (unix/Linux) remote host.'
                ' Files are staged to an ephemeral sandbox'
                f' ({Colors.yellow}~/eda.work/<uuid>.ssh/{Colors.off}),'
                ' run there, and artifacts fetched back.'
                f' Works with eda multi. Use {Colors.bcyan}--ssh-setup{Colors.off}'
                ' to source tool environments on the remote.'
                f' Example: {Colors.bgreen}eda sim --tool=vivado --ssh=user@hw1'
                f' --ssh-setup="source /opt/Xilinx/Vivado/2024.2/settings64.sh" mytarget'
                f'{Colors.off}'
            ), width=get_terminal_columns(), indent=4))


        print()
        print_base_help()
        lines = []
        if not self.args and tokens:
            print(f'  Unparsed args: {Colors.byellow}{" ".join(tokens)}')
            return

        if self.command_name:
            line = (
                f"{safe_emoji('🔧 ')}{Colors.cyan}Generic help for"
                f" command={Colors.byellow}{self.command_name}{Colors.cyan}"
            )
            if tool := get_class_tool_name(self):
                line += f" tool={Colors.byellow}{tool}"
            line += f" {Colors.off}(using '{self.__class__.__name__}')"
            lines.append(line)
        else:
            lines.append(
                f"{safe_emoji('🔧 ')}{Colors.cyan}Generic help:{Colors.off}"
            )

        # Attempt to run argparser on args, but don't error if it fails.
        unparsed = []
        if tokens:
            try:
                _, unparsed = self.run_argparser_on_list(tokens=tokens)
            except Exception:
                pass

        short_help_lines = util.get_argparser_short_help(
            parser=self.get_argparser(support_underscores=False)
        ).split('\n')
        short_help_lines.pop(0) # Strip first line w/ argparser prog name
        lines.extend(short_help_lines)

        # Include -G, +incdir+, +define+ help if this is a CommanDesign class:
        lines.append('')
        if isinstance(self, CommandDesign):
            lines.append(
                f"  {Colors.cyan}-G{Colors.byellow}<parameterName>{Colors.off}=" \
                + f"{Colors.yellow}<value>{Colors.off}"
            )
            lines.append(indent_me((
                " Add parameter to top level, support bit/int/string types only."
                " Example: -GDEPTH=8 (DEPTH treated as SV int/integer)."
                " -GENABLE=1 (ENABLED treated as SV bit/int/integer)."
                " -GName=eda (Name treated as SV string \"eda\")."
            )))

            lines.append(f"  {Colors.cyan}+define+{Colors.byellow}<defineName>{Colors.off}")
            lines.append(indent_me((
                " Add define w/out value to tool ahead of SV sources"
                " Example: +define+SIM_SPEEDUP"
            )))
            lines.append(
                f"  {Colors.cyan}+define+{Colors.byellow}<defineName>{Colors.off}=" \
                + f"{Colors.yellow}<value>{Colors.off}")
            lines.append(indent_me((
                " Add define w/ value to tool ahead of SV sources"
                " Example: +define+TECH_LIB=2 +define+FULL_NAME=\"E D A\""
            )))
            lines.append(f"  {Colors.cyan}+incdir+{Colors.byellow}PATH{Colors.off}")
            lines.append(indent_me((
                " Add path (absolute or relative) for include directories"
                " for SystemVerilog `include \"<some-file>.svh\""
                " Example: +incdir+../lib +incdir+$MY_REPO_ROOT/lib"
            )))
            lines.append('')

        for line in lines:
            print(line)

        if unparsed:
            print(f'  Unparsed args: {Colors.byellow}{" ".join(unparsed)}')

    def get_argparsers_args_list(self) -> list:
        '''Returns list of all args that we know about from eda_config, util, eda, and our self.args

        All items will include the -- prefix (--help, etc)'''
        return util.get_argparsers_args_list(parsers=[
            eda_config.get_argparser(),
            util.get_argparser(),
            get_argparser(),
            self.get_argparser(support_underscores=False)
        ])

    def pretty_str_known_args(self, command: str = '') -> str:
        '''Returns multiple line column organized string of all known args'''
        _command = command
        if not _command:
            _command = self.command_name

        _args_list = self.get_argparsers_args_list()
        _pretty_args_list = pretty_list_columns_manual(data=_args_list)
        return (f"Known args for command '{_command}' :\n"
                "  " + "\n  ".join(_pretty_args_list)
                )

    def warning_show_known_args(self, command: str = '') -> None:
        '''Print a helpful warning showing available args for this eda command (or commands)'''

        if not command:
            commands = [self.command_name]
        else:
            commands = command.split() # support for command="multi sim"

        _tool = self.safe_which_tool(commands[0]) # use first command if > 1 presented
        lines = []
        if _tool:
            lines.append(f"To see all args for command(s) {commands}, tool '{_tool}', run:")
        else:
            lines.append(f"To see all args for command(s) {commands}, run:")

        for cmd in commands:
            if _tool:
                lines.append(f"  eda {cmd} --tool={_tool} --help")
            else:
                lines.append(f"  eda {cmd} --help")

        lines.append(self.pretty_str_known_args(command=commands[-1])) # use last command if > 1
        util.warning("\n".join(lines))

    def error_ifarg(
            self, *msg, arg: str, error_code: int = status_constants.EDA_COMMAND_OR_ARGS_ERROR
    ) -> None:
        '''For errors involving an unknown --arg, they can be optionally disabled

        using CLI: --no-error-unknown-args, and this method arg='error-uknown-arg'

        Note if arg is not present in self.args, the error is enabled.
        '''
        if self.args.get(arg, True):
            self.error(*msg, error_code=error_code)
        else:
            util.warning(*msg)

    @staticmethod
    def get_top_name(name: str) -> str:
        '''Attempt to get the 'top' module name from a file, such as path/to/mine.sv will

        return "mine"'''
        return os.path.splitext(os.path.basename(name))[0]


    def incr_tool_error(self, count: int = 1) -> None:
        '''Increments tool_error_count (if exists) and util.artifacts'''
        tool = getattr(self, '_TOOL', '')
        if not tool:
            return
        setattr(self, 'tool_error_count', getattr(self, 'tool_error_count', 0) + count)
        util.artifacts.incr_tool_error(tool, count=count)


    def incr_tool_warning(self, count: int = 1) -> None:
        '''Increments tool_warning_count (if exists) and util.artifacts'''
        tool = getattr(self, '_TOOL', '')
        if not tool:
            return
        setattr(self, 'tool_warning_count', getattr(self, 'tool_warning_count', 0) + count)
        util.artifacts.incr_tool_warning(tool, count=count)


    def update_tool_warn_err_counts_from_log_lines(
            self, log_lines: list, bad_strings: list, warning_strings: list
    ) -> None:
        '''Given lines (list of str) from a log, update self.tool_[error|warning]_count values

        Since Tool class is not always a parent in Command/CommandDesign/CommandSim, we
        have to modfiy these member names safely for pylint.

        Derived classes may override this, especially if the Tool is known and reports
        the complete error/warning summary counts in a certain way.
        '''
        if not isinstance(self, Tool):
            return
        _tool = getattr(self, '_TOOL', 'unknown')
        for line in log_lines:
            if any(bad_str in line for bad_str in bad_strings):
                self.incr_tool_error(count=1)
            if any(warn_str in line for warn_str in warning_strings):
                self.incr_tool_warning(count=1)


    def get_command_line_args( # pylint: disable=dangerous-default-value
            self, remove_args: list = [], remove_args_startswith: list = []
    ) -> list:
        '''Returns a list of all the args if you wanted to re-run this command
        (excludes eda, command, target).'''

        # This will not set bool's that are False, does not add --no-<somearg>
        # nor --<somearg>=False (it's unclear which are 'store_true' v/ bool-action-kwargs)
        # This will not set str's that are empty
        # this will not set ints that are 0 (unless modified)
        # TODO(drew): we may want to revisit this if we tracked default arg values,
        # or perhaps if they have been "modified".
        ret = []
        for k,v in self.args.items():

            # Some args cannot be extracted and work, so omit these:
            if k in remove_args:
                continue
            if any(k.startswith(x) for x in remove_args_startswith):
                continue

            is_modified = self.modified_args.get(k, False)

            if isinstance(v, bool) and v:
                ret.append(f'--{k}')
            elif isinstance(v, int) and (bool(v) or is_modified):
                ret.append(f'--{k}={v}')
            elif isinstance(v, str) and v:
                ret.append(f'--{k}={v}')
            elif isinstance(v, list):
                for item in v:
                    if item or isinstance(item, (bool, str)):
                        # don't print bool/str that are blank.
                        ret.append(f'--{k}={item}') # lists append

        return ret


    def _apply_deps_file_defaults_if_present(self) -> None:
        '''Only runs if this is not a CommandDesign class.

        Looks at DEPS file in current directory, and will attempt to apply 'DEFAULTS' if present
        '''

        if isinstance(self, CommandDesign):
            return

        # Try to fish out DEFAULTS from ./DEPS if present, so we can apply args to self.
        _cache_none = {}
        data = None
        if self.config['deps_markup_supported']:
            deps = DepsFile(
                command_design_ref=self, target_path=str(Path('.')), cache=_cache_none
            )
            data = deps.data

        entry = None
        if data is not None and 'DEFAULTS' in data:
            entry = deps.get_entry(target_node=str(Path('./DEFAULTS')))
        if not entry:
            return

        target = str(Path('./DEFAULTS'))
        _, target_node = os.path.split(target)
        caller_info = deps.gen_caller_info(target)

        deps_processor = DepsProcessor(
            command_design_ref=self,
            deps_entry=entry,
            target=target,
            target_path=str(Path('.')),
            target_node=target_node,
            deps_file = deps.deps_file,
            caller_info = caller_info
        )
        _ = deps_processor.process_deps_entry()




class CommandDesign(Command): # pylint: disable=too-many-instance-attributes
    '''CommandDesign is the eda base class for command handlers that need to track files.

    This is the base class for CommandSim, CommandSynth, and others.'''

    # Used by for DEPS work_dir_add_srcs@ commands, by class methods:
    #   update_file_lists_for_work_dir(..), and resolve_target(..)
    _work_dir_add_srcs_path_string = '@EDA-WORK_DIR@'

    # Optionally error in self.process_tokens, derived classes can override.
    error_on_no_files_or_targets = False
    error_on_missing_top = False

    def __init__(self, config: dict):
        Command.__init__(self, config=config)
        self.args.update({
            'seed': seed.get_seed(style="urandom"),
            'top': '',
            'all-sv': True,
            'unprocessed-plusargs': [],
        })
        self.args_help.update({
            'seed':   'design seed, default is 31-bit non-zero urandom',
            'top':    'TOP level verilog/SV module or VHDL entity for this target',
            'all-sv': (
                'Maintain .sv and .v in single file list.'
                ' False: .sv flist separate from .v flist and separate compile(s)'
                ' True: .sv and .v files compiled together if possible'
            ),
            'unprocessed-plusargs': (
                'Args that began with +, but were not +define+ or +incdir+, +<name>, '
                ' or +<name>=<value>. These become tool dependent, for example "sim" commands will'
                ' treat as sim-plusargs'
            ),
        })
        self.defines = {}
        self.parameters = {}
        self.incdirs = []
        self.files = {}
        self.files_v = []
        self.files_sv = []
        self.files_vhd = []
        self.files_cpp = []
        self.files_sdc = []
        self.files_py = []
        self.files_makefile = []
        self.files_non_source = []
        self.files_caller_info = {}
        self.dep_shell_commands = [] # each list entry is a {}
        self.dep_work_dir_add_srcs = set() # key: tuple (target_path, target_node, filename)
        self.oc_root = util.get_oc_root()
        for (d,v) in self.config.get('defines', {}).items():
            self.defines[d] = v

        # cached_deps: key = abspath of DEPS markup file, value is a dict with
        # keys 'data' and 'line_numbers'
        self.cached_deps = {}

        # targets_dict: key = str(Path(target)), value is DepsFile caller_info, or path
        self.targets_dict = {}

        self.has_pre_compile_dep_shell_commands = False
        self.has_post_tool_dep_shell_commands = False

        self.top_details = {
            # used by CommandDesign to track last file added.
            'last_added_source_file':               '',
            # implies top was not set, we inferred from the final command line
            # DEPS target name
            'inferred_from_target_name':      False,
            # implies top was not set, we inferred from the final source file added,
            # based on a 'module' name we parsed.
            # also implies self.target was set from inferred top:
            'inferred_from_last_added_source_file': False,
        }


    def run_dep_commands(self) -> None:
        '''Run shell/peakrdl style commands from DEPS files, this is peformed before

        any tool compile step. These are deferred to maintain the deps ordering, and
        run in that order. Note this will NOT run any DEPS command marked with
        run-after-tool=True.
        '''
        self.run_dep_shell_commands()
        # Update any work_dir_add_srcs@ in our self.files, self.files_v, etc, b/c
        # self.args['work-dir'] now exists.
        self.update_file_lists_for_work_dir()
        # Link any non-sources to our work-dir:
        self.update_non_source_files_in_work_dir()


    def run_post_tool_dep_commands(self) -> None:
        '''Run shell style commands from DEPS files that have been marked with

        run-after-tool=True. Note these are skipped if any args like
        stop-before- or stop-after- are set.
        '''

        self.run_dep_shell_commands(filter_run_after_tool=True)


    def run_dep_shell_commands( # pylint: disable=too-many-branches,too-many-locals
            self, filter_run_after_tool: bool = False
    ) -> None:
        '''Runs collected shell command from DEPS files.

        There are two flavors of shell commands: with or without 'run-after-tool'
        set. The default is to run shell command before the compile step of any tool,
        by calling this method with default pre_compile=True before any tool runs
        (for generating code, etc). However, it may be useful to run shell commands
        after a tool is complete (check timing, coverage, etc).
        '''

        # Runs from self.args['work-dir']
        all_cmds_lists = []

        log_fnames_count = {} # count per target_node.

        filtered_dep_shell_commands = []
        for value in self.dep_shell_commands:
            if value['attributes']['run-after-tool'] == filter_run_after_tool:
                filtered_dep_shell_commands.append(value)


        for i, d in enumerate(filtered_dep_shell_commands):
            clist = util.ShellCommandList(d['exec_list'])
            log = clist.tee_fpath
            target_node = d["target_node"]
            if clist.tee_fpath is None:
                lognum = log_fnames_count.get(target_node, 0)
                log = f'{target_node}__shell_{lognum}.log' # auto log every shell command.
                clist.tee_fpath = log
                # In case some single target has N shell commands, give them unique log names.
                log_fnames_count.update({target_node: lognum + 1})
            all_cmds_lists += [
                [], # blank line
                # comment, where it came from, log to {node}__shell_{lognum}.log
                # (or tee name from DEPS.yml)
                [f'# command {i}: target: {d["target_path"]} : {target_node} --> {log}'],
            ]
            if not d['attributes']['run-from-work-dir']:
                all_cmds_lists.append([f'cd {d["target_path"]}'])

            # actual command (list or util.ShellCommandList)
            all_cmds_lists.append(clist)

            if not d['attributes']['run-from-work-dir']:
                all_cmds_lists.append([f'cd {os.path.abspath(self.args["work-dir"])}'])

            d['exec_list'] = clist # update to tee_fpath is set.

        if all_cmds_lists:
            if filter_run_after_tool:
                filename='post_tool_dep_shell_commands.sh'
                self.has_post_tool_dep_shell_commands = True
            else:
                filename='pre_compile_dep_shell_commands.sh'
                self.has_pre_compile_dep_shell_commands = True

            util.write_shell_command_file(
                dirpath=self.args['work-dir'], filename=filename,
                command_lists=all_cmds_lists
            )


        if all_cmds_lists and filter_run_after_tool and \
           any(self.args.get(x, False) for x in (
                "stop-before-compile",
                "stop-after-compile",
                "stop-after-elaborate"
                )):
            args_set = [key for key,value in self.args.items() if \
                        key.startswith('stop-') and value]
            util.info(f'Skipping DEPS run-after-tool commands due to args {args_set}')
            util.debug(f'Skipped commands: {filtered_dep_shell_commands=}')
            return

        for i, d in enumerate(filtered_dep_shell_commands):
            util.info(f'run_dep_shell_commands {i=}: {d=}')
            clist = util.ShellCommandList(d['exec_list'])
            tee_fpath=clist.tee_fpath
            if d['attributes']['run-from-work-dir']:
                run_from_dir = self.args['work-dir']
            else:
                # Run from the target's directory (not the `eda` caller $PWD)
                run_from_dir = d["target_path"]
                tee_fpath = os.path.abspath(os.path.join(self.args['work-dir'], tee_fpath))
            # NOTE(drew): shell=True subprocess call, can disable with self.config
            if sys.platform.startswith('win'):
                # for Windows, we run shell=True otherwise most built-in cmd.exe calls won't work.
                self.exec(run_from_dir, clist, tee_fpath=tee_fpath, shell=True)
            else:
                self.exec(run_from_dir, clist, tee_fpath=tee_fpath,
                          shell=self.config.get('deps_subprocess_shell', False))


    def update_file_lists_for_work_dir(self) -> None:
        '''Handles any source files that were creating by "shell" style commands in the

        work-dir, these need to be added via self.add_file(..) in the correct order
        of the dependencies. Basically, any files that were left with @EDA-WORK-DIR@ prefix
        need to be patched and added.
        '''
        if not self.dep_work_dir_add_srcs:
            return

        # If we encounter any @EDA-WORK_DIR@some_file.v in self.files, self.files_v, etc,
        # then replace it with: self.args['work-dir'] / some_file.v:
        _work_dir_add_srcs_path_string_len = len(self._work_dir_add_srcs_path_string)
        work_dir_abspath = os.path.abspath(self.args['work-dir'])
        for key in list(self.files.keys()): # list so it's not an iterator, updates self.files.
            if isinstance(key, str) and key.startswith(self._work_dir_add_srcs_path_string):
                new_key = os.path.join(work_dir_abspath, key[_work_dir_add_srcs_path_string_len :])
                self.files.pop(key)
                self.files[new_key] = True

        my_file_lists_list = [self.files_v, self.files_sv, self.files_vhd, self.files_cpp,
                              self.files_sdc, self.files_py, self.files_makefile]
        for my_file_list in my_file_lists_list:
            for i,value in enumerate(my_file_list):
                if value and isinstance(value, str) and \
                   value.startswith(self._work_dir_add_srcs_path_string):
                    new_value = os.path.join(
                        work_dir_abspath, value[_work_dir_add_srcs_path_string_len :]
                    )
                    my_file_list[i] = new_value
                    util.debug(f"file lists: replaced {value} with {new_value}")


    def update_non_source_files_in_work_dir(self) -> None:
        '''For non-source files (that are tracked as 'reqs' in DEPS markup files) these

        need to be copied or linked to the work-dir. For example, if some SV assumes it
        can $readmemh('file_that_is_here.txt') but we're running out of work-dir. Linking
        is the easy work-around vs trying to run-in-place of all SV files.

        Note that we also include .py and Makefile(s) in this.
        '''

        for fname in self.files_non_source + self.files_py + self.files_makefile:
            _, leaf_fname = os.path.split(fname)
            destfile = os.path.join(self.args['work-dir'], leaf_fname)
            relfname = os.path.relpath(fname)
            caller_info = self.files_caller_info[fname]
            if not os.path.exists(fname):
                util.info(f'{fname=} {self.files_caller_info=}')
                self.error(f'Non-source file (reqs?) {relfname=} does not exist from {caller_info}')
            elif not os.path.exists(destfile):
                util.debug(f'updating non-source file to work-dir: Linked {fname=} to {destfile=},',
                           f'from {caller_info}')
                if sys.platform == "win32":
                    shutil.copyfile(fname, destfile) # On Windows, fall back to copying
                else:
                    os.symlink(src=fname, dst=destfile)


    def update_top_if_inferred(self) -> None:
        '''Overridden from Command, uses self.top_details - attempts to overwrite

        self.args['top'] if it was previously set based on the first DEPS target name,
        so it more accurately reflects the top-module-name. You should only do this if the
        self.args['work-dir'] has already been determined.
        '''

        if not self.args['top']:
            return

        if not (self.top_details and \
                self.top_details['inferred_from_target_name'] and \
                self.top_details['last_added_source_file']):
            return

        # since work-dir should be set, we will change self.args['top'] if there
        # was a last-added source file.
        best_top_fname = self.top_details['last_added_source_file']

        if not best_top_fname:
            return

        best_top_name = self.get_top_name(best_top_fname)
        best_top = util.get_inferred_top_module_name(
            module_guess=best_top_name,
            module_fpath=best_top_fname
        )
        if not best_top:
            return

        util.info("--top was previously inferred from target name",
                  f"({self.top_details['inferred_from_target_name']}), overriding with:",
                  f"{best_top} (from file: {best_top_fname})")
        self.args['top'] = best_top


    def set_parameter(
            self, name: str, value, caller_info: str = '(CLI)',
            wrap_str_double_quotes: bool = True
    ) -> None:
        '''Safe wrapper for setting a parameter (can only set once, from CLI or DEPS)'''

        if name in self.parameters:
            util.debug("Parameter not set because it is already modified",
                       f"(via -G<Name>=<Value>): {name}={value}; orig value",
                       f'{self.parameters[name]} from {caller_info}')
        else:
            if isinstance(value, (bool, int)):
                value = int(value)
            elif isinstance(value, str):
                value = strip_outer_quotes(value.strip('\n'))
                if wrap_str_double_quotes:
                    value = '"' + value + '"'
            self.parameters[name] = value
            util.debug(f"Parameter (via -G<Name>=<Value>): {name}={value}",
                       f'from {caller_info}')


    def process_parameter_arg(
            self, text: str, pwd: str = os.getcwd()
    ) -> None:
        '''Retuns None, parses -G<Name>=<Value> adds to internal self.parameters.

        Note that not all tools will allow hierarchy path information when setting
        parameters. The Queta/Modelsim family can handle -G/Path/To/Name=Value,
        but other tools may crash.

        Also note - we do not currently accept [0] in the /Path/To/generate[1]/Name=Value
        '''

        # Deal with raw CLI/bash/powershell argparser, strip all outer quotes.
        text = strip_outer_quotes(text)
        if not pwd:
            pwd = ''

        if not text.startswith('-G'):
            self.error(f"Didn't understand -G parameter arg: '{text}'",
                       error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR)
            return

        text = text[2:] # strip leading -G
        m = re.match(rf'^({PARAMETER_NAME_RSTR})$', text)
        if m:
            k = m.group(1)
            util.warning(f"Parameter {k} has no value and will not be applied")
            return
        m = re.match(rf'^({PARAMETER_NAME_RSTR})\=(\S+)$', text)
        if not m:
            m = re.match(rf'^({PARAMETER_NAME_RSTR})\=(\"[^\"]*\")$', text)
        if m:
            k = m.group(1)
            v = m.group(2)
            # since this is coming from a CLI string, we have to guess at types,
            # for int or bool (not str) and convert bool to int:
            if not isinstance(v, (int, str)):
                self.error(f"Didn't understand -G parameter arg: name={k} value={v}",
                           f"value must be int/str type, from: '{text}'",
                           error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR)
                return
            try:
                v = int(v, 0)
            except ValueError:
                pass
            util.debug('set parameter Name={k} Value={v}')
            self.set_parameter(k, v)
        else:
            util.warning(f'Unable to set parameter from token (-G removed): {text}')



    def process_plusarg( # pylint: disable=too-many-branches
            self, plusarg: str, pwd: str = os.getcwd()
    ) -> str:
        '''Retuns str, parses a +define+, +incdir+, +key=value str; adds to internal.

        Adds to self.defines, self.incdirs,
        Or, adds to self.args['unprocessed-plusargs'] (list) and retuns value added
        to unprocessed-plusargs.
        '''

        # Since this may be coming from a raw CLI/bash/powershell argparser, we may have
        # args that come from shlex.quote(token), such as:
        #   token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
        # So we strip all outer ' or " on the plusarg:
        plusarg = strip_outer_quotes(plusarg)
        if not pwd:
            pwd = ''

        if plusarg.startswith('+define+'):
            # For defines, we do use os.path.expandvars in define values, and support some
            # %PWD% and %SEED% special replacement
            plusarg = plusarg[len('+define+'):]
            m = re.match(r'^(\w+)$', plusarg)
            if m:
                k = m.group(1)
                self.defines[k] = None
                util.debug(f"Defined {k}")
                return ''
            m = re.match(r'^(\w+)\=(\S+)$', plusarg)
            if not m:
                m = re.match(r'^(\w+)\=(\"[^\"]*\")$', plusarg)
            if m:
                k = m.group(1)
                v = m.group(2)
                if v and isinstance(v, str):
                    if v.startswith('%PWD%/'):
                        v = v.replace('%PWD%', os.path.abspath(pwd))
                    if v.startswith('%SEED%'):
                        v = v.replace('%SEED%', str(self.args.get('seed', 1)))
                    v = os.path.expandvars(v)
                self.defines[k] = v
                util.debug(f"Defined {k}={v}")
                return ''
            self.error(f"Didn't understand +define+: '{plusarg}'",
                       error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR)
            return ''

        if plusarg.startswith('+incdir+'):
            # For incdirs, we DO use os.path.expandvars
            plusarg = plusarg[len('+incdir+'):]
            m = re.match(r'^(\S+)$', plusarg)
            if m:
                incdir = os.path.expandvars(m.group(1))
                if incdir and not os.path.exists(os.path.abspath(incdir)):
                    util.warning(
                        f'Unable to resolve +incdir+ for "{incdir}" (path does not exist)'
                    )
                elif incdir and incdir not in self.incdirs:
                    self.incdirs.append(os.path.abspath(incdir))
                    util.debug(f"Added include dir '{os.path.abspath(incdir)}'")
                return ''
            self.error(f"Didn't understand +incdir+: '{plusarg}'",
                       error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR)
            return ''

        # remaining plusargs as stored in self.args['unprocessed-plusargs'] (list)
        # these also do NOT get os.path.expandvars
        if plusarg.startswith('+'):
            if not self.config.get('bare_plusarg_supported', False):
                self.error(f"bare plusarg(s) are not supported: {plusarg}'")
                return ''
            if plusarg not in self.args['unprocessed-plusargs']:
                self.args['unprocessed-plusargs'].append(plusarg)
                # For anything added to unprocessed-plusarg, we have to return it, to let
                # derived classes have the option to handle it
                return plusarg

        self.error(f"Didn't understand +plusarg: '{plusarg}'",
                   error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR)
        return ''


    def append_shell_commands(self, cmds : list) -> None:
        ''' Given a cmds (list), where each item is a dict with:

        { 'target_node': str,
          'target_path': str,
          'exec_list': list
        }
        add to this class's dep_shell_commands for deferred processing
        '''

        for entry in cmds:
            if entry is None or not isinstance(entry, dict):
                continue
            if entry in self.dep_shell_commands:
                # we've already run this exact command (target node, target path, exec list),
                # don't run it again
                continue

            assert 'exec_list' in entry, f'{entry=}'
            util.debug(f'adding - dep_shell_command: {entry=}')
            self.dep_shell_commands.append(entry)


    def append_work_dir_add_srcs(self, add_srcs: list, caller_info: str) -> None:
        '''Given add_srcs (list), where each item is a dict with:

        { 'target_node': str,
          'target_path': str,
          'file_list': list
        }

        adds files to set self.dep_work_dir_add_src, and call add_file on it, basically
        resolving adding source files that were generated and residing in the work-dir.
        '''
        for entry in add_srcs:
            if entry is None or not isinstance(entry, dict):
                continue

            work_dir_files = entry['file_list']
            for filename in work_dir_files:
                # Unfortunately, self.args['work-dir'] doesn't exist yet and hasn't been set,
                # so we'll add these files as '@EDA-WORK_DIR@' + filename, and have to replace
                # the EDA-WORK_DIR@ string later in our flow.
                filename_use = self._work_dir_add_srcs_path_string + filename
                dep_key_tuple = (
                    entry['target_path'],
                    entry['target_node'],
                    filename_use
                )
                if filename_use not in self.files:
                    util.debug(f'work_dir_add_srcs@ {dep_key_tuple=} added file {filename_use=}')
                    self.add_file(filename=filename_use, use_abspath=False, caller_info=caller_info)
                    # avoid duplicate calls, and keep a paper trail of which DEPS added
                    # files from the self.args['work-dir'] using this method.
                    self.dep_work_dir_add_srcs.add(dep_key_tuple) # add to set()
                elif dep_key_tuple not in self.dep_work_dir_add_srcs:
                    # we've already added the file so this dep was skipped for this one file.
                    util.warning(f'work_dir_add_srcs@ {dep_key_tuple=} but {filename_use=}',
                                 'is already in self.files (duplicate dependency on generated',
                                 'file?)')


    def resolve_target(
            self, target: str, no_recursion: bool = False, caller_info: str = ''
    ) -> bool:
        '''Returns True if target is found. Entry point for resolving a CLI path based target name.

        Will recursively call resolve_target_core, looking for a filename existing
        or DEPS markup file in that path and a target (key) in the markup table.

        This will populate all source file dependencies, commands, and other DEPS markup
        features as the target is resolved.
        '''
        util.debug(f"Entered resolve_target({target})")
        # self.target is a name we grab for the job (i.e. for naming work dir etc).  we don't want
        # the path prefix.

        self.target_path, self.target = os.path.split(target)

        if str(Path(target)) in self.targets_dict:
            # If we're encountered this target before, stop. We're not traversing again.
            return True

        self.targets_dict[str(Path(target))] = f'({self.target_path})'
        file_exists, fpath, forced_extension = files.get_source_file(target)
        if file_exists:
            # If the target is a file (we're at the root here processing CLI arg tokens)
            # and that file exists and has an extension, then there's no reason to go looking
            # in DEPS files, add the file and return True.
            _, file_ext = os.path.splitext(fpath)
            if forced_extension or file_ext:
                self.add_file(fpath, caller_info=caller_info,
                              forced_extension=forced_extension)
                return True

        return self.resolve_target_core(target, no_recursion, caller_info)

    def resolve_target_core( # pylint: disable=too-many-locals,too-many-branches
            self, target: str, no_recursion: bool, caller_info: str = '',
            error_on_not_found: bool = True
    ) -> bool:
        '''Returns True if target is found. recursive point for resolving path or DEPS markup
        target names.'''

        util.debug(f"Entered resolve_target_core({target=})")
        found_target = False
        target_path, target_node = os.path.split(target)

        deps, data, deps_file = None, None, None
        found_deps_file = False
        found_caller_info = ''

        if self.config['deps_markup_supported']:
            deps = DepsFile(
                command_design_ref=self, target_path=target_path, cache=self.cached_deps
            )
            deps_file = deps.deps_file
            data = deps.data

        # Continue if we have data, otherwise look for files other than DEPS.<yml|yaml>
        if data is not None:
            found_deps_file = True
            found_target = deps.lookup(target_node=target_node, caller_info=caller_info)

        if found_deps_file and found_target:

            found_caller_info = deps.gen_caller_info(target_node)
            entry = deps.get_entry(target_node=target_node)

            # For convenience, use an external class for this DEPS.yml table/dict
            # This could be re-used for any markup DEPS.json, DEPS.toml, DEPS.py, etc.
            deps_processor = DepsProcessor(
                command_design_ref = self,
                deps_entry = entry,
                target = target,
                target_path = target_path,
                target_node = target_node,
                deps_file = deps_file,
                # Update the caller_info for this DEPS.yml file's target_node, b/c we're now
                # examining this entry (target) in this deps_file.
                caller_info = deps.gen_caller_info(target_node)
            )

            # Process the target, and get new (unprocessed) deps entries list.
            # This updates self (for defines, incdirs, top, args, etc)
            # This will skip remaining deps in self.targets_dict
            deps_targets_to_resolve = deps_processor.process_deps_entry()
            util.debug(f'   ... for {target_node=} {deps_file=}, {deps_targets_to_resolve=}')

            # Recurse on the returned deps (ordered list), if they haven't already been traversed.
            for x in deps_targets_to_resolve:
                caller_info = deps.gen_caller_info(target_node)
                if x and isinstance(x, tuple):
                    # if deps_processor.process_deps_entry() gave us a tuple, it's an
                    # unprocessed 'command' that we kept in order until now. Append it.
                    assert len(x) == 2, \
                        f'command tuple {x=} must be len 2, {target_node=} {deps_file=}'
                    shell_commands_list, work_dir_add_srcs_list = x
                    self.append_shell_commands( cmds=shell_commands_list )
                    self.append_work_dir_add_srcs( add_srcs=work_dir_add_srcs_list,
                                                   caller_info=caller_info )

                elif x and x not in self.targets_dict:
                    self.targets_dict[str(Path(x))] = None # add it before processing.
                    file_exists, fpath, forced_extension = files.get_source_file(x)
                    if file_exists:
                        self.add_file(filename=fpath, caller_info=caller_info,
                                      forced_extension=forced_extension)
                    else:
                        util.debug(f'   ... Calling resolve_target_core({x=})')
                        found_target |= self.resolve_target_core(
                            x, no_recursion, caller_info=caller_info
                        )

        # Done with DEPS.yml if it existed.

        if not found_target:
            util.debug(f"Haven't been able to resolve {target=} via DEPS")
            known_file_extensions_for_source = []
            for x in ('verilog', 'systemverilog', 'vhdl', 'cpp'):
                known_file_extensions_for_source += self.config.get(
                    'file_extensions', {}).get(x, [])
            for e in known_file_extensions_for_source:
                try_file = target + e
                util.debug(f"Looking for file {try_file}")
                if os.path.exists(try_file):
                    self.add_file(try_file, caller_info=f'n/a::{target}::n/a')
                    found_target = True
                    found_caller_info = os.path.split(try_file)[0]
                    break # move on to the next target
            if not found_target and error_on_not_found: # if STILL not found_this_target...
                # allow this if --no-error-unknown-args:
                self.error_ifarg(
                    f"Unable to resolve {target=}",
                    arg='error-unknown-args',
                    error_code=status_constants.EDA_DEPS_TARGET_NOT_FOUND
                )

        # if we've found any target since being called, it means we found the one we were called for
        if found_target:
            self.targets_dict[str(Path(target))] = f'({found_caller_info})'
        return found_target

    def add_file( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
            self, filename: str, use_abspath: bool = True, add_to_non_sources: bool = False,
            caller_info: str = '', forced_extension: str = ''
    ) -> str:
        '''Given a filename, add it to one of self.files_sv or similar lists

        based on file extension or prefix directive.'''
        _, file_ext = os.path.splitext(filename)
        if use_abspath:
            file_abspath = os.path.abspath(filename)
        else:
            file_abspath = filename


        if file_abspath in self.files:
            util.debug(f"Not adding file {file_abspath}, already have it")
            return ''

        known_file_ext_dict = self.config.get('file_extensions', {})
        v_file_ext_list = known_file_ext_dict.get('verilog', [])
        sv_file_ext_list = known_file_ext_dict.get('systemverilog', [])
        vhdl_file_ext_list = known_file_ext_dict.get('vhdl', [])
        cpp_file_ext_list = known_file_ext_dict.get('cpp', [])
        sdc_file_ext_list = known_file_ext_dict.get('synth_constraints', [])
        dotf_file_ext_list = known_file_ext_dict.get('dotf', [])
        py_file_ext_list = known_file_ext_dict.get('python', [])
        makefile_ext_list = known_file_ext_dict.get('makefile', [])
        markup_file_ext_list = known_file_ext_dict.get('markup', [])

        if forced_extension:
            # If forced_extension='systemverilog', then use the first known extension for
            # it ('.sv', from eda_config_defaults.yml), which will pick it up in the if-elif
            # below.
            file_ext = known_file_ext_dict.get(forced_extension, [''])[0]
            util.debug(f"{forced_extension=} for {filename=} as type '{file_ext}'")


        if not add_to_non_sources and \
           file_ext in known_file_ext_dict.get('inferred_top', []):
            self.top_details['last_added_source_file'] = file_abspath

        if add_to_non_sources:
            self.files_non_source.append(file_abspath)
            util.debug(f"Added non-source file file {filename} as {file_abspath}")
        elif file_ext in v_file_ext_list and not self.args['all-sv']:
            self.files_v.append(file_abspath)
            util.debug(f"Added Verilog file {filename} as {file_abspath}")
        elif file_ext in sv_file_ext_list or \
             ((file_ext in v_file_ext_list) and self.args['all-sv']):
            self.files_sv.append(file_abspath)
            util.debug(f"Added SystemVerilog file {filename} as {file_abspath}")
        elif file_ext in vhdl_file_ext_list:
            self.files_vhd.append(file_abspath)
            util.debug(f"Added VHDL file {filename} as {file_abspath}")
        elif file_ext in cpp_file_ext_list:
            self.files_cpp.append(file_abspath)
            util.debug(f"Added C++ file {filename} as {file_abspath}")
        elif file_ext in sdc_file_ext_list:
            self.files_sdc.append(file_abspath)
            util.debug(f"Added SDC file {filename} as {file_abspath}")
        elif file_ext in dotf_file_ext_list:
            # a stray .f file as a source file, sure why not support it:
            dp = DepsProcessor(command_design_ref=self, deps_entry={}, target='',
                               target_path='', target_node='', deps_file='',
                               caller_info=caller_info)
            dp.apply_args(args_list=[f'-f={file_abspath}'])
            del dp
        elif file_ext in py_file_ext_list:
            self.files_py.append(file_abspath)
            util.debug(f"Added Python file {filename} as {file_abspath}")
        elif file_ext in makefile_ext_list or os.path.split(filename)[1] == 'Makefile':
            self.files_makefile.append(file_abspath)
            util.debug(f"Added Makefile {filename} as {file_abspath}")
        elif file_ext in markup_file_ext_list and os.path.split(filename)[1].startswith('DEPS.'):
            util.warning(f'DEPS markup file ({filename}, from {caller_info}) will be ingored,',
                         'these are implicitly found via a TARGET path')
        else:
            # unknown file extension. In these cases we link the file to the working directory
            # so it is available (for example, a .mem file that is expected to exist with relative
            # path)
            self.files_non_source.append(file_abspath)
            util.info(f"Added non-source file {filename} as {file_abspath}")

        self.files[file_abspath] = True
        self.files_caller_info[file_abspath] = caller_info
        return file_abspath

    def process_tokens( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
            self, tokens: list, process_all: bool = True,
            pwd: str = os.getcwd()
    ) -> list:
        '''Returns a list of unparsed args (self.args did not have these keys present)

        Entrypoint from eda.py calling a command handler obj.process_tokens(args).
        Dervied classes are expected to call this parent process_tokens to parse and handle
        their self.args dict'''

        util.debug(f'CommandDesign - process_tokens start - {tokens=}')

        # see if it's a flag/option like --debug, --seed <n>, etc
        # This returns all unparsed args, and doesn't error out due to process_all=False
        orig_tokens = tokens.copy()
        unparsed = Command.process_tokens(self, tokens, process_all=False, pwd=pwd)
        util.debug(f'CommandDesign - after Command.process_tokens(..) {unparsed=}')

        # deal with +define+, +incdir+, +(plusargName)+, or -GParameterName=Value:
        # consume it and remove from unparsed, walk the list, remove all items after we're done.
        remove_list = []
        for token in unparsed:
            # Since this is a raw argparser, we may have args that come from shlex.quote(token),
            # such as:
            #   token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
            # So we have to check for strings that have been escaped for shell with extra single
            # quotes.
            m = re.match(r"^\'?\+\w+", token)
            if m:
                # Copy and strip all outer ' or " on the plusarg:
                util.debug(f'plusarg found at {token=}')
                plusarg = strip_outer_quotes(token)
                self.process_plusarg(plusarg, pwd=pwd)
                remove_list.append(token)
                continue

            # Parameters in -G<word>=<something>
            # Parameters in -Gpath.to.<word>=<something>
            # Parameters in -G/path/to/<word>=<something>
            m = re.match(rf"^\'?\-G{PARAMETER_NAME_RSTR}\=.+", token)
            if m:
                # Copy and strip all outer ' or " on the text:
                util.debug(f'found parameter at {token=}')
                param = strip_outer_quotes(token)
                self.process_parameter_arg(param, pwd=pwd)
                remove_list.append(token)


        for x in remove_list:
            unparsed.remove(x)

        if self.error_on_no_files_or_targets and not unparsed:
            # derived classes can set error_on_no_files_or_targets=True
            # For example: CommandSim will error (requires files/targets),
            # CommandWaves does not (files/targets not required)
            # A nice-to-have: if someone ran: eda sim; without a file/target,
            # check the DEPS markup file for a single target, and if so run it
            # on the one and only target.
            if self.config['deps_markup_supported']:
                deps = DepsFile(
                    command_design_ref=self, target_path=os.getcwd(), cache=self.cached_deps
                )
                if deps.deps_file and deps.data:
                    all_targets = deps_data_get_all_targets(deps.data)
                    if all_targets:
                        target = all_targets[-1]
                        unparsed.append(target)
                        util.warning(f"For command '{self.command_name}' no files or targets were",
                                     f"presented at the command line, so using '{target}' from",
                                     f"{deps.deps_file}")
            if not unparsed:
                # If unparsed is still empty, then error.
                self.error(f"For command '{self.command_name}' no files or targets were",
                           f"presented at the command line: {orig_tokens}",
                           error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR)

        # by this point hopefully this is a target ... is it a simple filename?

        # Before we look for files, check for stray --some-arg in unparsed, we don't want to treat
        # these as potential targets if process_all=True, but someone might have a file named
        # --my_file.sv, so those are technically allowed until the tool would fail on them.
        possible_unparsed_args = [
            x for x in unparsed if x.startswith('--') and not os.path.isfile(x)
        ]
        if process_all and possible_unparsed_args:
            _tool = self.safe_which_tool()
            self.warning_show_known_args()
            self.error_ifarg(
                f"Didn't understand unparsed args: {possible_unparsed_args}, for command",
                f"'{self.command_name}', tool '{_tool}'",
                arg='error-unknown-args',
                error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR
            )

        remove_list = []
        last_potential_top_file = ('', '')   # (top, fpath)
        last_potential_top_target = ('', '') # (top, path/to/full-target-name)
        last_potential_top_isfile = False
        caller_info = ''
        for token in unparsed:
            file_exists, fpath, forced_extension = files.get_source_file(token)
            if file_exists:
                file_abspath = os.path.abspath(fpath)
                _, file_ext = os.path.splitext(file_abspath)
                if not forced_extension and not file_ext:
                    # This probably isn't a file we want to use
                    util.warning(f'looking for deps {token=}, found {file_abspath=}' \
                                 + ' but has no file extension, we will not add this file')
                    # do not consume it, it's probably a named target in DEPS.yml
                else:
                    self.add_file(filename=fpath, caller_info=caller_info,
                                  forced_extension=forced_extension)
                    if not self.args['top']:
                        known_file_ext_dict = self.config.get('file_extensions', {})
                        if forced_extension:
                            file_ext = known_file_ext_dict.get(forced_extension, [''])[0]
                        if file_ext in known_file_ext_dict.get('inferred_top', []):
                            # last cmd line arg was a filename that could have inferred top.
                            last_potential_top_isfile = True

                    remove_list.append(token)
                    continue # done with token, consume it, we added the file.

            # we appear to be dealing with a target name which needs to be resolved (usually
            # recursively)
            if token.startswith('-'):
                # We are not going to handle targets that start with a -, it's likely
                # an unparsed arg.
                continue
            if token.startswith(os.sep):
                target_name = token # if it's absolute path, don't prepend anything
            else:
                target_name = os.path.join(".", token) # prepend ./ to make it like: <path>/<file>

            util.debug(f'Calling self.resolve_target on {target_name=} ({token=})')
            if self.resolve_target(target_name, caller_info=caller_info):
                if not self.args['top']:
                    # last cmd line arg was a target that we'll likely use for inferred top.
                    last_potential_top_target = (self.get_top_name(target_name), target_name)
                    last_potential_top_isfile = False

                remove_list.append(token)
                continue # done with token

        for x in remove_list:
            unparsed.remove(x)

        # we were unable to figure out what this command line token is for...
        if process_all and unparsed:
            self.warning_show_known_args()
            self.error_ifarg(
                f"Didn't understand remaining args or targets {unparsed=} for command",
                f"'{self.command_name}'",
                arg='error-unknown-args',
                error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR
            )

        # handle a missing self.args['top'] with last filepath or last target:
        if not self.args.get('top', ''):
            top_path = ''
            if not last_potential_top_isfile and last_potential_top_target[0]:
                # If we have a target name from DEPS, prefer to use that.
                self.args['top'], top_path = last_potential_top_target
                util.info("--top not specified, inferred from target:",
                          f"{self.args['top']} ({top_path})")
                self.top_details['inferred_from_target_name'] = top_path

            else:
                best_top_fname = self.top_details['last_added_source_file']
                if best_top_fname:
                    last_potential_top_file = (self.get_top_name(best_top_fname), best_top_fname)

            if not self.args['top'] and last_potential_top_file[0]:
                # If we don't have a target name, and no top name yet, then go looking for the
                # module name in the final source file added.
                top_path = last_potential_top_file[1] # from tuple: (top, fpath)
                self.args['top'] = util.get_inferred_top_module_name(
                    module_guess=last_potential_top_file[0],
                    module_fpath=top_path
                )
                if self.args['top']:
                    util.info("--top not specified, inferred from final source file:",
                              f"{self.args['top']} ({top_path})")
                    # If top wasn't set, and we're using the final command-line 'arg' filename
                    # (not from DEPS.yml) we need to override self.target if that was set. Otherwise
                    # it won't save to the correct work-dir:
                    self.target = self.args['top']
                    self.targets_dict[str(Path(self.target_path) / self.target)] = (
                        f'({os.path.split(top_path)[-1]})'
                    )
                    self.top_details['inferred_from_last_added_source_file'] = top_path

        caller_info_lookup = str(Path(self.target_path) / self.target)
        util.info(f'{self.command_name}: top-most target name: {self.target}',
                  f'{self.targets_dict.get(caller_info_lookup, "")}')

        if self.error_on_missing_top and not self.args.get('top', ''):
            self.error("Did not get a --top or DEPS top, required to run command",
                       f"'{self.command_name}' for tool={self.args.get('tool', None)}",
                       error_code=status_constants.EDA_COMMAND_MISSING_TOP)

        if self.tool_changed_respawn:
            util.info(
                'CommandDesign: need to respawn due to tool change to',
                f'\'{self.tool_changed_respawn["tool"]}\' from',
                f'\'{self.tool_changed_respawn["orig_tool"]}\'',
                f'(from DEPS, {self.tool_changed_respawn["from"]})'
            )

        return unparsed


class ThreadStats:
    '''To avoid globals for two ints, keep a class holder so CommandParallel and
    CommandParallelWorker can share values'''

    done = 0
    started = 0

    def all_done(self) -> bool:
        '''Returns True if all started jobs are done'''
        return self.started == self.done

    def get_remaining(self) -> int:
        '''Returns remaining number of jobs'''
        return max(self.started - self.done, 0)

    def __repr__(self) -> str:
        '''Pretty print DONE/STARTED'''
        return f'{self.done}/{self.started}'


class CommandParallelWorker(threading.Thread):
    '''Helper class for a single threaded job, often run by a CommandParallel command
    handler class'''

    subprocess_popen_kwargs = subprocess_helpers.get_kwargs(
        work_dir='', shell=False, is_subprocess_popen=True
    )

    def __init__(
            self, n, work_queue: queue.Queue, done_queue: queue.Queue,
            threads_stats: ThreadStats
    ):
        threading.Thread.__init__(self)
        self.n = n
        self.work_queue = work_queue
        self.done_queue = done_queue
        self.stop_request = False
        self.job_name = ""
        self.proc = None
        self.pid = None
        self.last_timer_debug = 0
        self.threads_stats = threads_stats # ref to shared object
        util.debug(f"WORKER_{n}: START")

    def run(self):
        '''Runs this single job via threading. This is typically created and called by

        a CommandParallel class handler
        '''

        while True:
            # Get the work from the queue and expand the tuple
            i, command_list, job_name, _ = self.work_queue.get()
            self.job_name = job_name
            try:
                util.debug(f"WORKER_{self.n}: Running job {i}: {job_name}")
                util.debug(f"WORKER_{self.n}: Calling Popen")

                # Since this class tracks its own subprocess, we'll re-use some of the
                # subprocess_helpers to make sure this works cleanly in Windows, and has
                # linux process group IDs:
                proc = subprocess.Popen( # pylint: disable=consider-using-with
                    subprocess_helpers.get_command_list(command_list, shell=False),
                    **self.subprocess_popen_kwargs
                )
                self.proc = proc
                util.debug(f"WORKER_{self.n}: Opened process, PID={proc.pid}")
                self.pid = proc.pid

                # For better Ctrl-C behavior, we'll also track this PID in subprocess_helpers:
                subprocess_helpers.add_running_parent_pid(self.pid)

                self.threads_stats.started += 1
                while proc.returncode is None:
                    try:
                        if (time.time() - self.last_timer_debug) > 10:
                            util.debug(f"WORKER_{self.n}: Calling proc.communicate")
                        stdout, stderr = proc.communicate(timeout=0.5)
                        util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
                                   f"stderr:{stderr}")
                    except subprocess.TimeoutExpired:
                        if (time.time() - self.last_timer_debug) > 10:
                            util.debug(f"WORKER_{self.n}: Timer expired, stop_request=",
                                       f"{self.stop_request}")
                            self.last_timer_debug = time.time()
                    if self.stop_request:
                        util.debug(f"WORKER_{self.n}: got stop request, issuing SIGINT")
                        proc.send_signal(signal.SIGINT)
                        util.debug(f"WORKER_{self.n}: got stop request, calling proc.wait")
                        proc.wait()

                util.debug(f"WORKER_{self.n}: -- out of while loop")
                util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
                try:
                    util.debug(f"WORKER_{self.n}: Calling proc.communicate one last time")
                    stdout, stderr = proc.communicate(timeout=0.1)
                    util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
                               f"stderr:{stderr}")
                except subprocess.TimeoutExpired:
                    util.debug(f"WORKER_{self.n}: timeout waiting for communicate after loop?")
                except Exception as e:
                    util.debug(f"WORKER_{self.n}: timeout with exception {e=}")

                return_code = proc.poll()
                util.debug(f"WORKER_{self.n}: Finished job {i}: {job_name} with return code",
                           f"{return_code}")
                self.done_queue.put((i, job_name, return_code))

                # Remove this PID from the Ctrl-C cleaner:
                subprocess_helpers.remove_completed_parent_pid(self.pid)

                self.pid = None
                self.proc = None
                self.job_name = "<idle>"

            finally:
                util.debug(f"WORKER_{self.n}: -- in finally block")
                self.work_queue.task_done()
                self.threads_stats.done += 1


class CommandParallel(Command):
    '''Base class command handler for running multiple eda jobs, such as for child classes

    in commands.multi or commands.sweep
    '''
    def __init__(self, config: dict):
        Command.__init__(self, config=config)
        self.jobs = []
        self.jobs_ran = []
        self.jobs_status = []
        self.args.update({
            'parallel': 1,
            'threads': util.get_thread_budget(),
            'threads-per-child': 1,
            'slurm-srun': False,
        })

        thread_budget = util.get_thread_budget()

        self.args_help.update({
            'slurm-srun': (
                'Run each single job through slurm via: srun eda COMMAND ...;'
                ' consider using with --parallel=P (P > 1)'
            ),
            'parallel': (
                'Number of jobs to attempt to run in parallel, limited by --threads and'
                f' the sytem budget: {thread_budget}'
            ),
            'threads': (
                'Logical CPU core count for this "parallel" command. This is the total budget'
                ' for this job: and will limit the used value for --parallel=P, maximum value'
                f' of: {thread_budget}. If --use-max-threads, effectively uses: {thread_budget}'
            ),
            'threads-per-child': (
                'Value passed to "eda" single/child job for --threads=Value, limited to:'
                ' (--threads total budget value) // (--parallel value). If --use-max-threads,'
                f' attempts to use up to {thread_budget} // (--parallel value)'
            ),
        })

        self.worker_threads = []
        self.threads_stats = ThreadStats()

    def __del__(self): # pylint: disable=too-many-branches,too-many-statements
        util.debug(f"In Command.__del__, threads done/started: {self.threads_stats}")
        if self.threads_stats.all_done():
            return
        util.warning(f"Need to shut down {self.threads_stats.get_remaining()} worker threads...")
        for w in self.worker_threads:
            if w.proc:
                util.warning(f"Requesting stop of PID {w.pid}: {w.job_name}")
                w.stop_request = True
        for _ in range(10):
            util.debug(f"Threads done/started: {self.threads_stats}")
            if self.threads_stats.started == self.threads_stats.done:
                util.info("All threads done")
                return
            time.sleep(1)
        subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with

        util.debug("Scanning workers again")
        # Since these were called with Command.exec(..) which calls:
        # opencos.utils.subprocess_helpers.subprocess_run_background(..)
        # all jobs should have been spawned in:
        #   -- Linux: start_new_session=True
        #   -- Windows: creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
        # which gives us a nicely chained Linux PGID, or Windows PID we can CTRL_BREAK_EVENT
        # or taskill.
        # This means our first attempt can be more robust at killing the running parallel jobs:
        for w in self.worker_threads:
            if w.proc and w.pid:
                subprocess_helpers.kill_proc_tree(pid=w.pid)
        time.sleep(1)
        subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with

        # The rest is legacy cleanup, left although unlikely we hit this section of code.
        util.debug("Scanning workers again")
        for w in self.worker_threads:
            if w.proc:
                util.info(f"need to SIGINT WORKER_{w.n}, may need manual cleanup, check 'ps'")
                if w.pid:
                    try:
                        os.kill(w.pid, signal.SIGINT)
                    except (ProcessLookupError, PermissionError):
                        continue

        for _ in range(5):
            util.debug(f"Threads done/started: {self.threads_stats}")
            if self.threads_stats.all_done():
                util.info("All threads done")
                return
            time.sleep(1)
        subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
        util.debug("Scanning workers again")
        for w in self.worker_threads:
            if w.proc:
                util.info(f"need to TERM WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
                if w.pid:
                    try:
                        os.kill(w.pid, signal.SIGTERM)
                    except (ProcessLookupError, PermissionError):
                        continue
        for _ in range(5):
            util.debug(f"Threads done/started: {self.threads_stats}")
            if self.threads_stats.all_done():
                util.info("All threads done")
                return
            time.sleep(1)
        subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
        for w in self.worker_threads:
            if w.proc:
                util.info(f"need to KILL WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
                if w.pid:
                    # Windows compatible: signal.SIGKILL is not available, so we could use
                    # os.kill(PID, 9) but we'll be nice and do another SIGTERM
                    # os.kill(PID, 9)
                    try:
                        os.kill(w.pid, getattr(signal, 'SIGKILL', signal.SIGTERM))
                    except (ProcessLookupError, PermissionError):
                        continue
        util.stop_log()
        subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with


    def get_eda_child_thread_budget(self) -> int:
        '''Calls util.get_tool_args_threads(max, parallel), used as a limiter on sub-calls
        to eda --threads=CHILD_THREADS. If --use-max-threads is set, then attempts to maximize
        --threads-per-child even if set to a lower value.

        If running with --slurm-srun, the budgets are ignored and --threads-per-child is used
        (default 1)
        '''

        if self.args['slurm-srun']:
            return max(1, self.args.get('threads-per-child', 1))

        total_budget = util.get_thread_budget()
        if self.args['use-max-threads']:
            requested_threads_per_child = total_budget
        else:
            requested_threads_per_child = self.args.get('threads-per-child', 1)
            total_budget = min(self.args['threads'], util.get_thread_budget())

        return max(
            1,
            min(
                requested_threads_per_child,
                max(1, total_budget // self.args['parallel'])
            )
        )


    def update_artifacts_with_jobs_ran(self) -> None:
        '''Update util.artifacts if a sub-job had errors/warnings or tool errors/warnings'''

        error_summary_lines = []

        for j in self.jobs_ran:
            job_name = j['job-name']
            errors_info_json = os.path.join(
                self.args['eda-dir'], job_name, util.Artifacts.EDA_ERRORS_NAME
            )
            data = {}
            if os.path.exists(errors_info_json):
                with open(errors_info_json, 'r', encoding='utf-8', errors='replace') as f:
                    data = json.load(f)
                    # We might want to report this separately for jobs that had
                    # any errors/warnings/tool non-zero values?
                    if data['errors'] or data['warnings'] or data['tool_errors'] or \
                       data['tool_warnings']:
                        error_summary_lines.append([
                            job_name, f'({errors_info_json}):', flatten_dict_to_str(data)
                        ])
                        util.artifacts.job_errors_or_warnings[job_name] = data

        if error_summary_lines:
            error_summary_lines = pretty_2dlist_columns(error_summary_lines)
            util.info(
                f'Failing jobs error summary: (see {util.artifacts.eda_errors_json_filepath})'
            )
            for line in error_summary_lines:
                util.info(line, color=Colors.yellow)


    def run_jobs( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
            self, command: str
    ) -> None:
        '''Runs all "jobs" in self.jobs list, either serially or in parallel'''

        # this is where we actually run the jobs.  it's a messy piece of code and prob could use
        # refactoring but the goal was to share as much as possible (job start, end, pass/fail
        # judgement, etc) while supporting various mode combinations (parallel mode, verbose mode,
        # fancy mode, etc) and keeping the UI output functional and awesome sauce

        # walk targets to find the longest name, for display reasons
        longest_job_name = 0
        total_jobs = len(self.jobs)
        self.jobs_status = [None] * total_jobs
        for i in range(total_jobs):
            longest_job_name = max(longest_job_name, len(self.jobs[i]['name']))

        # Note the budgetting is ignored if using --slurm-srun
        srun = safe_shutil_which('srun')
        if not srun and self.args['slurm-srun']:
            self.error(
                '--slurm-srun is set, however "srun" is not found on this system, cannot run Slurm'
            )
            return

        if not self.args['slurm-srun']:
            max_budget = self.args.get('threads', util.get_thread_budget())
            if self.args['parallel'] > max_budget:
                util.warning(
                    f'--parallel={self.args["parallel"]} but will be limited to max budget:',
                    f'{max_budget}'
                )
                self.args['parallel'] = max_budget

            max_threads_per_child = self.get_eda_child_thread_budget()
            if self.args['threads-per-child'] > max_threads_per_child:
                util.warning(
                    f'--threads-per-child={self.args["threads-per-child"]} but will be limited to',
                    f'{max_threads_per_child} due to max budget: {max_budget},',
                    f'parallel: {self.args["parallel"]}'
                )
                self.args['threads-per-child'] = max_threads_per_child

        run_parallel = self.args['parallel'] > 1

        # figure out the width to print various numbers
        jobs_digits = len(f"{total_jobs}")

        job_done_return_code = None
        job_done_run_time = 0
        job_done_name = ''

        # run the jobs!
        running_jobs = {}
        passed_jobs = []
        failed_jobs = []
        workers = []
        jobs_complete = 0
        jobs_launched = 0
        num_parallel = min(len(self.jobs), self.args['parallel'])
        _, lines = shutil.get_terminal_size()
        # we will enter fancy mode if we are parallel and we can leave 6 lines of regular scrolling
        # output
        fancy_mode = all([util.args['fancy'], num_parallel > 1, num_parallel <= (lines-6)])
        multi_cwd = util.getcwd() + os.sep

        self.patch_jobs_for_duplicate_target_names()

        if run_parallel:
            # we are doing this multi-threaded
            util.info(f"Parallel: Running multi-threaded, starting {num_parallel} workers")
            work_queue = queue.Queue()
            done_queue = queue.Queue()
            for x in range(num_parallel):
                worker = CommandParallelWorker(
                    n=x, work_queue=work_queue, done_queue=done_queue,
                    threads_stats=self.threads_stats
                )
                # Setting daemon to True will let the main thread exit even though the workers are
                # blocking
                worker.daemon = True
                worker.start()
                self.worker_threads.append(worker)
                workers.append(x)
            if fancy_mode:
                # in fancy mode, we will take the bottom num_parallel lines to show state of workers
                util.fancy_start(fancy_lines=num_parallel)
                for x in range(num_parallel):
                    util.fancy_print(f"Starting worker {x}", x)

        while self.jobs or running_jobs:
            job_done = False
            job_done_quiet = False
            anything_done = False

            def sprint_job_line(job_number=0, job_name="", final=False, hide_stats=False,
                                info_str: str = 'INFO: [EDA] ' + safe_emoji('    ', ' ')):
                '''
                Try to make a pretty line for a "multi" style target line

                info_str looks a little odd b/c this is often printed directly, instead
                of using util.info(...), and the use of safe_emoji('    ', ' ') is for
                alignment with util.error(..)
                '''
                if jobs_complete == total_jobs:
                    pct = '100.0%' # 6 chars
                else:
                    pct = f'{(100 * (jobs_complete)) / total_jobs : 5.1f}%' # 6chars
                return ''.join([
                    info_str,
                    string_or_space(
                        f"[job {job_number:0{jobs_digits}d}/{total_jobs:0{jobs_digits}d} ",
                        final),
                    string_or_space("| pass ", hide_stats or final),
                    string_or_space(
                        f"{len(passed_jobs):0{jobs_digits}d}/{jobs_complete:0{jobs_digits}d} ",
                        hide_stats),
                    string_or_space(f"@ {pct}", hide_stats or final),
                    string_or_space("] ", final),
                    f"{command} {(job_name+' ').ljust(longest_job_name+3,'.')}"
                ])

            # for any kind of run (parallel or not, fancy or not, verbose or not) ...
            # can we launch a job?
            if self.jobs and (len(running_jobs) < num_parallel):
                # we are launching a job
                jobs_launched += 1
                anything_done = True
                job = self.jobs.pop(0)
                self.jobs_ran.append(job) # copy to self.jobs_ran (so it's not lost)
                if job['name'].startswith(multi_cwd):
                    job['name'] = job['name'][len(multi_cwd):]
                # in all but fancy mode, we will print this text at the launch of a job.  It may
                # get a newline below
                job_text = sprint_job_line(
                    jobs_launched, job['name'], hide_stats=run_parallel
                )
                command_list = job['command_list']
                cwd = util.getcwd()

                if run_parallel:
                    # multithreaded job launch: add to queue
                    worker = workers.pop(0)
                    running_jobs[str(jobs_launched)] = { 'name' : job['name'],
                                                         'number' : jobs_launched,
                                                         'worker' : worker,
                                                         'start_time' : time.time(),
                                                         'update_time' : time.time()}
                    work_queue.put((jobs_launched, command_list, job['name'], cwd))
                    suffix = "<START>"

                    if fancy_mode:
                        util.fancy_print(job_text + suffix, worker)
                    else:
                        util.print_foreground_color(job_text + suffix)
                else:
                    # single-threaded job launch, we are going to print out job info as we start
                    # each job... no newline. since non-verbose silences the job and prints only
                    # <PASS>/<FAIL> after the trailing "..." we leave here
                    util.print_foreground_color(job_text, end="")
                    job_done_number = jobs_launched
                    job_done_name = job['name']
                    job_start_time = time.time()
                    if util.args['verbose']:
                        # previous status line gets a \n, then job is run passing
                        # stdout/err, then print 'job_text' again with pass/fail
                        util.print_wrap("")
                        # run job, sending output to the console
                        _, _, job_done_return_code = self.exec(
                            cwd, command_list, background=False, stop_on_error=False, quiet=False
                        )
                        # reprint the job text previously printed before running job (and given
                        # "\n" after the trailing "...")
                    else:
                        # run job, swallowing output (hope you have a logfile)
                        _, _, job_done_return_code = self.exec(
                            cwd, command_list, background=True, stop_on_error=False, quiet=True
                        )
                        # in this case, we have the job start text (trailing "...", no newline)
                        # printed
                        job_done_quiet = True
                    job_done = True
                    job_done_run_time = time.time() - job_start_time
                    # Since we consumed the job, use the job['index'] to track the per-job status:

            if run_parallel:
                # parallel run, check for completed job
                if done_queue.qsize():
                    # we're collecting a finished job from a worker thread.  note we will only
                    # reap one job per iter of the big loop, so as to share job completion code
                    # at the bottom
                    anything_done = True
                    job_done = True
                    job_done_number, job_done_name, job_done_return_code = done_queue.get()
                    t = running_jobs[str(job_done_number)]
                    # in fancy mode, we need to clear the worker line related to this job.
                    if fancy_mode:
                        util.fancy_print("INFO: [EDA] Parallel: Worker Idle ...", t['worker'])
                    job_done_run_time = time.time() - t['start_time']
                    util.debug(
                        f"removing job #{job_done_number} from running jobs: {job_done_name}",
                        f"rc={job_done_return_code}"
                    )
                    del running_jobs[str(job_done_number)]
                    workers.append(t['worker'])

                # parallel run, update the UI on job status
                for _,t in running_jobs.items():
                    if (fancy_mode or (time.time() - t['update_time']) > 30):
                        t['update_time'] = time.time()
                        job_text = sprint_job_line(t['number'], t['name'], hide_stats=True)
                        suffix = f"<RUNNING: {sprint_time(time.time() - t['start_time'])}>"

                        if fancy_mode:
                            util.fancy_print(f"{job_text}{suffix}", t['worker'])
                        else:
                            util.print_foreground_color(job_text + suffix)

            # shared job completion code
            # single or multi-threaded, we can arrive here to harvest <= 1 jobs, and need
            # {job, return_code} valid, and we expect the start of a status line to have been
            # printed, ready for pass/fail
            if job_done:
                jobs_complete += 1
                this_job_failed = False
                if job_done_return_code is None or job_done_return_code:
                    if job_done_return_code == 124:
                        # bash uses 124 for bash timeout errors, if that was preprended to the
                        # command list.
                        suffix = (
                            f"<TOUT{safe_emoji(' ❌')}:"
                            f" {sprint_time(job_done_run_time)}>"
                        )
                        this_job_failed = True
                    else:
                        suffix = (
                            f"<FAIL{safe_emoji(' ❌')}:"
                            f" {sprint_time(job_done_run_time)}>"
                        )
                        this_job_failed = True
                    failed_jobs.append(job_done_name)

                    # Make sure our overall status flags this, in case we terminate in some other
                    # way, see below on 'this_job_failed (we call self.error)

                    if job_done_return_code is None:
                        self.status = max(self.status, 2)
                    else:
                        self.status = max(self.status, job_done_return_code)

                else:
                    suffix = (
                        f"<PASS{safe_emoji(' ✅')}:"
                        f" {sprint_time(job_done_run_time)}>"
                    )
                    passed_jobs.append(job_done_name)
                # we want to print in one shot, because in fancy modes that's all that we're allowed
                if job_done_quiet:
                    job_done_text = ""
                elif this_job_failed:
                    job_done_text = sprint_job_line(job_done_number, job_done_name,
                                                    info_str='')
                else:
                    job_done_text = sprint_job_line(job_done_number, job_done_name)

                if this_job_failed:
                    # Make sure this error is reported (overall) w/ self.error(...)
                    if not run_parallel:
                        print(f'{Colors.red}{suffix}') # end the line with <FAIL>
                        # Error with the job name
                        self.error(job["name"], do_exit=False)
                    else:
                        # Parallel uses fancy print, so re-print the whole line as error.
                        self.error(f"{job_done_text}{suffix}", do_exit=False)
                elif failed_jobs:
                    util.print_foreground_color(f"{job_done_text}{Colors.yellow}{suffix}")
                else:
                    util.print_foreground_color(f"{job_done_text}{Colors.green}{suffix}")
                self.jobs_status[job_done_number-1] = job_done_return_code

            if not anything_done:
                time.sleep(0.25) # if nothing happens for an iteration, chill out a bit

        if total_jobs:
            if len(passed_jobs) == total_jobs:
                emojitxt = safe_emoji('😀', ':)')
                util.info(sprint_job_line(final=True, job_name="jobs passed") + f"< {emojitxt} >",
                           start="")
            else:
                emojitxt = safe_emoji('😦', ':(')
                # Make sure this error is reported (overall), use self.error(...):
                self.error(
                    (
                        sprint_job_line(final=True, job_name="jobs passed", info_str='') +
                        f"< {emojitxt} >"
                    ), do_exit=False
                )
        else:
            util.info(f"Parallel: <{safe_emoji('❓ ')}No jobs found>")
        # Make sure all jobs have a set status:
        for i, rc in enumerate(self.jobs_status):
            if rc is None or not isinstance(rc, int):
                self.error(f'job {i=} {rc=} did not return a proper return code')
                self.jobs_status[i] = 2

        # if self.status > 0, then keep it non-zero, else set it if we still have running jobs.
        if self.status == 0:
            if self.jobs_status:
                self.status = max(self.jobs_status)
            # else keep at 0, empty list.
        util.fancy_stop()

        # Report jobs that failed to run:
        if self.jobs:
            self.error(f'multi: jobs that failed to run {len(self.jobs)}:', do_exit=False)
            pprint(self.jobs)

        # If in debug/verbose, report jobs that did run:
        if util.args['verbose'] or util.args['debug']:
            if self.jobs_ran:
                util.info(f'multi: jobs run {len(self.jobs)}:')
                pprint(self.jobs_ran)

        # For jobs that did run, if there were errors, collect that in
        # our artifacts (via util.artifacts.job_errors_or_warnings):
        self.update_artifacts_with_jobs_ran()


    @staticmethod
    def get_name_from_target(target: str) -> str:
        '''Given a target path, strip leftmost path separators to get a shorter string name'''
        return target.replace('../', '').lstrip('./').lstrip(os.path.sep)


    def update_args_list(self, args: list, tool: str) -> None:
        '''Modfies list args, using allow-listed known top-level args:

        --config-yml
        --eda-safe
        --tool

        Many args were consumed by eda before CommandParallel saw them
        (for commands like 'multi' or 'sweep'). Some are in self.config.
         We need to apply those eda level args to each single exec-command

        Returns a dict of everything we added
        '''
        added = {}
        if any(a.startswith('--config-yml') for a in self.config['eda_original_args']):
            cfg_yml_fname = self.config.get('config-yml', None)
            if cfg_yml_fname:
                added['config-yml'] = cfg_yml_fname
                args.append(f'--config-yml={cfg_yml_fname}')
        if '--eda-safe' in self.config['eda_original_args']:
            added['eda-safe'] = ''
            args.append('--eda-safe')
        if tool:
            # tool can be None, if so we won't add it to the command (assumes default from
            # config-yml auto load order)
            added['tool'] = tool
            args.append('--tool=' + tool)
        return added


    def get_unparsed_args_on_single_command(self, command: str, tokens: list) -> list:
        '''Returns a list of args that the single (non-multi) command cannot parse

        This will error on bad --args or -arg, such as:
          eda multi sim --seeeed=1
        is not a valid arg in CommandSim

        +arg=value, +arg+value will not be included in the return list, those are
        intended to be consumed by the single/job command downstream (anything starting
        with '+')

        Used by CommandMulti and CommandSweep.
        '''
        single_cmd_handler = self.config['command_handler'][command](config=self.config)
        single_cmd_parsed, single_cmd_unparsed = single_cmd_handler.run_argparser_on_list(
            tokens=tokens.copy(),
            apply_parsed_args=False,
        )
        util.debug(f'{self.command_name}: {command=} (namespace) {single_cmd_parsed=}')
        util.debug(f'{self.command_name}: {command=} (unparsed) {single_cmd_unparsed=}')

        # There should not be any single_cmd_unparsed args starting with '-'
        bad_remaining_args = [x for x in single_cmd_unparsed if x.startswith('-')]
        if bad_remaining_args:
            self.warning_show_known_args(command=f'{self.command_name} {command}')
            self.error_ifarg(
                f'for {self.command_name} {command=} the following args are unknown',
                f'{bad_remaining_args}',
                arg='error-unknown-args',
                error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR
            )

        # Remove unparsed args starting with '+', since those are commonly sent downstream to
        # single job (example, CommandSim plusargs).
        return [x for x in single_cmd_unparsed if not x.startswith('+')]


    def patch_jobs_for_duplicate_target_names(self) -> None:
        '''Examines list self.jobs, and if leaf target names are duplicate will
        patch each command's job-name to:
            --job-name=path.leaf.command[.tool]

        Also do for --force-logfile
        '''

        def get_job_arg(job_dict: dict, arg_name: str) -> str:
            '''Fishes the arg_name out of an entry in self.jobs'''
            for i, item in enumerate(job_dict['command_list']):
                if item.startswith(f'--{arg_name}='):
                    _, name = item.split(f'--{arg_name}=')
                    return name
                if item == f'--{arg_name}':
                    return job_dict['command_list'][i + 1]
            return ''

        def replace_job_arg(job_dict: dict, arg_name: str, new_value: str) -> bool:
            '''Replaces the arg_name's value in an entry in self.jobs'''
            for i, item in enumerate(job_dict['command_list']):
                if item.startswith(f'--{arg_name}='):
                    job_dict['command_list'][i] = f'--{arg_name}=' + new_value
                    return True
                if item == f'--{arg_name}':
                    job_dict['command_list'][i + 1] = new_value
                    return True
            return False


        job_names_count_dict = {}
        for job_dict in self.jobs:

            key = job_dict.get('job-name', '')
            if not key:
                self.error(f'{job_dict=} needs to have a --job-name= arg attached',
                           error_code=status_constants.EDA_COMMAND_OR_ARGS_ERROR)
            if key not in job_names_count_dict:
                job_names_count_dict[key] = 1
            else:
                job_names_count_dict[key] += 1

        for i, job_dict in enumerate(self.jobs):
            key = job_dict.get('job-name', '')
            if job_names_count_dict[key] < 2:
                continue

            tpath, _ = os.path.split(job_dict['target'])

            # prepend path information to job-name:
            patched_target_path = os.path.relpath(tpath).replace(os.sep, '_').lstrip('.')
            if patched_target_path:
                new_job_name = f'{patched_target_path}.{key}'
            else:
                continue # there's nothing to "patch", our job-name will be unchanged.

            replace_job_arg(job_dict, arg_name='job-name', new_value=new_job_name)
            job_dict['job-name'] = new_job_name

            # prepend path information to force-logfile (if present):
            force_logfile = get_job_arg(job_dict, arg_name='force-logfile')
            if force_logfile:
                left, right = os.path.split(force_logfile)
                new_force_logfile = os.path.join(left, f'{patched_target_path}.{right}')
                replace_job_arg(
                    job_dict, arg_name='force-logfile', new_value=new_force_logfile
                )
                util.debug(
                    f'Patched job {job_dict["name"]}: --force-logfile={new_force_logfile}'
                )


            self.jobs[i] = job_dict
            util.debug(f'Patched job {job_dict["name"]}: --job-name={new_job_name}')
