''' opencos.deps.deps_processor -- module is less about "parsing" and more about "processing"

a DEPS markup files targets (applying deps, reqs, commands, tags, incdirs, defines, etc) to a
CommandDesign ref object
'''

# pylint: disable=too-many-lines

import argparse
import copy
import glob
import os
from pathlib import Path

from opencos import files
from opencos import eda_config
from opencos.util import Colors, debug, info, warning, error, read_tokens_from_dot_f, \
    patch_args_for_dir, load_env_file
from opencos.utils import status_constants
from opencos.utils.str_helpers import dep_str2list
from opencos.utils.dict_helpers import dict_diff
from opencos.deps.deps_file import deps_target_get_deps_list
from opencos.deps.deps_commands import deps_commands_handler

from opencos.deps.defaults import SUPPORTED_TARGET_TABLE_KEYS, SUPPORTED_TAG_KEYS, \
    SUPPORTED_DEP_KEYS_BY_TYPE

class DepsProcessor: # pylint: disable=too-many-instance-attributes,too-many-public-methods
    '''DepsProcessor -- called by eda_base resolve_target_core(..)

    example usage:
    my_dp = DepsProcessor(
        command_design_ref = self, # aka, your CommandDesign obj
        deps_entry = <some-table-found-in-deps-data>,
        target = <target-str-we-were-looking-for>
        target_path = <filepath, for debug>
        target_node = <target leaf str, for debug>
        deps_file = <original DEPS.[markup-ext] for debug>
        caller_info = <str info for debug>
    )

    '''

    def __init__(self, command_design_ref, deps_entry: dict, target: str,
                 target_path: str, target_node: str, deps_file: str, caller_info: str):
        '''
        command_design_ref (eda.CommandDesign),
        deps_entry (dict, target in DEPS.yml file)
        target_node (str) -- key in DEPS.yml that got us the deps_entry, used for debug
        deps_file (str) -- file, used for debug
        caller_info (str) -- used for debug
        '''

        self.command_design_ref = command_design_ref
        self.deps_entry = deps_entry
        self.target = target
        self.target_path = target_path
        self.target_node = target_node # for debug
        self.deps_file = deps_file # for debug
        self.deps_dir, _ = os.path.split(deps_file)
        self.caller_info = caller_info

        # Check if it's a Command instead of CommandDesign:
        self.is_command_design = bool(
            getattr(command_design_ref, 'process_plusarg', None) and \
            getattr(command_design_ref, 'set_parameter', None) and \
            isinstance(getattr(command_design_ref, 'incdirs', None), list)
        )


        assert isinstance(deps_entry, dict), \
            f'{deps_entry=} for {target_node=} in {deps_file=} must be a dict'
        assert command_design_ref is not None, \
            'called DepsProcessor.__init__, but no ref to CommandDesign object (is None)'

        # named eda commands in the target:
        # If this deps_entry has a 'sim', 'build', etc command entry for this target, grab that
        # because it can set defines or other things specific to an eda command ('sim', for example)
        self.entry_eda_command = self.deps_entry.get(command_design_ref.command_name, {})

        # alias some of the self.command_design_ref values
        self.command_name = self.command_design_ref.command_name # str, for debug
        self.args         = self.command_design_ref.args         # dict
        self.config       = self.command_design_ref.config       # dict
        self.set_arg      = self.command_design_ref.set_arg      # method
        self.error        = self.command_design_ref.error        # method.

        # If there are expanded eda commands in
        # self.command_design_ref.config['command_handler'].keys(), then make note of that now.
        self.known_eda_commands = getattr(
            self.command_design_ref, 'config', {}
        ).get('command_handler', {}).keys()

        self._is_ok_deps_field_type(
            item=self.entry_eda_command, want_types=dict, name=self.command_name,
            msg='command entry in target'
        )


    def _defines_as_dict(self, defines) -> dict:
        '''Converts a dict/list/str defines obj to dict'''
        if isinstance(defines, dict):
            return defines

        if isinstance(defines, list):
            as_list = defines
        elif isinstance(defines, str):
            as_list = defines.split()
        else:
            assert False, \
                f'unknown type for at {self.caller_info=}: {defines=} {type(defines)=}'

        ret = {}
        for x in as_list:
            if x:
                if x.startswith('+define+'):
                    x = x[8:]
                parts = x.split('=')
                if parts[1:]:
                    ret[parts[0]] = '='.join(parts[1:])
                else:
                    ret[parts[0]] = None
        return ret


    def _plusargs_as_dict(self, plusargs) -> dict:
        '''Converts a dict/list/str plusargs obj to dict'''
        if isinstance(plusargs, dict):
            return plusargs

        if isinstance(plusargs, list):
            as_list = plusargs
        elif isinstance(plusargs, str):
            as_list = plusargs.split()
        else:
            assert False, \
                f'unknown type for at {self.caller_info=}: {plusargs=} {type(plusargs)=}'

        ret = {}
        for x in as_list:
            if x:
                if x.startswith('+'):
                    x = x[1:]
                parts = x.split('=')
                if parts[1:]:
                    ret[parts[0]] = '='.join(parts[1:])
                else:
                    ret[parts[0]] = ''
        return ret


    def _parameters_as_dict(self, parameters) -> dict:
        '''Converts a dict/list/str parameters obj to dict'''
        if isinstance(parameters, dict):
            return parameters

        if isinstance(parameters, list):
            as_list = parameters
        elif isinstance(parameters, str):
            as_list = parameters.split()
        else:
            assert False, \
                f'unknown type for at {self.caller_info=}: {parameters=} {type(parameters)=}'

        ret = {}
        for x in as_list:
            if x:
                if x.startswith('-G'):
                    x = x[2:]
                parts = x.split('=')
                if parts[1:]:
                    ret[parts[0]] = '='.join(parts[1:])
                else:
                    self.error(
                        f'At {self.caller_info}: parameter entry (non-Table, as markup',
                        f'Array/string) is not in the form Name=Value: "{x}"',
                       '\n    To see detailed DEPS schema, run:',
                       f'{Colors.byellow}eda deps-schema [--verbose]',
                       error_code=status_constants.EDA_DEPS_BAD_TYPE
                    )
        return ret


    def apply_defines(self, defines_dict: dict, check: bool = True) -> None:
        '''Given defines_dict, applies them to our self.command_design_ref obj'''
        if not self.is_command_design or not defines_dict:
            return
        if check and not self._is_ok_deps_field_type(defines_dict, (dict, list, str),
                                                     name='defines'):
            return
        if not isinstance(defines_dict, dict):
            defines_dict = self._defines_as_dict(defines_dict)

        for k,v in defines_dict.items():
            if v is None or v == '':
                self.command_design_ref.process_plusarg(f'+define+{k}')
            else:
                # %PWD% and %SEED% substiutions:
                if v and isinstance(v, str):
                    if v.startswith('%PWD%/') or v.startswith('"%PWD%/'):
                        v = v.replace('%PWD%', os.path.abspath(self.target_path))
                    if v.startswith('%SEED%') or v.startswith('"%SEED%'):
                        v = v.replace('%SEED%', str(self.args.get('seed', 1)))
                    # ENV var replacement:
                    v = os.path.expandvars(v)
                self.command_design_ref.process_plusarg(f'+define+{k}={v}')


    def apply_plusargs(self, plusargs_dict: dict, check: bool = True) -> None:
        '''Given plusarsg_dict, applies them to our self.command_design_ref obj'''
        if not self.is_command_design or not plusargs_dict:
            return
        if check and not self._is_ok_deps_field_type(plusargs_dict, (dict, list, str),
                                                     name='plusargs'):
            return

        if not isinstance(plusargs_dict, dict):
            plusargs_dict = self._plusargs_as_dict(plusargs_dict)

        for k,v in plusargs_dict.items():
            if v is None or v == '':
                self.command_design_ref.process_plusarg(f'+{k}')
            else:
                self.command_design_ref.process_plusarg(f'+{k}={v}')


    def apply_parameters(self, parameters_dict: dict, check: bool = True) -> None:
        '''Given parameters_dict, applies them to our self.command_design_ref obj'''
        if not self.is_command_design:
            return
        if check and not isinstance(parameters_dict, dict):
            self.error(f"{parameters_dict=} is not type dict, can't apply defines,",
                       f"in {self.caller_info}",
                       '\n    To see detailed DEPS schema, run:',
                       f'{Colors.byellow}eda deps-schema [--verbose]',
                       error_code=status_constants.EDA_DEPS_BAD_TYPE)
        for k,v in parameters_dict.items():
            if v is None or v == '' or not isinstance(v, (int, str, bool)):
                warning(f'parameter {k} has value: {v}, parameters must be bool/int/string types',
                        f'from {self.caller_info}')
                continue

            # since we are dealing with markup or a split markup string, we need to check
            # for integral types that may have been wrapped in strings:
            if isinstance(v, str):
                try:
                    v = int(v, 0)
                except ValueError:
                    pass
            self.command_design_ref.set_parameter(
                name=k, value=v, caller_info=self.caller_info
            )


    def apply_incdirs(self, incdirs_list:list) -> None:
        '''Given incdirs_list, applies them to our self.command_design_ref obj'''
        if not self.is_command_design:
            return
        if not isinstance(incdirs_list, (str, list)):
            self.error(f"{incdirs_list=} is not type str/list, can't apply incdirs",
                       f"in {self.caller_info}",
                       '\n    To see detailed DEPS schema, run:',
                       f'{Colors.byellow}eda deps-schema [--verbose]',
                       error_code=status_constants.EDA_DEPS_BAD_TYPE)

        incdirs_list = dep_str2list(incdirs_list)
        for x in incdirs_list:
            # Support both relative path to this DEPS file (current target) AND
            # if there is ENV var replacement. Try ENV var replacement first
            x_expanded = os.path.expandvars(x)
            if x_expanded and Path(x_expanded).is_absolute():
                # If abspath already, don't join it with DEPS file path:
                abspath = str(Path(x_expanded).resolve())
            elif x_expanded:
                # It's relative to DEPS file:
                abspath = os.path.abspath(os.path.join(self.target_path, x))
            else:
                # else, don't add blank incdirs
                abspath = ''

            if abspath and not os.path.exists(abspath):
                # Do warn if abspath doesn't exist
                warning(f'Unable to resolve incdir "{x}" ({abspath} does not exist)',
                        f'from {self.caller_info}')
            elif abspath and abspath not in self.command_design_ref.incdirs:
                self.command_design_ref.incdirs.append(abspath)
                debug(f'Added include dir "{abspath}" from {self.caller_info}')


    def _apply_args_check_tools(self, tokens: list, tagname: str) -> list:
        '''Helper for apply_args(list), returns list strips --tool args under certain conditions

        Basically, we want to see if a DEPS target want to set arg --tool, if so:
        Accept it:
          - this is not a tool class
          - tool was automatically chosen by eda.py's auto-tool-order (--tool not set at
             CLI)
          - accepting means set self.args['tool'], so we can respawn with this.
        Reject/Warn if:
          - previous tool was not automatically applied (--tool was set at CLI)
            and you're trying to change the --tool value.
          - This can happen if you have complicated DEPS targets trying to set the
            tool to different values.
        '''

        parser = argparse.ArgumentParser(
            prog='deps_processor --tool', add_help=False, allow_abbrev=False
        )
        parser.add_argument('--tool', default='')
        try:
            parsed, unparsed = parser.parse_known_args(tokens + [''])
            tokens2 = list(filter(None, unparsed))
        except argparse.ArgumentError:
            error('deps_processor --tool problem attempting to parse_known_args for:',
                  f'{tokens}, {self.caller_info} {tagname}')
            tokens2 = tokens

        _tool_class = 'tool' in self.args
        _orig_tool = self.args.get('tool', '')

        if not self.command_design_ref.auto_tool_applied and \
           _tool_class and parsed.tool and _orig_tool \
           and parsed.tool != _orig_tool:
            # tool arg present, --tool in this DEPS args, and tool already set.
            warning(
                f'Attempting to set --tool {parsed.tool} from DEPS',
                f'(file={self.deps_file}:{self.target_node})',
                f'however the tool was already chosen as: {_orig_tool}. The --tool arg will',
                f'not be applied from: {tokens}'
            )
        elif (self.command_design_ref.auto_tool_applied or not _tool_class) and parsed.tool:
            # tool arg wasn't present (not a Tool class), or it was auto-applied,
            # then add the arg anyway so we can later respawn with the correct tool.
            self.args['tool'] = parsed.tool
            debug(f'setting arg.tool to {parsed.tool=} from {self.caller_info}')

        # remove blanks, '--tool[=value| value]' removed.
        return [item for item in tokens2 if item != '']


    def apply_args( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
            self, args_list:list, tagname: str = ''
    ) -> list:
        '''Given args_list, applies them to our self.command_design_ref obj

        This will return unparsed args that weren't in the self.command_design_ref.args keys
        unparsed args will show up as eda.py warnings, but will not fail. Most callers do not
        use the unparsed args from this method.
        '''
        if tagname:
            tagname = f'{tagname=}'

        if not isinstance(args_list, (str, list)):
            self.error(f"{args_list=} is not type str/list, can't apply args",
                       f"in {self.caller_info} {tagname}",
                       '\n    To see detailed DEPS schema, run:',
                       f'{Colors.byellow}eda deps-schema [--verbose]',
                       error_code=status_constants.EDA_DEPS_BAD_TYPE)

        prev_args = copy.deepcopy(self.args)

        tokens = dep_str2list(args_list)

        # patch args relative to the DEPS (if self.deps_dir exists) so things like
        # --build-tcl=<file> for relative <file> works when calling targets from any directory.
        tokens = patch_args_for_dir(
            tokens=tokens, patch_dir=self.deps_dir, caller_info=self.caller_info
        )

        # We're going to run an ArgumentParser here, which is not the most efficient
        # thing to do b/c it runs on all of self.command_design_ref.args (dict) even
        # if we're applying a single token.

        # Since some args (util.py, eda_config.py, eda.py) can only be handled from command
        # line, it would be nice if -f or --input-file is handled from DEPS, so we'll special
        # case that now. Recursively resolve -f / --input-file.
        # Do similary for --env-file (also only supported in util.py)
        parser = argparse.ArgumentParser(
            prog='deps_processor -f/--input-file', add_help=False, allow_abbrev=False
        )
        parser.add_argument('--env-file', default=[], action='append',
                            help=(
                                "dotenv file(s) to pass ENV vars, (default: .env loaded first,"
                                " subsequent files' vars override .env"
                            ))
        parser.add_argument('-f', '--input-file', default=[], action='append',
                            help=(
                                'Input .f file to be expanded as eda args, defines, incdirs,'
                                ' files, or targets.'
                            ))
        try:
            parsed, unparsed = parser.parse_known_args(tokens + [''])
            tokens2 = list(filter(None, unparsed))
        except argparse.ArgumentError:
            error('deps_processor -f/--input-file, problem attempting to parse_known_args for:',
                  f'{tokens}, {self.caller_info} {tagname}')
            tokens2 = tokens

        if parsed.input_file:
            dotf_tokens = []
            for filepath in parsed.input_file:
                # Since this isn't command line, we have to assume the path is relative
                # to this DEPS file.
                if not os.path.isabs(filepath):
                    filepath = os.path.join(self.deps_dir, filepath)
                dotf_tokens.extend(read_tokens_from_dot_f(
                    filepath=filepath, caller_info=self.caller_info, verbose=True
                ))

            # put the .f files before the unparsed args.
            tokens2 = dotf_tokens + tokens2

            # recurse until we've resolved nested .f files.
            return self.apply_args(args_list=tokens2)

        if parsed.env_file:
            for env_file in parsed.env_file:
                load_env_file(env_file)

        # if no --input-file/--env-file values, keep parsing the remaining tokens2:
        tokens = tokens2

        # We have to special-case anything with --tool[=value] in tokens, otherwise
        # the user may think they were allowed to set --tool, but in our flow the Command handler
        # (self.command_design_ref) has already been chosen, so setting the tool can have
        # strange side-effects.
        _tool_class = 'tool' in self.args
        _orig_tool = self.args.get('tool', '')
        tokens = self._apply_args_check_tools(tokens=tokens, tagname=tagname)

        debug(f'deps_processor - custom apply_args with {tokens=}',
              f'from {self.caller_info} {tagname}')
        _, unparsed = self.command_design_ref.run_argparser_on_list(
            tokens=tokens
        )

        # Annoying, but check for plusargs in unparsed, and have referenced CommandDesign
        # or CommandSim class handle it with process_plusarg.
        for arg in list(unparsed):
            if arg.startswith('+') and self.is_command_design:
                self.command_design_ref.process_plusarg(plusarg=arg, pwd=self.target_path)
                unparsed.remove(arg)

        # For any leftover files, or targets, attempt to process those too:
        for arg in list(unparsed):
            # Since this isn't command line, we have to assume for files, the path is relative
            # to this DEPS file.
            target = self.correct_a_deps_target(target=arg, deps_dir=self.deps_dir)

            file_exists, fpath, forced_extension = files.get_source_file(target)
            if file_exists:
                _, file_ext = os.path.splitext(fpath)
                if forced_extension or file_ext:
                    self.command_design_ref.add_file(fpath, caller_info=self.caller_info,
                                                     forced_extension=forced_extension)
                    unparsed.remove(arg)

            else:
                if not os.path.isdir(target) and \
                   self.is_command_design and \
                   self.command_design_ref.resolve_target_core(
                       target=target, no_recursion=False, caller_info=self.caller_info,
                       error_on_not_found=False
                   ):
                    unparsed.remove(arg)

        if unparsed:
            # This is only a warning - because things like CommandFlist may not have every
            # one of their self.args.keys() set for a given target, such as a 'sim' target that
            # has --optimize, which is not an arg for CommandFlist. But we'd still like to get an
            # flist from that target.
            warning(f'For {self.command_design_ref.command_name}:' \
                    + f' in {self.caller_info} has unknown args {unparsed=}')

        if (self.command_design_ref.auto_tool_applied or not _tool_class) and \
            _orig_tool != self.args.get('tool', ''):
            # If there was an auto tool applied (tool class or not) then attempt to pick
            # a new sub-command-object with that tool.
            debug(f'deps_processor.apply_args: tool changed, {self.args["tool"]=}, will attempt',
                  f'to respawn the job using original args: {self.config["eda_original_args"]}')
            self.command_design_ref.tool_changed_respawn = {
                'tool': self.args['tool'],
                'orig_tool': _orig_tool,
                'from': self.caller_info,
            }

        diff_args = dict_diff(prev_args, self.args)
        if diff_args:
            args_list = [item for item in args_list if item != ''] # remove blanks
            info(f'{Colors.off}{Colors.yellow}{self.caller_info} {tagname}{Colors.info}:',
                 f'applying args for {args_list}: {Colors.cyan}{diff_args}')

        return unparsed

    def apply_reqs(self, reqs_list:list) -> None:
        '''Given reqs_list, applies them ot our self.command_design_ref obj'''
        for req in reqs_list:
            req_abspath = os.path.abspath(os.path.join(self.target_path, req))
            self.command_design_ref.add_file(
                req_abspath, use_abspath=False, add_to_non_sources=True,
                caller_info=self.caller_info
            )

    def process_deps_entry( # pylint: disable=too-many-branches
            self
    ) -> list:
        '''Main entry point (after creating DepsProcessor obj) to resolve a deps target

        Example usage:
        deps_processor = DepsProcessor(...)
        deps_targets_to_resolve = deps_processor.process_deps_entry()

        This will return a list of "deps" that haven't been traversed, but are needed by
        the deps_processor entry (the target we're trying to resolve).

        This method will apply all target features to the CommandDesign ref object as
        we traverse.

        NOTE: This does not yet support conditional inclusions based on defines,
         like the old DEPS files did with pattern:
            SOME_DEFINE ?  dep_if_define_present : dep_if_define_not_present
         I would like to deprecate that in favor of 'tags'. However, likely will need
         to walk the entire DEPS.yml once to populate all args/defines, and then re-
         walk them to add/prune the correct tag based dependencies, or rely on it being
         entirely top-down.
        '''

        # DEPS.yml entries have ordered keys, and process these in-order
        # with how the <target> defined it.
        remaining_deps_list = [] # deps items we find that are not yet processed.
        for key in self.deps_entry.keys():

            # Make sure DEPS target table keys are legal:
            if key not in SUPPORTED_TARGET_TABLE_KEYS and \
               key not in self.known_eda_commands:
                error(f'Unknown target {key=} in {self.caller_info},',
                      f' must be one of opencos.deps.defaults.{SUPPORTED_TARGET_TABLE_KEYS=}',
                      f' or an eda command: {self.known_eda_commands}')

            if key == 'tags' or 'tags' in self.entry_eda_command:
                remaining_deps_list += self.process_tags()
            elif key == 'defines' or 'defines' in self.entry_eda_command:
                self.process_defines()
            elif key == 'plusargs' or 'plusargs' in self.entry_eda_command:
                self.process_plusargs()
            elif key == 'parameters' or 'parameters' in self.entry_eda_command:
                self.process_parameters()
            elif key == 'incdirs' or 'incdirs' in self.entry_eda_command:
                self.process_incdirs()
            elif key == 'top' or 'top' in self.entry_eda_command:
                self.process_top()
            elif key == 'args' or 'args' in self.entry_eda_command:
                self.process_args()
            elif key == 'commands' or 'commands' in self.entry_eda_command:
                self.process_commands()
            elif key == 'reqs' or 'reqs' in self.entry_eda_command:
                self.process_reqs()

            elif any((key == 'deps', 'deps' in self.entry_eda_command,
                      (key == 'files' and 'deps' not in self.deps_entry),
                      ('files' in self.entry_eda_command and 'deps' not in self.entry_eda_command)
                      )):
                remaining_deps_list += self.process_deps_return_discovered_deps()

            if self.command_design_ref.tool_changed_respawn:
                # Stop now, and have eda.py respawn the command.
                return []

        # We return the list of deps that still need to be resolved (['full_path/some_target', ...])
        return remaining_deps_list

    def process_tags( # pylint: disable=too-many-statements,too-many-branches,too-many-locals
            self
    ) -> list:
        '''Returns List of added deps, applies tags (dict w/ details, if any) to
        self.command_design_ref.

        Tags are only supported as a Table within a target. Current we only support:
        'args', 'replace-config-tools', 'additive-config-tools', 'with-tools', 'with-args'.
        '''

        deps_tags_enables = self.config.get('dep_tags_enables', {})
        ret_deps_added_from_tags = []

        entry_tags = {} # from yml table
        entry_tags.update(self.deps_entry.get('tags', {}))
        for tagname,value in entry_tags.items():
            debug(f'process_tags(): {tagname=} in {self.caller_info}' \
                  + f' observed: {value=}')
            assert isinstance(value, dict), \
                f'{tagname=} {value=} value must be a dict for in {self.caller_info}'
            tags_dict_to_apply = value.copy()

            for key in value.keys():
                if key not in SUPPORTED_TAG_KEYS:
                    self.error(f'{tagname=} in {self.caller_info}:',
                               f'has unsupported {key=} {SUPPORTED_TAG_KEYS=}',
                               '\n    To see detailed DEPS schema, run:',
                               f'{Colors.byellow}eda deps-schema [--verbose]',
                               error_code=status_constants.EDA_DEPS_BAD_SCHEMA_KEY)

            enable_tags_matched = False
            disable_tags_matched = False
            if tagname in self.command_design_ref.args['enable-tags']:
                # tagname was force enabled by --enable-tags=tagname.
                debug(f'process_tags(): {tagname=} in {self.caller_info=}',
                      'will be enabled, matched in --enable-tags:',
                      f'{self.command_design_ref.args["enable-tags"]}')
                enable_tags_matched = True
            if tagname in self.command_design_ref.args['disable-tags']:
                # tagname was force disabled by --disable-tags=tagname.
                debug(f'process_tags(): {tagname=} in {self.caller_info=}',
                      'will be disabled, matched in disable-tags:',
                      f'{self.command_design_ref.args["disable-tags"]}')
                disable_tags_matched = True


            apply_tag_items_tools = False
            apply_tag_items_commands = False
            apply_tag_items_with_args = False

            tool = self.args.get('tool', None)

            if disable_tags_matched or enable_tags_matched:
                # skip checking with-tools or with-args, b/c we are already
                # force matched by tagname from --enable-tags or --disable-tags.
                pass
            else:

                with_tools = dep_str2list(value.get('with-tools', []))
                if with_tools and not deps_tags_enables.get('with-tools', None):
                    with_tools = []
                    warning(f'{tagname=} in {self.caller_info}:',
                            ' skipped due to with-tools disabled.')

                with_commands = dep_str2list(value.get('with-commands', []))
                if with_commands and not deps_tags_enables.get('with-commands', None):
                    with_commands = []
                    warning(f'{tagname=} in {self.caller_info}:',
                            ' skipped due to with-commands disabled.')

                with_args = value.get('with-args', {})
                if not isinstance(with_args, dict):
                    error(f'{tagname=} in {self.caller_info}:',
                          ' with-args must be a table (dict) of key-value pairs')
                if with_args and not deps_tags_enables.get('with-args', None):
                    with_args = {}
                    warning(f'{tagname=} in {self.caller_info}:',
                            ' skipped due to with-args disabled.')

                # check with-tools?
                if not with_tools:
                    apply_tag_items_tools = True # no with-tools present
                elif tool in with_tools:
                    apply_tag_items_tools = True # with-tools present and we matched.
                else:
                    # Each item of with-tools can also be in the form
                    # {tool (str)}:{TOOL.tool_version (str)}
                    # this matches Tool.get_full_tool_and_versions()
                    if getattr(self.command_design_ref, 'get_full_tool_and_versions', None):
                        tool_full_version = self.command_design_ref.get_full_tool_and_versions()
                        if tool_full_version and tool_full_version in with_tools:
                            apply_tag_items_tools = True

                # check with-commands?
                if not with_commands:
                    apply_tag_items_commands = True # no with-commands present
                elif getattr(self.command_design_ref, 'command_name', '') in with_commands:
                    apply_tag_items_commands = True # with-commands present and we matched.

                # check with-args?
                with_args_matched_list = []
                for k,v in with_args.items():
                    with_args_matched_list.append(False)
                    if not all((apply_tag_items_tools, apply_tag_items_commands)):
                        # If we didn't previously match with-tools (if with-tools was present),
                        # or match with-commands, then we may not match the args, b/c those are
                        # command + tool dependent in the Command handling class.
                        pass
                    elif k not in self.command_design_ref.args:
                        warning(f'{tagname=} in {self.caller_info}:',
                                f'with-args key {k} is not a valid arg for {tool=}')
                    elif not isinstance(v, type(self.command_design_ref.args[k])):
                        warning(f'{tagname=} in {self.caller_info}:',
                                f' with-args table key {k} value {v} (type {type(v)}) does not',
                                f' match type in args (type {self.command_design_ref.args[k]})')
                    elif self.command_design_ref.args[k] == v:
                        # set it as matched:
                        with_args_matched_list[-1] = True
                        debug(f'{tagname=} in {self.caller_info}:',
                              f' with-args table key {k} value {v} matched')
                    else:
                        debug(f'{tagname=} in {self.caller_info}:',
                              f'with-args table key {k} value {v} did not match args value: ',
                              f'{self.command_design_ref.args[k]}')

                if not with_args_matched_list:
                    apply_tag_items_with_args = True # no with-args set
                else:
                    apply_tag_items_with_args = all(with_args_matched_list)

            # Did we match all with-tools and with-args?
            if disable_tags_matched:
                apply_tag_items = False
            elif enable_tags_matched:
                apply_tag_items = True
            else:
                apply_tag_items = all([apply_tag_items_tools, apply_tag_items_with_args,
                                       apply_tag_items_commands])

            if not apply_tag_items:
                debug(f'process_tags(): {tagname=} in {self.caller_info}',
                      f'skipped for {tool=}, {with_args=}, {with_args_matched_list=}')
            elif apply_tag_items_tools or apply_tag_items_with_args:
                debug(f'process_tags(): {tagname=} in {self.caller_info=}',
                      f'applying tags for {tool=}, {with_args=}, {with_args_matched_list=},',
                      f'{tags_dict_to_apply.keys()=}')


            if apply_tag_items:
                # We have matched something (with-tools, etc).
                # apply these in the original order of the keys:
                for key in tags_dict_to_apply.keys():

                    if key == 'defines':
                        # apply defines:
                        self.apply_defines(value.get('defines', {}))

                    if key == 'plusargs':
                        # apply plusargs:
                        self.apply_plusargs(value.get('plusargs', {}))

                    elif key == 'parameters':
                        self.apply_parameters(value.get('parameters', {}))

                    elif key == 'incdirs':
                        # apply incdirs:
                        self.apply_incdirs(value.get('incdirs', []))

                    elif key == 'args':
                        # apply args
                        args_list = dep_str2list(value.get('args', []))
                        if args_list and not deps_tags_enables.get('args', None):
                            args_list = []
                            warning(f'{tagname=} in {self.caller_info=}:',
                                    ' skipped args due to args disabled.')
                        if args_list:
                            # This will apply knowns args to the target dep:
                            debug(f'{tagname=} in {self.caller_info=}:',
                                 f'applying args for {args_list=}')
                            self.apply_args(args_list, tagname=tagname)

                    elif key == 'reqs':
                        reqs_list = deps_target_get_deps_list(entry=value,
                                                              default_key='reqs',
                                                              target_node=self.target_node,
                                                              deps_file=self.deps_file)
                        self.apply_reqs(reqs_list)

                    elif key == 'deps' or \
                         (key == 'files' and 'deps' not in tags_dict_to_apply):

                        # apply deps (includes commands, stray +define+ +incdir+)
                        # treat the same way we treat self.process_deps_return_discovered_deps
                        deps_list = deps_target_get_deps_list(entry=value,
                                                              default_key='deps',
                                                              target_node=self.target_node,
                                                              deps_file=self.deps_file)
                        ret_deps_added_from_tags += self.get_remaining_and_apply_deps(deps_list)

                # for replace-config-tools or additive-config-tools from tags, these don't need to
                # handle in order of tags keys:

                # apply replace-config-tools
                # This will replace lists (compile-waivers).
                tool_config = value.get('replace-config-tools', {}).get(tool, None)
                ref_has_tool_config = isinstance(
                    getattr(self.command_design_ref, 'tool_config', None), dict
                )
                if tool_config and (not deps_tags_enables.get('replace-config-tools', None) or \
                                    not ref_has_tool_config):
                    tool_config = None
                    warning(f'{tagname=} in {self.caller_info}:',
                            'skipped replace-config-tools b/c it is disabled or not present for',
                            'this tool and command')
                if ref_has_tool_config and tool_config and isinstance(tool_config, dict):
                    # apply it to self.tool_config:
                    info(f'{tagname=} in {self.caller_info}:',
                         f'applying replace-config-tools for {tool=}: {tool_config}')
                    eda_config.merge_config(self.command_design_ref.tool_config, tool_config)
                    # Since we altered command_design_ref.tool_config, need to call update on it:
                    self.command_design_ref.update_tool_config()
                    debug(f'{tagname=} in {self.caller_info}:',
                          'Updated {self.command_design_ref.tool_config=}')

                # apply additive-config-tools
                # This will append to lists (compile-waivers)
                tool_config = value.get('additive-config-tools', {}).get(tool, None)
                if tool_config and (not deps_tags_enables.get('additive-config-tools', None) or \
                                    not ref_has_tool_config):
                    tool_config = None
                    warning(f'{tagname=} in {self.caller_info}:',
                            ' skipped additive-config-tools b/c it is disable or not present for',
                            'this tool and command')
                if ref_has_tool_config and tool_config and isinstance(tool_config, dict):
                    # apply it to self.tool_config:
                    info(f'{tagname=} in {self.caller_info}:',
                         f'applying additive-config-tools for {tool=}: {tool_config}')
                    eda_config.merge_config(self.command_design_ref.tool_config, tool_config,
                                            additive_strategy=True)
                    # Since we altered command_design_ref.tool_config, need to call update on it:
                    self.command_design_ref.update_tool_config()
                    debug(f'{tagname=} in {self.caller_info}:',
                          f'Updated {self.command_design_ref.tool_config=}')

        return ret_deps_added_from_tags

    def _is_ok_deps_field_type(
            self, item, want_types, name: str, must_be: str = '', msg: str = '',
            error_if_item_empty: bool = False
    ) -> bool:
        '''Returns false if we don't like the the DEPS markup entry, with error message'''
        if not isinstance(item, want_types):
            if msg:
                msg = f'{msg}, '
            if not must_be:
                if want_types == (dict, list, str):
                    must_be = (
                        'markup Table/dict, Array/list, or string (space or newline separated)'
                    )
                elif want_types == (list, str):
                    must_be = 'markup Array/list, or string (space or newline separated)'
                elif want_types is dict:
                    must_be = 'markup Table/dict'
                elif want_types is list:
                    must_be = 'markup Array/list'
                elif want_types is str:
                    must_be = 'markup string (space or newline separated)'
                else:
                    assert False, f'bad {want_types=}'

            # Since all checked types here are dict, list, str, if the value is empty, allow it and
            # move on (caller shouldn't apply it)
            if not item and not error_if_item_empty:
                # item empty, no error if empty:
                return True

            self.error(f'At {self.caller_info}: {msg}"{name}:" entry must be a {must_be},',
                       f'but instead got: type={type(item)}, {name}: {item}.',
                       '\n    To see detailed DEPS schema, run:',
                       f'{Colors.byellow}eda deps-schema [--verbose]',
                       error_code=status_constants.EDA_DEPS_BAD_TYPE)
            return False

        return True


    def process_defines(self) -> None:
        '''Returns None, applies defines (dict, if any) from self.deps_entry to
        self.command_design_ref.'''

        # Defines:
        # apply command specific defines, with higher priority than the a
        # deps_entry['sim']['defines'] entry,
        # do this with dict1.update(dict2):
        entry_defines = {}
        _got_defines = self.deps_entry.get('defines', {})
        _got_cmd_defines = self.entry_eda_command.get('defines', {})

        if not self._is_ok_deps_field_type(_got_defines, (dict, list, str), name='defines'):
            return
        if not self._is_ok_deps_field_type(
                _got_cmd_defines, (dict, list, str), name='defines',
                msg=f'command entry in target ("{self.command_name}:")'
        ):
            return

        entry_defines.update(self._defines_as_dict(_got_defines))
        entry_defines.update(self._defines_as_dict(_got_cmd_defines))

        self.apply_defines(entry_defines, check=False)


    def process_plusargs(self) -> None:
        '''Returns None, applies plusargs (dict, if any) from self.deps_entry to
        self.command_design_ref.

        These work w/ the same rules as defines (no value, or value int/str)
        '''

        # Plusargs:
        # apply command specific plusargs, with higher priority than the a
        # deps_entry['sim']['plusargs'] entry,
        # do this with dict1.update(dict2):
        entry_plusargs = {}
        _got_plusargs = self.deps_entry.get('plusargs', {})
        _got_entry_eda_command_plusargs = self.entry_eda_command.get('plusargs', {})

        if not self._is_ok_deps_field_type(_got_plusargs, (dict, list, str),
                                           name='plusargs'):
            return
        if not self._is_ok_deps_field_type(
                _got_entry_eda_command_plusargs, (dict, list, str), name='plusargs',
                msg=f'command entry in target ("{self.command_name}:")'
        ):
            return

        if _got_plusargs:
            entry_plusargs.update(self._plusargs_as_dict(_got_plusargs))
        if _got_entry_eda_command_plusargs:
            entry_plusargs.update(self._plusargs_as_dict(_got_entry_eda_command_plusargs))
        assert isinstance(entry_plusargs, dict), \
            f'{entry_plusargs=} for in {self.caller_info} must be a dict'

        self.apply_plusargs(entry_plusargs, check=False)


    def process_parameters(self) -> None:
        '''Returns None, applies parameters (dict, if any) from self.deps_entry to
        self.command_design_ref.'''

        # Parameters:
        # apply command specific parameters, with higher priority than the a
        # deps_entry['sim']['parameters'] entry,
        # do this with dict1.update(dict2):
        entry_parameters = {}
        _got_params = self.deps_entry.get('parameters', {})
        _got_entry_eda_command_params = self.entry_eda_command.get('parameters', {})

        if not self._is_ok_deps_field_type(_got_params, (dict, list, str),
                                           name='parameters'):
            return
        if not self._is_ok_deps_field_type(
                _got_entry_eda_command_params, (dict, list, str), name='parameters',
                msg=f'command entry in target("{self.command_name}:")'
        ):
            return

        if _got_params:
            entry_parameters.update(self._parameters_as_dict(_got_params))
        if _got_entry_eda_command_params:
            entry_parameters.update(self._parameters_as_dict(_got_entry_eda_command_params))
        assert isinstance(entry_parameters, dict), \
            f'{entry_parameters=} for in {self.caller_info} must be a dict'

        self.apply_parameters(entry_parameters, check=False)


    def process_incdirs(self) -> None:
        '''Returns None, applies incdirs (dict, if any) from self.deps_entry to
        self.command_design_ref.'''

        entry_incdirs = []
        _got_incdirs = self.entry_eda_command.get('incdirs', [])
        _got_entry_eda_command_incdirs = self.deps_entry.get('incdirs', [])
        if not self._is_ok_deps_field_type(
                _got_incdirs, want_types=(list, str), name='incdirs',
        ):
            return
        if not self._is_ok_deps_field_type(
                _got_entry_eda_command_incdirs, want_types=(list, str),
                name='incdirs', msg=f'command entry in target ("{self.command_name}:")'
        ):
            return

        # apply command specific incdirs, higher in the incdir list:
        if _got_incdirs:
            entry_incdirs = dep_str2list(_got_incdirs)
        if _got_entry_eda_command_incdirs:
            entry_incdirs += dep_str2list(_got_entry_eda_command_incdirs)
        assert isinstance(entry_incdirs, list), \
            f'{entry_incdirs=} for in {self.caller_info} must be a list'
        self.apply_incdirs(entry_incdirs)

    def process_top(self) -> None:
        '''Returns None, applies top (str, if any) from self.deps_entry to
        self.command_design_ref.'''

        if self.args['top'] != '':
            return # already set

        _got_top = self.deps_entry.get('top', '')            # target.top
        _got_cmd_top = self.entry_eda_command.get('top', '') # target.sim.top for example
        if not self._is_ok_deps_field_type(_got_top, str, name='top'):
            return
        if not self._is_ok_deps_field_type(_got_cmd_top, str, name='top',
                                          msg=f'command entry in target ("{self.command_name}:")'):
            return

        # For 'top', we overwrite it if not yet set.
        # the command specific 'top' has higher priority.
        entry_top = _got_cmd_top
        if entry_top == '':
            entry_top = _got_top

        if entry_top != '':
            if self.args['top'] == '':
                # overwrite only if unset - we don't want other deps overriding the topmost
                # target's setting for 'top'.
                self.set_arg('top', str(entry_top))

    def process_args(self) -> None:
        '''Returns None, applies args (list or str, if any) from self.deps_entry to
        self.command_design_ref.'''

        # for 'args', process each. command specific args take higher priority that target args.
        # run_argparser_on_list: uses argparse, which takes precedence on the last arg that is set,
        # so put the command specific args last.
        # Note that if an arg is already set, we do NOT update it
        _got_args = self.deps_entry.get('args', [])
        _got_cmd_args = self.entry_eda_command.get('args', [])
        if not self._is_ok_deps_field_type(
                _got_args, want_types=(list, str), name='args'
        ):
            return
        if not self._is_ok_deps_field_type(
                _got_cmd_args, want_types=(list, str), name='args',
                msg=f'command entry in target ("{self.command_name}:")'
        ):
            return

        args_list = []
        if _got_args:
            args_list = dep_str2list(_got_args)
        if _got_cmd_args:
            args_list += dep_str2list(_got_cmd_args)

        # for args_list, re-parse these args to apply them to self.args.
        if not args_list:
            return

        debug(f'in {self.caller_info}: {args_list=}')
        self.apply_args(args_list)

        # TODO(drew): Currently, I can't support changing the 'config' via an arg encountered in
        # DEPS.yml. This is prevented b/c --config-yml appears as a modifed arg no matter what
        # (and we don't let DEPS.yml override modifed args, otherwise a target would override the
        # user command line).


    def get_commands( # pylint: disable=dangerous-default-value
            self, commands: list = [], dep: str = ''
    ) -> (list, list):
        '''Returns tuple of (shell_commands_list, work_dir_add_srcs_list).

        Does not have side effects on self.command_design_ref.
        '''

        default_ret = [], []

        if not commands:
            # if we weren't passed commands, then get them from our target (self.deps_entry)
            commands = self.deps_entry.get('commands', [])

        if not self._is_ok_deps_field_type(commands, list, name='commands'):
            return default_ret

        if not commands: # No commands in this target
            return default_ret

        debug(f"Got {self.deps_entry=} for in {self.caller_info}, has {commands=}")
        shell_commands_list = [] # list of dicts
        work_dir_add_srcs_list = [] # list of dicts

        if not dep:
            # if we weren't passed a dep, then use our target_node (str key for our self.deps_entry)
            dep = self.target_node

        # Run handler for this to convert to shell commands in self.command_design_ref
        shell_commands_list, work_dir_add_srcs_list = deps_commands_handler(
            config=self.command_design_ref.config,
            eda_args=self.command_design_ref.args,
            dep=dep,
            deps_file=self.deps_file,
            target_node=self.target_node,
            target_path=self.target_path,
            commands=commands
        )

        return shell_commands_list, work_dir_add_srcs_list

    def process_commands( # pylint: disable=dangerous-default-value
            self, commands: list = [], dep: str = ''
    ) -> None:
        '''Returns None, handles commands (shell, etc) in the target that aren' in the 'deps' list.

        Applies these to self.command_design_ref.

        You can optionally call this with a commands list and a single dep, which we support for
        commands lists that exist within the 'deps' entry of a target.
        '''

        shell_commands_list, work_dir_add_srcs_list = self.get_commands(commands=commands, dep=dep)

        if shell_commands_list and \
           not self.is_command_design:
            warning(f'Not applying shell commands from {self.caller_info}, not supported',
                    'for this tool and command')
            return

        # add these commands lists to self.command_design_ref:
        # Process all shell_commands_list:
        # This will track each shell command with its target_node and target_path
        self.command_design_ref.append_shell_commands( cmds=shell_commands_list )
        # Process all work_dir_add_srcs_list:
        # This will track each added filename with its target_node and target_path
        self.command_design_ref.append_work_dir_add_srcs( add_srcs=work_dir_add_srcs_list,
                                                          caller_info=self.caller_info )


    def process_reqs(self) -> None:
        '''Process any 'reqs:' table in a DEPS markup entry'''
        reqs_list = deps_target_get_deps_list(entry=self.deps_entry,
                                              default_key='reqs',
                                              target_node=self.target_node,
                                              deps_file=self.deps_file)
        if not self._is_ok_deps_field_type(reqs_list, want_types=(list, str), name='reqs'):
            return
        if reqs_list:
            self.apply_reqs(reqs_list)


    def process_deps_return_discovered_deps(self) -> list:
        '''Returns list of deps targets to continue processing,

        -- iterates through 'deps' for this target (self.deps_entry['deps'])
           -- note this will also append 'files' if that table key + list/str exists.
        -- applies to self.command_design_ref
        '''

        # Get the list of deps from this entry (entry is a target in our DEPS.yml):
        deps = deps_target_get_deps_list(
            self.deps_entry,
            target_node=self.target_node,
            deps_file=self.deps_file
        )
        if not self._is_ok_deps_field_type(deps, want_types=(dict, list, str), name='deps'):
            return []
        return self.get_remaining_and_apply_deps(deps)

    def get_remaining_and_apply_deps(
            self, deps: list
    ) -> list:
        '''Given a list of deps, process what is supported in a "deps:" table in DEPS
        markup file.'''

        deps_targets_to_resolve = []

        # Process deps (list)
        for dep in deps:
            deps_targets_to_resolve.extend(
                self._get_remaining_and_apply_single_dep(dep)
            )
        return deps_targets_to_resolve


    def _get_remaining_and_apply_single_dep( # pylint: disable=too-many-branches
            self, dep: str
    ) -> list:
        '''Given a single dep, process is and return targets/files that need resolving'''

        deps_targets_to_resolve = []
        typ = type(dep)
        if typ not in SUPPORTED_DEP_KEYS_BY_TYPE:
            self.error(f'{self.target_node=} {dep=} in {self.deps_file=}:' \
                       + f'has unsupported {type(dep)=} {SUPPORTED_DEP_KEYS_BY_TYPE=}',
                       '\n    To see detailed DEPS schema, run:',
                       f'{Colors.byellow}eda deps-schema [--verbose]',
                       error_code=status_constants.EDA_DEPS_BAD_TYPE)

        for supported_values in SUPPORTED_DEP_KEYS_BY_TYPE.values():
            if '*' in supported_values:
                continue
            if typ in (dict, list) and any(k not in supported_values for k in dep):
                self.error(
                    f'{self.target_node=} {dep=} in {self.deps_file=}: has dict-key or',
                    f'list-item not in {SUPPORTED_DEP_KEYS_BY_TYPE[typ]=}',
                    '\n    To see detailed DEPS schema, run:',
                    f'{Colors.byellow}eda deps-schema [--verbose]',
                    error_code=status_constants.EDA_DEPS_BAD_TYPE
                )

        # In-line commands in the deps list, in case the results need to be in strict file
        # order for other deps
        if isinstance(dep, dict) and 'commands' in dep:

            commands = dep['commands']
            debug(f"Got commands {dep=} for in {self.caller_info}, {commands=}")

            assert isinstance(commands, list), \
                f'dep commands must be a list: {dep=} in {self.caller_info}'

            # For this, we need to get the returned commands (to keep strict order w/ other
            # deps)
            command_tuple = self.get_commands( commands=commands, dep=dep )
            # TODO(drew): it might be cleaner to return a dict instead of list, b/c those
            # are also ordered and we can pass type information, something like:
            deps_targets_to_resolve.append(command_tuple)


        elif isinstance(dep, str) and \
             any(dep.startswith(x) for x in ['+define+', '+incdir+']) and \
             self.is_command_design:
            # Note: we still support +define+ and +incdir in the deps list.
            # check for compile-time Verilog style plusarg, which are supported under targets
            # These are not run-time Verilog style plusargs comsumable from within the .sv:
            debug(f"Got plusarg (define, incdir) {dep=} for {self.caller_info}")
            self.command_design_ref.process_plusarg(plusarg=dep, pwd=self.target_path)

        else:
            # If we made it this far, dep better be a str type.
            assert isinstance(dep, str), f'{dep=} {type(dep)=} must be str'
            dep_path = self.correct_a_deps_target(target=dep, deps_dir=self.target_path)
            debug(f"Got dep {dep_path=} for in {self.caller_info}")

            if self.is_command_design and \
               dep_path in self.command_design_ref.targets_dict or \
               dep_path in deps_targets_to_resolve:
                debug(f" - already processed ({dep_path}), skipping")
            else:
                # This is where we support files/deps file wildcards via glob syntax.
                # If glob found none, we fall back to a single-file or target
                glob_added_from_dep_path = False
                if any(x in dep_path for x in ('*', '?', '[')):
                    try:
                        glob_list = glob.glob(dep_path)
                        for fpath in glob_list:
                            file_exists, _, _ = files.get_source_file(fpath)
                            if file_exists and fpath not in deps_targets_to_resolve:
                                debug(f" - raw file ({fpath}), from glob ({dep_path}) adding",
                                      "to return list...")
                                deps_targets_to_resolve.append(fpath) # append
                                glob_added_from_dep_path = True
                        if glob_list and not glob_added_from_dep_path:
                            self.error(f'No files were expanded from glob: {dep_path} in',
                                       f'{self.caller_info}',
                                       error_code=status_constants.EDA_DEPS_TARGET_NOT_FOUND)
                    except Exception as e:
                        self.error(f'Unable to add files via glob {dep_path}, in',
                                   f'{self.caller_info}exception {e}',
                                   error_code=status_constants.EDA_DEPS_TARGET_NOT_FOUND)

                if not glob_added_from_dep_path:
                    file_exists, _, _ = files.get_source_file(dep_path)
                    if file_exists:
                        debug(f" - raw file ({dep_path}), adding to return list...")
                        deps_targets_to_resolve.append(dep_path) # append, keeping file order.
                    else:
                        debug(f" - a target non-file ({dep_path}) needs to be resolved,",
                              "adding to return list...")
                        deps_targets_to_resolve.append(dep_path) # append, keeping file order.

        # We return the list of deps or files that still need to be resolved
        # (['full_path/some_target', ...])
        # items in this list are either:
        #  -- string (dep or file)
        #  -- tuple (unprocessed commands, in form: (shell_commands_list, work_dir_add_srcs_list))
        return deps_targets_to_resolve


    def correct_a_deps_target(self, target: str, deps_dir: str) -> str:
        '''Give a target/file in a deps: list, return a patched version

        - $VAR replacment
        - relative to current DEPS file dir (or not if it was abspath)
        '''
        if self.config['deps_expandvars_enable']:
            target = os.path.expandvars(target)
        if not os.path.isabs(target):
            target = os.path.join(deps_dir, target)
        return target
