import contextlib
import distro
import getpass
import multiprocessing
import os
import platform
import psutil
import socket
import re
import shlex
import shutil
import subprocess
import sys
import time
import packaging.version
import packaging.specifiers
from io import StringIO
import traceback
from datetime import datetime
from siliconcompiler import sc_open
from siliconcompiler import utils
from siliconcompiler import _metadata
from siliconcompiler.remote import Client
from siliconcompiler.schema import Schema
from siliconcompiler.scheduler import slurm
from siliconcompiler.scheduler import docker_runner
from siliconcompiler import NodeStatus, SiliconCompilerError
from siliconcompiler.flowgraph import _get_flowgraph_nodes, _get_flowgraph_execution_order, \
    _get_pruned_node_inputs, _get_flowgraph_entry_nodes, \
    _unreachable_steps_to_execute, _nodes_to_execute, \
    get_nodes_from, nodes_to_execute, _check_flowgraph
from siliconcompiler.tools._common import input_file_node_name
import lambdapdk
from siliconcompiler.tools._common import get_tool_task, record_metric
from siliconcompiler.scheduler import send_messages

try:
    import resource
except ModuleNotFoundError:
    resource = None


# callback hooks to help custom runners track progress
_callback_funcs = {}


def register_callback(hook, func):
    _callback_funcs[hook] = func


def _get_callback(hook):
    if hook in _callback_funcs:
        return _callback_funcs[hook]
    return None


# Max lines to print from failed node log
_failed_log_lines = 20


###############################################################################
class SiliconCompilerTimeout(Exception):
    ''' Minimal Exception wrapper used to raise sc timeout errors.
    '''
    def __init__(self, message):
        super(Exception, self).__init__(message)


def run(chip):
    '''
    See :meth:`~siliconcompiler.core.Chip.run` for detailed documentation.
    '''

    _check_display(chip)

    # Check required settings before attempting run()
    for key in (['option', 'flow'], ):
        if chip.get(*key) is None:
            raise SiliconCompilerError(
                f"{key} must be set before calling run()",
                chip=chip)

    org_jobname = chip.get('option', 'jobname')
    _increment_job_name(chip)

    # Re-init logger to include run info after setting up flowgraph.
    chip._init_logger(in_run=True)

    # Check if flowgraph is complete and valid
    flow = chip.get('option', 'flow')
    if not _check_flowgraph(chip, flow=flow):
        raise SiliconCompilerError(
            f"{flow} flowgraph contains errors and cannot be run.",
            chip=chip)

    copy_old_run_dir(chip, org_jobname)
    clean_build_dir(chip)
    _reset_flow_nodes(chip, flow, nodes_to_execute(chip, flow))
    __record_packages(chip)

    if chip.get('option', 'remote'):
        client = Client(chip)
        client.run()
    else:
        _local_process(chip, flow)

    # Merge cfgs from last executed tasks, and write out a final manifest.
    _finalize_run(chip)


###########################################################################
def _finalize_run(chip):
    '''
    Helper function to finalize a job run after it completes:
    * Clear any -arg_step/-arg_index values in case only one node was run.
    * Store this run in the Schema's 'history' field.
    * Write out a final JSON manifest containing the full results and history.
    '''

    # Clear scratchpad args since these are checked on run() entry
    chip.set('arg', 'step', None, clobber=True)
    chip.set('arg', 'index', None, clobber=True)

    # Store run in history
    chip.schema.record_history()

    # Storing manifest in job root directory
    filepath = os.path.join(chip.getworkdir(), f"{chip.design}.pkg.json")
    chip.write_manifest(filepath)

    # Update dashboard
    if chip._dash:
        chip._dash.update_manifest()

    send_messages.send(chip, 'summary', None, None)


def _increment_job_name(chip):
    '''
    Auto-update jobname if ['option', 'jobincr'] is True
    Do this before initializing logger so that it picks up correct jobname
    '''
    if not chip.get('option', 'clean'):
        return
    if chip.get('option', 'jobincr'):
        workdir = chip.getworkdir()
        if os.path.isdir(workdir):
            # Strip off digits following jobname, if any
            stem = chip.get('option', 'jobname').rstrip('0123456789')

            designdir = os.path.dirname(workdir)
            jobid = 0
            for job in os.listdir(designdir):
                m = re.match(stem + r'(\d+)', job)
                if m:
                    jobid = max(jobid, int(m.group(1)))
            chip.set('option', 'jobname', f'{stem}{jobid + 1}')


###########################################################################
def _check_display(chip):
    '''
    Automatically disable display for Linux systems without desktop environment
    '''
    if not chip.get('option', 'nodisplay') and sys.platform == 'linux' \
            and 'DISPLAY' not in os.environ and 'WAYLAND_DISPLAY' not in os.environ:
        chip.logger.warning('Environment variable $DISPLAY or $WAYLAND_DISPLAY not set')
        chip.logger.warning("Setting ['option', 'nodisplay'] to True")
        chip.set('option', 'nodisplay', True)


def _local_process(chip, flow):
    from_nodes = []
    extra_setup_nodes = {}

    if chip.get('option', 'clean') or not chip.get('option', 'from'):
        load_nodes = _get_flowgraph_nodes(chip, flow)
    else:
        for step in chip.get('option', 'from'):
            from_nodes.extend(
                [(step, index) for index in chip.getkeys('flowgraph', flow, step)])

        load_nodes = _nodes_to_execute(
            chip,
            flow,
            _get_flowgraph_entry_nodes(chip, flow),
            from_nodes,
            chip.get('option', 'prune'))

    for node_level in _get_flowgraph_execution_order(chip, flow):
        for step, index in node_level:
            if (step, index) not in load_nodes:
                continue
            if (step, index) in from_nodes:
                continue

            manifest = os.path.join(chip.getworkdir(step=step, index=index),
                                    'outputs',
                                    f'{chip.design}.pkg.json')
            if os.path.exists(manifest):
                # ensure we setup these nodes again
                try:
                    extra_setup_nodes[(step, index)] = Schema(manifest=manifest, logger=chip.logger)
                except Exception:
                    pass

    # Setup tools for all nodes to run.
    nodes = nodes_to_execute(chip, flow)
    all_setup_nodes = nodes + load_nodes + list(extra_setup_nodes.keys())
    for layer_nodes in _get_flowgraph_execution_order(chip, flow):
        for step, index in layer_nodes:
            if (step, index) in all_setup_nodes:
                node_kept = _setup_node(chip, step, index)
                if not node_kept and (step, index) in extra_setup_nodes:
                    del extra_setup_nodes[(step, index)]
                if (step, index) in extra_setup_nodes:
                    schema = extra_setup_nodes[(step, index)]
                    node_status = None
                    try:
                        node_status = schema.get('record', 'status', step=step, index=index)
                    except:  # noqa E722
                        pass
                    if node_status:
                        chip.set('record', 'status', node_status, step=step, index=index)

    def mark_pending(step, index):
        chip.set('record', 'status', NodeStatus.PENDING, step=step, index=index)
        for next_step, next_index in get_nodes_from(chip, flow, [(step, index)]):
            if chip.get('record', 'status', step=next_step, index=next_index) == \
                    NodeStatus.SKIPPED:
                continue

            # Mark following steps as pending
            chip.set('record', 'status', NodeStatus.PENDING, step=next_step, index=next_index)

    # Check if nodes have been modified from previous data
    for layer_nodes in _get_flowgraph_execution_order(chip, flow):
        for step, index in layer_nodes:
            # Only look at successful nodes
            if chip.get('record', 'status', step=step, index=index) not in \
                    (NodeStatus.SUCCESS, NodeStatus.SKIPPED):
                continue

            if not check_node_inputs(chip, step, index):
                # change failing nodes to pending
                mark_pending(step, index)
            elif (step, index) in extra_setup_nodes:
                # import old information
                chip.schema._import_journal(extra_setup_nodes[(step, index)])

    # Ensure pending nodes cause following nodes to be run
    for step, index in nodes:
        if chip.get('record', 'status', step=step, index=index) in \
                (NodeStatus.PENDING, NodeStatus.ERROR):
            mark_pending(step, index)

    # Clean nodes marked pending
    for step, index in nodes:
        if chip.get('record', 'status', step=step, index=index) == NodeStatus.PENDING:
            clean_node_dir(chip, step, index)

    # Check validity of setup
    chip.logger.info("Checking manifest before running.")
    check_ok = chip.check_manifest()

    # Check if there were errors before proceeding with run
    if not check_ok:
        raise SiliconCompilerError('Manifest check failed. See previous errors.', chip=chip)

    if chip._error:
        raise SiliconCompilerError(
            'Implementation errors encountered. See previous errors.',
            chip=chip)

    nodes_to_run = {}
    processes = {}
    local_processes = []
    _prepare_nodes(chip, nodes_to_run, processes, local_processes, flow)

    # Update dashboard before run begins
    if chip._dash:
        chip._dash.update_manifest()

    try:
        _launch_nodes(chip, nodes_to_run, processes, local_processes)
    except KeyboardInterrupt:
        # exit immediately
        sys.exit(0)

    if _get_callback('post_run'):
        _get_callback('post_run')(chip)

    _check_nodes_status(chip, flow)


def __is_posix():
    return sys.platform != 'win32'


###########################################################################
def _setup_node(chip, step, index, flow=None):
    preset_step = chip.get('arg', 'step')
    preset_index = chip.get('arg', 'index')
    preset_flow = chip.get('option', 'flow')

    if flow:
        chip.set('option', 'flow', flow)

    chip.set('arg', 'step', step)
    chip.set('arg', 'index', index)
    tool, task = get_tool_task(chip, step, index, flow=flow)

    # Run node setup.
    setup_ret = None
    try:
        setup_step = getattr(chip._get_task_module(step, index), 'setup', None)
    except SiliconCompilerError:
        setup_step = None
    if setup_step:
        try:
            chip.logger.info(f'Setting up node {step}{index} with {tool}/{task}')
            setup_ret = setup_step(chip)
        except Exception as e:
            chip.logger.error(f'Failed to run setup() for {tool}/{task}')
            raise e
    else:
        raise SiliconCompilerError(f'setup() not found for tool {tool}, task {task}', chip=chip)

    # Need to restore step/index, otherwise we will skip setting up other indices.
    chip.set('option', 'flow', preset_flow)
    chip.set('arg', 'step', preset_step)
    chip.set('arg', 'index', preset_index)

    if setup_ret is not None:
        chip.logger.warning(f'Removing {step}{index} due to {setup_ret}')
        chip.set('record', 'status', NodeStatus.SKIPPED, step=step, index=index)

        return False

    return True


def _check_version(chip, reported_version, tool, step, index):
    # Based on regex for deprecated "legacy specifier" from PyPA packaging
    # library. Use this to parse PEP-440ish specifiers with arbitrary
    # versions.
    _regex_str = r"""
        (?P<operator>(==|!=|<=|>=|<|>|~=))
        \s*
        (?P<version>
            [^,;\s)]* # Since this is a "legacy" specifier, and the version
                      # string can be just about anything, we match everything
                      # except for whitespace, a semi-colon for marker support,
                      # a closing paren since versions can be enclosed in
                      # them, and a comma since it's a version separator.
        )
        """
    _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE)

    normalize_version = getattr(chip._get_tool_module(step, index), 'normalize_version', None)
    # Version is good if it matches any of the specifier sets in this list.
    spec_sets = chip.get('tool', tool, 'version', step=step, index=index)
    if not spec_sets:
        return True

    for spec_set in spec_sets:
        split_specs = [s.strip() for s in spec_set.split(",") if s.strip()]
        specs_list = []
        for spec in split_specs:
            match = re.match(_regex, spec)
            if match is None:
                chip.logger.warning(f'Invalid version specifier {spec}. '
                                    f'Defaulting to =={spec}.')
                operator = '=='
                spec_version = spec
            else:
                operator = match.group('operator')
                spec_version = match.group('version')
            specs_list.append((operator, spec_version))

        if normalize_version is None:
            normalized_version = reported_version
            normalized_specs = ','.join([f'{op}{ver}' for op, ver in specs_list])
        else:
            try:
                normalized_version = normalize_version(reported_version)
            except Exception as e:
                chip.logger.error(f'Unable to normalize version for {tool}: {reported_version}')
                raise e
            normalized_spec_list = [f'{op}{normalize_version(ver)}' for op, ver in specs_list]
            normalized_specs = ','.join(normalized_spec_list)

        try:
            version = packaging.version.Version(normalized_version)
        except packaging.version.InvalidVersion:
            chip.logger.error(f'Version {reported_version} reported by {tool} does '
                              'not match standard.')
            if normalize_version is None:
                chip.logger.error('Tool driver should implement normalize_version().')
            else:
                chip.logger.error('normalize_version() returned '
                                  f'invalid version {normalized_version}')

            return False

        try:
            spec_set = packaging.specifiers.SpecifierSet(normalized_specs)
        except packaging.specifiers.InvalidSpecifier:
            chip.logger.error(f'Version specifier set {normalized_specs} '
                              'does not match standard.')
            return False

        if version in spec_set:
            return True

    allowedstr = '; '.join(spec_sets)
    chip.logger.error(f"Version check failed for {tool}. Check installation.")
    chip.logger.error(f"Found version {reported_version}, "
                      f"did not satisfy any version specifier set {allowedstr}.")
    return False


###########################################################################
def _runtask(chip, flow, step, index, exec_func, pipe=None, replay=False):
    '''
    Private per node run method called by run().

    The method takes in a step string and index string to indicate what
    to run.

    Note that since _runtask occurs in its own process with a separate
    address space, any changes made to the `self` object will not
    be reflected in the parent. We rely on reading/writing the chip manifest
    to the filesystem to communicate updates between processes.
    '''

    chip._init_codecs()

    chip._init_logger(step, index, in_run=True)

    chip.set('arg', 'step', step, clobber=True)
    chip.set('arg', 'index', index, clobber=True)

    chip.schema._start_journal()

    # Make record of sc version and machine
    __record_version(chip, step, index)
    # Record user information if enabled
    if chip.get('option', 'track', step=step, index=index):
        __record_usermachine(chip, step, index)

    # Start wall timer
    wall_start = time.time()
    __record_time(chip, step, index, wall_start, 'start')

    workdir = _setup_workdir(chip, step, index, replay)
    cwd = os.getcwd()
    os.chdir(workdir)

    chip._add_file_logger(os.path.join(workdir, f'sc_{step}{index}.log'))

    try:
        _setupnode(chip, flow, step, index, replay)

        exec_func(chip, step, index, replay)
    except Exception as e:
        print_traceback(chip, e)
        _haltstep(chip, chip.get('option', 'flow'), step, index)

    # return to original directory
    os.chdir(cwd)
    chip.schema._stop_journal()

    if pipe:
        pipe.send(chip._packages)


###########################################################################
def _haltstep(chip, flow, step, index, log=True):
    chip.set('record', 'status', NodeStatus.ERROR, step=step, index=index)
    chip.write_manifest(os.path.join("outputs", f"{chip.get('design')}.pkg.json"))

    if log:
        chip.logger.error(f"Halting step '{step}' index '{index}' due to errors.")
        send_messages.send(chip, "fail", step, index)
    sys.exit(1)


def _setupnode(chip, flow, step, index, replay):
    _hash_files(chip, step, index, setup=True)

    # Select the inputs to this node
    _select_inputs(chip, step, index)

    # Write manifest prior to step running into inputs
    chip.write_manifest(f'inputs/{chip.get("design")}.pkg.json')

    # Forward data
    _copy_previous_steps_output_data(chip, step, index, replay)

    # Check manifest
    if not _check_manifest_dynamic(chip, step, index):
        chip.logger.error("Fatal error in check_manifest()! See previous errors.")
        _haltstep(chip, flow, step, index)


###########################################################################
def _write_task_manifest(chip, tool, path=None, backup=True):
    suffix = chip.get('tool', tool, 'format')
    if suffix:
        manifest_path = f"sc_manifest.{suffix}"
        if path:
            manifest_path = os.path.join(path, manifest_path)

        if backup and os.path.exists(manifest_path):
            shutil.copyfile(manifest_path, f'{manifest_path}.bak')

        chip.write_manifest(manifest_path, abspath=True)


###########################################################################
def _setup_workdir(chip, step, index, replay):
    workdir = chip.getworkdir(step=step, index=index)

    if os.path.isdir(workdir) and not replay:
        shutil.rmtree(workdir)
    os.makedirs(workdir, exist_ok=True)
    os.makedirs(os.path.join(workdir, 'inputs'), exist_ok=True)
    os.makedirs(os.path.join(workdir, 'outputs'), exist_ok=True)
    os.makedirs(os.path.join(workdir, 'reports'), exist_ok=True)
    return workdir


def _select_inputs(chip, step, index, trial=False):

    flow = chip.get('option', 'flow')
    tool, _ = get_tool_task(chip, step, index, flow)
    sel_inputs = []

    select_inputs = getattr(chip._get_task_module(step, index, flow=flow),
                            '_select_inputs',
                            None)
    if select_inputs:
        log_handlers = None
        if trial:
            log_handlers = chip.logger.handlers.copy()
            chip.logger.handlers.clear()
        sel_inputs = select_inputs(chip, step, index)
        if log_handlers:
            chip.logger.handlers = log_handlers
    else:
        sel_inputs = _get_pruned_node_inputs(chip, flow, (step, index))

    if (step, index) not in _get_flowgraph_entry_nodes(chip, flow) and not sel_inputs:
        chip.logger.error(f'No inputs selected after running {tool}')
        _haltstep(chip, flow, step, index)

    if not trial:
        chip.set('record', 'inputnode', sel_inputs, step=step, index=index)

    return sel_inputs


def copy_output_file(chip, outfile, folder='inputs'):
    design = chip.get('design')

    if outfile.is_file() or outfile.is_symlink():
        if outfile.name == f'{design}.pkg.json':
            return
        utils.link_symlink_copy(outfile.path, f'{folder}/{outfile.name}')
    elif outfile.is_dir():
        shutil.copytree(outfile.path,
                        f'{folder}/{outfile.name}',
                        dirs_exist_ok=True,
                        copy_function=utils.link_symlink_copy)


def forward_output_files(chip, step, index):
    for in_step, in_index in chip.get('record', 'inputnode', step=step, index=index):
        in_workdir = chip.getworkdir(step=in_step, index=in_index)
        for outfile in os.scandir(f"{in_workdir}/outputs"):
            copy_output_file(chip, outfile, folder='outputs')


def _copy_previous_steps_output_data(chip, step, index, replay):
    '''
    Copy (link) output data from previous steps
    '''

    flow = chip.get('option', 'flow')
    if not _get_pruned_node_inputs(chip, flow, (step, index)):
        all_inputs = []
    elif not chip.get('record', 'inputnode', step=step, index=index):
        all_inputs = _get_pruned_node_inputs(chip, flow, (step, index))
    else:
        all_inputs = chip.get('record', 'inputnode', step=step, index=index)

    strict = chip.get('option', 'strict')
    tool, task = get_tool_task(chip, step, index)
    in_files = chip.get('tool', tool, 'task', task, 'input', step=step, index=index)
    for in_step, in_index in all_inputs:
        if chip.get('record', 'status', step=in_step, index=in_index) == NodeStatus.ERROR:
            chip.logger.error(f'Halting step due to previous error in {in_step}{in_index}')
            _haltstep(chip, flow, step, index)

        # Skip copying pkg.json files here, since we write the current chip
        # configuration into inputs/{design}.pkg.json earlier in _runstep.
        if not replay:
            in_workdir = chip.getworkdir(step=in_step, index=in_index)
            output_dir = os.path.join(in_workdir, "outputs")

            if not os.path.isdir(output_dir):
                chip.logger.error(
                    f'Unable to locate outputs directory for {in_step}{in_index}: {output_dir}')
                _haltstep(chip, flow, step, index)

            for outfile in os.scandir(output_dir):
                new_name = input_file_node_name(outfile.name, in_step, in_index)
                if strict:
                    if outfile.name not in in_files and new_name not in in_files:
                        continue

                copy_output_file(chip, outfile)

                if new_name in in_files:
                    # perform rename
                    os.rename(f'inputs/{outfile.name}', f'inputs/{new_name}')


def __read_std_streams(chip, quiet,
                       is_stdout_log, stdout_reader, stdout_print,
                       is_stderr_log, stderr_reader, stderr_print):
    '''
    Handle directing tool outputs to logger
    '''
    if not quiet:
        if is_stdout_log:
            for line in stdout_reader.readlines():
                stdout_print(line.rstrip())
        if is_stderr_log:
            for line in stderr_reader.readlines():
                stderr_print(line.rstrip())


############################################################################
# Chip helper Functions
############################################################################
def _getexe(chip, tool, step, index):
    path = chip.get('tool', tool, 'path', step=step, index=index)
    exe = chip.get('tool', tool, 'exe')
    if exe is None:
        return None

    syspath = os.getenv('PATH', os.defpath)
    if path:
        # Prepend 'path' schema var to system path
        syspath = utils._resolve_env_vars(chip, path, step, index) + os.pathsep + syspath

    fullexe = shutil.which(exe, path=syspath)

    return fullexe


def _get_run_env_vars(chip, tool, task, step, index, include_path):
    envvars = utils.get_env_vars(chip, step, index)
    for item in chip.getkeys('tool', tool, 'licenseserver'):
        license_file = chip.get('tool', tool, 'licenseserver', item, step=step, index=index)
        if license_file:
            envvars[item] = ':'.join(license_file)

    if include_path:
        path = chip.get('tool', tool, 'path', step=step, index=index)
        if path:
            envvars['PATH'] = path + os.pathsep + os.environ['PATH']
        else:
            envvars['PATH'] = os.environ['PATH']

        # Forward additional variables
        for var in ('LD_LIBRARY_PATH',):
            val = os.getenv(var, None)
            if val:
                envvars[var] = val

    return envvars


#######################################
def _makecmd(chip, tool, task, step, index, script_name='replay.sh', include_path=True):
    '''
    Constructs a subprocess run command based on eda tool setup.
    Creates a replay script in current directory.

    Returns:
        runnable command (list)
        printable command (str)
        command name (str)
        command arguments (list)
    '''

    fullexe = _getexe(chip, tool, step, index)

    is_posix = __is_posix()

    def parse_options(options):
        if not options:
            return []
        shlex_opts = []
        for option in options:
            option = option.strip()
            if (option.startswith("\"") and option.endswith("\"")) or \
               (option.startswith("'") and option.endswith("'")):
                # Make sure strings are quoted in double quotes
                shlex_opts.append(f'"{option[1:-1]}"')
            else:
                shlex_opts.extend(shlex.split(option, posix=is_posix))
        return shlex_opts

    # Add scripts files
    scripts = chip.find_files('tool', tool, 'task', task, 'script', step=step, index=index)

    cmdlist = [fullexe]
    cmdlist.extend(parse_options(chip.get('tool', tool, 'task', task, 'option',
                                          step=step, index=index)))
    cmdlist.extend(scripts)

    runtime_options = getattr(chip._get_task_module(step, index), 'runtime_options', None)
    if not runtime_options:
        runtime_options = getattr(chip._get_tool_module(step, index), 'runtime_options', None)
    if runtime_options:
        try:
            chip.schema._start_record_access()
            cmdlist.extend(parse_options(runtime_options(chip)))
            chip.schema._stop_record_access()
        except Exception as e:
            chip.logger.error(f'Failed to get runtime options for {tool}/{task}')
            raise e

    # Separate variables to be able to display nice name of executable
    cmd = os.path.basename(cmdlist[0])
    cmd_args = cmdlist[1:]
    print_cmd = " ".join([cmd, *cmd_args])
    cmdlist = [cmdlist[0]]
    for arg in cmd_args:
        if arg.startswith("\"") and arg.endswith("\""):
            # Remove quoting since subprocess will handle that for us
            cmdlist.append(arg[1:-1])
        else:
            cmdlist.append(arg)

    # create replay file
    with open(script_name, 'w') as f:
        # Ensure execution runs from the same directory
        replay_opts = {}
        work_dir = chip.getworkdir(step=step, index=index)
        if chip._relative_path:
            work_dir = os.path.relpath(work_dir, chip._relative_path)
        replay_opts["work_dir"] = work_dir
        replay_opts["exports"] = _get_run_env_vars(chip,
                                                   tool, task,
                                                   step, index,
                                                   include_path=include_path)
        replay_opts["executable"] = chip.get('tool', tool, 'exe')

        vswitch = chip.get('tool', tool, 'vswitch')
        if vswitch:
            replay_opts["version_flag"] = " ".join(vswitch)

        format_cmd = [replay_opts["executable"]]
        arg_test = re.compile(r'^[-+]')
        file_test = re.compile(r'^[/]')
        for cmdarg in cmd_args:
            add_new_line = len(format_cmd) == 1

            if arg_test.match(cmdarg) or file_test.match(cmdarg):
                add_new_line = True
            else:
                if not arg_test.match(format_cmd[-1]):
                    add_new_line = True

            if add_new_line:
                format_cmd.append(cmdarg)
            else:
                format_cmd[-1] += f' {cmdarg}'

        replay_opts["cmds"] = format_cmd

        f.write(utils.get_file_template("replay/replay.sh.j2").render(replay_opts))
        f.write("\n")

    os.chmod(script_name, 0o755)

    return cmdlist, print_cmd, cmd, cmd_args


def __get_stdio(chip, tool, task, flow, step, index):
    def get_file(io_type):
        suffix = chip.get('tool', tool, 'task', task, io_type, 'suffix',
                          step=step, index=index)
        destination = chip.get('tool', tool, 'task', task, io_type, 'destination',
                               step=step, index=index)

        io_file = None
        if destination == 'log':
            io_file = step + "." + suffix
        elif destination == 'output':
            io_file = os.path.join('outputs', chip.top() + "." + suffix)
        elif destination == 'none':
            io_file = os.devnull
        else:
            # This should not happen
            chip.logger.error(f'{io_type}/destination has no support for {destination}.')
            _haltstep(chip, flow, step, index)

        return io_file

    stdout_file = get_file('stdout')
    stderr_file = get_file('stderr')

    return stdout_file, stderr_file


def _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, run_func=None):
    '''
    Run executable (or copy inputs to outputs for builtin functions)
    '''

    flow = chip.get('option', 'flow')
    tool, task = get_tool_task(chip, step, index, flow)

    quiet = (
        chip.get('option', 'quiet', step=step, index=index) and
        not chip.get('option', 'breakpoint', step=step, index=index)
    )

    stdout_print = chip.logger.info
    stderr_print = chip.logger.error
    if chip.get('option', 'loglevel', step=step, index=index) == "quiet":
        stdout_print = chip.logger.error
        stderr_print = chip.logger.error

    # TODO: Currently no memory usage tracking in breakpoints, builtins, or unexpected errors.
    max_mem_bytes = 0
    cpu_start = time.time()

    stdout_file, stderr_file = __get_stdio(chip, tool, task, flow, step, index)
    is_stdout_log = chip.get('tool', tool, 'task', task, 'stdout', 'destination',
                             step=step, index=index) == 'log'
    is_stderr_log = chip.get('tool', tool, 'task', task, 'stderr', 'destination',
                             step=step, index=index) == 'log' and stderr_file != stdout_file

    chip.logger.info(f'Running in {workdir}')

    retcode = 0
    cmdlist = []
    cmd_args = []
    if run_func:
        logfile = None
        try:
            with open(stdout_file, 'w') as stdout_writer, \
                    open(stderr_file, 'w') as stderr_writer:
                if stderr_file == stdout_file:
                    stderr_writer.close()
                    stderr_writer = sys.stdout

                with contextlib.redirect_stderr(stderr_writer), \
                        contextlib.redirect_stdout(stdout_writer):
                    retcode = run_func(chip)
        except Exception as e:
            chip.logger.error(f'Failed in run() for {tool}/{task}: {e}')
            retcode = 1  # default to non-zero
            print_traceback(chip, e)
            chip._error = True
        finally:
            with sc_open(stdout_file) as stdout_reader, \
                    sc_open(stderr_file) as stderr_reader:
                __read_std_streams(chip,
                                   quiet,
                                   is_stdout_log, stdout_reader, stdout_print,
                                   is_stderr_log, stderr_reader, stderr_print)

            try:
                if resource:
                    # Since memory collection is not possible, collect the current process
                    # peak memory
                    max_mem_bytes = max(
                        max_mem_bytes,
                        1024 * resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
            except (OSError, ValueError, PermissionError):
                pass
    else:
        cmdlist, printable_cmd, _, cmd_args = _makecmd(chip, tool, task, step, index)

        ##################
        # Make record of tool options
        if cmd_args is not None:
            chip.set('record', 'toolargs',
                     ' '.join(f'"{arg}"' if ' ' in arg else arg for arg in cmd_args),
                     step=step, index=index)

        chip.logger.info('%s', printable_cmd)
        timeout = chip.get('option', 'timeout', step=step, index=index)
        logfile = step + '.log'
        if sys.platform in ('darwin', 'linux') and \
           chip.get('option', 'breakpoint', step=step, index=index):
            # When we break on a step, the tool often drops into a shell.
            # However, our usual subprocess scheme seems to break terminal
            # echo for some tools. On POSIX-compatible systems, we can use
            # pty to connect the tool to our terminal instead. This code
            # doesn't handle quiet/timeout logic, since we don't want either
            # of these features for an interactive session. Logic for
            # forwarding to file based on
            # https://docs.python.org/3/library/pty.html#example.
            with open(logfile, 'wb') as log_writer:
                def read(fd):
                    data = os.read(fd, 1024)
                    log_writer.write(data)
                    return data
                import pty  # Note: this import throws exception on Windows
                retcode = pty.spawn(cmdlist, read)
        else:
            with open(stdout_file, 'w') as stdout_writer, \
                    open(stdout_file, 'r', errors='replace_with_warning') as stdout_reader, \
                    open(stderr_file, 'w') as stderr_writer, \
                    open(stderr_file, 'r', errors='replace_with_warning') as stderr_reader:
                # if STDOUT and STDERR are to be redirected to the same file,
                # use a single writer
                if stderr_file == stdout_file:
                    stderr_writer.close()
                    stderr_reader.close()
                    stderr_writer = subprocess.STDOUT

                preexec_fn = None
                nice = None
                if __is_posix():
                    nice = chip.get('option', 'nice', step=step, index=index)

                    def set_nice():
                        os.nice(nice)

                    if nice:
                        preexec_fn = set_nice

                cmd_start_time = time.time()
                proc = subprocess.Popen(cmdlist,
                                        stdin=subprocess.DEVNULL,
                                        stdout=stdout_writer,
                                        stderr=stderr_writer,
                                        preexec_fn=preexec_fn)
                # How long to wait for proc to quit on ctrl-c before force
                # terminating.
                POLL_INTERVAL = 0.1
                MEMORY_WARN_LIMIT = 90
                try:
                    while proc.poll() is None:
                        # Gather subprocess memory usage.
                        try:
                            pproc = psutil.Process(proc.pid)
                            proc_mem_bytes = pproc.memory_full_info().uss
                            for child in pproc.children(recursive=True):
                                proc_mem_bytes += child.memory_full_info().uss
                            max_mem_bytes = max(max_mem_bytes, proc_mem_bytes)

                            memory_usage = psutil.virtual_memory()
                            if memory_usage.percent > MEMORY_WARN_LIMIT:
                                chip.logger.warn(
                                    f'Current system memory usage is {memory_usage.percent}%')

                                # increase limit warning
                                MEMORY_WARN_LIMIT = int(memory_usage.percent + 1)
                        except psutil.Error:
                            # Process may have already terminated or been killed.
                            # Retain existing memory usage statistics in this case.
                            pass
                        except PermissionError:
                            # OS is preventing access to this information so it cannot
                            # be collected
                            pass

                        # Loop until process terminates
                        __read_std_streams(chip,
                                           quiet,
                                           is_stdout_log, stdout_reader, stdout_print,
                                           is_stderr_log, stderr_reader, stderr_print)

                        if timeout is not None and time.time() - cmd_start_time > timeout:
                            chip.logger.error(f'Step timed out after {timeout} seconds')
                            utils.terminate_process(proc.pid)
                            raise SiliconCompilerTimeout(f'{step}{index} timeout')
                        time.sleep(POLL_INTERVAL)
                except KeyboardInterrupt:
                    kill_process(chip, proc, tool, 5 * POLL_INTERVAL, msg="Received ctrl-c. ")
                    _haltstep(chip, flow, step, index, log=False)
                except SiliconCompilerTimeout:
                    send_messages.send(chip, "timeout", step, index)
                    kill_process(chip, proc, tool, 5 * POLL_INTERVAL)
                    chip._error = True

                # Read the remaining
                __read_std_streams(chip,
                                   quiet,
                                   is_stdout_log, stdout_reader, stdout_print,
                                   is_stderr_log, stderr_reader, stderr_print)
                retcode = proc.returncode

    chip.set('record', 'toolexitcode', retcode, step=step, index=index)
    if retcode != 0:
        msg = f'Command failed with code {retcode}.'
        if logfile:
            if quiet:
                # Print last N lines of log when in quiet mode
                with sc_open(logfile) as logfd:
                    loglines = logfd.read().splitlines()
                    for logline in loglines[-_failed_log_lines:]:
                        chip.logger.error(logline)
                # No log file for pure-Python tools.
            msg += f' See log file {os.path.abspath(logfile)}'
        chip.logger.warning(msg)
        chip._error = True

    # Capture cpu runtime
    record_metric(chip, step, index, 'exetime', round((time.time() - cpu_start), 2),
                  source=None,
                  source_unit='s')

    # Capture memory usage
    record_metric(chip, step, index, 'memory', max_mem_bytes,
                  source=None,
                  source_unit='B')


def _post_process(chip, step, index):
    flow = chip.get('option', 'flow')
    tool, task = get_tool_task(chip, step, index, flow)
    func = getattr(chip._get_task_module(step, index, flow=flow), 'post_process', None)
    if func:
        try:
            chip.schema._start_record_access()
            func(chip)
            chip.schema._stop_record_access()
        except Exception as e:
            chip.logger.error(f'Failed to run post-process for {tool}/{task}.')
            print_traceback(chip, e)
            chip._error = True


def _check_logfile(chip, step, index, quiet=False, run_func=None):
    '''
    Check log file (must be after post-process)
    '''
    if run_func is None:
        tool, task = get_tool_task(chip, step, index)

        log_file = os.path.join(chip.getworkdir(step=step, index=index), f'{step}.log')
        matches = check_logfile(chip, step=step, index=index,
                                display=not quiet,
                                logfile=log_file)
        if 'errors' in matches:
            errors = chip.get('metric', 'errors', step=step, index=index)
            if errors is None:
                errors = 0
            errors += matches['errors']

            sources = [f'{step}.log']
            if chip.valid('tool', tool, 'task', task, 'regex', 'errors'):
                if chip.get('tool', tool, 'task', task, 'regex', 'errors',
                            step=step, index=index):
                    sources.append(f'{step}.errors')

            record_metric(chip, step, index, 'errors', errors, sources)
        if 'warnings' in matches:
            warnings = chip.get('metric', 'warnings', step=step, index=index)
            if warnings is None:
                warnings = 0
            warnings += matches['warnings']

            sources = [f'{step}.log']
            if chip.valid('tool', tool, 'task', task, 'regex', 'warnings'):
                if chip.get('tool', tool, 'task', task, 'regex', 'warnings',
                            step=step, index=index):
                    sources.append(f'{step}.warnings')

            record_metric(chip, step, index, 'warnings', warnings, sources)


def _executenode(chip, step, index, replay):
    workdir = chip.getworkdir(step=step, index=index)
    flow = chip.get('option', 'flow')
    tool, _ = get_tool_task(chip, step, index, flow)

    _pre_process(chip, step, index)

    if chip.get('record', 'status', step=step, index=index) == NodeStatus.SKIPPED:
        # copy inputs to outputs and skip execution
        forward_output_files(chip, step, index)

        send_messages.send(chip, "skipped", step, index)
    else:
        org_env = _set_env_vars(chip, step, index)

        run_func = getattr(chip._get_task_module(step, index, flow=flow), 'run', None)
        toolpath, version = _check_tool_version(chip, step, index, run_func)

        if version:
            chip.set('record', 'toolversion', version, step=step, index=index)

        if toolpath:
            chip.set('record', 'toolpath', toolpath, step=step, index=index)

        # Write manifest (tool interface) (Don't move this!)
        _write_task_manifest(chip, tool)

        send_messages.send(chip, "begin", step, index)

        _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, run_func)

        os.environ.clear()
        os.environ.update(org_env)

        _post_process(chip, step, index)

    _finalizenode(chip, step, index, replay)

    send_messages.send(chip, "end", step, index)


def _pre_process(chip, step, index):
    flow = chip.get('option', 'flow')
    tool, task = get_tool_task(chip, step, index, flow)
    func = getattr(chip._get_task_module(step, index, flow=flow), 'pre_process', None)
    if func:
        try:
            chip.schema._start_record_access()
            func(chip)
            chip.schema._stop_record_access()
        except Exception as e:
            chip.logger.error(f"Pre-processing failed for '{tool}/{task}'.")
            raise e
        if chip._error:
            chip.logger.error(f"Pre-processing failed for '{tool}/{task}'")
            _haltstep(chip, flow, step, index)


def _set_env_vars(chip, step, index):
    org_env = os.environ.copy()

    tool, task = get_tool_task(chip, step, index)

    chip.schema._start_record_access()

    os.environ.update(_get_run_env_vars(chip, tool, task, step, index, include_path=True))

    chip.schema._stop_record_access()

    return org_env


def _check_tool_version(chip, step, index, run_func=None):
    '''
    Check exe version
    '''

    flow = chip.get('option', 'flow')
    tool, task = get_tool_task(chip, step, index, flow)

    vercheck = not chip.get('option', 'novercheck', step=step, index=index)
    veropt = chip.get('tool', tool, 'vswitch')
    exe = _getexe(chip, tool, step, index)
    version = None
    if exe is not None:
        exe_path, exe_base = os.path.split(exe)
        if veropt:
            cmdlist = [exe]
            cmdlist.extend(veropt)
            proc = subprocess.run(cmdlist,
                                  stdin=subprocess.DEVNULL,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT,
                                  universal_newlines=True)
            if proc.returncode != 0:
                chip.logger.warning(f'Version check on {tool} failed with '
                                    f'code {proc.returncode}')

            parse_version = getattr(chip._get_tool_module(step, index, flow=flow),
                                    'parse_version',
                                    None)
            if parse_version is None:
                chip.logger.error(f'{tool}/{task} does not implement parse_version().')
                _haltstep(chip, flow, step, index)
            try:
                version = parse_version(proc.stdout)
            except Exception as e:
                chip.logger.error(f'{tool} failed to parse version string: {proc.stdout}')
                raise e

            chip.logger.info(f"Tool '{exe_base}' found with version '{version}' "
                             f"in directory '{exe_path}'")
            if vercheck and not _check_version(chip, version, tool, step, index):
                if proc.returncode != 0:
                    chip.logger.error(f"Tool '{exe_base}' responded with: {proc.stdout}")
                _haltstep(chip, flow, step, index)
        else:
            chip.logger.info(f"Tool '{exe_base}' found in directory '{exe_path}'")
    elif run_func is None:
        exe_base = chip.get('tool', tool, 'exe')
        chip.logger.error(f'Executable {exe_base} not found')
        _haltstep(chip, flow, step, index)
    return (exe, version)


def _hash_files(chip, step, index, setup=False):
    if chip._error:
        return

    flow = chip.get('option', 'flow')
    tool, task = get_tool_task(chip, step, index, flow)
    if chip.get('option', 'hash'):
        if not setup:
            # hash all outputs
            chip.hash_files('tool', tool, 'task', task, 'output',
                            step=step, index=index, check=False, verbose=False)
        else:
            for task_key in ('refdir', 'prescript', 'postscript', 'script'):
                chip.hash_files('tool', tool, 'task', task, task_key,
                                step=step, index=index, check=False,
                                allow_cache=True, verbose=False)

        # hash all requirements
        for item in set(chip.get('tool', tool, 'task', task, 'require', step=step, index=index)):
            args = item.split(',')
            sc_type = chip.get(*args, field='type')
            if 'file' in sc_type or 'dir' in sc_type:
                if chip.get(*args, field='pernode').is_never():
                    if not setup:
                        if chip.get(*args, field='filehash'):
                            continue
                    chip.hash_files(*args, check=False, allow_cache=True, verbose=False)
                else:
                    if not setup:
                        if chip.get(*args, field='filehash', step=step, index=index):
                            continue
                    chip.hash_files(*args, step=step, index=index,
                                    check=False, allow_cache=True, verbose=False)


def _finalizenode(chip, step, index, replay):
    if chip.schema._do_record_access():
        assert_required_accesses(chip, step, index)

    flow = chip.get('option', 'flow')
    tool, task = get_tool_task(chip, step, index, flow)
    quiet = (
        chip.get('option', 'quiet', step=step, index=index) and not
        chip.get('option', 'breakpoint', step=step, index=index)
    )
    run_func = getattr(chip._get_task_module(step, index, flow=flow), 'run', None)

    is_skipped = chip.get('record', 'status', step=step, index=index) == NodeStatus.SKIPPED

    if not is_skipped:
        _check_logfile(chip, step, index, quiet, run_func)

    # Report metrics
    for metric in ['errors', 'warnings']:
        val = chip.get('metric', metric, step=step, index=index)
        if val is not None:
            chip.logger.info(f'Number of {metric}: {val}')

    _hash_files(chip, step, index)

    # Capture wall runtime and cpu cores
    wall_end = time.time()
    __record_time(chip, step, index, wall_end, 'end')

    # calculate total time
    total_times = []
    for check_step, check_index in _get_flowgraph_nodes(chip, flow):
        total_time = chip.get('metric', 'totaltime', step=check_step, index=check_index)
        if total_time is not None:
            total_times.append(total_time)
    if total_times:
        total_time = max(total_times)
    else:
        total_time = 0.0

    walltime = wall_end - get_record_time(chip, step, index, 'starttime')
    record_metric(chip, step, index, 'tasktime', walltime,
                  source=None, source_unit='s')
    record_metric(chip, step, index, 'totaltime', total_time + walltime,
                  source=None, source_unit='s')
    chip.logger.info(f"Finished task in {round(walltime, 2)}s")

    # Save a successful manifest
    if not is_skipped:
        chip.set('record', 'status', NodeStatus.SUCCESS, step=step, index=index)
    chip.write_manifest(os.path.join("outputs", f"{chip.get('design')}.pkg.json"))

    if chip._error and not replay:
        _make_testcase(chip, step, index)

    # Stop if there are errors
    errors = chip.get('metric', 'errors', step=step, index=index)
    if errors and not chip.get('option', 'continue', step=step, index=index):
        # TODO: should we warn if errors is not set?
        chip.logger.error(f'{tool} reported {errors} errors during {step}{index}')
        _haltstep(chip, flow, step, index)

    if chip._error:
        _haltstep(chip, flow, step, index)

    if chip.get('option', 'strict'):
        assert_output_files(chip, step, index)


def _make_testcase(chip, step, index):
    # Import here to avoid circular import
    from siliconcompiler.issue import generate_testcase

    generate_testcase(
        chip,
        step,
        index,
        archive_directory=chip.getworkdir(),
        include_pdks=False,
        include_specific_pdks=lambdapdk.get_pdks(),
        include_libraries=False,
        include_specific_libraries=lambdapdk.get_libs(),
        hash_files=chip.get('option', 'hash'),
        verbose_collect=False)


def assert_output_files(chip, step, index):
    flow = chip.get('option', 'flow')
    tool, task = get_tool_task(chip, step, index, flow)

    if tool == 'builtin':
        return

    outputs = os.listdir(f'{chip.getworkdir(step=step, index=index)}/outputs')
    outputs.remove(f'{chip.design}.pkg.json')

    output_files = chip.get('tool', tool, 'task', task, 'output',
                            step=step, index=index)

    if set(outputs) != set(output_files):
        raise SiliconCompilerError(
            f'Output files set {output_files} for {step}{index} does not match generated '
            f'outputs: {outputs}',
            chip=chip)


def assert_required_accesses(chip, step, index):
    flow = chip.get('option', 'flow')
    jobname = chip.get('option', 'jobname')
    tool, task = get_tool_task(chip, step, index, flow)

    if tool == 'builtin':
        return

    gets = chip.schema._get_record_access()
    logfile = os.path.join(
        chip.getworkdir(jobname=jobname, step=step, index=index),
        f'{step}.log')

    with sc_open(logfile) as f:
        for line in f:
            if line.startswith(Schema._RECORD_ACCESS_IDENTIFIER):
                key = line[len(Schema._RECORD_ACCESS_IDENTIFIER):].strip().split(',')
                if chip.valid(*key, check_complete=True):
                    gets.add(tuple(key))

    def get_value(*key):
        if chip.get(*key, field='pernode').is_never():
            return chip.get(*key)
        else:
            return chip.get(*key, step=step, index=index)

    getkeys = set()
    # Remove keys with empty values
    for key in set(sorted(gets)):
        if get_value(*key):
            getkeys.add(key)

    # Remove keys that dont matter
    exempt = [
        ('design',),
        ('arg', 'step'), ('arg', 'index'),
        ('option', 'jobname'), ('option', 'flow'), ('option', 'strict'), ('option', 'builddir'),
        ('option', 'quiet'),
        ('tool', tool, 'exe'),
        ('tool', tool, 'task', task, 'require'),
        ('tool', tool, 'task', task, 'threads'),
        ('flowgraph', flow, step, index, 'tool'), ('flowgraph', flow, step, index, 'task'),
        ('flowgraph', flow, step, index, 'taskmodule')]
    for key in chip.getkeys('metric'):
        exempt.append(('metric', key))
    for key in chip.getkeys('tool', tool, 'task', task, 'report'):
        exempt.append(('tool', tool, 'task', task, 'report', key))

    # Get exempted keys from task
    func = getattr(chip._get_task_module(step, index, flow=flow), 'exempt_keys', None)
    if func:
        # No need for try / except since this must work properly
        exempt.extend(func(chip))

    required = set(
        [tuple(key.split(',')) for key in chip.get('tool', tool, 'task', task, 'require',
                                                   step=step, index=index)])

    for key in set(exempt):
        if key in getkeys:
            getkeys.remove(key)
        if key in required:
            required.remove(key)

    excess_require = required.difference(getkeys)
    if True:
        for key in sorted(excess_require):
            chip.logger.error(f"{step}{index} does not require requirement: {','.join(key)}")
    missing_require = getkeys.difference(required)
    for key in sorted(missing_require):
        chip.logger.error(f"{step}{index} has an unexpressed requirement: "
                          f"{','.join(key)} = {get_value(*key)}")

    if missing_require:
        raise SiliconCompilerError(
            f'Requirements for {step}{index} does not match access list: '
            f'{", ".join([",".join(key) for key in sorted(missing_require)])}',
            chip=chip)


def _reset_flow_nodes(chip, flow, nodes_to_execute):
    # Reset flowgraph/records/metrics by probing build directory. We need
    # to set values to None for steps we may re-run so that merging
    # manifests from _runtask() actually updates values.

    def clear_node(step, index):
        # Reset metrics and records
        for metric in chip.getkeys('metric'):
            _clear_metric(chip, step, index, metric)
        for record in chip.getkeys('record'):
            _clear_record(chip, step, index, record, preserve=[
                'remoteid',
                'status',
                'pythonpackage'])

    # Mark all nodes as pending
    for step, index in _get_flowgraph_nodes(chip, flow):
        chip.set('record', 'status', NodeStatus.PENDING, step=step, index=index)

    should_resume = not chip.get('option', 'clean')
    for step, index in _get_flowgraph_nodes(chip, flow):
        stepdir = chip.getworkdir(step=step, index=index)
        cfg = f"{stepdir}/outputs/{chip.get('design')}.pkg.json"

        if not os.path.isdir(stepdir) or (
                (step, index) in nodes_to_execute and not should_resume):
            # If stepdir doesn't exist, we need to re-run this task. If
            # we're not running with -resume, we also re-run anything
            # in the nodes to execute.
            clear_node(step, index)
        elif os.path.isfile(cfg):
            try:
                old_status = Schema(manifest=cfg).get('record', 'status', step=step, index=index)
                if old_status:
                    chip.set('record', 'status', old_status, step=step, index=index)
            except Exception:
                # unable to load so leave it default
                pass
        else:
            chip.set('record', 'status', NodeStatus.ERROR, step=step, index=index)

    for step in chip.getkeys('flowgraph', flow):
        all_indices_failed = True
        for index in chip.getkeys('flowgraph', flow, step):
            if chip.get('record', 'status', step=step, index=index) == NodeStatus.SUCCESS:
                all_indices_failed = False

        if should_resume and all_indices_failed:
            # When running with -resume, we re-run any step in flowgraph that
            # had all indices fail.
            for index in chip.getkeys('flowgraph', flow, step):
                if (step, index) in nodes_to_execute:
                    clear_node(step, index)


def _prepare_nodes(chip, nodes_to_run, processes, local_processes, flow):
    '''
    For each node to run, prepare a process and store its dependencies
    '''

    # Call this in case this was invoked without __main__
    multiprocessing.freeze_support()

    init_funcs = set()
    for (step, index) in nodes_to_execute(chip, flow):
        node = (step, index)

        if chip.get('record', 'status', step=step, index=index) != NodeStatus.PENDING:
            continue

        nodes_to_run[node] = _get_pruned_node_inputs(chip, flow, (step, index))

        exec_func = _executenode

        if chip.get('option', 'scheduler', 'name', step=step, index=index) == 'slurm':
            # Defer job to compute node
            # If the job is configured to run on a cluster, collect the schema
            # and send it to a compute node for deferred execution.
            init_funcs.add(slurm.init)
            exec_func = slurm._defernode
        elif chip.get('option', 'scheduler', 'name', step=step, index=index) == 'docker':
            # Run job in docker
            init_funcs.add(docker_runner.init)
            exec_func = docker_runner.run
            local_processes.append((step, index))
        else:
            local_processes.append((step, index))

        process = {
            "child_pipe": None,
            "parent_pipe": None,
            "proc": None
        }
        process["parent_pipe"], process["child_pipe"] = multiprocessing.Pipe()
        process["proc"] = multiprocessing.Process(
            target=_runtask,
            args=(chip, flow, step, index, exec_func),
            kwargs={"pipe": process["child_pipe"]})

        processes[node] = process

    for init_func in init_funcs:
        init_func(chip)


def _check_node_dependencies(chip, node, deps, deps_was_successful):
    had_deps = len(deps) > 0
    step, index = node
    tool, _ = get_tool_task(chip, step, index)

    # Clear any nodes that have finished from dependency list.
    for in_step, in_index in list(deps):
        in_status = chip.get('record', 'status', step=in_step, index=in_index)
        if NodeStatus.is_done(in_status):
            deps.remove((in_step, in_index))
        if in_status == NodeStatus.SUCCESS:
            deps_was_successful[node] = True
        if NodeStatus.is_error(in_status):
            # Fail if any dependency failed for non-builtin task
            if tool != 'builtin':
                deps.clear()
                chip.set('record', 'status', NodeStatus.ERROR, step=step, index=index)
                return

    # Fail if no dependency successfully finished for builtin task
    if had_deps and len(deps) == 0 \
            and tool == 'builtin' and not deps_was_successful.get(node):
        chip.set('record', 'status', NodeStatus.ERROR, step=step, index=index)


def _launch_nodes(chip, nodes_to_run, processes, local_processes):
    running_nodes = {}
    max_parallel_run = chip.get('option', 'scheduler', 'maxnodes')
    max_cores = utils.get_cores(chip)
    max_threads = utils.get_cores(chip)
    if not max_parallel_run:
        max_parallel_run = utils.get_cores(chip)

    # clip max parallel jobs to 1 <= jobs <= max_cores
    max_parallel_run = max(1, min(max_parallel_run, max_cores))

    def allow_start(node):
        if node not in local_processes:
            # using a different scheduler, so allow
            return True, 0

        if len(running_nodes) >= max_parallel_run:
            return False, 0

        # Record thread count requested
        step, index = node
        tool, task = get_tool_task(chip, step, index)
        requested_threads = chip.get('tool', tool, 'task', task, 'threads',
                                     step=step, index=index)
        if not requested_threads:
            # not specified, marking it max to be safe
            requested_threads = max_threads
        # clamp to max_parallel to avoid getting locked up
        requested_threads = max(1, min(requested_threads, max_threads))

        if requested_threads + sum(running_nodes.values()) > max_cores:
            # delay until there are enough core available
            return False, 0

        # allow and record how many threads to associate
        return True, requested_threads

    deps_was_successful = {}

    if _get_callback('pre_run'):
        _get_callback('pre_run')(chip)

    while len(nodes_to_run) > 0 or len(running_nodes) > 0:
        changed = _process_completed_nodes(chip, processes, running_nodes)

        # Check for new nodes that can be launched.
        for node, deps in list(nodes_to_run.items()):
            # TODO: breakpoint logic:
            # if node is breakpoint, then don't launch while len(running_nodes) > 0

            _check_node_dependencies(chip, node, deps, deps_was_successful)

            if chip.get('record', 'status', step=node[0], index=node[1]) == NodeStatus.ERROR:
                del nodes_to_run[node]
                continue

            # If there are no dependencies left, launch this node and
            # remove from nodes_to_run.
            if len(deps) == 0:
                dostart, requested_threads = allow_start(node)

                if dostart:
                    if _get_callback('pre_node'):
                        _get_callback('pre_node')(chip, *node)

                    chip.set('record', 'status', NodeStatus.RUNNING, step=node[0], index=node[1])
                    changed = True

                    processes[node]["proc"].start()
                    del nodes_to_run[node]
                    running_nodes[node] = requested_threads

        # Check for situation where we have stuff left to run but don't
        # have any nodes running. This shouldn't happen, but we will get
        # stuck in an infinite loop if it does, so we want to break out
        # with an explicit error.
        if len(nodes_to_run) > 0 and len(running_nodes) == 0:
            raise SiliconCompilerError(
                'Nodes left to run, but no running nodes. From/to may be invalid.', chip=chip)

        if chip._dash and changed:
            # Update dashboard if the manifest changed
            chip._dash.update_manifest()

        if len(running_nodes) == 1:
            # if there is only one node running, just join the thread
            running_node = list(running_nodes.keys())[0]
            processes[running_node]["proc"].join()
        elif len(running_nodes) > 1:
            # if there are more than 1, join the first with a timeout
            running_node = list(running_nodes.keys())[0]
            processes[running_node]["proc"].join(timeout=0.1)


def _process_completed_nodes(chip, processes, running_nodes):
    changed = False
    for node in list(running_nodes.keys()):
        if not processes[node]["proc"].is_alive():
            step, index = node
            manifest = os.path.join(chip.getworkdir(step=step, index=index),
                                    'outputs',
                                    f'{chip.design}.pkg.json')
            chip.logger.debug(f'{step}{index} is complete merging: {manifest}')
            if os.path.exists(manifest):
                chip.schema.read_journal(manifest)

            if processes[node]["parent_pipe"] and processes[node]["parent_pipe"].poll(1):
                try:
                    packages = processes[node]["parent_pipe"].recv()
                    if isinstance(packages, dict):
                        chip._packages.update(packages)
                except:  # noqa E722
                    pass

            del running_nodes[node]
            if processes[node]["proc"].exitcode > 0:
                status = NodeStatus.ERROR
            else:
                status = chip.get('record', 'status', step=step, index=index)
                if not status or status == NodeStatus.PENDING:
                    status = NodeStatus.ERROR

            chip.set('record', 'status', status, step=step, index=index)

            changed = True

            if _get_callback('post_node'):
                _get_callback('post_node')(chip, *node)

    return changed


def _check_nodes_status(chip, flow):
    def success(node):
        return chip.get('record', 'status', step=node[0], index=node[1]) in \
            (NodeStatus.SUCCESS, NodeStatus.SKIPPED)

    unreachable_steps = _unreachable_steps_to_execute(chip, flow, cond=success)
    if unreachable_steps:
        raise SiliconCompilerError(
            f'These final steps could not be reached: {list(unreachable_steps)}', chip=chip)


#######################################
def __record_version(chip, step, index):
    chip.set('record', 'scversion', _metadata.version, step=step, index=index)
    chip.set('record', 'pythonversion', platform.python_version(), step=step, index=index)


#######################################
def __record_packages(chip):
    try:
        from pip._internal.operations.freeze import freeze
    except:  # noqa E722
        freeze = None

    if freeze:
        # clear record
        chip.set('record', 'pythonpackage', [])

        for pkg in freeze():
            chip.add('record', 'pythonpackage', pkg)


#######################################
def __record_time(chip, step, index, record_time, timetype):
    formatted_time = datetime.fromtimestamp(record_time).strftime('%Y-%m-%d %H:%M:%S')

    if timetype == 'start':
        key = 'starttime'
    elif timetype == 'end':
        key = 'endtime'
    else:
        raise ValueError(f'{timetype} is not a valid time record')

    chip.set('record', key, formatted_time, step=step, index=index)


def get_record_time(chip, step, index, timetype):
    return datetime.strptime(
        chip.get('record', timetype, step=step, index=index),
        '%Y-%m-%d %H:%M:%S').timestamp()


#######################################
def _get_cloud_region():
    # TODO: add logic to figure out if we're running on a remote cluster and
    # extract the region in a provider-specific way.
    return 'local'


#######################################
def __record_usermachine(chip, step, index):
    machine_info = _get_machine_info()
    chip.set('record', 'platform', machine_info['system'], step=step, index=index)

    if machine_info['distro']:
        chip.set('record', 'distro', machine_info['distro'], step=step, index=index)

    chip.set('record', 'osversion', machine_info['osversion'], step=step, index=index)

    if machine_info['kernelversion']:
        chip.set('record', 'kernelversion', machine_info['kernelversion'], step=step, index=index)

    chip.set('record', 'arch', machine_info['arch'], step=step, index=index)

    chip.set('record', 'userid', getpass.getuser(), step=step, index=index)

    chip.set('record', 'machine', platform.node(), step=step, index=index)

    chip.set('record', 'region', _get_cloud_region(), step=step, index=index)

    try:
        for interface, addrs in psutil.net_if_addrs().items():
            if interface == 'lo':
                # don't consider loopback device
                continue

            if not addrs:
                # skip missing addrs
                continue

            use_addr = False
            for addr in addrs:
                if addr.family == socket.AF_INET:
                    if not addr.address.startswith('127.'):
                        use_addr = True
                    break

            if use_addr:
                ipaddr = None
                macaddr = None
                for addr in addrs:
                    if not ipaddr and addr.family == socket.AF_INET:
                        ipaddr = addr.address
                    if not ipaddr and addr.family == socket.AF_INET6:
                        ipaddr = addr.address
                    if not macaddr and addr.family == psutil.AF_LINK:
                        macaddr = addr.address

                chip.set('record', 'ipaddr', ipaddr, step=step, index=index)
                chip.set('record', 'macaddr', macaddr, step=step, index=index)
                break
    except:  # noqa E722
        chip.logger.warning('Could not find default network interface info')


#######################################
def _get_machine_info():
    system = platform.system()
    if system == 'Darwin':
        lower_sys_name = 'macos'
    else:
        lower_sys_name = system.lower()

    if system == 'Linux':
        distro_name = distro.id()
    else:
        distro_name = None

    if system == 'Darwin':
        osversion, _, _ = platform.mac_ver()
    elif system == 'Linux':
        osversion = distro.version()
    else:
        osversion = platform.release()

    if system == 'Linux':
        kernelversion = platform.release()
    elif system == 'Windows':
        kernelversion = platform.version()
    elif system == 'Darwin':
        kernelversion = platform.release()
    else:
        kernelversion = None

    arch = platform.machine()

    return {'system': lower_sys_name,
            'distro': distro_name,
            'osversion': osversion,
            'kernelversion': kernelversion,
            'arch': arch}


def print_traceback(chip, exception):
    chip.logger.error(f'{exception}')
    trace = StringIO()
    traceback.print_tb(exception.__traceback__, file=trace)
    chip.logger.error("Backtrace:")
    for line in trace.getvalue().splitlines():
        chip.logger.error(line)


def kill_process(chip, proc, tool, poll_interval, msg=""):
    TERMINATE_TIMEOUT = 5
    interrupt_time = time.time()
    chip.logger.info(f'{msg}Waiting for {tool} to exit...')
    while proc.poll() is None and \
            (time.time() - interrupt_time) < TERMINATE_TIMEOUT:
        time.sleep(5 * poll_interval)
    if proc.poll() is None:
        chip.logger.warning(f'{tool} did not exit within {TERMINATE_TIMEOUT} '
                            'seconds. Terminating...')
        utils.terminate_process(proc.pid)


def get_check_node_keys(chip, step, index):
    tool, task = get_tool_task(chip, step, index)

    # Collect keys to check for changes
    required = chip.get('tool', tool, 'task', task, 'require', step=step, index=index)

    tool_task_key = ('tool', tool, 'task', task)
    for key in ('option', 'threads', 'prescript', 'postscript', 'refdir', 'script',):
        required.append(",".join([*tool_task_key, key]))

    for env_key in chip.getkeys(*tool_task_key, 'env'):
        required.append(",".join([*tool_task_key, 'env', env_key]))

    return set(sorted(required))


def check_node_inputs(chip, step, index):
    from siliconcompiler import Chip  # import here to avoid circular import

    if chip.get('option', 'clean'):
        return True

    def get_file_time(path):
        times = [os.path.getmtime(path)]
        if os.path.isdir(path):
            for path_root, _, files in os.walk(path):
                for path_end in files:
                    times.append(os.path.getmtime(os.path.join(path_root, path_end)))

        return max(times)

    # Load previous manifest
    input_manifest = None
    in_cfg = f"{chip.getworkdir(step=step, index=index)}/inputs/{chip.design}.pkg.json"
    if os.path.exists(in_cfg):
        input_manifest_time = get_file_time(in_cfg)
        input_manifest = Schema(manifest=in_cfg, logger=chip.logger)

    if not input_manifest:
        # No manifest found so assume okay
        return True

    flow = chip.get('option', 'flow')
    input_flow = input_manifest.get('option', 'flow')

    # Assume modified if flow does not match
    if flow != input_flow:
        return False

    input_chip = Chip('<>')
    input_chip.schema = input_manifest
    # Copy over useful information from chip
    input_chip.logger = chip.logger
    input_chip._packages = chip._packages

    tool, task = get_tool_task(chip, step, index)
    input_tool, input_task = get_tool_task(input_chip, step, index)

    # Assume modified if tool or task does not match
    if tool != input_tool or task != input_task:
        return False

    # Check if inputs changed
    new_inputs = set(_select_inputs(chip, step, index, trial=True))
    if set(input_chip.get('record', 'inputnode', step=step, index=index)) != new_inputs:
        chip.logger.warning(f'inputs to {step}{index} has been modified from previous run')
        return False

    # Collect keys to check for changes
    required = get_check_node_keys(chip, step, index)
    required.update(get_check_node_keys(input_chip, step, index))

    def print_warning(key, extra=None):
        if extra:
            chip.logger.warning(f'[{",".join(key)}] ({extra}) in {step}{index} has been modified '
                                'from previous run')
        else:
            chip.logger.warning(f'[{",".join(key)}] in {step}{index} has been modified '
                                'from previous run')

    # Check if keys have been modified
    for check_key in required:
        key = check_key.split(',')

        if not chip.valid(*key) or not input_chip.valid(*key):
            print_warning(key)
            return False

        check_step = step
        check_index = index
        if chip.get(*key, field='pernode').is_never():
            check_step = None
            check_index = None

        sc_type = chip.get(*key, field='type')
        if 'file' in sc_type or 'dir' in sc_type:
            if chip.get('option', 'hash') and input_chip.get('option', 'hash'):
                check_hash = chip.hash_files(*key, update=False, check=False,
                                             verbose=False, allow_cache=True,
                                             step=check_step, index=check_index)
                prev_hash = input_chip.get(*key, field='filehash',
                                           step=check_step, index=check_index)

                if check_hash != prev_hash:
                    print_warning(key, "file hash")
                    return False
            else:
                # check timestamps on current files
                for check_file in chip.find_files(*key, step=check_step, index=check_index):
                    if get_file_time(check_file) > input_manifest_time:
                        print_warning(key, "timestamp")
                        return False

                # check values
                for field in ('value', 'package'):
                    check_val = chip.get(*key, field=field, step=check_step, index=check_index)
                    prev_val = input_chip.get(*key, field=field, step=check_step, index=check_index)

                    if check_val != prev_val:
                        print_warning(key)
                        return False
        else:
            check_val = chip.get(*key, step=check_step, index=check_index)
            prev_val = input_chip.get(*key, step=check_step, index=check_index)

            if check_val != prev_val:
                print_warning(key)
                return False

    return True


###########################################################################
def check_logfile(chip, jobname=None, step=None, index='0',
                  logfile=None, display=True):
    '''
    Checks logfile for patterns found in the 'regex' parameter.

    Reads the content of the task's log file and compares the content found
    with the task's 'regex' parameter. The matches are stored in the file
    '<step>.<suffix>' in the current directory. The matches are logged
    if display is set to True.

    Args:
        jobname (str): Job directory name. If None, :keypath:`option, jobname` is used.
        step (str): Task step name ('syn', 'place', etc). If None, :keypath:`arg, step` is used.
        index (str): Task index. Default value is 0. If None, :keypath:`arg, index` is used.
        logfile (str): Path to logfile. If None, the default task logfile is used.
        display (bool): If True, logs matches.

    Returns:
        Dictionary mapping suffixes to number of matches for that suffix's
        regex.

    Examples:
        >>> chip.check_logfile(step='place')
        Searches for regex matches in the place logfile.
    '''

    # Using manifest to get defaults

    flow = chip.get('option', 'flow')

    if jobname is None:
        jobname = chip.get('option', 'jobname')
    if step is None:
        step = chip.get('arg', 'step')
        if step is None:
            raise ValueError("Must provide 'step' or set ['arg', 'step']")
    if index is None:
        index = chip.get('arg', 'index')
        if index is None:
            raise ValueError("Must provide 'index' or set ['arg', 'index']")
    if logfile is None:
        logfile = os.path.join(chip.getworkdir(jobname=jobname, step=step, index=index),
                               f'{step}.log')

    tool, task = get_tool_task(chip, step, index, flow=flow)

    # Creating local dictionary (for speed)
    # chip.get is slow
    checks = {}
    matches = {}
    for suffix in chip.getkeys('tool', tool, 'task', task, 'regex'):
        regexes = chip.get('tool', tool, 'task', task, 'regex', suffix, step=step, index=index)
        if not regexes:
            continue

        checks[suffix] = {}
        checks[suffix]['report'] = open(f"{step}.{suffix}", "w")
        checks[suffix]['args'] = regexes
        matches[suffix] = 0

    # Order suffixes as follows: [..., 'warnings', 'errors']
    ordered_suffixes = list(filter(lambda key:
                                   key not in ['warnings', 'errors'], checks.keys()))
    if 'warnings' in checks:
        ordered_suffixes.append('warnings')
    if 'errors' in checks:
        ordered_suffixes.append('errors')

    # Looping through patterns for each line
    with sc_open(logfile) as f:
        line_count = sum(1 for _ in f)
        right_align = len(str(line_count))
        for suffix in ordered_suffixes:
            # Start at the beginning of file again
            f.seek(0)
            for num, line in enumerate(f, start=1):
                string = line
                for item in checks[suffix]['args']:
                    if string is None:
                        break
                    else:
                        string = utils.grep(chip, item, string)
                if string is not None:
                    matches[suffix] += 1
                    # always print to file
                    line_with_num = f'{num: >{right_align}}: {string.strip()}'
                    print(line_with_num, file=checks[suffix]['report'])
                    # selectively print to display
                    if display:
                        if suffix == 'errors':
                            chip.logger.error(line_with_num)
                        elif suffix == 'warnings':
                            chip.logger.warning(line_with_num)
                        else:
                            chip.logger.info(f'{suffix}: {line_with_num}')

    for suffix in ordered_suffixes:
        checks[suffix]['report'].close()

    return matches


def copy_old_run_dir(chip, org_jobname):
    from_nodes = []
    flow = chip.get('option', 'flow')
    for step in chip.get('option', 'from'):
        from_nodes.extend(
            [(step, index) for index in chip.getkeys('flowgraph', flow, step)])
    from_nodes = set(from_nodes)
    if not from_nodes:
        # Nothing to do
        return

    if org_jobname == chip.get('option', 'jobname'):
        return

    # Copy nodes forward
    org_nodes = set(_nodes_to_execute(
        chip,
        flow,
        _get_flowgraph_entry_nodes(chip, flow),
        from_nodes,
        chip.get('option', 'prune')))

    copy_nodes = org_nodes.difference(from_nodes)

    def copy_files(from_path, to_path):
        shutil.copytree(from_path, to_path,
                        dirs_exist_ok=True,
                        copy_function=utils.link_copy)

    for step, index in copy_nodes:
        copy_from = chip.getworkdir(jobname=org_jobname, step=step, index=index)
        copy_to = chip.getworkdir(step=step, index=index)

        if not os.path.exists(copy_from):
            continue

        chip.logger.info(f'Importing {step}{index} from {org_jobname}')
        copy_files(copy_from, copy_to)

    # Copy collect directory
    copy_from = chip._getcollectdir(jobname=org_jobname)
    copy_to = chip._getcollectdir()
    if os.path.exists(copy_from):
        copy_files(copy_from, copy_to)

    # Modify manifests to correct jobname
    for step, index in copy_nodes:
        # rewrite replay files
        replay_file = f'{chip.getworkdir(step=step, index=index)}/replay.sh'
        if os.path.exists(replay_file):
            # delete file as it might be a hard link
            os.remove(replay_file)
            chip.set('arg', 'step', step)
            chip.set('arg', 'index', index)
            tool, task = get_tool_task(chip, step, index)
            _makecmd(chip, tool, task, step, index, script_name=replay_file)
            chip.unset('arg', 'step')
            chip.unset('arg', 'index')

        for io in ('inputs', 'outputs'):
            manifest = f'{chip.getworkdir(step=step, index=index)}/{io}/{chip.design}.pkg.json'
            if os.path.exists(manifest):
                schema = Schema(manifest=manifest)
                # delete file as it might be a hard link
                os.remove(manifest)
                schema.set('option', 'jobname', chip.get('option', 'jobname'))
                with open(manifest, 'w') as f:
                    schema.write_json(f)


def clean_node_dir(chip, step, index):
    node_dir = chip.getworkdir(step=step, index=index)
    if os.path.isdir(node_dir):
        shutil.rmtree(node_dir)


def clean_build_dir(chip):
    if chip.get('record', 'remoteid'):
        return

    if chip.get('arg', 'step'):
        return

    if chip.get('option', 'clean') and not chip.get('option', 'from'):
        # If no step or nodes to start from were specified, the whole flow is being run
        # start-to-finish. Delete the build dir to clear stale results.
        cur_job_dir = chip.getworkdir()
        if os.path.isdir(cur_job_dir):
            shutil.rmtree(cur_job_dir)

        return

    if chip.get('option', 'from'):
        # Remove stale outputs that will be rerun
        for step, index in nodes_to_execute(chip):
            clean_node_dir(chip, step, index)

    all_nodes = set(_get_flowgraph_nodes(chip, flow=chip.get('option', 'flow')))
    old_nodes = __collect_nodes_in_workdir(chip)
    node_mismatch = old_nodes.difference(all_nodes)
    if node_mismatch:
        # flow has different structure so clear whole
        cur_job_dir = chip.getworkdir()
        shutil.rmtree(cur_job_dir)


def __collect_nodes_in_workdir(chip):
    workdir = chip.getworkdir()
    if not os.path.isdir(workdir):
        return set()

    collect_dir = chip._getcollectdir()

    nodes = []
    for step in os.listdir(workdir):
        step_dir = os.path.join(workdir, step)

        if step_dir == collect_dir:
            continue

        if not os.path.isdir(step_dir):
            continue

        for index in os.listdir(step_dir):
            if os.path.isdir(os.path.join(step_dir, index)):
                nodes.append((step, index))

    return set(nodes)


###########################################################################
def _check_manifest_dynamic(chip, step, index):
    '''Runtime checks called from _runtask().

    - Make sure expected inputs exist.
    - Make sure all required filepaths resolve correctly.
    '''
    error = False

    flow = chip.get('option', 'flow')
    tool, task = get_tool_task(chip, step, index, flow=flow)

    required_inputs = chip.get('tool', tool, 'task', task, 'input', step=step, index=index)
    input_dir = os.path.join(chip.getworkdir(step=step, index=index), 'inputs')
    for filename in required_inputs:
        path = os.path.join(input_dir, filename)
        if not os.path.exists(path):
            chip.logger.error(f'Required input {filename} not received for {step}{index}.')
            error = True

    all_required = chip.get('tool', tool, 'task', task, 'require', step=step, index=index)
    for item in all_required:
        keypath = item.split(',')
        if not chip.valid(*keypath):
            chip.logger.error(f'Cannot resolve required keypath {keypath}.')
            error = True
        else:
            paramtype = chip.get(*keypath, field='type')
            is_perstep = not chip.get(*keypath, field='pernode').is_never()
            if ('file' in paramtype) or ('dir' in paramtype):
                for val, check_step, check_index in chip.schema._getvals(*keypath):
                    if is_perstep:
                        if check_step is None:
                            check_step = Schema.GLOBAL_KEY
                        if check_index is None:
                            check_index = Schema.GLOBAL_KEY
                    abspath = chip.find_files(*keypath,
                                              missing_ok=True,
                                              step=check_step, index=check_index)
                    unresolved_paths = val
                    if not isinstance(abspath, list):
                        abspath = [abspath]
                        unresolved_paths = [unresolved_paths]
                    for i, path in enumerate(abspath):
                        if path is None:
                            unresolved_path = unresolved_paths[i]
                            chip.logger.error(f'Cannot resolve path {unresolved_path} in '
                                              f'required file keypath {keypath}.')
                            error = True

    return not error


#######################################
def _clear_metric(chip, step, index, metric, preserve=None):
    '''
    Helper function to clear metrics records
    '''

    # This function is often called in a loop; don't clear
    # metrics which the caller wants to preserve.
    if preserve and metric in preserve:
        return

    flow = chip.get('option', 'flow')
    tool, task = get_tool_task(chip, step, index, flow=flow)

    chip.unset('metric', metric, step=step, index=index)
    chip.unset('tool', tool, 'task', task, 'report', metric, step=step, index=index)


#######################################
def _clear_record(chip, step, index, record, preserve=None):
    '''
    Helper function to clear record parameters
    '''

    # This function is often called in a loop; don't clear
    # records which the caller wants to preserve.
    if preserve and record in preserve:
        return

    if chip.get('record', record, field='pernode').is_never():
        chip.unset('record', record)
    else:
        chip.unset('record', record, step=step, index=index)
