import os
import inspect
from shutil import copy2
from subprocess import call
from collections import namedtuple
try:
    import numpy as np
except ImportError:
    np = None
try:
    import matplotlib.pyplot as plt
except ImportError:
    plt = None
else:
    import matplotlib.ticker as mticker


def postprocessor(script):
    """
    This is the key function that controls composition
    of the previously rendered scenes. It first applies
    overlays if requested, and then copies/moves/composes
    individual frames so that an initial set of
    scene1...png, sceneX...png files is converted
    into movie_name...png files that can then
    be merged into a file using ffmpeg.
    :param script: Script instance, the master object controlling the movie layout
    :return: None
    """
    try:
        layout_dirs = script.directives['layout']  # layout controls composition of panels
    except KeyError:
        layout_dirs = None
        nrows = 1
        ncols = 1
    else:
        nrows = int(layout_dirs['rows']) if 'rows' in layout_dirs.keys() else 1
        ncols = int(layout_dirs['columns']) if 'columns' in layout_dirs.keys() else 1
    check_overlays(script)
    num_base_scenes = len([sc for sc in script.scenes if not sc.is_overlay and not sc.is_after])
    if num_base_scenes < nrows*ncols:
        print(f"Please note: grid is supposed to be {nrows} by {ncols}, but only {num_base_scenes} base scenes were "
              "found, this might be intended but double-check your `$ layout` settings")
    elif num_base_scenes > nrows*ncols:
        print("Some scenes were not included in the grid definition, please check your `$ layout` settings")
    for scene in sorted(script.scenes, key=lambda sc: int(sc.is_overlay), reverse=True):
        if scene.contour:
            for fr in range(scene.total_frames):
                img = f"{scene.name}-{fr}.png"
                call_moly(f'{script.convert} {img} -canny 0x1+10%+40% -negate -transparent white -blur 0x0.75 '
                          f'border.png; composite border.png {img} {img}', log=script.log)
        for action in scene.actions:  # due to sorting, scenes without dependencies should be composed first
            if 'add_overlay' in action.action_type:
                for ovl in action.overlays.keys():
                    if 'scene' in action.overlays[ovl].keys():
                        ovl_scene_name = action.overlays[ovl]['scene']
                        ovl_scene = [sc for sc in script.scenes if sc.name == ovl_scene_name][0]
                        overlay_res = calc_ores(action, ovl, range(ovl_scene.total_frames), True)
                        for fr, ores in zip(range(ovl_scene.total_frames), overlay_res):
                            call_moly(f'{script.convert} {ovl_scene_name}-{fr}.png -resize {ores[0]}x{ores[1]} '
                                      f'{ovl}-{scene.name}-{fr + action.initframe}.png', log=script.log)
                compose_overlay(action)

    if nrows * ncols == 1:  # simplest case: one scene
        scene = script.scenes[0].name
        for fr in range(np.max([sc.total_frames for sc in script.scenes])):
            if not (script.restart and os.path.exists(f'{script.name}-{fr}.png')):
                os.rename(f'{scene}-{fr}.png', f'{script.name}-{fr}.png')

    elif layout_dirs and nrows * ncols > 1:  # here we parse multiple scenes
        # if one has less frames than the other, copy last frame (N-n) times to make counts equal:
        if not all([sc.total_frames == script.scenes[0].total_frames for sc in script.scenes if not sc.is_overlay]):
            equalize_frames(script)
        labels_matrix = [[] for _ in range(nrows)]
        all_scenes = [sc.name for sc in script.scenes]
        positions = {}
        for scene in all_scenes:
            try:
                positions[tuple(int(x) for x in script.directives[scene]['position'].split(','))] = scene
            except KeyError:
                raise ValueError(f'The position for scene {scene} in the global layout is not specified')
        for r in range(nrows):
            for c in range(ncols):
                try:
                    scene_name = positions[(r, c)]
                except KeyError:
                    labels_matrix[r].append('')
                else:
                    labels_matrix[r].append(scene_name)
        def build_convert_command(frame):
            command = ''
            for r in range(nrows):
                command += ' \\( '
                for c in range(ncols):
                    command += f'{labels_matrix[r][c]}-{frame}.png '
                command += ' +append \\) '
            command += ' -append '
            return command

        for fr in range(script.scenes[0].total_frames):
            convert_command = build_convert_command(fr)
            call_moly(f'{script.convert} {convert_command}{script.name}-{fr}.png',
                      log=script.log, quiet=True)

    process_audio(script)


def check_overlays(script):
    """
    Ensures that dependent overlays (i.e. Scenes
    that will be rendered atop other Scenes)
    are properly defined, and calculates level
    of nesting for subsequent ordering of
    rounds of Scene composition
    :param script: Script instance, the master object controlling the movie layout
    :return: None
    """

    def return_dependencies(scn):
        dps = []
        for action in scn.actions:
            if 'add_overlay' in action.action_type:
                for ovl in action.overlays.keys():
                    if 'scene' in action.overlays[ovl].keys():
                        ovl_scene_name = action.overlays[ovl]['scene']
                        try:
                            dps.append([sc for sc in script.scenes if sc.name == ovl_scene_name][0])
                        except IndexError:
                            raise RuntimeError(f"In action {action.description}, scene={ovl_scene_name} is not a valid "
                                               "scene identifier")
        return dps

    all_deps = []
    for scene in script.scenes:
        all_deps.extend(return_dependencies(scene))
    indeps = [sc for sc in script.scenes if sc not in all_deps]
    for indep in indeps:
        deps = return_dependencies(indep)
        incr = 1
        while deps:
            new_deps = []
            for dep in deps:
                dep.is_overlay += incr
                new_deps.extend(return_dependencies(dep))
            deps = new_deps[:]
            incr += 1


def gen_fig(action):
    """
    Responsible for frame-by-frame generation of files
    associated with (a) external images and (b)
    on-the-fly generated matplotlib plots
    (essentially any graphics that is not rendered
    by TCL/VMD)
    :param action: Action, object to extract info from
    :return: None
    """
    if 'show_figure' in action.action_type:
        gen_fig_show(action)
    if 'add_overlay' in action.action_type:
        for ovl in action.overlays.keys():
            gen_ovl(action, ovl)


def add_master_overlays(script):
    if 'layout' in script.directives.keys():
        resolution = get_resolution(script)
    else:
        resolution = [max([scene.resolution[0] for scene in script.scenes if not scene.is_overlay]),
                      max([scene.resolution[1] for scene in script.scenes if not scene.is_overlay])]
    for ovl_action in script.master_overlays:
        ovl_action.scene.resolution = resolution
        gen_ovl(ovl_action, list(ovl_action.overlays.keys())[0], master=True)
        compose_overlay(ovl_action, master=True)


def get_resolution(script):
    layout_dirs = script.directives['layout']  # layout controls composition of panels
    nrows = int(layout_dirs['rows']) if 'rows' in layout_dirs.keys() else 1
    ncols = int(layout_dirs['columns']) if 'columns' in layout_dirs.keys() else 1
    xres = 0
    yres = 0
    for row in range(nrows):
        scenes_in_row_width = sum([sc.resolution[0] for sc in script.scenes if sc.position[0] == row])
        xres = max([xres, scenes_in_row_width])
    for col in range(ncols):
        scenes_in_col_height = sum([sc.resolution[1] for sc in script.scenes if sc.position[1] == col])
        yres = max([yres, scenes_in_col_height])
    return xres, yres


def gen_fig_show(action):
    """
    Generates a static figure whem action is `show_figure`
    :param action: The Action object to draw data from
    :return: None
    """
    convert = action.scene.script.convert
    x_res, y_res = action.scene.resolution
    if 'figure' in action.parameters.keys():
        fig_file = action.parameters['figure']
        fig_file = action.scene.script.check_path(fig_file)
        for fr in range(action.initframe, action.initframe + action.framenum):
            call_moly(f'{convert} {fig_file} -resize {x_res}x{y_res} {action.scene.name}-{fr}.png',
                      log=action.scene.script.log)
    elif 'datafile' in action.parameters.keys():
        df = action.parameters['datafile']
        df = action.scene.script.check_path(df)
        data_simple_plot(action, df, 'spl')
        for fr in range(action.initframe, action.initframe + action.framenum):
            fig_file = f'{action.scene.name}-{fr}.png'
            call_moly(f'{convert} spl-{fig_file} -resize {x_res}x{y_res} {fig_file}',
                      log=action.scene.script.log)


def gen_ovl(action, ovl, master=False):
    """
    Handles the generation of dynamic overlays (figures,
    text, plots, movies) as requested by `add_overlay`
    :param action: The Action object to draw data from
    :param ovl: str, key for the Action.overlays dictionary
    :return: None
    """
    if "scene" not in action.overlays[ovl].keys():
        print(f"Rendering overlays for scene {action.scene.name}, currently: {ovl}")
    convert = action.scene.script.convert
    scene = action.scene.name
    res = action.scene.resolution
    frames, actions = calc_frames_ud(action, ovl, master)
    overlay_res = calc_ores(action, ovl, frames, default=True, master=master)
    if 'figure' in action.overlays[ovl].keys():
        fig_file = action.overlays[ovl]['figure']
        if fig_file not in ['f', 'false', 'n', 'no', '0']:
            fig_file = action.scene.script.check_path(fig_file)
            for fr, ores in zip(frames, overlay_res):
                ovl_file = f'{ovl}-{scene}-{fr}.png'
                call_moly(f'{convert} {fig_file} -resize {ores[0]}x{ores[1]} {ovl_file}',
                          log=action.scene.script.log)
    elif 'datafile' in action.overlays[ovl].keys():
        df = action.overlays[ovl]['datafile']
        if df not in ['f', 'false', 'n', 'no', '0']:
            df = action.scene.script.check_path(df)
            data_simple_plot(action, df, ovl, master)
            for fr, ores in zip(frames, overlay_res):
                fig_file = f'{ovl}-{scene}-{fr}.png'
                call_moly(f'{convert} {fig_file} -resize {ores[0]}x{ores[1]} {fig_file}',
                          log=action.scene.script.log)
    elif 'text' in action.overlays[ovl].keys():
        plt.rcParams['mathtext.default'] = 'regular'
        text = action.overlays[ovl]['text']
        scaling_text = ''
        tsize = int(32 * np.sqrt(int(res[0]) * int(res[1])) / (10 ** 3))
        try:
            tsize = int(float(action.overlays[ovl]['textsize']) * tsize)
        except KeyError:
            pass
        textcolors = {'red': 'B22222', 'blue': '2B6CC4', 'green': '008000', 'yellow': 'FFCF48',
                      'orange': 'FF8C00', 'purple': '8F509D'}
        try:
            box_params_raw = action.overlays[ovl]['textbox']
        except KeyError:
            box_params = {}
            text_origin = 0
        else:  # TODO check for correct formatting
            if box_params_raw in ['f', 'false', 'n', 'no', '0']:
                box_params = {}
                text_origin = 0
            else:
                box_params = {'boxstyle': "round", 'ec': (0.0, 0.0, 0.0), 'fc': (1.0, 1.0, 1.0), 'alpha': 0.8,
                              'lw': tsize*0.1}
                text_origin = 0.002 * tsize
                if box_params_raw not in ['t', 'true', 'y', 'yes', '1']:
                    box_params_raw = box_params_raw.split(',')
                    param_maps = {'style': 'boxstyle', 'background_color': 'fc', 'line_color': 'ec'}
                    for param_pair in box_params_raw:
                        boxkey, boxval = param_pair.split(':')
                        bkey = param_maps[boxkey] if boxkey in param_maps.keys() else boxkey
                        try:
                            box_params[bkey] = eval(boxval)
                        except NameError:
                            box_params[bkey] = boxval
        try:
            color = action.overlays[ovl]['textcolor']
        except KeyError:
            tcolor = 'black'
        else:
            if color in ['f', 'false', 'n', 'no', '0']:
                tcolor = 'black'
            else:
                try:
                    tcolor = f"#{textcolors[color.lower()]}"
                except KeyError:
                    tcolor = color

        try:
            animation_frames = [float(x) for x in action.overlays[ovl]['dataframes'].split(':')]
            arr = np.linspace(animation_frames[0], animation_frames[1], len(frames))
        except KeyError:
            if '[' in text and ']' in text and '[]' not in text:
                scaling_text = text.split('[')[1].split(']'[0])[0]
                try:
                    _ = [int(x) for x in action.parameters['frames'].split(':')]
                except (KeyError, AttributeError):
                    raise RuntimeError("When the [scaling_factor] syntax is used, an 'animate' action"
                                       "has to specify 'frames' simultaneously")
                else:
                    arr = calc_frames_arr(actions)
            else:
                arr = np.arange(len(frames))
        try:
            fig = plt.figure(figsize=[r / 100 for r in res])
        except RuntimeError:
            plt.switch_backend('agg')
            fig = plt.figure(figsize=[r / 100 for r in res])
        text_dict = {'ha': 'left', 'va': 'bottom', 'ma': 'center', 'clip_on': False, 'size': tsize, 'color': tcolor,
                     'wrap': True}
        if box_params:
            text_dict.update({'bbox': box_params})
        if '[' in text and ']' in text:
            try:
                dpt = action.overlays[ovl]['decimal_points']
            except KeyError:
                dpt = '3'
            for fr in frames:
                scaler = float(scaling_text) if scaling_text else 1.0
                formatted_val = f'{arr[fr - action.initframe] * scaler:.{dpt}f}'
                newtext = text.replace(f'[{scaling_text}]', formatted_val)
                fig_file = f'{ovl}-{scene}-{fr}.png'
                plt.clf()
                _ = fig.text(text_origin, text_origin, newtext, **text_dict)
                plt.savefig(fig_file, transparent=True)
        else:
            templ_file = f'{ovl}-{scene}-x.png'
            plt.clf()
            if '\\n' in text:
                text = text.replace('\\n', '\n')
            _ = fig.text(text_origin, text_origin, text, **text_dict)
            plt.savefig(templ_file, transparent=True)
            for fr in frames:
                fig_file = f'{ovl}-{scene}-{fr}.png'
                copy2(templ_file, fig_file)
            os.remove(templ_file)
    elif 'movie' in action.overlays[ovl].keys():
        basename = f'{ovl}-{scene}-%d.png'
        movie = action.overlays[ovl]['movie']
        try:
            start = action.overlays[ovl]['from'].rstrip('s')
        except KeyError:
            start = 0
        try:
            time = float(action.overlays[ovl]['length'])
        except KeyError:
            time = action.framenum / action.scene.script.fps
        fps = float(action.framenum) / time
        time += 1 / fps
        timespec = f"-ss {start} -t {time} "
        tmp_basename = f'tmp_{basename}'
        call_moly(f'{action.scene.script.ffmpeg} -i {movie} {timespec} -vf fps={fps} {tmp_basename} -hide_banner',
                  log=action.scene.script.log)
        initframe = frames[0]
        totalframes = len(frames)
        for i, ores, n in zip(range(totalframes, 0, -1), overlay_res[::-1], range(totalframes)):
            orig = f'tmp_{ovl}-{scene}-{i}.png'
            modd = f'{ovl}-{scene}-{i + initframe - 1}.png'
            if n % 10 == 0:
                print(f'processing frame {n} of movie {movie}')
            call_moly(f'{convert} {orig} -resize {ores[0]}x{ores[1]} {modd}', log=action.scene.script.log)
            os.remove(orig)


def show_grid(action):
    # TODO adjust
    convert = action.scene.script.convert
    frame = action.initframe - 1
    scene = action.scene.name
    call_moly(f'{convert} {scene}-{frame}.png \\( +clone -colorspace gray -fx '
              '"(i==int(w/10)||i==2*int(w/10)||i==3*int(w/10)||i==4*int(w/10)||i==5*int(w/10)||'
              'i==6*int(w/10)||i==7*int(w/10)||i==8*int(w/10)||i==9*int(w/10)||'
              'j==int(h/10)||j==2*int(h/10)||j==3*int(h/10)||j==4*int(h/10)||j==5*int(h/10)||'
              'j==6*int(h/10)||j==7*int(h/10)||j==8*int(h/10)||j==9*int(h/10))?0:1" \\)'
              f' -compose darken -composite grid-{scene}-{frame}.png')


def calc_ores(action, ovl, frames, default=False, master=False):
    """
    Calculates per-frame overlay resolutions
    in case relative size is scaled linearly
    :param action: Action or SimultaneousAction instance
    :param ovl: str, name of the overlay
    :param frames: iterable, enumerates all frames overlay should apply to
    :param default: bool, whether a default should be provided or error raised when relative_size is not defined
    :return: list of 2-element lists: [[x_res, y_res] for frame in frames]
    """
    res = action.scene.resolution
    _, actions = calc_frames_ud(action, ovl, master)
    if not any('relative_size' in act.overlays[ovl].keys() for act in actions
               if hasattr(act, 'overlays') and ovl in act.overlays.keys()):
        if default:
            overlay_res = [[r for r in res] for _ in range(len(frames))]
        else:
            raise RuntimeError("With add_overlay, 'relative_size=... has to be specified")
    else:
        sizes = extract_value_actions(actions, ovl, 'relative_size')
        overlay_res = [[r*val[0] for r in res] for val in sizes]
    return overlay_res


def equalize_frames(script):
    """
    If individual scenes have different frame counts,
    this function appends the corresponding last figure
    to the shorter scenes
    :param script: Script instance, the master object controlling the movie layout
    :return: None
    """
    nframes = [sc.total_frames for sc in script.scenes]
    names = [sc.name for sc in script.scenes]
    highest = max(nframes)
    for n, nf in enumerate(nframes):
        if nf < highest:
            for i in range(nf, highest):
                copy2(f'{names[n]}-{nf - 1}.png', f'{names[n]}-{i}.png')


def compose_overlay(action, master=False):
    """
    Introduces the overlay provided that the picture
    ('overlay-frame_name.png') was already produced
    and properly scaled by gen_fig
    :param action: Action or SimultaneousAction, object to extract data from
    :param master: bool, whether to compose with the final frames (for master_overlays)
    :return: None
    """
    assert hasattr(action, 'overlays') and isinstance(action.overlays, dict)
    scene = action.scene.name
    res = action.scene.resolution
    for ovl in action.overlays.keys():
        frames, actions = calc_frames_ud(action, ovl, master)
        try:
            sigmoid = action.overlays[ovl]['sigmoid']
        except KeyError:
            sigmoid = False
        else:
            sigmoid = True if sigmoid.lower() in ['true', 't', 'y', 'yes', '1'] else False
        if sigmoid:
            sgm_frames_init = int(0.05 * action.framenum)
            sgm_frames_final = int(0.05 * actions[-1].framenum)
            x_init = np.linspace(-5, 0, sgm_frames_init)
            x_final = np.linspace(0, 5, sgm_frames_final)
            sgm_init = 1 / (1 + np.exp(-x_init))
            sgm_final = 1 / (1 + np.exp(-x_final))
            opacity = np.concatenate((sgm_init, np.ones(len(frames) - (sgm_frames_init + sgm_frames_final)),
                                      sgm_final[::-1]))
        else:
            opacity = np.ones(len(frames))
        try:
            center = True if action.overlays[ovl]['center'].lower() in ['true', 't', 'y', 'yes', '1'] else False
        except KeyError:
            center = False
        if not any('angle' in act.overlays[ovl].keys() for act in actions
                   if hasattr(act, 'overlays') and ovl in act.overlays.keys()):
            angles = [0] * len(frames)
        else:
            angles = [x[0] for x in extract_value_actions(actions, ovl, 'angle', default=0)]
        if not any('origin' in act.overlays[ovl].keys() for act in actions
                   if hasattr(act, 'overlays') and ovl in act.overlays.keys()):
            origin_frac = [[0, 0] for _ in range(len(frames))]
        else:
            origin_frac = extract_value_actions(actions, ovl, 'origin')
        transp_bg = True if 'transparent_background' in action.overlays[ovl].keys() \
                            and action.overlays[ovl]['transparent_background'] in ['true', 't', 'y', 'yes', '1'] else False
        if not any('alpha' in act.overlays[ovl].keys() for act in actions
                   if hasattr(act, 'overlays') and ovl in act.overlays.keys()):
            alpha = np.ones(np.sum([act.framenum for act in actions]).astype(int))
        else:
            alpha = [x[0] for x in extract_value_actions(actions, ovl, 'alpha', default=1)]
        origin_px = [[int(r * o) for r, o in zip(res, ofr)] for ofr in origin_frac]
        for fr, opa, al, opx, ang in zip(frames, opacity, alpha, origin_px, angles):
            if fr % 10 == 0:
                print(f'composing frame {fr} of scene {action.scene.name} with {ovl}')
            fig_file = f'{ovl}-{scene}-{fr}.png'
            if master:
                target_fig = f'{action.scene.script.name}-{fr}.png'
            else:
                target_fig = f'{scene}-{fr}.png'
            opa *= al
            if transp_bg:
                call_moly(f'{action.scene.script.convert} {fig_file} -transparent white {fig_file}',
                          log=action.scene.script.log)
            if opa != 1:
                call_moly(f'{action.scene.script.convert} {fig_file} -alpha set -channel a -evaluate multiply {opa} '
                          f'+channel {fig_file}', log=action.scene.script.log)
            grav = 'SouthWest' if not center else 'Center'
            opx[1] = opx[1] if not center else -1 * opx[1]
            if 'text' in action.overlays[ovl].keys():
                call_moly(f'{action.scene.script.convert} {fig_file} -trim +repage {fig_file}', log=action.scene.script.log)
            if ang > 0.0000001 or ang < -0.0000001:
                xang = ang % 360
                call_moly(f'{action.scene.script.convert} -background "rgba(0,0,0,0)" -rotate "{xang}" {fig_file} {fig_file}')
            call_moly(f'{action.scene.script.compose} -colorspace sRGB -gravity {grav} -compose atop -geometry +{opx[0]}+{opx[1]} '
                      f'{fig_file} {target_fig} -colorspace sRGB {target_fig}', log=action.scene.script.log)


def extract_value_actions(actions, ovl, value, var_type=float, default=0):
    """
    Takes a list of Actions that are spanned by a given overlay and calculates
    a frame-by-frame list of values of a given property (specified by `value`)
    so that intermediate Actions can modify the original value
    :param actions: list of Actions, actions affected by the overlay
    :param ovl: str, alias of the overlay
    :param value: str, name/keyword of the property to extract
    :param var_type: type, variable type (to be checked for compatibility of conversion)
    :param default: any, default value if not specified for 1st action
    :return: list of tuples, length equal to num of frames affected by the overlay
    """
    result = []
    lastval = default

    def generator(lin, start, end, nfr, abruptness=1):
        if lin:
            return np.linspace(start, end, nfr)
        else:
            sigm = np.array(1 / (1 + np.exp(-abruptness * np.linspace(-5, 5, nfr))))
            return start + (end - start) * sigm

    for act in actions:
        try:
            dir = act.overlays[ovl][value]
        except (KeyError, AttributeError):
            for _ in range(act.framenum):
                result.append(lastval)
        else:
            linear = [True if '::' in d else False for d in dir.split(',')]
            dir.replace('::', ':')
            dirs = [(d.split(':')[0], d.split(':')[-1]) for d in dir.split(',')]
            [check_if_convertible(q, var_type, value) for d in dirs for q in d]
            vals = [generator(l, var_type(d[0]), var_type(d[1]), act.framenum) for d, l in zip(dirs, linear)]
            for x in zip(*vals):
                result.append(x)
            if len(result) > 0:
                lastval = result[-1]
            else:  # special case if we have an add_overlay with length 0
                lastval = [generator(l, var_type(d[0]), var_type(d[1]), 1)[0] for d, l in zip(dirs, linear)]
    return result


def data_simple_plot(action, datafile, basename, master=False, save_to_file=True):
    """
    Creates a set of customizable 1D line plots
    or 2D hexbin plots based on a provided data file
    to e.g. accompany the display of an
    animated trajectory
    :param action: Action or SimultaneousAction, object to extract data from
    :param datafile: str, file containing the data to be plotted
    :param basename: str, base name of the image to be produced (e.g. 'overlay1')
    :param master: bool, whether corresponds to a master_overlay rather than a regular overlay
    :param save_to_file: bool, whether the plot is saved (for movie-making) or displayed (for preview)
    :return: None
    """
    res = action.scene.resolution
    frames, actions = calc_frames_ud(action, basename, master)
    try:
        asp_ratio = float(action.parameters['aspect_ratio'])
    except KeyError:
        asp_ratio = res[0] / res[1]
    _data_simple_plot(datafile, basename, action.scene.name, frames, save_to_file, asp_ratio, action, action.initframe,
                      actions)


def _data_simple_plot(datafile, basename=None, scenename=None, frames=None, save_to_file=False, asp_ratio=1,
                      action=None, initframe=0, actions=None):
    """
    Removes all dependencies on molywood.action to allow for external calls
    """
    font = {'size': 18}
    plt.rc('font', **font)
    plt.rc('axes', linewidth=2)
    plt.rcParams['figure.figsize'] = [4.8 * np.sqrt(asp_ratio), 4.8 / np.sqrt(asp_ratio)]
    os.environ.pop("XDG_SESSION_TYPE", None)
    try:
        if save_to_file:
            plt.clf()
        else:
            plt.close('all')
        fig = plt.figure()
    except RuntimeError:
        plt.switch_backend('agg')
        if save_to_file:
            plt.clf()
        else:
            plt.close('all')
        fig = plt.figure()
    datasets = read_datasets(datafile)
    ax = fig.add_subplot(1, 1, 1)
    datasets, custom_settings = process_dset(datasets)
    arr = get_arr(action, datasets, actions, basename, frames) if save_to_file else np.arange(np.max([len(dset['data']) for dset in datasets]))
    frames = frames if save_to_file else None
    static_datasets = get_static_datasets(datasets)
    dynamic_datasets = get_dynamic_datasets(datasets)
    custom_settings = plot_static(ax, static_datasets, custom_settings)
    plot_dynamic(ax, dynamic_datasets, frames, arr, basename, scenename, initframe, custom_settings, save_to_file)


def get_arr(action, datasets, actions, basename, frames):
    try:
        animation_frames = [int(x) for x in action.overlays[basename]['dataframes'].split(':')]
        arr = np.linspace(animation_frames[0], animation_frames[1], len(frames)).astype(int)
    except KeyError:
        try:
            arr = np.loadtxt(action.overlays[basename]['dataframes_from_file']).reshape(-1).astype(int)
        except KeyError:
            if action is not None:
                try:
                    _ = [int(x) for x in action.parameters['frames'].split(':') if x != "last"]
                except (KeyError, AttributeError):
                    maxlen = np.max([len(dset['data']) for dset in datasets])
                    arr = np.linspace(0, maxlen-1, len(frames), dtype=int)
                else:
                    maxlen = np.max([len(dset['data']) for dset in datasets])
                    arr = calc_frames_arr(actions, maxlen=maxlen)
            else:
                arr = np.arange(np.max([len(dset['data']) for dset in datasets]))
        else:
            if len(arr) != len(frames):
                arr = [arr[int(i)] for i in np.linspace(0, len(arr) - 1, len(frames))]
    return arr


def read_datasets(datafile):
    datasets = []
    flag_setup = False
    raw_data = [line.strip() for line in open(datafile) if line.strip()]
    for line in raw_data:
        if not flag_setup and (line.startswith('#') or line.startswith('!')):
            flag_setup = True
            datasets.append({'setup': [], 'data': []})
        if not datasets:
            datasets.append({'setup': [], 'data': []})
        if flag_setup and not (line.startswith('#') or line.startswith('!')):
            flag_setup = False
        if flag_setup:
            datasets[-1]['setup'].append(line)
        else:
            datasets[-1]['data'].append([float(x) for x in line.split()])
    return datasets


def process_dset(datasets):
    custom = {}
    custom['xmin'], custom['ymin'], custom['xmax'], custom['ymax'] = np.inf, np.inf, -np.inf, -np.inf
    custom['xlim_set'] = False
    custom['ylim_set'] = False
    custom['labels'] = False
    custom['legend'] = False
    for dset in datasets:
        dset['data'] = np.array(dset['data'])
        # processing labels (can be placed with any dataset)
        try:
            custom['labels'] = [x.strip() for x in dset['setup'] if x.strip().startswith('#')][0]
        except IndexError:
            pass
        else:
            custom['labels'] = custom['labels'].strip('#').strip().split(';')
        if not custom['labels']:
            custom['labels'] = ['Time', 'Value']
        try:
            mpl_kw = [x.strip() for x in dset['setup'] if x.strip().startswith('!')][0]
        except IndexError:
            mpl_kw = {}
        else:
            mpl_kw = {x.split('=')[0]: x.split('=')[1] for x in mpl_kw.strip('!').strip().split()}
            for kw in mpl_kw.keys():
                try:
                    mpl_kw[kw] = eval(mpl_kw[kw])
                except:
                    pass
        # processing xlim/ylim (looking for the absolute lowest/highest)
        if 'xlim' not in mpl_kw.keys():
            if dset['data'].shape[1] < 6:
                custom['xmin'] = np.min(dset['data'][:, 0]) if np.min(dset['data'][:, 0]) < custom['xmin'] else custom[
                    'xmin']
                custom['xmax'] = np.max(dset['data'][:, 0]) if np.max(dset['data'][:, 0]) > custom['xmax'] else custom[
                    'xmax']
            else:
                custom['xmin'] = np.min(dset['data'][0, :]) if np.min(dset['data'][:, 0]) < custom['xmin'] else custom[
                    'xmin']
                custom['xmax'] = np.max(dset['data'][0, :]) if np.max(dset['data'][:, 0]) > custom['xmax'] else custom[
                    'xmax']
            custom['xlim_set'] = False
        else:
            custom['xmin'], custom['xmax'] = mpl_kw['xlim']
            custom['xlim_set'] = True
        if 'ylim' not in mpl_kw.keys():
            if dset['data'].shape[1] < 6:
                custom['ymin'] = np.min(dset['data'][:, 1]) if np.min(dset['data'][:, 1]) < custom['ymin'] else custom[
                    'ymin']
                custom['ymax'] = np.max(dset['data'][:, 1]) if np.max(dset['data'][:, 1]) > custom['ymax'] else custom[
                    'ymax']
            else:
                custom['ymin'] = np.min(dset['data'][1:, :]) if np.min(dset['data'][:, 1]) < custom['ymin'] else custom[
                    'ymin']
                custom['ymax'] = np.max(dset['data'][1:, :]) if np.max(dset['data'][:, 1]) > custom['ymax'] else custom[
                    'ymax']
            custom['ylim_set'] = False
        else:
            custom['ymin'], custom['ymax'] = mpl_kw['ylim']
            custom['ylim_set'] = True
        xspread = custom['xmax'] - custom['xmin']
        yspread = custom['ymax'] - custom['ymin']
        custom['xmarg'] = 0.0 if custom['xlim_set'] else 0.1 * xspread
        custom['ymarg'] = 0.0 if custom['ylim_set'] else 0.025 * yspread
        dset['plottype'], dset['2D'], dset['accumulate'], dset['dynamic'], dset['metadyn'], mpl_kw = identify_plot_type(mpl_kw, dset['data'].shape)
        # removing extras
        dset['mpl'] = {'lw': 2.5}
        dset['mpl'].update(mpl_kw)
        # checking for legend
        if 'label' in dset['mpl']:
            custom['legend'] = True
        # TODO edit data into xgrid/ygrid and data if multiple lines
        if 'contourf' in dset['plottype']:
            dset['xgrid'] = dset['data'][0]
            dset['ygrid'] = dset['data'][1]
            dset['data'] = dset['data'][2:]
    return datasets, custom


def identify_plot_type(mpl_kw, datashape):
    """
    Decide: chosen plot primitives, 2D?, accumulate?, dynamic?, metadynamics?
    Standards (precedence):
    1) Explicit keywords win: `kind`, `2d` (bool), `mode` (static|grow|evolve),
       `metadyn`/`metadynamics`, `heatmap`, `contour/contourf`, `hist`.
    2) Legacy sugar: `accumulate` implies mode=grow; `dynamic` implies mode=evolve.
    3) Auto kind: grids/default-2D → contourf; otherwise line/plot.
    Supported data shapes:
    - Point/line data: (N, C) with C >= 2 → scatter/plot; if C == 3 and kind=hist → 2D hist.
    - 2D grids: (NX, NY) → contourf (static surface).
    - Evolving 2D grids: (F, NX, NY) → contourf (dynamic surface).
    - Histograms: 1D or 2D (C == 3) point data; plot_static/plot_dynamic will call `ax.hist`.
    Legal output combinations (chosen_plots, two_dim):
    - ['plot'] + two_dim=False → line plot on (x, y) columns; accumulates if mode=grow.
    - ['scatter'] + two_dim=False → scatter on (x, y); used when kind=scatter.
    - ['plot','scatter'] + two_dim=False → line plus moving dot.
    - ['hist'] + two_dim=False/True → 1D or 2D hist depending on column count.
    - ['contourf'] + two_dim=True → filled contour from grid data or metadynamics.
    - Metadynamics: metadyn flag pushes contourf (2D) or plot (1D); dynamic follows the explicit `dynamic` kwarg when provided.
    Outputs feed plot_static/plot_dynamic: `plot`/`scatter` draw lines/dots, `hist` uses `ax.hist`,
      `contourf` uses `ax.contourf`.
    Returns: (chosen_plots, two_dim, accumulate, dynamic, metadyn)
    """
    plots = ['scatter', 'plot', 'contourf', 'hist']

    # --- helpers ------------------------------------------------------------
    def norm_key(s): return str(s).strip().lower()

    def as_bool(v):
        if isinstance(v, bool): return v
        if v is None: return False
        return str(v).strip().lower() in {"1","true","t","yes","y","on"}

    def get_kw(*names, default=None):
        for n in names:
            if n in mpl_kw: return mpl_kw[n]
            # allow case-insensitive keys
            for k in mpl_kw.keys():
                if norm_key(k) == norm_key(n): return mpl_kw[k]
        return default

    # normalize primary switches
    kind_raw = get_kw("kind", default="auto")
    kind = norm_key(kind_raw)
    dims_raw = get_kw("2d", default="auto")
    ds2d = norm_key(dims_raw)
    if ds2d == "auto":
        dims = "auto"
    else:
        if as_bool(ds2d):
            dims = 2
        else:
            dims = 1

    mode_raw = get_kw("mode", default="")
    mode = norm_key(mode_raw)
    if mode not in {"static","grow","evolve"}:
        # accept legacy booleans as sugar
        if as_bool(get_kw("accumulate")):
            mode = "grow"
        elif as_bool(get_kw("dynamic")):
            mode = "evolve"
        else:
            mode = "static"

    # metadynamics flag (kind beats explicit)
    metadyn = (kind == "meta") or as_bool(get_kw("metadyn", "metadynamics"))

    # --- infer dims if needed ----------------------------------------------
    shape_len = len(datashape) if datashape is not None else 0
    ncols = datashape[-1] if (shape_len >= 2) else 1
    nrows = datashape[0] if (shape_len >= 1) else 0

    def infer_dims():
        # explicit nudges
        if as_bool(get_kw("heatmap")): return 2
        if kind in {"contour","contourf","meta"}: return 2 if kind != "meta" else None
        if kind in {"hist"}:
            # 1D hist typically 2 cols (x, w?) or 1; 2D hist usually >=3 cols or grids
            if shape_len == 3: return 2
            if ncols >= 3: return 2
            return 1
        if shape_len == 3:  # (F, NX, NY): evolving 2D surface
            return 2
        if shape_len == 2 and ncols <= 2:
            # generic point data → default to 1D (x→y) unless user asks otherwise
            return 1
        return 1

    if dims == "auto":
        d = infer_dims()
        # meta without explicit dims: try to infer from columns
        if metadyn and d is None:
            if ncols >= 6 or shape_len == 3: dims = 2
            else: dims = 1
        else:
            dims = d

    two_dim = bool(dims == 2)

    # --- choose primitives --------------------------------------------------
    chosen = []

    def push(p):
        if p in plots and p not in chosen: chosen.append(p)

    # auto kind selection (grid → contourf; else line)
    if kind == "auto":
        if shape_len == 3 or (shape_len == 2 and nrows > 4 and ncols > 4 and two_dim):
            kind_pick = "contour"
        else:
            kind_pick = "line"
    else:
        kind_pick = kind
    if metadyn:
        # metadynamics visualization primitive
        push("contourf" if two_dim else "plot")
    else:
        if kind_pick in {"line","plot"}:
            push("plot")
        elif kind_pick in {"scatter"}:
            push("scatter")
        elif kind_pick in {"bar", "hist", "histogram"}:
            push("hist")
        elif kind_pick in {"contour", "contourf", "heatmap"}:
            push("contourf")
        else:  # fallback
            push("plot")

    if 'hist' in chosen and shape_len >= 2 and datashape[1] == 3:
        two_dim = True
    # --- flags: accumulate & dynamic ---------------------------------------
    accumulate = (mode == "grow") or (kind_pick == "hist" and as_bool(get_kw("cumulative")))
    dynamic_kw = get_kw("dynamic", default=None)
    if dynamic_kw is not None:
        dynamic = as_bool(dynamic_kw)
    else:
        dynamic = (mode in {"grow", "evolve"}) or metadyn

    # guard: avoid nonsense combos
    if two_dim and "plot" in chosen and "scatter" not in chosen and not metadyn and kind_pick != "hist":
        # a plain 2D "plot" is unusual; prefer scatter for point clouds or contourf for grids
        if shape_len >= 2 and ncols == 2:
            chosen = [p for p in chosen if p != "plot"]
            push("scatter")

    return chosen, two_dim, accumulate, dynamic, bool(metadyn), mpl_kw


def get_static_datasets(datasets):
    static_datasets = []
    for dset in datasets:
        if not dset['dynamic']:
            static_datasets.append(dset)
    return static_datasets


def get_dynamic_datasets(datasets):
    dynamic_datasets = []
    for dset in datasets:
        if dset['dynamic']:
            dynamic_datasets.append(dset)
    return dynamic_datasets


def plot_static(ax, static_datasets, custom_settings):
    custom_settings['skip_axis_setup'] = False
    for dset in static_datasets:
        if 'scatter' in dset['plottype']:
            kwargs = filter_kwargs("scatter", dset['mpl'])
            ax.scatter(dset['data'][:, 0], dset['data'][:, 1], zorder=-5, **kwargs)
        if 'plot' in dset['plottype']:
            kwargs = filter_kwargs("plot", dset['mpl'])
            ax.plot(dset['data'][:, 0], dset['data'][:, 1], zorder=-10, **kwargs)
        if 'contourf' in dset['plottype']:
            kwargs = filter_kwargs("contourf", dset['mpl'])
            xsh = len(set(list(dset['xgrid'])))
            ysh = len(set(list(dset['ygrid'])))
            X = dset['xgrid'].reshape(xsh, ysh)
            Y = dset['ygrid'].reshape(xsh, ysh)
            Z0 = dset['data'].reshape(xsh, ysh)
            ax.contourf(X, Y, Z0, zorder=-20, **kwargs)
            ax.set_xlim(X[0,0], X[-1,-1])
            ax.set_ylim(Y[0,0], Y[-1,-1])
            custom_settings['skip_axis_setup'] = True
        if 'hist' in dset['plottype']:
            nbins_default = 20
            nbins = dset['mpl'].pop('bins', nbins_default)
            if not dset['mpl']['2d']:
                data = dset['data'][:,-1]
                kwargs = filter_kwargs("hist", dset['mpl'])
                if not custom_settings['xlim_set']:
                    ymin, ymax = float(np.min(data)), float(np.max(data))
                else:
                    ymin, ymax = custom_settings['ymin'], custom_settings['ymax']
                edges = np.linspace(ymin, ymax, nbins + 1)
                widths = np.diff(edges)
                dens, _ = np.histogram(data, bins=edges, density=True)
                centers = 0.5 * (edges[:-1] + edges[1:])
                kwargs = filter_kwargs("hist", dset['mpl'])
                ax.bar(centers, dens, width=widths, align='center', zorder=4, ec='C0', fc='w', **kwargs)
                span = ymax - ymin
                ax.set_xlim(edges[0] - 0.05 * span, edges[-1] + 0.05 * span)
                custom_settings['skip_axis_setup'] = True
            else:
                data = dset['data']
                smooth = dset['mpl'].pop('smooth', 0)
                if smooth > 0:
                    from scipy.ndimage import gaussian_filter
                kwargs = filter_kwargs("contourf", dset['mpl'])
                levels_default = 8
                levels = kwargs.pop('levels', levels_default)
                if not custom_settings['xlim_set']:
                    xmin, xmax = float(np.min(data[:, 1])), float(np.max(data[:, 1]))
                else:
                    xmin, xmax = custom_settings['xmin'], custom_settings['xmax']
                if not custom_settings['ylim_set']:
                    ymin, ymax = float(np.min(data[:, 2])), float(np.max(data[:, 2]))
                else:
                    ymin, ymax = custom_settings['ymin'], custom_settings['ymax']
                custom_settings['skip_axis_setup'] = True
                xedges = np.linspace(xmin, xmax, nbins + 1)
                yedges = np.linspace(ymin, ymax, nbins + 1)
                xbins = xedges[:-1] + 0.5 * (xedges[1] - xedges[0])
                ybins = yedges[:-1] + 0.5 * (yedges[1] - yedges[0])
                C, _, _ = np.histogram2d(data[:, 1], data[:, 2], bins=(xedges, yedges), density=True)
                C = np.nan_to_num(C, nan=0)
                if smooth > 0:
                    C = gaussian_filter(C, smooth)
                ax.contourf(xbins, ybins, C, levels=levels, zorder=4, **kwargs)
                ax.set_xlim(xedges[0], xedges[-1])
                ax.set_ylim(yedges[0], yedges[-1])
    return custom_settings


def plot_dynamic(ax, dynamic_datasets, frames, arr, basename, scenename, initframe, custom_settings, save_to_file):
    handles = {}
    if not save_to_file:
        arr = np.linspace(0, len(arr)-0.6, 200).astype(int)
    for i in range(len(dynamic_datasets)):
        dset = dynamic_datasets[i]
        if dset['metadyn']:
            if dset['2D']:
                dynamic_datasets[i], custom_settings = process_2D_metadyn(dset, custom_settings, arr)
            else:
                dynamic_datasets[i], custom_settings = process_1D_metadyn(dset, custom_settings, arr)
    for n, dset in enumerate(dynamic_datasets):
        data = dset['data']
        val0 = data[0]
        colorize_scatter = dset['2D'] and dset['accumulate']
        if 'scatter' in dset['plottype']:
            # if 1D, col1 is time; if 2D, col1 is X
            kwargs = filter_kwargs("scatter", dset['mpl'])
            scatter_args = {'zorder': 5, **kwargs}
            if colorize_scatter:
                # use the chosen cmap to color points along the dataset from 0→1
                scatter_args.pop('c', None)
                scatter_args.pop('color', None)
                scatter_args['vmin'], scatter_args['vmax'] = 0, 1
                colors_full = np.linspace(0, 1, len(data))
                scatter_args['c'] = [colors_full[0]]
                handles[(n, 'scatter_colors')] = colors_full
            handles[(n, 'scatter')] = ax.scatter(val0[0], val0[1], **scatter_args)
        if 'plot' in dset['plottype']:
            if not dset['accumulate']:
                kwargs = filter_kwargs("plot", dset['mpl'])
                handles[(n, 'plot')] = ax.plot(val0, zorder=4, **kwargs)[0]
            else:
                kwargs = filter_kwargs("plot", dset['mpl'])
                handles[(n, 'plot')] = ax.plot([val0[0], val0[0]], [val0[1], val0[1]], zorder=4, **kwargs)[0]
        if 'contourf' in dset['plottype']:
            # this case needs an X/Y grid + levels + cbar
            # kwargs = filter_kwargs(ax.pcolormesh, dset['mpl'])
            # xsh = len(set(dset['xgrid']))
            # ysh = len(set(dset['ygrid']))
            # if not custom_settings['xlim_set'] and not custom_settings['xlim_set']:
            #     ax.set_xlim(np.min(dset['xgrid']), np.max(dset['xgrid']))
            #     ax.set_ylim(np.min(dset['ygrid']), np.max(dset['ygrid']))
            #     custom_settings['skip_axis_setup'] = True
            # handles[(n, 'contourf')] = ax.pcolormesh(dset['xgrid'].reshape(xsh, ysh), dset['ygrid'].reshape(xsh, ysh),
            #                                          val0.reshape(xsh, ysh), zorder=2, **kwargs)
            kwargs = filter_kwargs("contourf", dset['mpl'])
            xsh = len(set(list(dset['xgrid'])))
            ysh = len(set(list(dset['ygrid'])))
            X = dset['xgrid'].reshape(xsh, ysh)
            Y = dset['ygrid'].reshape(xsh, ysh)
            Z0 = val0.reshape(xsh, ysh)
            if not custom_settings['xlim_set'] and not custom_settings['ylim_set']:
                ax.set_xlim(np.min(X), np.max(X))
                ax.set_ylim(np.min(Y), np.max(Y))
                custom_settings['skip_axis_setup'] = True
            # choose levels (from kwargs if provided, else let mpl pick then cache)
            levels_default = 8
            levels = kwargs.pop('levels', levels_default)
            cs = ax.contourf(X, Y, Z0, levels=levels, zorder=2, **kwargs)
            # cache chosen levels to keep them consistent across frames
            cached_levels = cs.levels
            handles[(n, 'contourf')] = {'cs': cs, 'X': X, 'Y': Y, 'levels': cached_levels}
        if 'hist' in dset['plottype']:
            # this case needs a bincount + cbar
            nbins_default = 20
            nbins = dset['mpl'].pop('bins', nbins_default)
            if dset['2D']:
                smooth = dset['mpl'].pop('smooth', 0)
                if smooth > 0:
                    from scipy.ndimage import gaussian_filter
                kwargs = filter_kwargs("contourf", dset['mpl'])
                levels_default = 8
                levels = kwargs.pop('levels', levels_default)
                if not custom_settings['xlim_set']:
                    xmin, xmax = float(np.min(data[:, 1])), float(np.max(data[:, 1]))
                else:
                    xmin, xmax = custom_settings['xmin'], custom_settings['xmax']
                if not custom_settings['ylim_set']:
                    ymin, ymax = float(np.min(data[:, 2])), float(np.max(data[:, 2]))
                else:
                    ymin, ymax = custom_settings['ymin'], custom_settings['ymax']
                xedges = np.linspace(xmin, xmax, nbins + 1)
                yedges = np.linspace(ymin, ymax, nbins + 1)
                xbins = xedges[:-1] + 0.5 * (xedges[1] - xedges[0])
                ybins = yedges[:-1] + 0.5 * (yedges[1] - yedges[0])
                Z0 = np.zeros((nbins, nbins), float)
                cs = ax.contourf(xbins, ybins, Z0.T, levels=levels, zorder=4, **kwargs)
                cached_levels = cs.levels
                handles[(n, 'hist')] = {'cs': cs, 'X': xbins, 'Y': ybins, 'levels': cached_levels}
                ax.set_xlim(xedges[0], xedges[-1])
                ax.set_ylim(yedges[0], yedges[-1])
                custom_settings['skip_axis_setup'] = True
            else:
                smooth = dset['mpl'].pop('smooth', 0)
                if smooth > 0:
                    from scipy.ndimage import gaussian_filter1d
                if not custom_settings['xlim_set']:
                    ymin, ymax = float(np.min(data[:, 1])), float(np.max(data[:, 1]))
                else:
                    ymin, ymax = custom_settings['ymin'], custom_settings['ymax']
                edges = np.linspace(ymin, ymax, nbins + 1)
                widths = np.diff(edges)
                dens, _ = np.histogram(data[:, 1], bins=edges, density=True)
                centers = 0.5 * (edges[:-1] + edges[1:])
                kwargs = filter_kwargs("hist", dset['mpl'])
                bars = ax.bar(centers, np.zeros_like(widths), width=widths, align='center', zorder=4, ec='C0', fc='w', **kwargs)
                handles[(n, 'hist')] = {'bars': bars, 'edges': edges, 'widths': widths}
                custom_settings['skip_axis_setup'] = True
                if not custom_settings['ylim_set']:
                    ax.set_ylim(-0.05 * np.max(dens), 1.05 * np.max(dens))
                else:
                    ax.set_ylim(custom_settings['ymin'] - custom_settings['ymarg'], custom_settings['ymax'] + custom_settings['ymarg'])
                span = ymax - ymin
                ax.set_xlim(edges[0] - 0.05 * span, edges[-1] + 0.05 * span)

    axis_setup(ax, custom_settings)
    frames = frames if frames is not None else list(range(len(arr)))
    for nfr, fr in enumerate(frames):
        for n, dset in enumerate(dynamic_datasets):
            count = fr - initframe
            data = dset['data']
            if 'scatter' in dset['plottype']:
                # if 1D, col1 is time; if 2D, col1 is X
                colorize_scatter = dset['2D'] and dset['accumulate']
                colors_full = handles.get((n, 'scatter_colors'))
                if not dset['accumulate']:
                    handles[(n, 'scatter')].set_offsets(data[arr[count]:arr[count]+1, :2])
                    if colorize_scatter and colors_full is not None:
                        idx = min(arr[count], len(colors_full) - 1)
                        color_val = np.array([colors_full[idx]])
                        handles[(n, 'scatter')].set_array(color_val)
                        handles[(n, 'scatter')].set_clim(0, 1)
                else:
                    npts = min(arr[count], len(data))
                    handles[(n, 'scatter')].set_offsets(data[:npts,:2])
                    if colorize_scatter and colors_full is not None:
                        colors = colors_full[:npts]
                        handles[(n, 'scatter')].set_array(colors)
                        if colors.size:
                            handles[(n, 'scatter')].set_clim(0, 1)
            if 'plot' in dset['plottype']:
                if not dset['accumulate']:
                    # this case needs an X grid
                    yminc, ymaxc = ax.get_ylim()
                    if dset['metadyn']:
                        if not custom_settings['ylim_set']:
                            ydata = data[count] - np.min(data[count])
                            currmaxy = np.min([np.max(ydata), dset['mpl']['emax']])
                            if yminc != -0.75:
                                ax.set_ylim(-0.75, ymaxc)
                    else:
                        ydata = data[arr[count]]
                        currmaxy = np.max(ydata)
                    if not custom_settings['xlim_set']:
                        if 1.1 * np.min(ydata) < yminc or np.min(ydata) < 1.1 * yminc:
                            ax.set_ylim(1.1 * np.min(ydata), ymaxc)
                            yminc = 1.1 * np.min(ydata)
                            if nfr % 10 == 3:
                                retick(ax)
                    if not custom_settings['ylim_set']:
                        if 1.1 * currmaxy > ymaxc or currmaxy > 1.1 * ymaxc:
                            ax.set_ylim(yminc, 1.1 * currmaxy)
                            if nfr % 10 == 3:
                                retick(ax)
                    handles[(n, 'plot')].set_data(data[0], ydata)
                else:
                    # adding just one line segment
                    handles[(n, 'plot')].set_data(data[:arr[count],0], data[:arr[count],1])
            if 'contourf' in dset['plottype']:
                # this case needs an X/Y grid + levels + cbar
                # handles[(n, 'contourf')].set_array(data[arr[count]].reshape(xsh, ysh))
                h = handles[(n, 'contourf')]
                if dset['metadyn']:
                    Zt = data[count].reshape(h['X'].shape) - np.min(data[count].reshape(h['X'].shape))
                    if np.max(Zt) - np.min(Zt) > dset['mpl']['emax']:
                        clevels = np.linspace(np.min(Zt), np.min(Zt) + dset['mpl']['emax'], dset['mpl']['levels'])
                    else:
                        clevels = np.linspace(np.min(Zt), np.max(Zt)+0.1, dset['mpl']['levels'])
                else:
                    Zt = data[arr[count]].reshape(h['X'].shape)
                    clevels = h['levels']
                for coll in h['cs'].collections:
                    coll.remove()
                kwargs = filter_kwargs("contourf", dset['mpl'])
                _ = kwargs.pop('levels')
                h['cs'] = ax.contourf(h['X'], h['Y'], Zt, levels=clevels, zorder=2, **kwargs)
            if 'hist' in dset['plottype']:
                # this case needs a bincount + cbar
                if dset['2D']:
                    sel = (data[:, 0] <= arr[count])
                    C, _, _ = np.histogram2d(data[sel, 1], data[sel, 2], bins=(xedges, yedges), density=True)
                    C = np.nan_to_num(C, nan=0)
                    if smooth > 0:
                        C = gaussian_filter(C, smooth)
                    h = handles[(n, 'hist')]
                    for coll in h['cs'].collections:
                        coll.remove()
                    kwargs = filter_kwargs("contourf", dset['mpl'])
                    _ = kwargs.pop('levels')
                    h['cs'] = ax.contourf(h['X'], h['Y'], C, levels=levels, zorder=2, **kwargs)
                    # handles[(n, 'hist')].set_array(C.reshape(len(xbins), len(ybins)))
                    # handles[(n, 'hist')].set_clim(0, np.max(C))
                else:
                    sel = np.sum((data[:, 0] <= arr[count]).astype(int))
                    edges = handles[(n, 'hist')]['edges']
                    dens, _ = np.histogram(data[:sel, 1], bins=edges, density=True)
                    if smooth > 0:
                        dens = gaussian_filter1d(dens, smooth)
                    yminc, ymaxc = ax.get_ylim()
                    if not custom_settings['ylim_set']:
                        if 1.1 * np.max(dens) > ymaxc or np.max(dens) < 1.1 * ymaxc:
                            ax.set_ylim(yminc, 1.1 * np.max(dens))
                            if nfr % 25 == 3:
                                retick(ax)
                    for rect, h in zip(handles[(n, 'hist')]['bars'].patches, dens):
                        rect.set_height(h)
        do_plotting(ax, basename, scenename, fr, save_to_file, len(frames))


def retick(ax):
    ax.yaxis.set_major_locator(mticker.AutoLocator())


def filter_kwargs(func, kwargs):
    mpl_kwargs = {
        "plot": ["color", "c", "linestyle", "ls", "linewidth", "lw", "drawstyle", "marker", "markersize", "ms",
                 "markerfacecolor", "mfc", "markeredgecolor", "mec", "markeredgewidth", "mew", "markevery", "fillstyle",
                 "alpha", "zorder", "label", "clip_on", "rasterized"],
        "scatter": ["c", "color", "cmap", "norm", "vmin", "vmax", "s", "marker", "linewidths", "edgecolors",
                    "facecolors", "alpha", "label", "zorder", "rasterized", "transform"],
        "hist": ["width", "bottom", "align", "orientation", "color", "c", "edgecolor", "facecolor", "fill", "hatch",
                 "yerr", "xerr", "error_kw", "ecolor", "capsize", "alpha", "linewidth", "lw", "label", "zorder", "log"],
        "contourf": ["levels", "cmap", "norm", "vmin", "vmax", "colors", "extend", "linewidths", "lw", "linestyles",
                     "hatches", "antialiased", "alpha", "label", "zorder"]
    }
    valid = mpl_kwargs[func]
    return {k: v for k, v in kwargs.items() if k in valid}


def axis_setup(ax, customs):
    if not customs['skip_axis_setup']:
        ax.set_xlim(customs['xmin'] - customs['xmarg'], customs['xmax'] + customs['xmarg'])
        ax.set_ylim(customs['ymin'] - customs['ymarg'], customs['ymax'] + customs['ymarg'])
    ax.set_xlabel(customs['labels'][0])
    ax.set_ylabel(customs['labels'][1])
    if customs['legend']:
        ax.legend(loc='upper right')


def process_1D_metadyn(dset, custom, arr):
    data = dset['data']
    nfr = len(arr)
    biasfactor = data[-1, -1]

    def gau(x, r, s, h):
        return h * np.exp(-((x - r)**2 / (2 * s**2)))

    if not custom['xlim_set']:
        custom['xmin'], custom['xmax'] = np.min(data[:, 1]), np.max(data[:, 1])
        custom['xmarg'] = 0.1 * (custom['xmax'] - custom['xmin'])
    grid = np.linspace(custom['xmin'], custom['xmax'], 100)
    new_dset = np.zeros((nfr + 2, len(grid)))
    pmf_accumulated = np.zeros_like(grid)
    new_dset[0] = grid
    ckpts = np.linspace(0, data[-1, 0], nfr+1)
    prstep = 0
    for mtdstep in range(len(data)):
        dframe = data[mtdstep]
        ctime = dframe[0]
        if ctime > ckpts[prstep]:
            prstep += 1
            for step in np.arange(ckpts[prstep-1], ckpts[prstep]):
                new_dset[1+int(prstep)] = - (biasfactor/(biasfactor-1)) * pmf_accumulated
        pmf_accumulated += gau(grid, dframe[1], dframe[2], dframe[3])
        pmf_accumulated = pmf_accumulated - np.min(pmf_accumulated)
    dset['data'] = new_dset
    return dset, custom


def process_2D_metadyn(dset, custom, arr):
    data = dset['data']
    nfr = len(arr)
    biasfactor = data[-1, -1]

    def gau2d(x, y, r1, r2, s1, s2, h1):
        return h1 * np.exp(-(((x - r1)**2) / (2 * s1**2) + ((y - r2)**2) / (2 * s2**2)))

    if not custom['xlim_set']:
        custom['xmin'], custom['xmax'] = np.min(data[:, 1]), np.max(data[:, 1])
    if not custom['ylim_set']:
        custom['ymin'], custom['ymax'] = np.min(data[:, 2]), np.max(data[:, 2])
    xgrid, ygrid = np.meshgrid(np.linspace(custom['xmin'], custom['xmax'], 50), np.linspace(custom['ymin'], custom['ymax'], 50))

    new_dset = np.zeros((nfr + 2, xgrid.shape[0] * xgrid.shape[1]))
    ckpts = np.linspace(0, data[-1, 0], nfr+1)
    pmf_accumulated = np.zeros_like(xgrid)
    prstep = 0
    for mtdstep in range(len(data)):
        dframe = data[mtdstep]
        crstep = dframe[0]
        if crstep > ckpts[prstep]:
            prstep += 1
            for step in np.arange(ckpts[prstep-1], ckpts[prstep]):
                new_dset[1+int(prstep)] = (- (biasfactor/(biasfactor-1)) * pmf_accumulated).reshape(xgrid.shape[0] * xgrid.shape[1])
        pmf_accumulated += gau2d(xgrid, ygrid, dframe[1], dframe[2], dframe[3], dframe[4], dframe[5])
        pmf_accumulated = pmf_accumulated - np.min(pmf_accumulated)
    dset['data'] = new_dset
    dset['xgrid'] = xgrid.reshape(-1)
    dset['ygrid'] = ygrid.reshape(-1)
    return dset, custom


def do_plotting(ax, basename, scenename, fr, save_to_file, maxframe):
    xticks, xlim = ax.get_xticks(), ax.get_xlim()  # prevents ticks/scaling from changing from frame to frame
    yticks, ylim = ax.get_yticks(), ax.get_ylim()  # which would normally happen with tight_layout
    xticks = [xt for xt in xticks if xlim[0] < xt < xlim[1]]
    yticks = [yt for yt in yticks if ylim[0] < yt < ylim[1]]
    ax.set_xticks(xticks)
    ax.set_yticks(yticks)
    fig = ax.figure
    plt.tight_layout(pad=0.5)
    if save_to_file:
        plt.savefig(f'{basename}-{scenename}-{fr}.png')
    else:
        # --- LIVE PREVIEW (non-blocking) ---
        # Make sure interactive mode is on (do this once in your outer routine too)
        if not plt.isinteractive():
            plt.ion()
        # Draw pending updates without blocking the GUI loop
        fig.canvas.draw_idle()
        fig.canvas.flush_events()
        # A tiny pause yields control to the GUI event loop (works across backends)
        plt.pause(2.0/maxframe)


def calc_frames_ud(action, ovl, master=False):
    """
    If an action can have modes up (u), down (d) or up-down (ud),
    this function will match 'ups' with 'downs' to return the
    frame numbers and Actions covered by the current action's span
    :param action: Action, contains at least one add_overlay
    :param ovl: str, name of the overlay
    :return: iterator, enumerates frames spanned by the action
             list, contains Actions affected by the overlay
    """
    try:
        mode = action.overlays[ovl]['mode']
    except KeyError:
        mode = 'ud'
    else:
        if mode not in ['u', 'd', 'n', 'ud']:
            raise RuntimeError(f"In {action.description}, mode should be 'u', 'd', 'n' or 'ud'")
    if mode == 'ud' or master:
        return range(action.initframe, action.initframe + action.framenum), [action]
    elif mode == 'u':
        action_index = action.scene.actions.index(action)
        try:
            alias = action.overlays[ovl]['alias']
        except KeyError:
            return range(action.initframe, action.scene.total_frames), action.scene.actions[action_index:]
        else:
            for act in action.scene.actions[action_index:]:
                try:
                    ovls = list(act.overlays.keys())
                except AttributeError:
                    ovls = []
                for ov in ovls:
                    try:
                        als = act.overlays[ov]['alias']
                    except KeyError:
                        pass
                    else:
                        try:
                            md = act.overlays[ov]['mode']
                        except KeyError:
                            pass
                        else:
                            if als == alias and md == 'd':
                                action_index_last = action.scene.actions.index(act)
                                return range(action.initframe, act.initframe + act.framenum), \
                                    action.scene.actions[action_index:action_index_last + 1]
            return range(action.initframe, action.scene.total_frames), action.scene.actions[action_index:]
    elif mode in 'nd':
        return range(0), []


def calc_frames_arr(actions, maxlen=None):
    arrs = []
    for ac in actions:
        try:
            animation_frames = [int(x) for x in ac.parameters['frames'].split(':')]
        except KeyError:
            animation_frames = 2 * [arrs[-1][-1]]
        except ValueError:
            if maxlen is not None:
                animation_frames = [int(a) if a != "last" else maxlen for a in ac.parameters['frames'].split(':')]
            else:
                raise RuntimeError("When used to guide overlays, 'animate' has to specify numerical values for frames, "
                                   "using 'last' is disallowed")
        if ac.framenum > 0:
            arrs.append(np.linspace(animation_frames[0], animation_frames[1], ac.framenum).astype(int))
    return np.concatenate(arrs)


def process_audio(scr):
    """
    Processes directives for the `add_audio` Action,
    yielding a list of relevant variables like starting
    poits, segment length, name of the file etc. then
    sorts the segments according to their starting times
    :param scr: The top-level Script object
    :return: None
    """
    for scene in scr.scenes:
        for action in scene.actions:
            if 'add_audio' in action.action_type:
                try:
                    audiofile = action.parameters['audiofile']
                except KeyError:
                    raise RuntimeError("To add a soundtrack, specify the audio file using 'audiofile=...'")
                else:
                    if not os.path.exists(os.path.expanduser(audiofile)):
                        raise RuntimeError(f"Audio file {audiofile} was not found in your filesystem")
                try:
                    starttime = float(action.parameters['from'])
                except KeyError:
                    starttime = 0
                try:
                    length = float(action.parameters['length'])
                except KeyError:
                    length = 3600
                try:
                    volume = float(action.parameters['volume'])
                except KeyError:
                    volume = 1.0
                try:
                    fin = float(action.parameters['fade_in'])
                except KeyError:
                    fin = 1.0
                try:
                    fout = float(action.parameters['fade_out'])
                except KeyError:
                    fout = 1.0
                tp = namedtuple("audio", "audiofile, movie_inittime, file_inittime, length, volume, fadein, fadeout")
                scr.audio.append(tp(audiofile, action.initframe/scr.fps, starttime, length, volume, fin, fout))
    scr.audio.sort(key=lambda x: x[1])


def call_moly(arg, answer=False, log=False, quiet=False):
    """
    A wrapper to consistently take care of all external calls
    :param arg: str, a command to be run from the command line
    :param answer: bool, whether to return the captured output
    :param log: bool, whether to log the command to a file
    :return: str, captured output from the command
    """
    quiet_comm = ' > /dev/null 2>&1' if quiet else ''
    result = call(arg + quiet_comm, shell=True)
    if log:
        with open('command_line_log.moly', 'a') as outfile:
            outfile.write(arg + '\n')
    if answer:
        return result


def check_if_convertible(string, object_type, param_name):
    """
    Checks if the user-specified value can be converted
    to the desired data type
    :param string: user-specified value
    :param object_type: what type should 'string' be convertible to
    :param param_name: name of the parameter in question
    :return: None
    """
    try:
        _ = object_type(string)
    except ValueError:
        raise RuntimeError(f"'{param_name}' must be {object_type}, instead '{string}' was given")
