import json
import os
import gc
import threading
import socket
import time
from importlib.metadata import version
from contextlib import ExitStack
from ewokscore import Task
import psutil
from pathlib import Path
import h5py
import hdf5plugin
import numpy
import logging
from ewokscore import missing_data
import matplotlib.image
from pyFAI import version as pyFAIVersion
from silx.io.h5py_utils import open_item as open_item_silx
from ewoksid02.utils.blissdata import (
    copy_group_excluding_dataset,
)
from ewoksid02.utils.io import (
    KEY_BEAMSTOP_MASK_FILE,
    KEY_BEAMSTOP_MASK_FOLDER,
    KEY_DARK_FILE,
    KEY_DARK_FOLDER,
    KEY_DETECTOR_MASK_FILE,
    KEY_DETECTOR_MASK_FOLDER,
    KEY_FLAT_FILE,
    KEY_FLAT_FOLDER,
    KEY_WINDOW_FILE,
    KEY_WINDOW_FOLDER,
    get_isotime,
    refactor_stream_name_raw,
    refactor_stream_name_interpreted,
    parse_titleextension_template,
    serialize_h5py_task,
    deserialize_h5py_task,
)

from ewoksid02.utils.blissdata import (
    LIMA_URL_TEMPLATE_ID02,
    # do_continue_pipeline,
    load_scan,
    _slice_dataset_online,
    _slice_dataset_offline,
    does_scan_contain_subscan2,
    # get_streams_subscan1,
    # _get_streams_subscan2,
    _get_new_slice_limits,
    _get_stream_names_from_acquisition_chain,
)

lock = threading.Lock()

PYFAI_PROCESSES = ["norm", "gaps", "2scat", "cave", "azim", "ave", "caving"]
TRUSAXS_PROCESSES = ["scalers", "dispatch", "debug"]
ALL_PROCESSES = PYFAI_PROCESSES + TRUSAXS_PROCESSES

PROCESSING_TYPE_TASK = {
    "norm": "ewoksid02.tasks.normalizationtask.NormalizationTask",
    "gaps": "ewoksid02.tasks.cavingtask.CavingGapsTask",
    "2scat": "ewoksid02.tasks.secondaryscatteringtask.SecondaryScatteringTask",
    "cave": "ewoksid02.tasks.cavingtask.CavingBeamstopTask",
    "azim": "ewoksid02.tasks.azimuthaltask.AzimuthalTask",
    "ave": "ewoksid02.tasks.averagetask.AverageTask",
    "scalers": "ewoksid02.tasks.scalerstask.ScalersTask",
}

KEYS_FLOAT = [
    "Center_1",
    "Center_2",
    "Dummy",
    "DDummy",
    "PSize_1",
    "PSize_2",
    "SampleDistance",
    "WaveLength",
]

KEYS_INT = [
    "BSize_1",
    "BSize_2",
    "Offset_1",
    "Offset_2",
    "RasterOrientation",
]

HEADERS_KEY_EXPOSURE_TIME = "HSTime"
HEADERS_KEY_MONITOR_0 = "HSI0"  # Monitor for beam intensity before the sample
HEADERS_KEY_MONITOR_0_FACTOR = "HSI0Factor"
HEADERS_KEY_MONITOR_1 = "HSI1"  # Monitor for beam intensity after the sample
HEADERS_KEY_MONITOR_1_FACTOR = "HSI1Factor"
HEADERS_KEY_SOT = "ShutterOpeningTime"
HEADERS_KEY_SCT = "ShutterClosingTime"
MAP_DETECTORS_LIMA = {
    "eiger2": "ESRF-ID02",
    "waxs": "instrument",
    "default": "ESRF-ID02",
    "eiger500k": "ESRF-ID02",  # TODO ???
}
DETECTOR_LIMA_DEFAULT = MAP_DETECTORS_LIMA.get("default")

SLOW_COUNTER_TIMER = "sampling_timer:epoch"


CHUNK_SIZE_3D = (1, 200, 200)

INFO_COMMON = {"h5path": "entry_0000"}

MAX_SLICE_SIZE = 50
LOG_LEVEL_DEFAULT = "warning"
LIMA_INDEX_NUMBER_FORMAT_ID02 = "%02d"
MEM_USAGE_START = None

# Global logger at ewoksid02.tasks.id02processingtask
logger = logging.getLogger("ewoksid02")
logger.propagate = True


class ID02ProcessingTask(
    Task,
    optional_input_names=[
        "detector_name",
        "scan_memory_url",
        "beacon_host",
        "reading_node",
        "filename_data",  # Bliss master file for a dataset
        "filename_lima",
        "scan_nb",
        "subscan",
        "headers",
        "dataset_signal",
        "dataset_variance",
        "dataset_sigma",
        "datatype",
        "lima_url_template",
        "lima_url_template_args",
        "log_level",
        "processing_filename",
        "processing_subtitle",
        "subtitle",
        "do_process",
        "do_save",
        "save_variance",
        "save_sigma",
        "save_metadata",
        "index_range",  # Global range, do not propagate
        "index_range_last",  # Dynamic range, propagate and change every loop
        "max_slice_size",
        "loop_nb",
        "info",
        "info_history",
        "gc_collect",
        "lima_index_number_format",
        "save_in_gallery",
    ],
    output_names=[
        "index_range_last",
        "loop_nb",
        "dataset_signal",
        "dataset_variance",
        "dataset_sigma",
        "continue_pipeline",
        "info_history",
    ],
):
    """This class contains processing support methods and saving methods in the ID02 SAXS pipeline.
    It extends the `ID02LoopTask` class and provides additional functionality for handling metadata, processing flags,
    and saving processed data to HDF5 files.This class is designed to be used as part of the ID02 pipeline.It does not contain a process method, that has to be implemented in the child class.

    Optional Inputs:
        - detector_name (str): Name of the detector used for data acquisition. This is the only mandatory input.
        - scan_memory_url (str): URL for accessing scan memory in online processing.
        - beacon_host (str): Host and port to plug blissdata to the correct beacon server. Only for online processing.
        - reading_node (bool): Flag to indicate if the task should read data from the node.
        - filename_data (str): Path to the dataset file (Master file, Nexus writer) for offline processing.
        - filename_lima (str): Path to the first Lima file, the only place where some detector metadata can be found.
        - scan_nb (int): Scan number for identifying the dataset.
        - subscan (int): Subscan number for processing. Default is `1`.
        - headers (dict): Only for Online processing. Dictionary containing headers information.
        - max_slice_size (int): Maximum number of frames to process in one iteration. Default is `20`.
        - dataset_signal (numpy.ndarray): Signal dataset to be processed.
        - dataset_variance (numpy.ndarray): Variance dataset to be processed.
        - dataset_sigma (numpy.ndarray): Sigma dataset to be processed.
        - datatype (str): Datatype to be used to save the 2D data. Default and recommended is float32.
        - lima_url_template (str): Format string to locate the Lima file and the path to the data inside that file.
        - lima_url_template_args (dict): Dictionary to format the lima_url_template.
        - log_level (str): Logging level for the task. Default is `"warning"`.
        - processing_filename (str): Full path to the (new) output file.
        - processing_subtitle (str): Additional subtitle for the processing task.
        - subtitle (str): Subtitle for the processing task to be added to the output filename.
        - do_process (bool): Flag to enable or disable processing. Default is `True`.
        - do_save (bool): Flag to enable or disable saving of processed data. Default is `True`.
        - save_variance (bool): Flag to enable or disable saving of variance dataset. Default is `False`.
        - save_sigma (bool): Flag to enable or disable saving of sigma dataset. Default is `True`.
        - save_metadata (bool): Flag to enable or disable saving of metadata. Default is `True`.
        - last_index_read (int): Index of the last frame read in the dataset. Default is `0`.
        - range_index_read (list): Range of indices to read from the dataset. This parameter is not propagated to the next task.
        - loop_nb (int): Current loop iteration number. Default is `0`.
        - info (dict): Additional metadata to save.
        - info_history (dict): Additional metadata to propagate and save, creating a history of processing.
        - gc_collect (bool): Manually collect garbage at the end of every task.
        - lima_index_number_format (str): format to find the first Lima file (02%d by default)
    Outputs:
        - last_index_read (int): Updated index of the last frame read.
        - loop_nb (int): Updated loop iteration number.
        - dataset_signal (numpy.ndarray): Processed signal dataset.
        - dataset_variance (numpy.ndarray): Processed variance dataset.
        - dataset_sigma (numpy.ndarray): Processed sigma dataset.
        - continue_pipeline (bool): Flag to indicate whether the pipeline should continue.
        - info_history (dict): Additional metadata to propagate and save, creating a history of processing.
    """

    class Benchmark:
        """A context manager for benchmarking."""

        def __init__(self, nb_frames, benchmark_name="processing"):
            self.nb_frames = nb_frames
            self.benchmark_name = benchmark_name
            self.bench_total_s = 0.0
            self.bench_per_frame_ms = 0.0

        def __enter__(self):
            self.start = time.perf_counter()
            return self

        def __exit__(self, exc_type, exc_value, traceback):
            self.end = time.perf_counter()
            self.bench_total_s = self.end - self.start
            if self.nb_frames > 0:
                self.bench_per_frame_ms = self.bench_total_s / self.nb_frames * 1000
            else:
                self.bench_per_frame_ms = 0

    def run(self, processing_type: str = "debug"):
        self.processing_type = processing_type
        self._pid = os.getpid()
        self._process = psutil.Process()
        self._set_log_level(
            log_level=self.get_input_value("log_level", LOG_LEVEL_DEFAULT)
        )
        self.detector_name = self.get_input_value("detector_name", None)
        self.scan_memory_url = self.get_input_value("scan_memory_url", None)
        self.beacon_host = self.get_input_value(
            "beacon_host", os.environ.get("BEACON_HOST")
        )
        self.filename_data = self.get_input_value("filename_data", None)
        self.filename_lima = self.get_input_value("filename_lima", None)
        self.subscan = self.get_input_value("subscan", 1)
        self.max_slice_size = self.get_input_value("max_slice_size", MAX_SLICE_SIZE)
        self.loop_nb = self.get_input_value("loop_nb", 0)
        self.headers = self.get_input_value("headers", None)
        self.index_range = self.get_input_value("index_range", None)
        self.index_range_last = self.get_input_value("index_range_last", None)
        self.links = {"source": {}, "destination": {}}

        self._set_input_parameters()
        self._load_headers()
        self._load_streams()
        self._log_allocated_memory()
        self._set_datasets()

        # Check-point to avoid data overwritting (only for offline processing)
        self.processing_filename = self.get_input_value("processing_filename", None)
        if not self.scan_memory_url and self.processing_filename:
            if self.loop_nb == 1 and os.path.exists(self.processing_filename):
                self.log_error(
                    f"Offline processing, processing file {self.processing_filename} already exist. Data will not be saved. Choose another name! Workflow is canceled!"
                )
                self.outputs.continue_pipeline = False

        if self.outputs.continue_pipeline is False:
            return False
        if self.processing_type == "debug":
            return

        self.do_process = self.get_input_value("do_process", True)
        self.do_save = self.get_input_value("do_save", True)
        if self.do_process:
            self.process()
            if self.do_save and self.processing_filename:
                self.save()
            elif self.do_save and not self.processing_filename:
                raise ValueError(
                    f"Processing filename for {self.processing_type} is not set."
                )
            else:
                self.log_warning("Save flag was set to False, data will not be saved")
        else:
            self.log_warning(msg=f"Processing {self.processing_type} will be skipped.")
            self.outputs.info_history = self.get_input_value("info_history", [])

        if self.get_input_value("save_in_gallery", False):
            self._save_in_gallery()

        if self.get_input_value("gc_collect", True):
            gc.collect()

    def _set_log_level(self, log_level="warning"):
        if not isinstance(log_level, str):
            return
        if log_level.lower() == "info":
            logger.setLevel(logging.INFO)
        elif log_level.lower() == "warning":
            logger.setLevel(logging.WARNING)
        elif log_level.lower() == "error":
            logger.setLevel(logging.ERROR)
        elif log_level.lower() == "debug":
            logger.setLevel(logging.DEBUG)

    def log_debug(self, msg):
        self._log(level="debug", msg=msg)

    def log_info(self, msg):
        self._log(level="info", msg=msg)

    def log_warning(self, msg):
        self._log(level="warning", msg=msg)

    def log_error(self, msg):
        self._log(level="error", msg=msg)

    def _log(self, level, msg):
        msg = f"Loop #{self.loop_nb}: {self.__class__.__name__}: (PID: {self._pid}): {msg}"
        logger.__getattribute__(level)(msg)

    def _log_allocated_memory(self):
        memory_info = self.get_memory_info()
        mem_usage_GB = memory_info["used"]
        total_mem_GB = memory_info["total"]
        available_mem_GB = memory_info["available"]

        if available_mem_GB / total_mem_GB < 0.1:
            mem_message = "Low memory available"
            color_prefix = "\033[91m"
        elif available_mem_GB / total_mem_GB < 0.3:
            mem_message = "Medium memory available"
            color_prefix = "\033[93m"
        else:
            mem_message = "Sufficient memory available"
            color_prefix = "\033[92m"
        color_suffix = "\033[0m"

        global MEM_USAGE_START
        if MEM_USAGE_START is None:
            memory_delta = 0.0
            MEM_USAGE_START = mem_usage_GB
        else:
            memory_delta = mem_usage_GB - MEM_USAGE_START

        logger.info(
            f"{color_prefix}Loop #{self.loop_nb}: {self.__class__.__name__}: (PID: {self._pid}): Memory: {mem_usage_GB:.2f}GB used, increased by {memory_delta:.2f}GB; {available_mem_GB:.2f}GB available. {mem_message}{color_suffix}"
        )

    def get_memory_info(self):
        # Return memory info in GBs
        return {
            "used": self._process.memory_info().rss / 1e9,
            "total": psutil.virtual_memory().total / 1e9,
            "available": psutil.virtual_memory().available / 1e9,
        }

    def _save_in_gallery(self): ...

    def _log_benchmark(self, bench):
        self.log_info(
            f"Benchmark. Total ({bench.nb_frames}). {bench.benchmark_name}: {bench.bench_total_s:.2f} s. Per frame: {bench.bench_per_frame_ms:.2f} ms"
        )

    def _set_input_parameters(self):
        # self.outputs.last_index_read = self.last_index_read
        # self.outputs.loop_nb = self.loop_nb

        if self.scan_memory_url:
            if self.scan_memory_url.startswith("esrf:scan"):
                # We trust this is an online processing
                if self.detector_name is None:
                    raise ValueError("Online processing requires a detector_name")
                if not self.beacon_host:
                    raise ValueError("Online processing requires a beacon_host")
                self.scan = load_scan(
                    scan_memory_url=self.scan_memory_url, beacon_host=self.beacon_host
                )
                self.scan_info = self.scan.info

            elif Path(self.scan_memory_url).is_file():
                # We trust this is a fake online processing, recreating streams from a file
                if self.detector_name is None:
                    raise ValueError("Fake online processing requires a detector_name")
                with h5py.File(self.scan_memory_url, "r") as f:
                    self.scan_info = json.loads(f["scan_info"][()].decode())

            self.filename_data = self.filename_data or self.scan_info["filename"]
            if self.filename_lima is None:
                lima_index_number_format = self.get_input_value(
                    "lima_index_number_format", LIMA_INDEX_NUMBER_FORMAT_ID02
                )
                self.filename_lima = f"{self.scan_info['images_path'].format(img_acq_device=self.detector_name)}{lima_index_number_format % 0}.h5"

            self.scan_nb = self.scan_info["scan_nb"]
            self.links["source"].update(
                {
                    "datasignal": f"{self.filename_data}::{self.scan_nb}.{self.subscan}/instrument/{self.detector_name}/data",
                    "datavariance": None,
                    "datasigma": None,
                    "metadata_counters": f"{self.filename_data}::/{self.scan_nb}.1/measurement",
                    "metadata_counters_subscan2": f"{self.filename_data}::/{self.scan_nb}.2/measurement",
                    "metadata_detector": f"{self.filename_lima}::entry_0000/{MAP_DETECTORS_LIMA.get(self.detector_name, DETECTOR_LIMA_DEFAULT)}/{self.detector_name}",
                    "metadata_headers": f"{self.filename_lima}::entry_0000/{MAP_DETECTORS_LIMA.get(self.detector_name, DETECTOR_LIMA_DEFAULT)}/{self.detector_name}/header",
                    "metadata_titleextension": None,
                    "ewoks": None,
                }
            )

        elif self.filename_data:
            # This is offline processing, but there are two types
            self.scan_nb = self.get_input_value("scan_nb", None)
            self.subscan = self.get_input_value("subscan", 1)
            self.scan = None
            self.scan_memory_url = None
            self.beacon_host = None
            with open_item_silx(self.filename_data, "/", "r") as file_input:
                if "entry_0000/PyFAI" in file_input:
                    # We trust this is an offline processing from an already processed file
                    self.log_info(
                        "Looks like an offline processing from PROCESSED data"
                    )
                    self.detector_name = file_input["entry_0000/detector_name"][
                        ()
                    ].decode()
                    self.filename_lima = None
                    self.links["source"].update(
                        {
                            "datasignal": f"{self.filename_data}::/entry_0000/PyFAI/result_{self.processing_type}/data",
                            "datavariance": f"{self.filename_data}::/entry_0000/PyFAI/result_{self.processing_type}/data_variance",
                            "datasigma": f"{self.filename_data}::/entry_0000/PyFAI/result_{self.processing_type}/data_errors",
                            "metadata_counters": f"{self.filename_data}::/entry_0000/PyFAI/MCS/raw/subscan_1",
                            "metadata_counters_subscan2": f"{self.filename_data}::/entry_0000/PyFAI/MCS/raw/subscan_2",
                            "metadata_detector": f"{self.filename_data}::/entry_0000/PyFAI/{self.detector_name}",
                            "metadata_headers": f"{self.filename_data}::/entry_0000/PyFAI/parameters",
                            "metadata_titleextension": f"{self.filename_data}::/entry_0000/PyFAI/parameters/TitleExtension",
                            "ewoks": f"{self.filename_data}::/entry_0000/ewoks",
                        }
                    )
                elif (
                    self.scan_nb is not None
                    and f"{self.scan_nb}.{self.subscan}" in file_input
                ):
                    # We trust this is an offline processing from a RAW_DATA file
                    if self.detector_name is None:
                        raise ValueError(
                            "Offline processing from RAW_DATA requires a detector_name"
                        )

                    collection_name = file_input[
                        f"{self.scan_nb}.{self.subscan}/sample/name"
                    ][()].decode()
                    self.filename_lima = (
                        Path(self.filename_data).parent
                        / f"{collection_name}_{self.detector_name}_{self.scan_nb:05}_00.h5"
                    )
                    if not self.filename_lima.exists():
                        raise FileNotFoundError(
                            f"Lima file could not be found: {self.filename_lima}"
                        )
                    self.filename_lima = str(self.filename_lima)
                    self.links["source"].update(
                        {
                            "datasignal": f"{self.filename_data}::/{self.scan_nb}.{self.subscan}/instrument/{self.detector_name}/data",
                            "datavariance": None,
                            "datasigma": None,
                            "metadata_counters": f"{self.filename_data}::/{self.scan_nb}.1/measurement",
                            "metadata_counters_subscan2": f"{self.filename_data}::/{self.scan_nb}.2/measurement",
                            "metadata_detector": f"{self.filename_lima}::/entry_0000/{MAP_DETECTORS_LIMA.get(self.detector_name, DETECTOR_LIMA_DEFAULT)}/{self.detector_name}",
                            "metadata_headers": f"{self.filename_lima}::/entry_0000/{MAP_DETECTORS_LIMA.get(self.detector_name, DETECTOR_LIMA_DEFAULT)}/{self.detector_name}/header",
                            "metadata_titleextension": None,
                            "ewoks": None,
                        }
                    )
                elif self.scan_nb is None:
                    raise ValueError(
                        "Offline processing requires a scan number: scan_nb"
                    )
                else:
                    raise ValueError(
                        f"The format in {self.filename_data} is not valid for offline processing."
                    )

    def _load_headers(self) -> None:
        if self.headers is not None:
            return

        self.headers = {}
        file_headers, h5path_headers = self.links["source"]["metadata_headers"].split(
            "::"
        )
        with open_item_silx(file_headers, h5path_headers, "r") as headers_group:
            for key_header in headers_group:
                value_header = headers_group[key_header][()]
                if isinstance(value_header, bytes):
                    value_header = value_header.decode()
                self.headers[key_header] = str(value_header)

    def _load_streams(self) -> None:
        self.streams_subscan1 = []
        self.streams_subscan2 = []

        if self.scan:
            acquisition_chains = self.scan.info.get("acquisition_chain")
            if does_scan_contain_subscan2(scan=self.scan):
                acquisition_chain_subscan1 = acquisition_chains.get("mcs")
                acquisition_chain_subscan2 = acquisition_chains.get("sampling_timer")

                # Subscan1
                if acquisition_chain_subscan1:
                    stream_names_subscan1 = [
                        s
                        for s in _get_stream_names_from_acquisition_chain(
                            acq_chain=acquisition_chain_subscan1, include_images=False
                        )
                        if s in self.scan.streams
                    ]
                else:
                    stream_names_subscan1 = []

                stream_names_subscan1_raw = [
                    refactor_stream_name_raw(stream_name=s, cut_name=True)
                    for s in stream_names_subscan1
                ]
                stream_names_subscan1_interpreted = [
                    refactor_stream_name_interpreted(stream_name=s)
                    for s in stream_names_subscan1
                ]

                for stream_name, stream_name_raw_, stream_name_interpreted in zip(
                    stream_names_subscan1,
                    stream_names_subscan1_raw,
                    stream_names_subscan1_interpreted,
                ):
                    if stream_names_subscan1_raw.count(stream_name_raw_) > 1:
                        stream_name_raw = refactor_stream_name_raw(
                            stream_name=stream_name,
                            detector_name=self.detector_name,
                            cut_name=False,
                        )
                    else:
                        stream_name_raw = stream_name_raw_

                    stream_subscan1_info = {
                        "stream": self.scan.streams[stream_name],
                        "name": stream_name,
                        "name_raw": stream_name_raw,
                        "name_interpreted": stream_name_interpreted,
                    }
                    self.streams_subscan1.append(stream_subscan1_info)

                # Subscan2
                if acquisition_chain_subscan2:
                    stream_names_subscan2 = [
                        s
                        for s in _get_stream_names_from_acquisition_chain(
                            acq_chain=acquisition_chain_subscan2, include_images=False
                        )
                        if s in self.scan.streams
                    ]
                else:
                    stream_names_subscan2 = []

                stream_names_subscan2_raw = [
                    refactor_stream_name_raw(stream_name=s, cut_name=True)
                    for s in stream_names_subscan2
                ]
                stream_names_subscan2_interpreted = [
                    refactor_stream_name_interpreted(stream_name=s)
                    for s in stream_names_subscan2
                ]

                for stream_name, stream_name_raw_, stream_name_interpreted in zip(
                    stream_names_subscan2,
                    stream_names_subscan2_raw,
                    stream_names_subscan2_interpreted,
                ):
                    if stream_names_subscan2_raw.count(stream_name_raw_) > 1:
                        stream_name_raw = refactor_stream_name_raw(
                            stream_name=stream_name, cut_name=False
                        )
                    else:
                        stream_name_raw = stream_name_raw_

                    stream_subscan2_info = {
                        "stream": self.scan.streams[stream_name],
                        "name": stream_name,
                        "name_raw": stream_name_raw,
                        "name_interpreted": stream_name_interpreted,
                    }
                    self.streams_subscan2.append(stream_subscan2_info)
            else:
                # One subscan1, no subscan2
                acquisition_chain_subscan1 = next(iter(acquisition_chains.values()))
                acquisition_chain_subscan2 = None

                stream_names_subscan1 = [
                    s
                    for s in _get_stream_names_from_acquisition_chain(
                        acq_chain=acquisition_chain_subscan1, include_images=False
                    )
                    if s in self.scan.streams
                ]
                stream_names_subscan1_raw = [
                    refactor_stream_name_raw(stream_name=s, cut_name=True)
                    for s in stream_names_subscan1
                ]
                stream_names_subscan1_interpreted = [
                    refactor_stream_name_interpreted(stream_name=s)
                    for s in stream_names_subscan1
                ]

                for stream_name, stream_name_raw_, stream_name_interpreted in zip(
                    stream_names_subscan1,
                    stream_names_subscan1_raw,
                    stream_names_subscan1_interpreted,
                ):
                    if stream_names_subscan1_raw.count(stream_name_raw_) > 1:
                        stream_name_raw = refactor_stream_name_raw(
                            stream_name=stream_name,
                            cut_name=False,
                        )
                    else:
                        stream_name_raw = stream_name_raw_

                    stream_subscan1_info = {
                        "stream": self.scan.streams[stream_name],
                        "name": stream_name,
                        "name_raw": stream_name_raw,
                        "name_interpreted": stream_name_interpreted,
                    }
                    self.streams_subscan1.append(stream_subscan1_info)

        elif self.scan_memory_url and Path(self.scan_memory_url).is_file():
            ...
            # with h5py.File(self.scan_memory_url, "r") as f:
            #     self.scan_info = json.loads(f["scan_info"][()].decode())
            #     nb_points = self.scan_info["npoints"]
            #     for stream_name in f["streams"]:
            #         stream = f["streams"][stream_name]
            #         if len(stream) == nb_points:
            #             self.streams[stream_name] = stream[:]
            #         else:
            #             self.streams_subscan2[stream_name] = stream[:]
        else:
            # The available counters are either in the RAW master file or in the HS32 arrays in the PROCESSED file
            link_counters = self.links["source"]["metadata_counters"]
            if link_counters is not None:
                filename, h5path_subscan1 = link_counters.split("::")
                _, h5path_subscan2 = self.links["source"][
                    "metadata_counters_subscan2"
                ].split("::")
                params = {
                    "filename": filename,
                    "name": "/",
                    "retry_timeout": 0.1,
                }

                with open_item_silx(**params) as root:
                    if h5path_subscan1 in root:
                        group_subscan1 = root[h5path_subscan1]
                        for dset_name in group_subscan1:
                            if group_subscan1[dset_name].ndim == 3:
                                # Skip the detector
                                continue

                            stream_info = {
                                "stream": group_subscan1[dset_name][:],
                                "name": dset_name,
                                "name_raw": dset_name,
                                "name_interpreted": dset_name,
                            }
                            self.streams_subscan1.append(stream_info)

                    if h5path_subscan2 in root:
                        group_subscan2 = root[h5path_subscan2]
                        for dset_name in group_subscan2:
                            stream_info = {
                                "stream": group_subscan2[dset_name][:],
                                "name": dset_name,
                                "name_raw": dset_name,
                                "name_interpreted": dset_name,
                            }
                            self.streams_subscan2.append(stream_info)

    def _get_new_datasets(self) -> dict:
        filename_data, h5path_datasignal = self.links["source"]["datasignal"].split(
            "::"
        )

        out = {
            "dataset_signal": numpy.array([]),
            "dataset_variance": numpy.array([]),
            "dataset_sigma": numpy.array([]),
            "index_range": None,
        }

        if self.scan_memory_url and self.scan_memory_url.startswith("esrf:scan"):
            # Reading online
            # - dataset_signal comes from the blissdata stream
            # - dataset_variance = empty
            # - dataset_sigma = empty

            if self.processing_type in PYFAI_PROCESSES:
                stream_name = f"{self.detector_name}:image"
            else:
                stream_name = "mcs:epoch"

            indew_range_new = _get_new_slice_limits(
                stream_name=stream_name,
                scan=self.scan,
                scan_memory_url=self.scan_memory_url,
                beacon_host=self.beacon_host,
                index_range=self.index_range,
                index_range_last=self.index_range_last,
                max_slice_size=self.max_slice_size,
            )
            if indew_range_new is None:
                return out

            ptdata_signal = _slice_dataset_online(
                stream_name=stream_name,
                detector_name=self.detector_name,
                scan=self.scan,
                scan_memory_url=self.scan_memory_url,
                beacon_host=self.beacon_host,
                lima_url_template=self.get_input_value(
                    "lima_url_template", LIMA_URL_TEMPLATE_ID02
                ),
                lima_url_template_args=self.get_input_value(
                    "lima_url_template_args", {}
                ),
                subscan=self.subscan,
                index_range=indew_range_new,
                start_from_memory=True,
            )

            if ptdata_signal["dataset"] is not None:
                out["dataset_signal"] = ptdata_signal["dataset"]
                out["index_range"] = ptdata_signal["index_range"]

        else:
            # Reading offline
            # - dataset_signal comes from the RAW_DATA or PROCESSED_DATA
            # - dataset_variance = comes only from PROCESSED_DATA
            # - dataset_sigma = comes only from PROCESSED_DATA

            filename_data, h5path_datasignal = self.links["source"]["datasignal"].split(
                "::"
            )
            indew_range_new = _get_new_slice_limits(
                filename_data=self.filename_data,
                h5path=h5path_datasignal,
                index_range=self.index_range,
                index_range_last=self.index_range_last,
                max_slice_size=self.max_slice_size,
            )
            if indew_range_new is None:
                return out

            ptdata_signal = _slice_dataset_offline(
                filename_data=filename_data,
                h5path_to_data=h5path_datasignal,
                index_range=indew_range_new,
            )
            if ptdata_signal["dataset"] is not None:
                out["dataset_signal"] = ptdata_signal["dataset"]
                out["index_range"] = ptdata_signal["index_range"]

            if self.links["source"]["datavariance"] is not None:
                filename_data, h5path_datavariance = self.links["source"][
                    "datavariance"
                ].split("::")
                ptdata_variance = _slice_dataset_offline(
                    filename_data=filename_data,
                    h5path_to_data=h5path_datavariance,
                    index_range=indew_range_new,
                )
                if ptdata_variance["dataset"] is not None:
                    out["dataset_variance"] = ptdata_variance["dataset"]

            if self.links["source"]["datasigma"] is not None:
                filename_data, h5path_datasigma = self.links["source"][
                    "datasigma"
                ].split("::")
                ptdata_sigma = _slice_dataset_offline(
                    filename_data=filename_data,
                    h5path_to_data=h5path_datasigma,
                    index_range=indew_range_new,
                )
                if ptdata_sigma["dataset"] is not None:
                    out["dataset_sigma"] = ptdata_sigma["dataset"]

        nb_frames_read = len(out["dataset_signal"])
        index_range_sliced = out["index_range"]
        if nb_frames_read > 0:
            self.log_info(f"""
                    \n\tIncoming ({nb_frames_read} frames) in the range {index_range_sliced[0]} -> {index_range_sliced[-1]},
                """)
        else:
            self.log_info("""
                    \n\tNo more data to read. End of the workflow.
                """)

        return out

    def _set_datasets(self) -> None:
        reading_node = self.get_input_value("reading_node", False)
        dataset_signal = self.get_input_value("dataset_signal", None)
        dataset_variance = self.get_input_value("dataset_variance", None)
        dataset_sigma = self.get_input_value("dataset_sigma", None)

        self.outputs.continue_pipeline = True
        if reading_node or dataset_signal is None:
            # Get new incoming data
            ptdata = self._get_new_datasets()
            dataset_signal = ptdata["dataset_signal"]
            dataset_variance = ptdata["dataset_variance"]
            dataset_sigma = ptdata["dataset_sigma"]
            index_range_new = ptdata["index_range"]

            if len(dataset_signal) == 0:
                self.outputs.continue_pipeline = False
                self.outputs.loop_nb = self.loop_nb
                self.outputs.index_range_last = self.index_range_last
                return
            self.loop_nb += 1

            # We define here the actual index limits because we read the data from streams
            self.index_range_last = index_range_new

        if (
            dataset_signal is not None
            and len(dataset_signal) > 0
            and self.index_range_last is None
        ):
            # Only possible if the data was sent as inputs without any index_range_last
            self.index_range_last = [0, len(dataset_signal)]

        self.outputs.loop_nb = self.loop_nb
        self.outputs.index_range_last = self.index_range_last
        self.outputs.dataset_signal = dataset_signal
        self.outputs.dataset_variance = dataset_variance
        self.outputs.dataset_sigma = dataset_sigma
        self.dataset_signal = dataset_signal
        self.dataset_variance = dataset_variance
        self.dataset_sigma = dataset_sigma

    def _new_loop(self):
        self.loop_nb += 1

        memory_used = self.get_memory_info()["used"]
        global MEM_USAGE_START
        if self.loop_nb == 1:
            memory_delta = 0.0
            MEM_USAGE_START = memory_used
        else:
            memory_delta = memory_used - MEM_USAGE_START

        if memory_delta == 0.0:
            color_prefix = "\033[93m"
        elif memory_delta < 0.0:
            color_prefix = "\033[92m"
        else:
            color_prefix = "\033[91m"
        color_suffix = "\033[0m"

        logger.info(
            f"{color_prefix}Loop #{self.loop_nb}: {self.__class__.__name__}: (PID: {self._pid}): \
                Memory Delta: {memory_delta:.2f}GB{color_suffix}"
        )

    def process(self):
        self.bench_process = None
        self.bench_saving = None
        self.processing_params = {}
        if self.dataset_signal.size == 0:
            self.log_warning(
                f"Skipping processing {self.processing_type} due to empty array."
            )
            self.do_save = False
            return False
        return True

    def _set_output_variables(self):
        # Write some h5 paths
        if not self.processing_filename:
            return

        if self.processing_type in PYFAI_PROCESSES:
            nxprocess = "PyFAI"
        elif self.processing_type in TRUSAXS_PROCESSES:
            nxprocess = "TRUSAXS"
        else:
            nxprocess = "other"

        self.links["destination"].update(
            {
                "nxdata": f"{self.processing_filename}::/entry_0000/{nxprocess}/result_{self.processing_type}",
                "datasignal": f"{self.processing_filename}::/entry_0000/{nxprocess}/result_{self.processing_type}/data",
                "datavariance": f"{self.processing_filename}::/entry_0000/{nxprocess}/result_{self.processing_type}/data_variance",
                "datasigma": f"{self.processing_filename}::/entry_0000/{nxprocess}/result_{self.processing_type}/data_errors",
                "metadata_detector": f"{self.processing_filename}::/entry_0000/{nxprocess}/{self.detector_name}",
                "metadata_headers": f"{self.processing_filename}::/entry_0000/{nxprocess}/parameters",
                "metadata_mcs": f"{self.processing_filename}::/entry_0000/{nxprocess}/MCS",
                "metadata_tfg": f"{self.processing_filename}::/entry_0000/{nxprocess}/TFG",
                "metadata_counters": None,
                "ewoks": f"{self.processing_filename}::/entry_0000/ewoks",
            }
        )

    def get_parameter(self, key: str, to_integer: bool = False, defaut=None):
        value = self.get_input_value(key=key)
        if value == missing_data.MISSING_DATA:
            # Try to get it from header
            value = self.get_from_headers(
                key=key,
                to_integer=to_integer,
                default=defaut,
            )
        return value

    def get_headers_filename(self, folder_key: str, file_key: str):
        folder = self.get_from_headers(
            key=folder_key,
        )
        file_name = self.get_from_headers(
            key=file_key,
        )
        if folder is None or file_name is None:
            return

        filename = os.path.join(folder, file_name)
        if not os.path.exists(filename):
            self.log_warning(f"{filename} from headers could not be found.")
            return
        return filename

    def get_mask_beamstop_filename(self):
        return self.get_headers_filename(
            folder_key=KEY_BEAMSTOP_MASK_FOLDER,
            file_key=KEY_BEAMSTOP_MASK_FILE,
        )

    def get_mask_gaps_filename(self):
        return self.get_headers_filename(
            folder_key=KEY_DETECTOR_MASK_FOLDER,
            file_key=KEY_DETECTOR_MASK_FILE,
        )

    def get_flat_filename(self):
        return self.get_headers_filename(
            folder_key=KEY_FLAT_FOLDER,
            file_key=KEY_FLAT_FILE,
        )

    def get_dark_filename(self):
        return self.get_headers_filename(
            folder_key=KEY_DARK_FOLDER,
            file_key=KEY_DARK_FILE,
        )

    def get_mask_window(self):
        return self.get_headers_filename(
            folder_key=KEY_WINDOW_FOLDER,
            file_key=KEY_WINDOW_FILE,
        )

    def save(self):
        self.log_debug("Saving processed data...")
        self._set_output_variables()
        with ExitStack() as stack:
            if (
                not self.processing_filename
                or not self.do_save
                or self.dataset_signal.size == 0
            ):
                return False

            # Create the file and root groups
            self._create_processing_file()

            # Append data to the nexus data group
            stack.enter_context(lock)
            self.bench_saving = self.Benchmark(
                nb_frames=len(self.dataset_signal), benchmark_name="saving"
            )
            stack.enter_context(self.bench_saving)
            self._update_id02_data(
                stack=stack,
            )
            if self.get_input_value("save_metadata", True):
                self._update_id02_metadata(stack)

        self._log_benchmark(self.bench_saving)
        self._save_benchmark(self.bench_process)
        self._save_benchmark(self.bench_saving)
        return True

    def _create_processing_file(self):
        if self.scan_memory_url:
            # With online processing, loop_nb=1 doesnt have to be the beginning of processing
            # Processing files are created if they dont exist, never overwrite, and only if there is self.dataset_signal
            if os.path.exists(self.processing_filename):
                return

            if self.dataset_signal is None:
                return
        else:
            # With offline processing, it will create the file in the first loop, since there will always be dataset_signal
            # Comment for dispatching
            if self.loop_nb == 1 and os.path.exists(self.processing_filename):
                self.log_error(
                    f"Offline processing, processing file {self.processing_filename} already exist. Data will not be saved. Choose another name!"
                )
                self.outputs.repeat = False
                return

            if os.path.exists(self.processing_filename):
                return

            if self.loop_nb > 1:
                return

        # Create directories if needed
        os.makedirs(os.path.dirname(self.processing_filename), exist_ok=True)

        self.log_info(f"Creating file: {self.processing_filename}")
        with ExitStack() as stack:
            root_group = stack.enter_context(
                open_item_silx(filename=self.processing_filename, name="/", mode="w")
            )
            root_group = self.write_root_group(root_group=root_group)

            if self.processing_type in PYFAI_PROCESSES:
                title = self.links["destination"]["nxdata"]
            elif self.processing_type in TRUSAXS_PROCESSES:
                title = "TFG metadata collection"
            else:
                title = ""

            # Entry group
            entry_group = self.create_h5_group(
                h5_parent_group=root_group,
                h5_group_name="entry_0000",
                title=title,
                NX_class="NXentry",
            )
            if self.processing_type in PYFAI_PROCESSES:
                entry_group.attrs["default"] = self.links["destination"][
                    "nxdata"
                ].split("::")[-1]

            entry_group["start_time"] = str(get_isotime())
            if self.processing_type in PYFAI_PROCESSES:
                entry_group["detector_name"] = self.detector_name

            # Configuration group
            configuration_group = self.create_h5_group(
                h5_parent_group=entry_group,
                h5_group_name="configuration",
                NX_class="NXnote",
            )
            configuration_group["type"] = "text/json"
            configuration_group["data"] = json.dumps(
                self.get_headers(), indent=2, separators=(",\r\n", ": ")
            )

            # PyFAI Nexus group
            if self.processing_type in PYFAI_PROCESSES:
                process_group = self.create_h5_group(
                    h5_parent_group=entry_group,
                    h5_group_name="PyFAI",
                    NX_class="NXprocess",
                    default=self.links["destination"]["nxdata"].split("::")[-1],
                )
                process_group["date"] = str(get_isotime())
                process_group["processing_type"] = self.processing_type
                process_group["program"] = "pyFAI"
                process_group["version"] = pyFAIVersion
            elif self.processing_type in TRUSAXS_PROCESSES:
                process_group = self.create_h5_group(
                    h5_parent_group=entry_group,
                    h5_group_name="TRUSAXS",
                    NX_class="NXinstrument",
                )
                process_group["date"] = str(get_isotime())
                process_group["processing_type"] = self.processing_type
                process_group["program"] = "TruSAXS"
            else:
                return

            # MCS group
            metadata_mcs_output = self.create_h5_group(
                h5_parent_group=process_group,
                h5_group_name="MCS",
                NX_class="NXcollection",
            )
            metadata_mcs_output["device"] = "bliss"

            # HS32 N(name), Z(zero), F(factor) arrays
            nb_pins = self.get_HS32_number_pins()
            HS32N_array = numpy.array(
                [
                    self.get_pin_name_from_index(index_pin=index_pin)
                    for index_pin in range(nb_pins)
                ],
                dtype=h5py.string_dtype(encoding="utf-8"),
            )
            HS32Z_array = numpy.array(
                [
                    self.get_pin_zerovalue_from_index(index_pin=index_pin)
                    for index_pin in range(nb_pins)
                ]
            )
            HS32F_array = numpy.array(
                [
                    self.get_pin_factorvalue_from_index(index_pin=index_pin)
                    for index_pin in range(nb_pins)
                ]
            )
            metadata_mcs_output.create_dataset(
                name="HS32N",
                data=HS32N_array,
                dtype=h5py.string_dtype(encoding="utf-8"),
            )

            metadata_mcs_output.create_dataset(
                name="HS32Z",
                data=HS32Z_array,
                dtype="float64",
            )
            metadata_mcs_output.create_dataset(
                name="HS32F",
                data=HS32F_array,
                dtype="float64",
            )

            # HSI0Factor, HSI1Factor
            for key in ["HSI0Factor", "HSI1Factor"]:
                value = self.get_from_headers(
                    key=key,
                )
                if value is not None:
                    metadata_mcs_output.create_dataset(
                        name=key, data=value, dtype="float64"
                    )

            # HSI0, HSI1, HSTime
            for key in ["HSI0", "HSI1", "HSTime"]:
                pin_name = self.get_from_headers(key=key)
                for index_pin in range(nb_pins):
                    if self.get_pin_name_from_index(index_pin=index_pin) == pin_name:
                        metadata_mcs_output.create_dataset(
                            name=key, data=index_pin + 1, dtype="int64"
                        )
                        break

            # ShutterTime
            for key in ["ShutterOpeningTime", "ShutterClosingTime"]:
                value = self.get_from_headers(key=key)
                if value is None:
                    value = 0.0
                metadata_mcs_output.create_dataset(
                    name=key, data=value, dtype="float64"
                )

            # Parameters (header) group
            metadata_headers_output = self.create_h5_group(
                h5_parent_group=process_group,
                h5_group_name="parameters",
                NX_class="NXcollection",
            )
            headers = self.get_headers()
            for key, value in headers.items():
                if key in KEYS_FLOAT:
                    metadata_headers_output.create_dataset(
                        name=key, data=value, dtype="float64"
                    )
                elif key in KEYS_INT:
                    metadata_headers_output.create_dataset(
                        name=key, data=value, dtype="int64"
                    )
                elif key == "TitleExtension":
                    continue
                else:
                    value = str(value)
                    metadata_headers_output.create_dataset(
                        name=key,
                        data=value,
                        dtype=h5py.string_dtype(encoding="utf-8"),
                    )

            # TFG group
            metadata_tfg_output = self.create_h5_group(
                h5_parent_group=process_group,
                h5_group_name="TFG",
                NX_class="NXcollection",
            )
            metadata_tfg_output["device"] = "bliss"

            for key in ["HMStartEpoch", "HMStartTime"]:
                if key in headers:
                    value = str(headers[key])
                    metadata_tfg_output.create_dataset(
                        name=key, data=value, dtype=h5py.string_dtype(encoding="utf-8")
                    )
                else:
                    self.log_warning(f"Key {key} not found in headers")

            # Type some static information
            # info_list = [{**INFO_COMMON}]
            # info_list += self.processing_info()
            # for info_item in info_list:
            #     h5group = info_item.get("h5path")
            #     if h5group in root_group:
            #         root_group[h5group][info_item["name"]] = str(info_item["value"])

            if self.processing_type in PYFAI_PROCESSES:
                self._link_ewoks_history(
                    root_group_destination=root_group,
                    stack=stack,
                )

    def get_HS32_number_pins(self):
        """
        Get the number of HS32 pins, which will determine the shape of metadata arrays
        """
        return self.get_from_headers(key="HS32Len", to_integer=True)

    def get_pin_name_from_index(self, index_pin: int):
        """
        Retrieve the name of the pin that is going to be saved in the
        array HS32V at position [:,index]
        No need to use scalers_keys dictionary
        index goes from 0 -> (self.get_hs32_number_pins() - 1)
        """
        return self.get_from_headers(key=f"HS32N{(index_pin + 1):02}")

    def get_pin_zerovalue_from_index(self, index_pin: int):
        return self.get_from_headers(key=f"HS32Z{(index_pin + 1):02}")

    def get_pin_factorvalue_from_index(self, index_pin: int):
        return self.get_from_headers(key=f"HS32F{(index_pin + 1):02}")

    def _update_id02_data(
        self,
        stack: ExitStack,
    ):
        file = stack.enter_context(
            open_item_silx(filename=self.processing_filename, name="/", mode="a")
        )
        # Three datasets: data, data_variance, data_errors
        _, path_nxdata_output = self.links["destination"]["nxdata"].split("::")
        _, path_data_output = self.links["destination"]["datasignal"].split("::")
        if path_nxdata_output not in file:
            nexus_data_grp = self.create_h5_group(
                h5_parent_group=file,
                h5_group_name=path_nxdata_output,
                NX_class="NXdata",
                default=path_data_output,
                signal="data",
            )
        else:
            nexus_data_grp = file[path_nxdata_output]
        self._update_dataset(
            added_dataset=self.outputs.dataset_signal,
            h5_group=nexus_data_grp,
            h5_dataset_name="data",
            index_read=self.index_range_last,
        )
        if self.get_input_value("save_variance", False):
            self._update_dataset(
                added_dataset=self.outputs.dataset_variance,
                h5_group=nexus_data_grp,
                h5_dataset_name="data_variance",
                index_read=self.index_range_last,
            )
        if self.get_input_value("save_sigma", True):
            self._update_dataset(
                added_dataset=self.outputs.dataset_sigma,
                h5_group=nexus_data_grp,
                h5_dataset_name="data_errors",
                index_read=self.index_range_last,
            )

    def _update_id02_metadata(self, stack: ExitStack):
        # Update MCS group
        if self.processing_type not in ALL_PROCESSES:
            self.log_error(f"Processing type {self.processing_type} not valid")
            return

        root_group_destination = stack.enter_context(
            open_item_silx(filename=self.processing_filename, name="/", mode="a")
        )

        _, h5path_mcs_output = self.links["destination"]["metadata_mcs"].split("::")
        _, h5path_tfg_output = self.links["destination"]["metadata_tfg"].split("::")
        _, h5path_headers_output = self.links["destination"]["metadata_headers"].split(
            "::"
        )
        _, h5path_detector = self.links["destination"]["metadata_detector"].split("::")
        _, h5path_nxdata_output = self.links["destination"]["nxdata"].split("::")
        _, h5path_ewoks = self.links["destination"]["ewoks"].split("::")

        metadata_mcs_output = root_group_destination.require_group(h5path_mcs_output)
        metadata_tfg_output = root_group_destination.require_group(h5path_tfg_output)
        metadata_headers_output = root_group_destination.require_group(
            h5path_headers_output
        )
        nexus_data_grp_destination = None
        if self.processing_type in PYFAI_PROCESSES:
            nexus_data_grp_destination = root_group_destination.require_group(
                h5path_nxdata_output
            )

        self._update_subscan1(metadata_mcs_output)
        self._update_subscan2(metadata_mcs_output)
        # self._copy_streams(metadata_mcs_output)
        self._update_HS32C(metadata_mcs_output)
        self._update_HS32V(metadata_mcs_output)
        self._update_exposuretime(metadata_mcs_output)
        self._update_deltatime(metadata_tfg_output, nexus_data_grp_destination)
        self._update_intensities(metadata_mcs_output)

        # Save the processing params (only once except for normalization values)
        if self.processing_type in PYFAI_PROCESSES:
            self._save_processing_params(root_group=root_group_destination)

        # Update TitleExtension
        self.update_title_extension(
            parameters_group_destination=metadata_headers_output,
        )

        # Update NexusDetector with metadata from the RAW_DATA file (has to be accesible). To be done only once
        if (
            self.processing_type in PYFAI_PROCESSES
            and h5path_detector not in root_group_destination
        ):
            filename_input, h5path_metadata_detector_input = self.links["source"][
                "metadata_detector"
            ].split("::")

            if not os.path.exists(filename_input):
                return

            try:
                file_input = stack.enter_context(h5py.File(filename_input, "r"))
            except Exception:
                return

            if h5path_metadata_detector_input in file_input:
                metadata_detector_input = file_input[h5path_metadata_detector_input]
                if (
                    metadata_detector_input is not None
                    and len(metadata_detector_input) > 0
                ):
                    metadata_detector_output = self.create_h5_group(
                        h5_parent_group=root_group_destination,
                        h5_group_name=h5path_detector,
                        NX_class="NXdetector",
                    )

                    self.update_nexus_detector_group(
                        nxdetector_group_destination=metadata_detector_output,
                        nxdetector_group_source=metadata_detector_input,
                    )

    def _update_HS32C(self, metadata_mcs_output: h5py.Group):
        HS32C_array = self.get_HS32C_array()
        self._update_dataset(
            added_dataset=HS32C_array,
            h5_group=metadata_mcs_output,
            h5_dataset_name="HS32C",
            index_read=self.index_range_last,
        )

    def _update_HS32V(self, metadata_mcs_output: h5py.Group):
        HS32V_array = self.get_HS32V_array()
        self._update_dataset(
            added_dataset=HS32V_array,
            h5_group=metadata_mcs_output,
            h5_dataset_name="HS32V",
            index_read=self.index_range_last,
        )

    def _update_exposuretime(self, metadata_mcs_output: h5py.Group):
        exposuretime_values, slice_init, slice_end = self._get_exposuretime_values()
        self._update_dataset(
            added_dataset=exposuretime_values,
            h5_group=metadata_mcs_output,
            h5_dataset_name="ExposureTime",
            index_read=[slice_init, slice_end],
        )

    def _update_deltatime(
        self, metadata_tfg_output: h5py.Group, nexus_data_grp_destination: h5py.Group
    ):
        stream_deltatime = self.get_stream_deltatime()
        deltatime, slice_init, slice_end = self._read_from_stream(
            stream_object=stream_deltatime.get("stream"),
            slice_init=self.index_range_last[0],
            slice_end=self.index_range_last[-1],
        )

        self._update_dataset(
            added_dataset=deltatime,
            h5_group=metadata_tfg_output,
            h5_dataset_name="delta_time",
            index_read=[slice_init, slice_end],
        )

        if self.processing_type in PYFAI_PROCESSES:
            self._update_dataset(
                added_dataset=deltatime,
                h5_group=nexus_data_grp_destination,
                h5_dataset_name="t",
                unit="s",
                index_read=[slice_init, slice_end],
            )

    def _update_intensities(self, metadata_mcs_output: h5py.Group):
        exposuretime_values, slice_init, slice_end = self._get_exposuretime_values()
        stream_intensity0 = self.get_stream_monitor_0()

        intensity0_values, monitor0_slice_init, monitor0_slice_end = (
            self._read_from_stream(
                stream_object=stream_intensity0.get("stream"),
                slice_init=slice_init,
                slice_end=slice_end,
            )
        )

        intensity0_factor = self.get_from_headers(key=HEADERS_KEY_MONITOR_0_FACTOR)
        intensity0uncor = intensity0_values * intensity0_factor
        sot = self.get_from_headers(key=HEADERS_KEY_SOT, default=0.0)
        sct = self.get_from_headers(key=HEADERS_KEY_SCT, default=0.0)
        intensity0shutcor = (
            intensity0uncor
            * (exposuretime_values - sot + sct)
            / (exposuretime_values - sot)
        )

        self._update_dataset(
            added_dataset=intensity0shutcor,
            h5_group=metadata_mcs_output,
            h5_dataset_name="Intensity0ShutCor",
            index_read=[monitor0_slice_init, monitor0_slice_end],
        )
        self._update_dataset(
            added_dataset=intensity0uncor,
            h5_group=metadata_mcs_output,
            h5_dataset_name="Intensity0UnCor",
            index_read=[monitor0_slice_init, monitor0_slice_end],
        )

        stream_intensity1 = self.get_stream_monitor_1()
        intensity1_values, monitor1_slice_init, monitor1_slice_end = (
            self._read_from_stream(
                stream_object=stream_intensity1.get("stream"),
                slice_init=slice_init,
                slice_end=slice_end,
            )
        )
        intensity1_factor = self.get_from_headers(key=HEADERS_KEY_MONITOR_1_FACTOR)
        intensity1uncor = intensity1_values * intensity1_factor
        intensity1shutcor = (
            intensity1uncor
            * (exposuretime_values - sot + sct)
            / (exposuretime_values - sot)
        )

        self._update_dataset(
            added_dataset=intensity1shutcor,
            h5_group=metadata_mcs_output,
            h5_dataset_name="Intensity1ShutCor",
            index_read=[monitor1_slice_init, monitor1_slice_end],
        )

        self._update_dataset(
            added_dataset=intensity1uncor,
            h5_group=metadata_mcs_output,
            h5_dataset_name="Intensity1UnCor",
            index_read=[monitor1_slice_init, monitor1_slice_end],
        )

    def update_nexus_detector_group(
        self,
        nxdetector_group_destination: h5py.Group = None,
        nxdetector_group_source: h5py.Group = None,
    ):
        for name, item in nxdetector_group_source.items():
            if isinstance(item, h5py.Group):
                # Recursively copy subgroups
                new_subgroup = nxdetector_group_destination.create_group(name)
                copy_group_excluding_dataset(item, new_subgroup, "data")
            elif isinstance(item, h5py.Dataset):
                if name != "data":  # Skip the excluded dataset
                    nxdetector_group_source.copy(
                        name, nxdetector_group_destination, name=name
                    )

    def get_headers(self):
        if not self.headers:
            self.log_error(
                "Headers was not initialized! Maybe after using .run() method"
            )
        return self.headers

    def get_from_headers(self, key: str, to_integer: bool = False, default=None):
        if not self.headers:
            self.log_error(
                "Headers was not initialized! Maybe after using .run() method"
            )
            return

        value = self.headers.get(key, default)
        if value is None:
            return

        if isinstance(value, bytes):
            value = value.decode("UTF-8")
        try:
            value = float(value)
            if to_integer:
                return int(value)
            return value
        except Exception:
            return value

    def get_streams_HS32_scalers(self) -> list:
        hs32_names = self.get_HS32_names()
        streams_scalers = []

        for key_name in hs32_names.values():
            if self.scan_memory_url:
                # In blissdata streams, they appear as scalers:{name}
                stream_name = f"scalers:{key_name}"
            else:
                # In the file, they are saved with the raw refactored name (without scalers)
                stream_name = key_name
            stream_info = self.get_stream_info(
                name=stream_name,
            )
            streams_scalers.append(stream_info)
        return streams_scalers

    def get_streams_HS32_raw(self) -> list:
        hs32_names = self.get_HS32_names()
        streams_raw = []

        for key_name in hs32_names.values():
            if self.scan_memory_url:
                # In blissdata streams, they appear as mcs:{name}_raw
                stream_name = f"mcs:{key_name}_raw"
            else:
                # In the file, they are saved with the raw refactored name (without mcs)
                stream_name = f"{key_name}_raw"
            stream_info = self.get_stream_info(
                name=stream_name,
            )
            streams_raw.append(stream_info)
        return streams_raw

    def get_HS32_factors(self) -> dict:
        nb_pins = self.get_HS32_number_pins()
        factors = {}
        for index_pin in range(nb_pins):
            factors[index_pin] = self.get_from_headers(key=f"HS32F{(index_pin + 1):02}")
        return factors

    def get_HS32_zeros(self) -> dict:
        nb_pins = self.get_HS32_number_pins()
        zeros = {}
        for index_pin in range(nb_pins):
            zeros[index_pin] = self.get_from_headers(key=f"HS32Z{(index_pin + 1):02}")
        return zeros

    def get_HS32_names(self) -> dict:
        nb_pins = self.get_HS32_number_pins()
        names = {}
        for index_pin in range(nb_pins):
            names[index_pin] = self.get_from_headers(key=f"HS32N{(index_pin + 1):02}")
        return names

    def get_stream_slow_timer(self) -> dict:
        for stream_info in self.streams_subscan2:
            if "epoch" in stream_info["name"]:
                return stream_info

    def get_stream_fast_timer(self) -> dict:
        return self.get_stream_info(name="epoch", prefix="mcs")

    def get_stream_exposuretime(self) -> dict:
        """
        In blissdata, this stream is (normally) called scalers:time (in subscan1)
        In the header, there is key "HSTime" pointing to the string 'time'
        """
        key_name = self.get_from_headers(key=HEADERS_KEY_EXPOSURE_TIME)
        stream_time_name = f"scalers:{key_name}"
        return self.get_stream_info(
            name=stream_time_name,
            subscan_2=False,
        )

    def get_stream_exposuretime_raw(self) -> dict:
        """
        In blissdata, this stream is (normally) called mcs:time_raw (in subscan1)
        In the header, there is key "HSTime" pointing to the string 'time'
        """
        key_name = self.get_from_headers(key=HEADERS_KEY_EXPOSURE_TIME)
        stream_time_name = f"mcs:{key_name}_raw"
        return self.get_stream_info(
            name=stream_time_name,
            subscan_2=False,
        )

    def get_stream_deltatime(self) -> dict:
        if self.scan_memory_url:
            stream_name = "mcs:elapsed_time"
        else:
            stream_name = "elapsed_time"

        stream_info = self.get_stream_info(
            name=stream_name,
        )
        if not stream_info:
            stream_name = "mcs_elapsed_time"
            stream_info = self.get_stream_info(
                name=stream_name,
            )
        return stream_info

    def get_stream_info(
        self,
        index_pin: int = None,
        name: str = None,
        header_key_pin: str = None,
        prefix: str = "",
        suffix: str = "",
        subscan_2: bool = False,
    ) -> dict:
        if name is None:
            if header_key_pin is not None:
                name = self.get_from_headers(key=header_key_pin)
            elif index_pin is not None:
                name = self.get_pin_name_from_index(index_pin=index_pin)
        if name is None:
            return

        if prefix:
            name = f"{prefix}:{name}"
        if suffix:
            name = f"{name}_{suffix}"

        name_raw = refactor_stream_name_raw(stream_name=name, cut_name=False)
        name_raw_cut = refactor_stream_name_raw(stream_name=name, cut_name=True)
        name_interpreted = refactor_stream_name_interpreted(stream_name=name)

        if subscan_2:
            streams = self.streams_subscan2
        else:
            streams = self.streams_subscan1

        for stream_info in streams:
            if name == stream_info["name"]:
                return stream_info

        for stream_info in streams:
            if name_raw == stream_info["name_raw"]:
                return stream_info

        for stream_info in streams:
            if name_raw_cut == stream_info["name_raw"]:
                return stream_info

        for stream_info in streams:
            if name_interpreted == stream_info["name_interpreted"]:
                return stream_info

    def _read_from_stream(
        self,
        stream_object,
        slice_init: int,
        slice_end: int,
        datatype: str = "float32",
    ) -> tuple:
        nb_available_frames = len(stream_object)
        if slice_init > nb_available_frames:
            self.log_error(
                f"Not enough frames in {stream_object} ({nb_available_frames}). Requested init: {slice_init}"
            )
            return
        if slice_end > nb_available_frames:
            self.log_warning(
                f"Not enough frames in {stream_object} ({nb_available_frames}). Requested end: {slice_end}"
            )
            slice_end = nb_available_frames

        try:
            stream_values = stream_object[slice_init:slice_end]
            return stream_values.astype(datatype), slice_init, slice_end
        except Exception as e:
            self.log_error(f"{stream_object} could not be sliced: {e}")

    def read_from_stream_interpolate(
        self,
        stream_object,
        stream_slice=None,
        datatype: str = "float32",
    ) -> numpy.ndarray:
        """
        stream_object is a sliceable object (numpy.ndarray or blissdata stream object)
        Valid for all streams, from subscan1 and subscan2
        """
        if stream_slice is None:
            slice_init = 0
            slice_end = len(stream_object)
        else:
            slice_init, slice_end = stream_slice

        stream_values = stream_object[slice_init:slice_end]
        stream_values = stream_values.astype(datatype)

        # 2) Read all the values in the slow epoch counter
        slow_epoch_stream = self.get_stream_slow_timer()
        if slow_epoch_stream is None:
            self.log_debug("Slow epoch stream could not be found.")
            return stream_values

        slow_epoch_stream = slow_epoch_stream.get("stream")

        # Both epoch must be read with double precision
        slow_epoch_values, slice_init, slice_end = self._read_from_stream(
            stream_object=slow_epoch_stream,
            slice_init=0,
            slice_end=len(slow_epoch_stream),
            datatype="float64",
        )
        if slow_epoch_values is None:
            self.log_error(f"Stream {slow_epoch_stream} could not be read.")
            return stream_values

        # 3) Match the slow streams
        if len(slow_epoch_values) != len(stream_values):
            nb_slow_frames = min(len(slow_epoch_values), len(stream_values))
            slow_epoch_values = slow_epoch_values[0:nb_slow_frames]
            stream_values = stream_values[0:nb_slow_frames]

        # 4) Read the fast epoch counter
        fast_epoch_stream = self.get_stream_fast_timer().get("stream")
        if fast_epoch_stream is None:
            self.log_error("Fast epoch stream could not be found.")
            return stream_values

        fast_epoch_values, slice_init, slice_end = self._read_from_stream(
            stream_object=fast_epoch_stream,
            slice_init=0,
            slice_end=self.index_range_last[-1],
            datatype="float64",
        )

        if fast_epoch_values is None:
            self.log_error(f"Stream {fast_epoch_stream} could not be read.")
        elif fast_epoch_values.size == 0:
            self.log_error(f"Stream {fast_epoch_stream} is empty.")
            return

        # 5) Interpolate data, len(interpdata) = len(fast_epoch_values)
        try:
            return numpy.interp(fast_epoch_values, slow_epoch_values, stream_values)
        except Exception as e:
            self.log_error(f"Error during numpy interpolation: {e}")
            return

    def get_HS32C_array(self):
        nb_hs32_pins = self.get_HS32_number_pins()
        new_HS32C_array = numpy.full(
            (self.index_range_last[1] - self.index_range_last[0], nb_hs32_pins),
            fill_value=-1,
            dtype="float64",
        )

        streams_raw = self.get_streams_HS32_raw()
        for index_pin, stream_info in enumerate(streams_raw):
            if not stream_info:
                continue
            stream_values, slice_init, slice_end = self._read_from_stream(
                stream_object=stream_info.get("stream"),
                slice_init=self.index_range_last[0],
                slice_end=self.index_range_last[-1],
            )
            if stream_values is not None:
                new_HS32C_array[0 : len(stream_values), index_pin] = stream_values

        return new_HS32C_array

    def get_HS32V_array(
        self,
    ):
        nb_hs32_pins = self.get_HS32_number_pins()
        new_HS32V_array = numpy.zeros(
            (self.index_range_last[1] - self.index_range_last[0], nb_hs32_pins),
            dtype="float64",
        )

        streams_scalers = self.get_streams_HS32_scalers()
        for index_pin, stream_info in enumerate(streams_scalers):
            if not stream_info:
                continue
            stream_values, slice_init, slice_end = self._read_from_stream(
                stream_object=stream_info.get("stream"),
                slice_init=self.index_range_last[0],
                slice_end=self.index_range_last[-1],
            )
            if stream_values is not None:
                new_HS32V_array[0 : len(stream_values), index_pin] = stream_values

        return new_HS32V_array

    def _get_exposuretime_values(
        self,
    ) -> tuple:
        # Go first for the already normalized time values
        stream_exposuretime = self.get_stream_exposuretime()
        exposuretime, slice_init, slice_end = self._read_from_stream(
            stream_object=stream_exposuretime.get("stream"),
            slice_init=self.index_range_last[0],
            slice_end=self.index_range_last[-1],
        )

        if exposuretime is None:
            # Try to go to the raw time
            stream_exposuretime_raw = self.get_stream_exposuretime_raw()
            exposuretime, slice_init, slice_end = self._read_from_stream(
                stream_object=stream_exposuretime_raw.get("stream"),
                slice_init=self.index_range_last[0],
                slice_end=self.index_range_last[-1],
            )

            if exposuretime is None:
                self.log_error("Exposure time could not be read.")
                return None, None, None

            factor_exposuretime = self.get_factor_exposuretime()
            if factor_exposuretime is None:
                self.log_error("Default exposure time factor 1.0 will be used")
                factor_exposuretime = 1.0
            exposuretime *= factor_exposuretime
        return exposuretime, slice_init, slice_end

    # def _get_deltatime_values(self) -> tuple:
    #     stream_deltatime = self.get_stream_deltatime()
    #     deltatime = self._read_from_stream(
    #         stream_object=stream_deltatime.get("stream"),
    #         slice_init=self.index_range_last[-1],
    #         slice_end=self.index_range_last[-1],
    #     )
    #     return deltatime

    def get_stream_monitor_0(self) -> dict:
        key_name = self.get_from_headers(key=HEADERS_KEY_MONITOR_0)
        stream_name = f"scalers:{key_name}"
        return self.get_stream_info(name=stream_name)

    def get_stream_monitor_1(self) -> dict:
        key_name = self.get_from_headers(key=HEADERS_KEY_MONITOR_1)
        stream_name = f"scalers:{key_name}"
        return self.get_stream_info(name=stream_name)

    def get_factor(
        self, index_pin: int = None, pin_name: str = None, header_key_pin: str = None
    ):
        if index_pin is not None:
            return self.get_from_headers(key=f"HS32F{(index_pin + 1):02}")
        if pin_name is None:
            nb_pins = self.get_HS32_number_pins()
            pin_name = self.get_from_headers(key=header_key_pin)
        for index_pin in range(nb_pins):
            if self.get_pin_name_from_index(index_pin=index_pin) == pin_name:
                return self.get_from_headers(key=f"HS32F{(index_pin + 1):02}")

    def get_factor_exposuretime(self):
        return self.get_factor(header_key_pin=HEADERS_KEY_EXPOSURE_TIME)

    def update_title_extension(
        self,
        parameters_group_destination: h5py.Group,
    ):
        titleextension_template = self.get_from_headers(key="TitleExtensionTemplate")
        if not titleextension_template:
            self.log_warning("There is no TitleExtensionTemplate in the header.")
            return

        title_extension_parsed, title_extension_formats = parse_titleextension_template(
            titleextension_template
        )
        title_extension_values = {}

        # We want to slice all the streams at once, and then build the array of strings
        for titleextension_dict in title_extension_formats:
            stream_name = titleextension_dict["stream_name"]
            stream_values = None
            title_extension_values[stream_name] = numpy.full(
                shape=(self.index_range_last[1],), fill_value=numpy.nan
            )

            # Try first from subscan2
            stream_info = self.get_stream_info(name=stream_name, subscan_2=True)
            if stream_info:
                stream_values = self.read_from_stream_interpolate(
                    stream_object=stream_info.get("stream"),
                    stream_slice=None,
                )
            else:
                stream_info = stream_info or self.get_stream_info(
                    name=stream_name, subscan_2=False
                )
                if stream_info is None:
                    self.log_error(
                        f"{stream_name} stream from TitleExtension could not be found"
                    )
                    continue
                stream_values, slice_init, slice_end = self._read_from_stream(
                    stream_object=stream_info.get("stream"),
                    slice_init=0,
                    slice_end=self.index_range_last[-1],
                )
            if stream_values is None:
                self.log_error(f"Stream {stream_info} could not be read")
                continue

            title_extension_values[stream_name] = stream_values

        # Now we go index by index, building the string cells
        # The title extensions will always be the size of last index (from 0 -> last_frame)
        new_title_extensions = numpy.full(
            shape=(self.index_range_last[1],),
            fill_value="",
            dtype=h5py.string_dtype(encoding="utf-8"),
        )
        for index in range(len(new_title_extensions)):
            format_index = {
                stream_name: title_extension_values[stream_name][index]
                for stream_name in title_extension_values
            }

            new_title_extensions[index] = title_extension_parsed.format(**format_index)

        self._update_dataset(
            added_dataset=new_title_extensions,
            h5_group=parameters_group_destination,
            h5_dataset_name="TitleExtension",
            index_read=[0, self.index_range_last[1]],
        )

    def _update_subscan1(
        self,
        mcs_group_destination: h5py.Group,
    ):
        """
        Updates both raw/subscan_1 and interpreted with the values of subscan_1 (equivalent)
        """
        raw_group = mcs_group_destination.require_group("raw")
        subscan1 = raw_group.require_group("subscan_1")
        interpreted_grp = mcs_group_destination.require_group("interpreted")

        for stream_info in self.streams_subscan1:
            stream_name_raw = stream_info.get("name_raw")
            stream_name_interpreted = stream_info.get("name_interpreted")
            stream = stream_info.get("stream")

            stream_values, slice_init, slice_end = self._read_from_stream(
                stream_object=stream,
                slice_init=self.index_range_last[0],
                slice_end=self.index_range_last[-1],
            )

            self._update_dataset(
                added_dataset=stream_values,
                h5_group=subscan1,
                h5_dataset_name=stream_name_raw,
                index_read=[slice_init, slice_end],
            )
            self._update_dataset(
                added_dataset=stream_values,
                h5_group=interpreted_grp,
                h5_dataset_name=stream_name_interpreted,
                index_read=[slice_init, slice_end],
            )

    def _update_subscan2(
        self,
        mcs_group: h5py.Group,
    ):
        if not self.streams_subscan2:
            return

        interpreted_grp = mcs_group.require_group("interpreted")
        raw_grp = mcs_group.require_group("raw")

        for stream_info in self.streams_subscan2:
            stream_name_raw = stream_info.get("name_raw")
            stream_name_interpreted = stream_info.get("name_interpreted")
            stream = stream_info.get("stream")

            # Read without any interpolation to send to raw-subscan2
            stream_values, slice_init, slice_end = self._read_from_stream(
                stream_object=stream,
                slice_init=0,
                slice_end=len(stream),
            )
            if stream_values is None:
                continue

            subscan2_grp = raw_grp.require_group("subscan_2")
            if stream_name_raw not in subscan2_grp:
                dset_raw_subscan2 = subscan2_grp.create_dataset(
                    name=stream_name_raw,
                    shape=stream_values.shape,
                    maxshape=(None,),
                    dtype=stream_values.dtype,
                )
            else:
                dset_raw_subscan2 = subscan2_grp[stream_name_raw]
            dset_raw_subscan2.resize((len(stream_values),))
            dset_raw_subscan2[:] = stream_values

            # Send the interpolated values to interpreted group
            interpolated_values = self.read_from_stream_interpolate(
                stream_object=stream,
                stream_slice=None,
            )
            if interpolated_values is None:
                continue

            if stream_name_interpreted not in interpreted_grp:
                dset_interpreted = interpreted_grp.create_dataset(
                    name=stream_name_interpreted,
                    shape=(0,),
                    maxshape=(None,),
                    dtype=interpolated_values.dtype,
                )
            else:
                dset_interpreted = interpreted_grp[stream_name_interpreted]
            dset_interpreted.resize((len(interpolated_values),))
            dset_interpreted[:] = interpolated_values

    def create_h5_group(
        self,
        h5_parent_group: h5py.Group,
        h5_group_name: str,
        title: str = None,
        **kwargs,
    ) -> h5py.Group:
        """
        Unified method to create a group in a HDF5 file with additional attributes.

        h5_parent_group: h5py.Group - Parent group where the new group will be created
        h5_group_name: str - Name of the new group
        title: str - Title of the group
        kwargs: dict - Additional arguments to add as attributes in the group
        """
        if h5_group_name in h5_parent_group:
            self.log_warning(
                f"Group {h5_group_name} already exists in {h5_parent_group.name}"
            )
            return h5_parent_group[h5_group_name]

        h5_group = h5_parent_group.create_group(h5_group_name)
        if title:
            h5_group["title"] = title
        for key, value in kwargs.items():
            h5_group.attrs[key] = value
        return h5_group

    def _update_dataset(
        self,
        added_dataset: numpy.ndarray,
        h5_group: h5py.Group,
        h5_dataset_name: str,
        index_read: tuple = None,
        **kwargs,
    ) -> None:
        """
        Update a dataset in a HDF5 file with new data.
        It will create the dataset if it does not exist.

        added_dataset: numpy.ndarray - Array with the new data. It has to contain an additional dimension to the data
        h5_group: h5py.Group - Group in the HDF5 file where the dataset is located
        h5_dataset_name: str - Name of the dataset in h5_group
        kwargs: dict - Additional arguments to add as attributes in the dataset
        """
        if added_dataset is None:
            return

        if not isinstance(added_dataset, numpy.ndarray):
            self.log_error(f"Added dataset is not a numpy array. {type(added_dataset)}")
            return

        nb_new_frames = len(added_dataset)
        if nb_new_frames == 0:
            return

        if index_read is None:
            slice_init = 0
            slice_end = nb_new_frames
        else:
            slice_init, slice_end = index_read

        self.log_debug(
            f"Updating dataset {h5_dataset_name} with {nb_new_frames} new frames"
        )

        ndim = added_dataset.ndim
        if h5_dataset_name not in h5_group:
            if ndim == 3:
                interpretation = "image"
                dtype = self.get_input_value("datatype", "float32")
                compression = hdf5plugin.Bitshuffle(cname="lz4")
                chunks = CHUNK_SIZE_3D
            elif ndim == 2:
                interpretation = "spectrum"
                dtype = "float64"
                compression = None
                chunks = None
            elif ndim == 1:
                interpretation = "scalar"
                dtype = added_dataset.dtype
                compression = None
                chunks = None
                if added_dataset.dtype.kind == "U":
                    dtype = h5py.string_dtype(encoding="utf-8")
                    added_dataset = added_dataset.astype(
                        h5py.string_dtype(encoding="utf-8")
                    )
            dset = h5_group.create_dataset(
                name=h5_dataset_name,
                shape=(0,) + added_dataset.shape[1:ndim],
                maxshape=(None,) + added_dataset.shape[1:ndim],
                chunks=chunks,
                dtype=dtype,
                compression=compression,
            )
            dset.attrs["interpretation"] = interpretation
            for key, value in kwargs.items():
                dset.attrs[key] = value
        else:
            dset = h5_group[h5_dataset_name]

        # The added dataset can be:
        #   - New frames to append between index_range_last limits (normal loop procedure)
        #   - Rewrite the whole dataset, posibly with more frames (subscan2 interpolation)
        dset_current_nb_frames = len(dset)
        if slice_end <= dset_current_nb_frames:
            # No resize is needed
            ...
        else:
            dset_new_shape = (slice_end, *dset.shape[1:ndim])
            dset.resize(dset_new_shape)

        try:
            dset[slice_init:slice_end] = added_dataset
            self.log_debug(
                f"Dataset {h5_dataset_name} updated with {len(added_dataset)} frames"
            )
        except Exception as e:
            self.log_error(f"""{e}: Failed while saving {h5_dataset_name}. \
                Incoming dataset shape: {added_dataset.shape}, \
                hdf5 dset shape={dset.shape}, {added_dataset.shape=} \
                limits: {slice_init} -> {slice_end}
                """)

    def write_root_group(self, root_group: h5py.Group) -> h5py.Group:
        self.log_debug("Creating root group")
        root_group.attrs.update(
            {
                "HDF5_Version": h5py.version.hdf5_version,
                "NX_class": "NXroot",
                "creator": "ewoksid02",
                "file_name": str(self.processing_filename),
                "file_time": get_isotime(),
                "default": "entry_0000",
            }
        )
        return root_group

    def _save_benchmark(
        self,
        bench,
    ):
        if not self.do_save:
            return

        benchmark_name = bench.benchmark_name
        total_time = bench.bench_total_s
        time_per_frame = bench.bench_per_frame_ms
        nb_frames = bench.nb_frames

        with ExitStack() as stack:
            if not self.processing_filename or not self.do_save:
                return

            # Append data to the nexus data group
            stack.enter_context(lock)
            root_group_destination = stack.enter_context(
                open_item_silx(filename=self.processing_filename, name="/", mode="a")
            )

            _, h5path_ewoks_destination = self.links["destination"]["ewoks"].split("::")
            ewoks_group = root_group_destination[h5path_ewoks_destination]

            if "last_process" not in ewoks_group:
                return

            title = ewoks_group["last_process/title"][()].decode()
            h5path_ewoks_benchmark = (
                f"{h5path_ewoks_destination}/history/{title}/benchmark"
            )

            if h5path_ewoks_benchmark not in root_group_destination:
                bench_grp = self.create_h5_group(
                    h5_parent_group=root_group_destination,
                    h5_group_name=h5path_ewoks_benchmark,
                    title="Benchmark",
                    NX_class="NXdata",
                    default=h5path_ewoks_benchmark,
                    signal="data",
                    # axes=["steps"],
                    axes=["index"],
                )
                steps = bench_grp.create_dataset(
                    name="steps",
                    shape=(0,),
                    maxshape=(None,),
                    dtype=h5py.string_dtype(encoding="utf-8"),
                )
                indexes = bench_grp.create_dataset(
                    name="index",
                    shape=(0,),
                    maxshape=(None,),
                )
                data = bench_grp.create_dataset(
                    name="data",
                    shape=(0,),
                    maxshape=(None,),
                )
                data.attrs.update(
                    {
                        "interpretation": "spectrum",
                    }
                )
                data_errors = bench_grp.create_dataset(
                    name="data_errors",
                    shape=(0,),
                    maxshape=(None,),
                )
                data_errors.attrs.update(
                    {
                        "interpretation": "spectrum",
                    }
                )
            else:
                bench_grp = root_group_destination[h5path_ewoks_benchmark]
                steps = bench_grp["steps"]
                indexes = bench_grp["index"]
                data = bench_grp["data"]
                data_errors = bench_grp["data_errors"]

            h5path_ewoks_benchmark_step = (
                f"{h5path_ewoks_destination}/history/{title}/benchmark/{benchmark_name}"
            )
            if h5path_ewoks_benchmark_step not in root_group_destination:
                bench_grp_step = self.create_h5_group(
                    h5_parent_group=root_group_destination,
                    h5_group_name=h5path_ewoks_benchmark_step,
                    title=benchmark_name,
                    NX_class="NXdata",
                    default=h5path_ewoks_benchmark,
                )
            else:
                bench_grp_step = root_group_destination[h5path_ewoks_benchmark_step]

            if f"{benchmark_name}_loop_nb" not in bench_grp_step:
                loop_dset = bench_grp_step.create_dataset(
                    name=f"{benchmark_name}_loop_nb",
                    dtype="int32",
                    shape=(0,),
                    maxshape=(None,),
                )
                perframe_dset = bench_grp_step.create_dataset(
                    name=f"{benchmark_name}_per_frame",
                    dtype="float32",
                    shape=(0,),
                    maxshape=(None,),
                )
                perframe_mean_dset = bench_grp_step.create_dataset(
                    name=f"{benchmark_name}_per_frame_mean",
                    dtype="float32",
                    data=time_per_frame,
                )
                perframe_std_dset = bench_grp_step.create_dataset(
                    name=f"{benchmark_name}_per_frame_std",
                    dtype="float32",
                    data=0.0,
                )
                nbframes_dset = bench_grp_step.create_dataset(
                    name=f"{benchmark_name}_nb_frames",
                    dtype="int32",
                    shape=(0,),
                    maxshape=(None,),
                )
                total_dset = bench_grp_step.create_dataset(
                    name=f"{benchmark_name}_total",
                    dtype="float32",
                    shape=(0,),
                    maxshape=(None,),
                )
                accumulated_dset = bench_grp_step.create_dataset(
                    name=f"{benchmark_name}_accumulated",
                    dtype="float32",
                    shape=(0,),
                    maxshape=(None,),
                )
                steps.resize((steps.shape[0] + 1,))
                steps[-1] = benchmark_name
                indexes.resize((indexes.shape[0] + 1,))
                indexes[-1] = len(indexes)
                data.resize((data.shape[0] + 1,))
                data_errors.resize((data_errors.shape[0] + 1,))
            else:
                loop_dset = bench_grp_step[f"{benchmark_name}_loop_nb"]
                perframe_dset = bench_grp_step[f"{benchmark_name}_per_frame"]
                perframe_mean_dset = bench_grp_step[f"{benchmark_name}_per_frame_mean"]
                perframe_std_dset = bench_grp_step[f"{benchmark_name}_per_frame_std"]
                nbframes_dset = bench_grp_step[f"{benchmark_name}_nb_frames"]
                total_dset = bench_grp_step[f"{benchmark_name}_total"]
                accumulated_dset = bench_grp_step[f"{benchmark_name}_accumulated"]

            # Append new data
            loop_dset.resize((loop_dset.shape[0] + 1,))
            perframe_dset.resize((perframe_dset.shape[0] + 1,))
            nbframes_dset.resize((nbframes_dset.shape[0] + 1,))
            total_dset.resize((total_dset.shape[0] + 1,))
            accumulated_dset.resize((accumulated_dset.shape[0] + 1,))

            loop_dset[-1] = self.loop_nb
            perframe_dset[-1] = time_per_frame
            nbframes_dset[-1] = nb_frames
            total_dset[-1] = total_time
            if len(accumulated_dset) == 1:
                accumulated_dset[-1] = total_time
            else:
                accumulated_dset[-1] = accumulated_dset[-2] + total_time

            mean_value = perframe_dset[:].mean()
            std_value = perframe_dset[:].std()
            perframe_mean_dset[()] = mean_value
            perframe_std_dset[()] = std_value

            step_names = [s.decode() for s in steps]
            index_benchmark = next(
                (i for i, s in enumerate(step_names) if s == benchmark_name), None
            )
            if index_benchmark is not None:
                data[index_benchmark] = mean_value
                data_errors[index_benchmark] = std_value

    def _save_processing_params(self, root_group: h5py.Group):
        # Do it once for non-numpy arrays (live parameters)
        _, h5path_ewoks_destination = self.links["destination"]["ewoks"].split("::")
        ewoks_group = root_group[h5path_ewoks_destination]

        if "last_process" not in ewoks_group:
            return

        title = ewoks_group["last_process/title"][()].decode()
        h5path_ewoks_info = f"{h5path_ewoks_destination}/history/{title}"

        ewoks_info_group = root_group[h5path_ewoks_info]
        params_group_name = "parameters"
        if params_group_name in ewoks_info_group:
            return

        info_history_complete = self.outputs.info_history
        if info_history_complete:
            info_history = self.outputs.info_history[-1]
        else:
            info_history = None

        if h5path_ewoks_info not in root_group:
            self.log_error(f"{h5path_ewoks_info} not found in {root_group}")
            return

        params_group = self.create_h5_group(
            h5_parent_group=ewoks_info_group,
            h5_group_name=params_group_name,
            NX_class="NXcollection",
        )

        params_info = {}
        for key, value in self.processing_params.items():
            if isinstance(value, numpy.ndarray):
                continue
                incoming_length = value.shape[0]
                if key not in params_group:
                    dset = params_group.create_dataset(
                        name=key, shape=(incoming_length,), maxshape=(None,), data=value
                    )
                else:
                    dset = params_group[key]
                    current_length = len(dset)
                    new_length = current_length + incoming_length
                    dset.resize((new_length,))
                    dset[current_length:new_length] = value
                params_info[key] = dset[()]
                continue

            if isinstance(value, dict):
                params_info[key] = json.dumps(value)
                for k, v in value.items():
                    params_info[k] = str(v)
            else:
                try:
                    params_info[key] = str(value)
                except Exception:
                    params_info[key] = json.dumps(value.as_dict())
                    for k, v in value.items():
                        params_info[k] = v

        for key, value in params_info.items():
            params_group[key] = value

        info_history["parameters"] = params_info
        info_history_complete = self.outputs.info_history[:-1]
        info_history_complete.append(info_history)
        self.outputs.info_history = info_history_complete

    def _link_ewoks_history(
        self, root_group_destination: h5py.Group, stack: ExitStack = None
    ):
        _, h5path_ewoks_history_destination = self.links["destination"]["ewoks"].split(
            "::"
        )
        if self.links["source"].get("ewoks") is not None:
            filename_source, h5path_ewoks_source = self.links["source"]["ewoks"].split(
                "::"
            )
        else:
            filename_source = None
            h5path_ewoks_source = None

        ewoks_destination = self.create_h5_group(
            h5_parent_group=root_group_destination,
            h5_group_name=h5path_ewoks_history_destination,
            NX_class="NXcollection",
        )
        ewoks_history = self.create_h5_group(
            h5_parent_group=ewoks_destination,
            h5_group_name="history",
            NX_class="NXcollection",
        )

        info_history_complete = self.get_input_value("info_history", []).copy()
        info_history_previous = None
        if info_history_complete:
            info_history_previous = info_history_complete[-1]

        title_format = "{index:02} - {class_name}"
        info_history = {
            "index": 0,
            "task_identifier": f"{self.__module__}.{self.__class__.__name__}",
            "class": self.__class__.__name__,
            "processing_type": self.processing_type,
            "datetime": str(get_isotime()),
            "version": version("ewoksid02"),
            "host": socket.gethostname(),
        }
        info_history["title"] = title_format.format(
            index=info_history["index"], class_name=info_history["class"]
        )
        if info_history_previous is not None:
            # The previous node send the recent historic information
            info_history["index"] = info_history_previous["index"] + 1
            info_history["title"] = title_format.format(
                index=info_history["index"], class_name=info_history["class"]
            )
        else:
            # No previous node with historic information
            if filename_source and h5path_ewoks_source:
                # This is an ewoks PROCESSED file
                root_group_source = stack.enter_context(
                    open_item_silx(filename=filename_source, name="/", mode="r")
                )
                if h5path_ewoks_source not in root_group_source:
                    # Use ewoksid02 version 1.0 onwards
                    self.log_warning(
                        f"{filename_source} is an ewoks PROCESSED_DATA file without ewoks group. Historical index will be set to 0"
                    )
                else:
                    ewoks_source = root_group_source[h5path_ewoks_source]
                    if "last_process" not in ewoks_source:
                        self.log_warning(
                            "No index could be found in ewoks group. Historical index will be set to 0"
                        )
                    else:
                        # Collect all the previous history and send to the next task
                        history_group = ewoks_source["history"]
                        for historic_task in history_group:
                            historic_task_serialized = serialize_h5py_task(
                                h5py_group=history_group[historic_task]
                            )
                            info_history_complete.append(historic_task_serialized)
                        info_history["index"] = info_history_complete[-1]["index"] + 1
                        info_history["title"] = title_format.format(
                            index=info_history["index"],
                            class_name=info_history["class"],
                        )
            else:
                # This is not an ewoks PROCESSED file (no historic information)
                ...

        # Add a new process into history group
        info_history_complete.append(info_history)

        for historic_task in info_history_complete:
            historic_task_group = self.create_h5_group(
                h5_parent_group=ewoks_history,
                h5_group_name=historic_task["title"],
                NX_class="NXcollection",
            )
            deserialize_h5py_task(
                h5dict=historic_task,
                h5py_parent=historic_task_group,
            )

        # Link to last_process group
        title = info_history["title"]
        ewoks_destination["last_process"] = h5py.SoftLink(f"history/{title}")

        # Send to the next node
        self.outputs.info_history = info_history_complete

    def processing_info(self) -> list:
        return []

    def _get_filename_gallery(self):
        processed_directory = Path(self.processing_filename).parent
        gallery_directory = processed_directory / "gallery"
        gallery_directory.mkdir(exist_ok=True)
        filename_png = (
            gallery_directory / Path(self.processing_filename).with_suffix(".png").name
        )
        return str(filename_png)

    def _save_png_to_gallery(self, array: numpy.ndarray) -> str:
        processed_directory = Path(self.processing_filename).parent
        gallery_directory = processed_directory / "gallery"
        gallery_directory.mkdir(exist_ok=True)
        filename_png = (
            gallery_directory / Path(self.processing_filename).with_suffix(".png").name
        )
        filename_png = str(filename_png)
        try:
            matplotlib.image.imsave(
                fname=filename_png,
                arr=array,
                cmap="viridis",
                dpi=100,
                origin="lower",
            )
            return filename_png
        except Exception as e:
            self.log_error(f"Failed to save image: {e}")
            return ""
