# TODO add glob to loadlist to load generic lists --> good idea I think and offer various sorting algos including natsort
# a tool to load lists that is smarter than the TA list loader

import os
import traceback
from functools import partial
from epyseg.tools.early_stopper_class import early_stop
from epyseg.tools.logger import TA_logger # logging
import glob
import shutil
import os
from natsort import natsorted

from epyseg.utils.commontools import execute_chained_functions_and_save_as_tiff

logger = TA_logger()


def create_list(input_folder, save=False, output_name=None, extensions=['*.tif', '*.tiff', '*.jpg', '*.jpeg', '*.png', '*.bmp', '*.tga', '*.lif'], sort_type='natsort'):
    """
    Creates a list of file paths in the specified input folder based on the given extensions.

    Parameters:
        - input_folder: The path to the input folder where the files are located.
        - save: A flag indicating whether to save the list to a file (default is False).
        - output_name: The name of the output file (default is None, which generates a file named 'list.lst' in the input folder).
        - extensions: A list of file extensions to consider when creating the list (default is a list of common image extensions).
        - sort_type: The type of sorting to apply to the list of file paths (default is 'natsort').

    Returns:
        A list of file paths.

    Examples:
        Suppose we have an input folder named 'images' with the following files:
        - image1.jpg
        - image2.png
        - image3.tif

        >>> create_list('.',extensions=['.py'])
        ['./__init__.py', './commontools.py', './loadlist.py']
    """

    lst = []  # Initialize an empty list to store file paths

    if not extensions:
        logger.error('Extensions are not defined --> list cannot be created')
        return lst

    path = input_folder  # Assign the input_folder path to the 'path' variable

    if not path.endswith('/') and not path.endswith('\\') and not '*' in path and not os.path.isfile(path):
        path += '/'  # Append a trailing slash to the path if it doesn't have one

    if not '*' in path:
        # Iterate over the extensions and append matching file paths to the list
        for ext in extensions:
            if not ext.startswith('*'):
                ext = '*' + ext  # Prepend '*' to the extension if it doesn't have one
            lst += glob.glob(path + ext)
    else:
        lst += glob.glob(path)  # Append all file paths in the path to the list

    if sort_type == 'natsort':
        lst = natsorted(lst)  # Apply natural sorting to the list of file paths

    if save:
        if output_name is None:
            output_name = os.path.join(input_folder, 'list.lst')  # Generate the default output file name
        save_list_to_file(lst, output_name)  # Save the list to the specified output file

    return lst  # Return the list of file paths


def get_resume_list(input_list, resume_value):
    """
    Retrieves a resumed list starting from the specified resume_value.

    Parameters:
        - input_list: The input list to retrieve the resume list from.
        - resume_value: The value in the list to resume from.

    Returns:
        A new list starting from the resume_value.

    Examples:
        >>> files = ['file1.jpg', 'file2.png', 'file3.tif', 'file4.jpg']
        >>> get_resume_list(files, 'file2.png')
        ['file2.png', 'file3.tif', 'file4.jpg']
    """

    try:
        resume_index = input_list.index(resume_value)
        resume_list = input_list[resume_index:]
        return resume_list
    except ValueError:
        return []



def loadlist(txtfile, always_prefer_local_directory_if_file_exists=True, filter_existing_files_only=False,
             smart_local_folder_detection=True, skip_hashtag_commented_lines=True, skip_empty_lines=True):
    """
    Loads a list from a text file with optional filtering and processing options.

    Parameters:
        - txtfile: The path to the text file to load the list from.
        - always_prefer_local_directory_if_file_exists: If True, files located in the .lst/.txt file containing folder will be preferred if they exist.
        - filter_existing_files_only: If True, filters the list to include only existing files.
        - smart_local_folder_detection: If True, applies local paths to the common root path in the list for smarter list opening.
        - skip_hashtag_commented_lines: If True, skips lines starting with '#' (commented lines) in the list.
        - skip_empty_lines: If True, skips empty lines and lines containing spaces only.

    Returns:
        The loaded list with the specified filtering and processing options.

    Examples:
        >>> list_file = 'loadlist.py'
        >>> loadlist(list_file, always_prefer_local_directory_if_file_exists=True, filter_existing_files_only=False)[:2]
        ['import os', 'import traceback']
    """

    if not os.path.exists(txtfile) or not os.path.isfile(txtfile):
        if not '*' in txtfile and not os.path.isdir(txtfile):
            return None
        else:
            lst = create_list(txtfile, save=False)  # Create a list using create_list() if txtfile is a directory or contains wildcard characters ('*')
    else:
        with open(txtfile) as f:
            if not skip_hashtag_commented_lines:
                lst = [line.rstrip() for line in f]  # Read all lines from the text file and remove trailing whitespaces
            else:
                lst = [line.rstrip() for line in f if not line.rstrip().startswith('#')]  # Exclude lines starting with '#' (commented lines)
        if skip_empty_lines:
            lst = [line.rstrip() for line in lst if not line.strip() == '']  # Exclude lines containing only spaces or empty lines

    if always_prefer_local_directory_if_file_exists:
        logger.debug('Files located in the .lst/.txt file containing folder will be preferred if they exist.')

        list_file_directory = os.path.dirname(txtfile)  # Get the directory of the txtfile

        if not smart_local_folder_detection:
            lst = [os.path.join(list_file_directory, os.path.basename(line)) if os.path.isfile(
                os.path.join(list_file_directory, os.path.basename(line))) else line for line in lst]
            # Prepend the list items with the list_file_directory if the file exists in the directory, otherwise keep the item as it is
        else:
            common_path = os.path.commonprefix(lst)  # Find the common root path of all items in the list
            if not os.path.isdir(common_path):
                common_path = os.path.dirname(common_path)

            if common_path != '':
                relative_paths = [os.path.relpath(path, common_path) for path in lst]  # Get the relative paths to the common_path
                lst = [os.path.join(list_file_directory, rel) if os.path.isfile(
                    os.path.join(list_file_directory, rel)) else line for rel, line in zip(relative_paths, lst)]
                # Prepend the list_file_directory to the relative paths if the file exists in the directory, otherwise keep the item as it is

    if filter_existing_files_only:
        logger.debug('Checking list for existing files')
        lst = [line for line in lst if os.path.isfile(line)]  # Filter the list to include only existing files

    return lst

def TA_smart_pairing_list(TA_list, corresponding_list_of_files_to_inject_or_folder, stop_on_error=False):
    """
    Performs smart pairing of two lists, TA_list and corresponding_list_of_files_to_inject_or_folder, based on certain conditions.

    Parameters:
        - TA_list: The list to be paired.
        - corresponding_list_of_files_to_inject_or_folder: The list of files or folder to be paired with TA_list.
        - stop_on_error: If True, stops the execution if the input lists/folders don't match.

    Returns:
        The paired list as a result of smart pairing.

    # Examples:
    #     >>> ta_list = 'ta_list.txt'
    #     >>> files_folder = 'files_folder'
    #     >>> TA_smart_pairing_list(ta_list, files_folder, stop_on_error=True)
    #     [('files_folder/file1.jpg', 'file1.jpg'), ('files_folder/file2.png', 'file2.png'), ('files_folder/file3.tif', 'file3.tif')]
    """

    if TA_list.lower().endswith('.txt') or TA_list.lower().endswith('.lst'):
        TA_list = loadlist(TA_list)  # Load the list from a text file if TA_list ends with '.txt' or '.lst'

    if os.path.isdir(corresponding_list_of_files_to_inject_or_folder):
        # If corresponding_list_of_files_to_inject_or_folder is a directory, assume same name as for input
        # corresponding_list_of_files_to_inject_or_folder
        matching_list = []
        for file in TA_list:
            filename0_without_path = os.path.basename(file)  # Get the filename without the path
            matching_list.append(os.path.join(corresponding_list_of_files_to_inject_or_folder, filename0_without_path))
            # Create the matching list by joining the corresponding_list_of_files_to_inject_or_folder with the filename without the path
    elif corresponding_list_of_files_to_inject_or_folder.lower().endswith('.txt') or \
            corresponding_list_of_files_to_inject_or_folder.lower().endswith('.lst'):
        matching_list = loadlist(corresponding_list_of_files_to_inject_or_folder)
        # Load the matching list from a text file if corresponding_list_of_files_to_inject_or_folder ends with '.txt' or '.lst'

    # Try zipping and print correspondence and if it fails, try smart matching
    if len(matching_list) != len(TA_list):
        if stop_on_error:
            print('Input lists/folders don\'t match', len(TA_list), len(matching_list))
            return
        else:
            # TODO: Implement rescue strategy for matching when lengths don't match
            pass

    zipped = list(zip(matching_list, TA_list))  # Zip the matching_list and TA_list together
    return zipped


def TA_injector(TA_list, corresponding_list_of_files_to_inject_or_folder, generic_TA_name_for_injected=None, stop_on_error=True, move_file=False, simulate_only=False):
    """
    Performs injection of files from the corresponding_list_of_files_to_inject_or_folder into the destination folders specified by the TA_list.

    Parameters:
        - TA_list: The list specifying the destination folders for injection.
        - corresponding_list_of_files_to_inject_or_folder: The list of files or folder to be injected.
        - generic_TA_name_for_injected: The generic name to be used for the injected files. If None, the original filename will be used.
        - stop_on_error: If True, stops the execution if the input lists/folders don't match.
        - move_file: If True, moves the files instead of copying them.
        - simulate_only: If True, simulates the injection without actually performing any file operations.

    Returns:
        None

    # Examples:
    #     >>> ta_list = ['folder1', 'folder2']
    #     >>> files_folder = 'files_folder'
    #     >>> TA_injector(ta_list, files_folder, generic_TA_name_for_injected='injected.jpg', stop_on_error=False, move_file=True, simulate_only=False)
    #     move files_folder/file1.jpg --> folder1/injected.jpg
    #     move files_folder/file2.png --> folder2/injected.jpg
    """

    zipped = TA_smart_pairing_list(TA_list, corresponding_list_of_files_to_inject_or_folder)
    if zipped is None:
        # Nothing to do, maybe return an error
        return

    for matching_things in zipped:
        file_to_copy = matching_things[0]
        file_to_copy_without_path = os.path.basename(file_to_copy)
        file = matching_things[-1]
        filename0_without_path = os.path.basename(file)
        filename0_without_ext = os.path.splitext(filename0_without_path)[0]
        path = os.path.dirname(file)
        destination_folder = os.path.join(path, filename0_without_ext)
        destination_name = os.path.join(destination_folder, file_to_copy_without_path if generic_TA_name_for_injected is None else generic_TA_name_for_injected)

        # Make sure the folder exists for the copy
        if not simulate_only:
            os.makedirs(os.path.dirname(destination_folder), exist_ok=True)

        if not move_file:
            print('copy', file_to_copy, '-->', destination_name)
            if not simulate_only:
                shutil.copy2(file_to_copy, destination_name)
        else:
            print('move', file_to_copy, '-->', destination_name)
            if not simulate_only:
                shutil.move(file_to_copy, destination_name)



# creates based on the main TA list a list of files with alternative names and returns it
# NB will only change name if the file with the alternate name exists --> I think this is the behaviour I want but ok
def smart_TA_list(main_list, name_to_search_first, alternative_name_if_name_to_search_first_does_not_exist=None):
    """
    Creates a smart list by combining the main_list with the specified name_to_search_first and an alternative name if the name_to_search_first doesn't exist.

    Parameters:
        - main_list: The main list of paths.
        - name_to_search_first: The name to search first in each path.
        - alternative_name_if_name_to_search_first_does_not_exist: The alternative name to use if the name_to_search_first doesn't exist.

    Returns:
        The smart list generated by combining the main_list with the specified names.

    Examples:
        >>> main_list = ['path1/file.txt', 'path2/image.png', 'path3/document.doc']
        >>> name_to_search_first = 'data.txt'
        >>> alternative_name_if_name_to_search_first_does_not_exist = 'backup.txt'
        >>> smart_TA_list(main_list, name_to_search_first, alternative_name_if_name_to_search_first_does_not_exist)
        ['path1/data.txt', 'path2/data.txt', 'path3/data.txt']
    """

    lst = [os.path.join(os.path.splitext(line)[0], name_to_search_first) for line in main_list]
    # Generate a list by joining the path without the extension with the name_to_search_first

    if alternative_name_if_name_to_search_first_does_not_exist is not None:
        lst = [line if os.path.isfile(line) or not os.path.isfile(os.path.join(os.path.dirname(line), alternative_name_if_name_to_search_first_does_not_exist)) else os.path.join(os.path.dirname(line), alternative_name_if_name_to_search_first_does_not_exist) for line in lst]
        # Check if the line is a file or if the alternative name doesn't exist in the same directory.
        # If it's a file or the alternative name exists, keep the line as it is.
        # Otherwise, replace the line with the alternative name.

    return

def save_list_to_file(lst, filename, col_separator='\t'):
    """
    Saves the given list to a file.

    Parameters:
        - lst: The list to be saved.
        - filename: The name of the file to save the list.
        - col_separator: The separator character to use for items in the list.

    Returns:
        The input list if successfully saved.

    Raises:
        ValueError: If the list is empty.

    # Examples:
    #     >>> lst = ['item1', 'item2', 'item3']
    #     >>> filename = 'output.txt'
    #     >>> save_list_to_file(lst, filename, col_separator='\t')
    #     ['item1', 'item2', 'item3']
    """

    if lst is not None:
        with open(filename, 'w') as f:
            for item in lst:
                if not isinstance(item, list) and not isinstance(item, tuple):
                    f.write("%s\n" % item)
                    # Write each item in the list to a new line in the file.
                else:
                    f.write("%s\n" % col_separator.join([str(i) for i in item]))
                    # If the item is a list, join its elements with the col_separator and write to the file.

        return lst
    else:
        raise ValueError('Empty list --> cannot save it to a file')
        # Raise an error if the list is empty.


# this gives the optimal folder name for saving images without requiring other stuff, from various models --> use that for ensemble tools
def smart_output_folder_name(lst):
    """
    Generates the smartest name for the output folder based on the given list of paths.

    Parameters:
        - lst: The list of paths.

    Returns:
        The list of smart folder names.

    """

    # TODO no clue what  i wanted to do here, maybe just return the commaon path ?

    # Get the common path among all the paths in the list
    common_path = os.path.commonprefix(lst)

    if not os.path.isdir(common_path):
        common_path = os.path.dirname(common_path)

    if common_path == '':
        return None

    # Replace slashes and backslashes in the common path with underscores to create a meaningful name
    smart_name = [os.path.splitext(name.replace(common_path, '').replace('/', '_').replace('\\', '_'))[0] for name in lst]

    # Remove leading underscores from the folder names
    smart_name = [name if not name.startswith('_') else name[1:] for name in smart_name]

    return smart_name

# can create a list of files with the same name across several folders generated by different models for example --> can then be used for ensemble or averaging...
def get_transversal_list(root_path):
    """
    Generates a transversal list of files by traversing through folders.

    Parameters:
        - root_path: The root path to start the traversal.

    Returns:
        The transversal list of files.

    # Examples:
    #     >>> root_path = '/E/Sample_images/Consensus_learning/gray/CARE/'
    #     >>> get_transversal_list(root_path)
    #     [['/E/Sample_images/Consensus_learning/gray/CARE/folder1/file1.txt',
    #       '/E/Sample_images/Consensus_learning/gray/CARE/folder2/file1.txt'],
    #      ['/E/Sample_images/Consensus_learning/gray/CARE/folder1/file2.txt',
    #       '/E/Sample_images/Consensus_learning/gray/CARE/folder2/file2.txt']]
    """

    transverse_list = []

    # Get the list of folders within the root path
    folders = os.listdir(root_path)
    folders = [os.path.join(root_path, folder) for folder in folders]
    folders = natsorted(folders)

    # Get the list of files in the first folder
    files_in_first_folder = os.listdir(folders[0])

    # Traverse through the files in the first folder
    for file in files_in_first_folder:
        multifolder_list = []

        # Check if the file exists in other folders and add them to the list
        for folder in folders:
            final_file = os.path.join(folder, file)
            if os.path.isfile(final_file):
                try:
                    multifolder_list.append(final_file)
                except:
                    print('invalid file', final_file)

        transverse_list.append(multifolder_list)

    return transverse_list


def list_processor(lst, processing_fn, multithreading=True, progress_callback=None, use_save_execution_if_chained=False, name_processor_function_for_saving=None):
    """
    Processes a list using a specified processing function.

    Parameters:
        - lst: The list to process.
        - processing_fn: The processing function to apply to each element of the list.
        - multithreading: Boolean value indicating whether to use multithreading.
        - progress_callback: Callback function to track the progress of the processing.
        - use_save_execution_if_chained: Boolean value indicating whether to use save execution if the processing function is chained.
        - name_processor_function_for_saving: Name of the processor function for saving.

    Returns:
        None

    # Examples:
    #     >>>lst = loadlist('/E/Sample_images/sample_images_PA/trash_test_mem/mini_empty/list.lst')
    #     >>>def output_file_name(input_file_name):
    #     ...    return smart_name_parser(input_file_name, 'full_no_ext') + 'inverted2.tif'
    #     >>>chained_functions = [Img, invert, elastic_deform]
    #     >>>list_processor(lst=lst, processing_fn=chained_functions, multithreading=True, use_save_execution_if_chained=True,progress_callback=None, name_processor_function_for_saving=output_file_name)

    """

    from tqdm import tqdm
    import multiprocessing
    from multiprocessing import Pool
    from timeit import default_timer as timer

    start = timer()

    nb_procs = multiprocessing.cpu_count() - 1
    if nb_procs <= 0:
        nb_procs = 1

    if multithreading:
        print('using', nb_procs, 'processors')

    if isinstance(processing_fn, list):
        processing_fn = partial(execute_chained_functions_and_save_as_tiff, function_to_chain_iterable=processing_fn,
                                output_file_name=name_processor_function_for_saving, reverse=False)

    if multithreading:
        import gc
        gc.collect()
        with Pool(processes=nb_procs) as pool:
            for i, _ in enumerate(tqdm(pool.imap_unordered(processing_fn, lst), total=len(lst))):
                if early_stop.stop:
                    pool.close()
                    pool.join()
                    return
                if progress_callback is not None:
                    try:
                        from qtpy.QtWidgets import QProgressBar
                        if isinstance(progress_callback, QProgressBar):
                            progress_callback.setValue(int((i / len(lst)) * 100))
                        else:
                            progress_callback.emit(int((i / len(lst)) * 100))
                    except:
                        traceback.print_exc()
            pool.close()
            pool.join()
    else:
        for i, lst_elm in enumerate(lst):
            result = processing_fn(lst_elm)
            if early_stop.stop:
                return
            if progress_callback is not None:
                try:
                    from qtpy.QtWidgets import QProgressBar
                    if isinstance(progress_callback, QProgressBar):
                        progress_callback.setValue(int((i / len(lst)) * 100))
                    else:
                        progress_callback.emit(int((i / len(lst)) * 100))
                except:
                    traceback.print_exc()

    print('total time', timer() - start)


if __name__ == '__main__':

    if True:
        transversal_list =get_transversal_list('/E/Sample_images/Consensus_learning/gray/CARE/')
        print(transversal_list)
        # for each can do an avg for example

        import sys
        sys.exit(0)


    if True:
        import sys
        models = [
            'CARE_model_trained_MAE_in_epyseg_29-1_50steps_per_epoch_not_outstanding_though/CARE_model-0.h5',  # 0 #####
            'CARE_model_trained_MSE_error_instead_MAE_in_epyseg_29-1/CARE_model-0.h5', #1
            'CARESEG3D_models_trained_on_colab_201214/CARESEG3D_model-0.h5', #2
            'CARESEG_another_normal_training_201216_colab_but_gave_crap/CARESEG_model-0.h5', # 3 # not bad in fact especially when there is an amazing amount of noise because it is an amazing denoiser...
            'CARESEG/CARESEG_model-0.h5', #4
            'CARESEG_first_correct_test/CARESEG_model-0-0-0.h5', #5
            ## 'CARESEG_first_correct_test/CARESEG_model-0-0-4.h5', #6
            'CARESEG_first_successful_retrain_on_CARE_data_very_very_good_surface_proj_but_not_for_all/CARESEG_model-0-0.h5', #7
            # 'CARESEG_first_successful_retrain_on_CARE_data_very_very_good_surface_proj_but_not_for_all/CARESEG_model-0-1.h5', #8
            # 'CARESEG_first_successful_retrain_on_CARE_data_very_very_good_surface_proj_but_not_for_all/CARESEG_model-0-2.h5', #9
            # 'CARESEG_first_successful_retrain_on_CARE_data_very_very_good_surface_proj_but_not_for_all/CARESEG_model-0-3.h5',#10
            # 'CARESEG_first_successful_retrain_on_CARE_data_very_very_good_surface_proj_but_not_for_all/CARESEG_model-0-4.h5',#11
            'CARESEG_masked_data_and_masked_loss_to_prevent_identity_v1_210107_good/CARESEG_model-0.h5',#12
            'CARESEG_masked_data_and_partial_masked_loss_to_limit_identity_but_not_remove_it_v1_210107_outstanding_on_some_just_good_on_others/CARESEG_model-0.h5',
            #####
            # 13
            'CARESEG_masked_data_and_partial_masked_loss_to_limit_identity_but_not_remove_it_v2_zRoll_stack_210108_quite_good_but_a_bit_low_signal/CARESEG_model-0.h5',
            ##### doesn't seem to require extra black frames and denoiser sucks because it has a lot of noise --> see the best of all
            ##########DO NOT USE NEO######## 'CARESEG_masked_data_and_partial_masked_loss_to_limit_identity_but_not_remove_it_v2_zRoll_stack_210108_quite_good_but_a_bit_low_signal/complete_model_with_3_outputs.h5', # NEO
            # 14
            'CARESEG_my_loss_mae_first_chan_dice_other_chans/CARESEG_model-0.h5',#15
            'CARESEG_RETRAINED_NEW_LOSS_DICE_201215/CARESEG_model-0.h5',#16
            'CARESEG_retrained_normally_201216/CARESEG_model-0.h5',  # 17 #####
            'CARESEG_second_correct_tested_210219_outstanding_denoise/CARESEG_model-0.h5',  # 18 #####
            'CARESEG_second_successful_retrain_on_CARE_data_better_than_previous_need_longer_training_and_more_diverse_one_too/CARESEG_model-0.h5',#19
            'CARESEG_shuffle_n_roll_models_trained_on_colab_201215/CARESEG_model-0.h5',#20
            'CARESEG_test_combined_loss_properly_made_with_reduce_mean_220105/CARESEG_model-0.h5',#21
            'CARESEG_test_combined_mae_jaccard_loss_properly_made_with_reduce_mean_220105_terrible_no_clue_why/CARESEG_model-0.h5',# 22
            'CARESEG_trained_different_losses_for_channels_mae_bce/CARESEG_model-0.h5', # 23
            'CARESEG_trained_on_CARE_data_with_dilation_good/CARESEG_model-0.h5',# 24
            'CARESEG_Zroll_models_trained_on_colab_201215/CARESEG_model-0.h5',# 25
            'CARE_trained_on_its_data_and_mine_not_working_on_its/CARE_model-0.h5',# 26
            'CARE_training_on_its_own_data_with_CARESEG/CARESEG_model-0-0.h5',# 27
            # 'CARE_training_on_its_own_data_with_CARESEG/CARESEG_model-0-4.h5',# 28
            'retrained_CARE_my_soft_fixed_normalization_not_outstanding_but_improving/CARE_model-0.h5',# 29
            'CARE_finally_working_retrain_with_CARE_normalization/CARE_model-0.h5',  # 30
            'CARE_models_trained_on_colab_201214/CARE_model-0.h5',  # 31 #####
            'CARE_masked_data_and_partial_masked_loss_to_limit_identity_but_not_remove_it_v1_210120_not_outstanding_CARESEG_is_better/CARE_model-0.h5',# 32
        ]

        for idx, model in enumerate(models):
            models[idx] = os.path.join('/E/models/my_model/bckup/', model)

        print(models)

        smart_output_folder_name(models)

        # get smart_output_folder_name

        sys.exit(0)


    if False:
        import sys

        # THIS CODE COPIES METADATA FROM ONE FILE TO ANOTHER
        # CAN I USE LEICA DATA TODO STUFF
        # I may still need a time file at least for a while until I change my other tools but that could be useful still
        # TODO --> make this its own function or class !!!!
        # do a generic executer within this class and say whether it can be Mted or not ... --> Could be useful
        list_file = [
            '/E/Sample_images/sample_images_denoise_manue/210402_EcadKI_mel_female_26hAPF_pupae_from_old_tube/predict/predict_model_nb_0_not_best_but_very_good_still/list.lst']  # done 3D
        list_file.append(
            '/E/Sample_images/sample_images_denoise_manue/210324_ON_suz_22h45_armGFP_line2/predict/predict_model_nb_0/mini_list.lst')
        list_file.append(
            '/E/Sample_images/sample_images_denoise_manue/fullset_Manue/Ovipositors/21-03/GT/predict/predict_model_nb_3/list.lst')
        list_file.append(
            '/E/Sample_images/sample_images_denoise_manue/210312_armGFP_line2_suz_39h30APF/predict/predict_model_nb_0/list.lst')

        list_file.append(
            '/E/Sample_images/sample_images_denoise_manue/fullset_Manue/Ovipositors/25-1/GT/predict/predict_model_nb_3/list.lst')
        list_file.append(
            '/E/Sample_images/sample_images_denoise_manue/fullset_Manue/Ovipositors/070219/predict/predict_model_nb_0/list.lst')
        list_file.append(
            '/E/Sample_images/sample_images_denoise_manue/fullset_Manue/Ovipositors/20-2/predict/predict_model_nb_0/list.lst')
        list_file.append(
            '/E/Sample_images/sample_images_denoise_manue/fullset_Manue/Ovipositors/210219/predict/predict_model_nb_0/list.lst')
        list_file.append(
            '/E/Sample_images/sample_images_denoise_manue/fullset_Manue/Ovipositors/27-02/predict/predict_model_nb_0/list.lst')
        list_file.append(
            '/E/Sample_images/sample_images_denoise_manue/210409_EcadKI_ovipo_female_around_40hAPF/predict/predict_model_nb_4/list.lst')
        # list_file = []
        list_file.append('/E/Sample_images/sample_images_denoise_manue/210121_armGFP_suz_line2_47h30_APF/predict/before_crash/predict_model_nb_0/mini_list.lst')

        # list_file = []
        list_file.append(
            '/E/Sample_images/sample_images_denoise_manue/201104_armGFP_different_lines_tila/predict/predict_model_nb_0/mini_list.lst')  # TODO

        # HACK FOR THE LAST BECAUSE IT IS INCORRECT!!!
        # zipped = TA_smart_pairing_list('/E/Sample_images/sample_images_denoise_manue/201104_armGFP_different_lines_tila/predict/predict_model_nb_0/registered/splitted/list2_matching.lst', '/E/Sample_images/sample_images_denoise_manue/201104_armGFP_different_lines_tila/list1.lst')

        # map 2 lists
        for lst in list_file:

            parent_folder = heightmap_folder = lst.split('/predict/')[0]
            # print(os.path.exists(heightmap_folder))
            if not os.path.exists(parent_folder):
                print('error', parent_folder)

            # print(parent_folder)

            # zipped = TA_smart_pairing_list(lst, parent_folder)

            # HACK FOR THE LAST BECAUSE IT IS INCORRECT!!!
            # zipped = TA_smart_pairing_list('/E/Sample_images/sample_images_denoise_manue/201104_armGFP_different_lines_tila/predict/predict_model_nb_0/registered/splitted/list2_matching.lst', '/E/Sample_images/sample_images_denoise_manue/201104_armGFP_different_lines_tila/list1.lst')
            zipped = TA_smart_pairing_list(lst, parent_folder)
            # print(zipped)
            for files in zipped:
                print('transferring metadata from', files[0], 'to', files[1])
                _transfer_voxel_size_metadata(files[0], files[1])#.replace('.tif','.lsm')
                # pass
        sys.exit(0)

    if True:
        # TODO add glob


        import sys
        # copy and inject height maps to the file
        # TA_injector
        simulate_only = False
        generic_TA_name = 'height_map.tif'

        list_file = ['/E/Sample_images/sample_images_denoise_manue/210402_EcadKI_mel_female_26hAPF_pupae_from_old_tube/predict/predict_model_nb_0_not_best_but_very_good_still/list.lst']  # done 3D

        list_file.append('/E/Sample_images/sample_images_denoise_manue/210324_ON_suz_22h45_armGFP_line2/predict/predict_model_nb_0/mini_list.lst')
        list_file.append('/E/Sample_images/sample_images_denoise_manue/fullset_Manue/Ovipositors/21-03/GT/predict/predict_model_nb_3/list.lst')
        list_file.append('/E/Sample_images/sample_images_denoise_manue/210312_armGFP_line2_suz_39h30APF/predict/predict_model_nb_0/list.lst')
        list_file.append('/E/Sample_images/sample_images_denoise_manue/fullset_Manue/Ovipositors/25-1/GT/predict/predict_model_nb_3/list.lst')
        list_file.append('/E/Sample_images/sample_images_denoise_manue/fullset_Manue/Ovipositors/070219/predict/predict_model_nb_0/list.lst')
        list_file.append('/E/Sample_images/sample_images_denoise_manue/fullset_Manue/Ovipositors/20-2/predict/predict_model_nb_0/list.lst')
        list_file.append('/E/Sample_images/sample_images_denoise_manue/fullset_Manue/Ovipositors/210219/predict/predict_model_nb_0/list.lst')
        list_file.append('/E/Sample_images/sample_images_denoise_manue/fullset_Manue/Ovipositors/27-02/predict/predict_model_nb_0/list.lst')
        list_file.append('/E/Sample_images/sample_images_denoise_manue/210409_EcadKI_ovipo_female_around_40hAPF/predict/predict_model_nb_4/list.lst')
        list_file.append(
            '/E/Sample_images/sample_images_denoise_manue/210121_armGFP_suz_line2_47h30_APF/predict/before_crash/predict_model_nb_0/mini_list.lst')
        # TODO hack the last one
        # list_file = []
        # list_file.append('/E/Sample_images/sample_images_denoise_manue/201104_armGFP_different_lines_tila/predict/predict_model_nb_0/registered/splitted/mini_list.lst')
        list_file.append('/E/Sample_images/sample_images_denoise_manue/201104_armGFP_different_lines_tila/predict/predict_model_nb_0/mini_list.lst')

        for lst in list_file:
            # TA_injector('/E/Sample_images/sample_images_denoise_manue/210402_EcadKI_mel_female_26hAPF_pupae_from_old_tube/predict/predict_model_nb_0_not_best_but_very_good_still/list.lst', '/E/Sample_images/sample_images_denoise_manue/210402_EcadKI_mel_female_26hAPF_pupae_from_old_tube/predict/height_map_test1', simulate_only=simulate_only, generic_TA_name_for_injected=generic_TA_name)
            heightmap_folder = lst.split('/predict/')[0]+'/predict/height_maps/'
            # print(os.path.exists(heightmap_folder))
            if not os.path.exists(heightmap_folder):
                print(heightmap_folder)
            # TA_injector(list, '/E/Sample_images/sample_images_denoise_manue/210402_EcadKI_mel_female_26hAPF_pupae_from_old_tube/predict/height_map_test1', simulate_only=simulate_only, generic_TA_name_for_injected=generic_TA_name)
            TA_injector(lst, heightmap_folder, simulate_only=simulate_only, generic_TA_name_for_injected=generic_TA_name)
        sys.exit(0)

    # TODO transform this into tests (just create some tmp files)
    print(loadlist('/E/Sample_images/sample_images_denoise_manue/200722_armGFP_suz_ON_47hAPF/predict/predict_model_nb_4/list.lst'))
    print(loadlist('/E/Sample_images/sample_images_PA/trash_test_mem/mini/list.lst'))
    print(loadlist('/E/Sample_images/sample_images_PA/trash_test_mem/mini/list.lst', always_prefer_local_directory_if_file_exists=False))
    print(loadlist('/E/Sample_images/sample_images_PA/trash_test_mem/mini/list.lst', always_prefer_local_directory_if_file_exists=False, filter_existing_files_only=True))
    print(loadlist('/E/Sample_images/sample_images_PA/trash_test_mem/mini/list.lst', always_prefer_local_directory_if_file_exists=True, filter_existing_files_only=True))
    print(loadlist('/E/Sample_images/sample_images_PA/trash_test_mem/mini/list_smart.lst', always_prefer_local_directory_if_file_exists=True, smart_local_folder_detection=False))
    print(loadlist('/E/Sample_images/sample_images_PA/trash_test_mem/mini/list_smart.lst',                  always_prefer_local_directory_if_file_exists=True))
    print(loadlist('/E/Sample_images/sample_images_PA/trash_test_mem/mini/list_smart.lst')) # this is the default behavior and I do really love it
    print(loadlist('/E/Sample_images/sample_images_PA/trash_test_mem/mini/list_smart.lst', always_prefer_local_directory_if_file_exists=True, smart_local_folder_detection=False, filter_existing_files_only=True))

    tmp_lst = loadlist('/E/Sample_images/sample_images_PA/trash_test_mem/mini/list.lst', always_prefer_local_directory_if_file_exists=True, filter_existing_files_only=True)
    print(smart_TA_list(tmp_lst, 'tutu.tif'))
    print(smart_TA_list(tmp_lst, 'handCorrection.tif', alternative_name_if_name_to_search_first_does_not_exist=None))
    print(smart_TA_list(tmp_lst, 'handCorrection.tif', alternative_name_if_name_to_search_first_does_not_exist='handCorrection.png'))
    # I think that works as I want
    print(smart_TA_list(tmp_lst, 'handCorrection2.png', alternative_name_if_name_to_search_first_does_not_exist='handCorrection.png'))

    save_list_to_file(smart_TA_list(tmp_lst, 'handCorrection.tif', alternative_name_if_name_to_search_first_does_not_exist='handCorrection.png'), '/E/Sample_images/sample_images_PA/trash_test_mem/mini/list_masks.lst')

    print('final list',create_list('/E/Sample_images/sample_images_denoiseg/train/raw/', save=True))
