
from ...utils import (
    generate_options_set, generating_locally, generating_on_ps, 
    uploading_with_ps_account, uploading_to_existing_ps_dataset, 
    can_resume_prior_upload, virtual_dataset_empty, PropertyNotSetError, 
    connect_pennsieve_client, get_dataset_id, get_access_token,
    PennsieveActionNoPermission, PennsieveDatasetCannotBeFound,
    EmptyDatasetError, LocalDatasetMissingSpecifiedFiles,
    PennsieveUploadException, create_request_headers, check_forbidden_characters_ps, get_users_dataset_list,
    PennsieveDatasetNameInvalid, PennsieveDatasetNameTaken, PennsieveAccountInvalid, TZLOCAL, GenerateOptionsNotSet,
    PennsieveDatasetFilesInvalid
)
from ..permissions import pennsieve_get_current_user_permissions
from os.path import isdir, isfile, getsize
from ..metadata import create_high_level_manifest_files, get_auto_generated_manifest_files, manifest, subjects, samples, code_description, dataset_description, performances, resources, sites, submission, readme_changes, METADATA_UPLOAD_PS_PATH, create_high_lvl_manifest_files_existing_ps_starting_point
from ..upload_manifests import get_upload_manifests
from .. import logger

main_curate_progress_message = ""
main_curate_status = ""

# -*- coding: utf-8 -*-

### Import required python modules
import platform
import os
from os import listdir, makedirs, mkdir, walk, rename
from os.path import (
    isdir,
    isfile,
    join,
    splitext,
    basename,
    exists,
    expanduser,
    dirname,
    getsize,
    abspath,
)
import pandas as pd
import time
from timeit import default_timer as timer
from datetime import timedelta
import shutil
import subprocess
import gevent
import pathlib
import requests
from datetime import datetime
from openpyxl import load_workbook
from openpyxl.styles import PatternFill
# from utils import connect_pennsieve_client, get_dataset_id, create_request_headers, TZLOCAL, get_users_dataset_list
# from manifest import create_high_lvl_manifest_files_existing_ps_starting_point, create_high_level_manifest_files, get_auto_generated_manifest_files
# from errors import PennsieveUploadException
from .manifestSession import UploadManifestSession
from ...constants import PENNSIEVE_URL
from ..dataset_importing import import_pennsieve_dataset

# from pysodaUtils import (
#     check_forbidden_characters_ps
# )

# from organizeDatasets import import_pennsieve_dataset


### Global variables
curateprogress = " "
curatestatus = " "
curateprintstatus = " "
total_dataset_size = 1
curated_dataset_size = 0
start_time = 0
uploaded_folder_counter = 0
current_size_of_uploaded_files = 0
generated_dataset_id = None
# the pennsieve python client used for uploading dataset files 
client = None 

userpath = expanduser("~")
configpath = join(userpath, ".pennsieve", "config.ini")
submitdataprogress = " "
submitdatastatus = " "
submitprintstatus = " "
total_file_size = 1
uploaded_file_size = 0
start_time_bf_upload = 0
start_submit = 0
metadatapath = join(userpath, "SODA", "SODA_metadata")
ps_recognized_file_extensions = [
    ".cram",
    ".jp2",
    ".jpx",
    ".lsm",
    ".ndpi",
    ".nifti",
    ".oib",
    ".oif",
    ".roi",
    ".rtf",
    ".swc",
    ".abf",
    ".acq",
    ".adicht",
    ".adidat",
    ".aedt",
    ".afni",
    ".ai",
    ".avi",
    ".bam",
    ".bash",
    ".bcl",
    ".bcl.gz",
    ".bin",
    ".brik",
    ".brukertiff.gz",
    ".continuous",
    ".cpp",
    ".csv",
    ".curv",
    ".cxls",
    ".czi",
    ".data",
    ".dcm",
    ".df",
    ".dicom",
    ".doc",
    ".docx",
    ".e",
    ".edf",
    ".eps",
    ".events",
    ".fasta",
    ".fastq",
    ".fcs",
    ".feather",
    ".fig",
    ".gif",
    ".h4",
    ".h5",
    ".hdf4",
    ".hdf5",
    ".hdr",
    ".he2",
    ".he5",
    ".head",
    ".hoc",
    ".htm",
    ".html",
    ".ibw",
    ".img",
    ".ims",
    ".ipynb",
    ".jpeg",
    ".jpg",
    ".js",
    ".json",
    ".lay",
    ".lh",
    ".lif",
    ".m",
    ".mat",
    ".md",
    ".mef",
    ".mefd.gz",
    ".mex",
    ".mgf",
    ".mgh",
    ".mgh.gz",
    ".mgz",
    ".mnc",
    ".moberg.gz",
    ".mod",
    ".mov",
    ".mp4",
    ".mph",
    ".mpj",
    ".mtw",
    ".ncs",
    ".nd2",
    ".nev",
    ".nex",
    ".nex5",
    ".nf3",
    ".nii",
    ".nii.gz",
    ".ns1",
    ".ns2",
    ".ns3",
    ".ns4",
    ".ns5",
    ".ns6",
    ".nwb",
    ".ogg",
    ".ogv",
    ".ome.btf",
    ".ome.tif",
    ".ome.tif2",
    ".ome.tif8",
    ".ome.tiff",
    ".ome.xml",
    ".openephys",
    ".pdf",
    ".pgf",
    ".png",
    ".ppt",
    ".pptx",
    ".ps",
    ".pul",
    ".py",
    ".r",
    ".raw",
    ".rdata",
    ".rh",
    ".rhd",
    ".sh",
    ".sldasm",
    ".slddrw",
    ".smr",
    ".spikes",
    ".svg",
    ".svs",
    ".tab",
    ".tar",
    ".tar.gz",
    ".tcsh",
    ".tdm",
    ".tdms",
    ".text",
    ".tif",
    ".tiff",
    ".tsv",
    ".txt",
    ".vcf",
    ".webm",
    ".xlsx",
    ".xml",
    ".yaml",
    ".yml",
    ".zip",
    ".zsh",
]

myds = ""
initial_bfdataset_size = 0
upload_directly_to_bf = 0
initial_bfdataset_size_submit = 0
renaming_files_flow = False

total_files = 0 # the total number of files in a given dataset that need to be uploaded to Pennsieve
total_bytes_uploaded = 0 # current number of bytes uploaded to Pennsieve in the upload session
total_upload_size = 0 # total number of bytes to upload to Pennsieve in the upload session

forbidden_characters = '<>:"/\|?*'
forbidden_characters_bf = '\/:*?"<>'

# a global that tracks the amount of files that have been uploaded in an upload session;
# is reset once the session ends by success, or failure (is implicitly reset in case of Pennsieve Agent freeze by the user closing SODA)
main_curation_uploaded_files = 0

DEV_TEMPLATE_PATH = join(dirname(__file__), "..", "file_templates")

# once pysoda has been packaged with pyinstaller
# it becomes nested into the pysodadist/api directory
PROD_TEMPLATE_PATH = join(dirname(__file__), "..", "..", "file_templates")
TEMPLATE_PATH = DEV_TEMPLATE_PATH if exists(DEV_TEMPLATE_PATH) else PROD_TEMPLATE_PATH




ums = UploadManifestSession()





def open_file(file_path):
    """
    Opening folder on all platforms
    https://stackoverflow.com/questions/6631299/python-opening-a-folder-in-explorer-nautilus-mac-thingie

    Args:
        file_path: path of the folder (string)
    Action:
        Opens file explorer window to the given path
    """

    if platform.system() == "Windows":
        subprocess.Popen(f"explorer /select,{str(file_path)}")
    elif platform.system() == "Darwin":
        subprocess.Popen(["open", file_path])
    else:
        subprocess.Popen(["xdg-open", file_path])



def folder_size(path):
    """
    Provides the size of the folder indicated by path

    Args:
        path: path of the folder (string)
    Returns:
        total_size: total size of the folder in bytes (integer)
    """
    total_size = 0

    for path, dirs, files in walk(path):
        for f in files:
            fp = join(path, f)
            total_size += getsize(fp)
    return total_size


def path_size(path):
    """
    Returns size of the path, after checking if it's a folder or a file
    Args:
        path: path of the file/folder (string)
    Returns:
        total_size: total size of the file/folder in bytes (integer)
    """
    return folder_size(path) if isdir(path) else getsize(path)


def create_folder_level_manifest(jsonpath, jsondescription):
    """
    Function to create manifest files for each SPARC folder.
    Files are created in a temporary folder

    Args:
        datasetpath: path of the dataset (string)
        jsonpath: all paths in json format with key being SPARC folder names (dictionary)
        jsondescription: description associated with each path (dictionary)
    Action:
        Creates manifest files in xslx format for each SPARC folder
    """
    global total_dataset_size
    local_timezone = TZLOCAL()

    try:
        shutil.rmtree(metadatapath) if isdir(metadatapath) else 0
        makedirs(metadatapath)
        folders = list(jsonpath.keys())

        if "main" in folders:
            folders.remove("main")
        # In each SPARC folder, generate a manifest file
        for folder in folders:
            if jsonpath[folder] != []:
                # Initialize dataframe where manifest info will be stored
                df = pd.DataFrame(
                    columns=[
                        "filename",
                        "timestamp",
                        "description",
                        "file type",
                        "Additional Metadata",
                    ]
                )
                # Get list of files/folders in the the folder
                # Remove manifest file from the list if already exists
                folderpath = join(metadatapath, folder)
                allfiles = jsonpath[folder]
                alldescription = jsondescription[folder + "_description"]

                countpath = -1
                for pathname in allfiles:
                    countpath += 1
                    if basename(pathname) in ["manifest.csv", "manifest.xlsx"]:
                        allfiles.pop(countpath)
                        alldescription.pop(countpath)

                # Populate manifest dataframe
                filename, timestamp, filetype, filedescription = [], [], [], []
                countpath = -1
                for paths in allfiles:
                    if isdir(paths):
                        key = basename(paths)
                        alldescription.pop(0)
                        for subdir, dirs, files in os.walk(paths):
                            for file in files:
                                gevent.sleep(0)
                                filepath = pathlib.Path(paths) / subdir / file
                                mtime = filepath.stat().st_mtime
                                lastmodtime = datetime.fromtimestamp(mtime).astimezone(
                                    local_timezone
                                )
                                timestamp.append(
                                    lastmodtime.isoformat()
                                    .replace(".", ",")
                                    .replace("+00:00", "Z")
                                )
                                full_filename = filepath.name

                                if folder == "main":  # if file in main folder
                                    filename.append(
                                        full_filename
                                    ) if folder == "" else filename.append(
                                        join(folder, full_filename)
                                    )
                                else:
                                    subdirname = os.path.relpath(
                                        subdir, paths
                                    )  # gives relative path of the directory of the file w.r.t paths
                                    if subdirname == ".":
                                        filename.append(join(key, full_filename))
                                    else:
                                        filename.append(
                                            join(key, subdirname, full_filename)
                                        )

                                fileextension = splitext(full_filename)[1]
                                if (
                                    not fileextension
                                ):  # if empty (happens e.g. with Readme files)
                                    fileextension = "None"
                                filetype.append(fileextension)
                                filedescription.append("")
                    else:
                        gevent.sleep(0)
                        countpath += 1
                        filepath = pathlib.Path(paths)
                        file = filepath.name
                        filename.append(file)
                        mtime = filepath.stat().st_mtime
                        lastmodtime = datetime.fromtimestamp(mtime).astimezone(
                            local_timezone
                        )
                        timestamp.append(
                            lastmodtime.isoformat()
                            .replace(".", ",")
                            .replace("+00:00", "Z")
                        )
                        filedescription.append(alldescription[countpath])
                        if isdir(paths):
                            filetype.append("folder")
                        else:
                            fileextension = splitext(file)[1]
                            if (
                                not fileextension
                            ):  # if empty (happens e.g. with Readme files)
                                fileextension = "None"
                            filetype.append(fileextension)

                df["filename"] = filename
                df["timestamp"] = timestamp
                df["file type"] = filetype
                df["description"] = filedescription

                makedirs(folderpath)
                # Save manifest as Excel sheet
                manifestfile = join(folderpath, "manifest.xlsx")
                df.to_excel(manifestfile, index=None, header=True)
                wb = load_workbook(manifestfile)
                ws = wb.active

                blueFill = PatternFill(
                    start_color="9DC3E6", fill_type="solid"
                )
                greenFill = PatternFill(
                    start_color="A8D08D", fill_type="solid"
                )
                yellowFill = PatternFill(
                    start_color="FFD965", fill_type="solid"
                )
                ws['A1'].fill = blueFill
                ws['B1'].fill = greenFill
                ws['C1'].fill = greenFill
                ws['D1'].fill = greenFill
                ws['E1'].fill = yellowFill

                wb.save(manifestfile)
                total_dataset_size += path_size(manifestfile)
                jsonpath[folder].append(manifestfile)

        return jsonpath

    except Exception as e:
        raise e


def return_new_path(topath):
    """
    This function checks if a folder already exists and in such cases,
    appends (1) or (2) etc. to the folder name

    Args:
        topath: path where the folder is supposed to be created (string)
    Returns:
        topath: new folder name based on the availability in destination folder (string)
    """

    if not exists(topath):
        return topath

    i = 1
    while True:
        if not exists(topath + " (" + str(i) + ")"):
            return topath + " (" + str(i) + ")"
        i += 1


def return_new_path_replace(topath):
    """
    This function checks if a folder already exists and in such cases,
    replace the existing folder (this is the opposite situation to the function return_new_path)

    Args:
        topath: path where the folder is supposed to be created (string)
    Returns:
        topath: new folder name based on the availability in destination folder (string)
    """

    if not exists(topath):
        return topath
    i = 1
    while True:
        if not exists(topath + " (" + str(i) + ")"):
            return topath + " (" + str(i) + ")"
        i += 1


def time_format(elapsed_time):
    mins, secs = divmod(elapsed_time, 60)
    hours, mins = divmod(mins, 60)
    return "%dh:%02dmin:%02ds" % (hours, mins, secs)


def mycopyfileobj(fsrc, fdst, length=16 * 1024 * 16):
    """
    Helper function to copy file

    Args:
        fsrc: source file opened in python (file-like object)
        fdst: destination file accessed in python (file-like object)
        length: copied buffer size in bytes (integer)
    """
    global curateprogress
    global total_dataset_size
    global curated_dataset_size
    global main_generated_dataset_size

    while True:
        buf = fsrc.read(length)
        if not buf:
            break
        gevent.sleep(0)
        fdst.write(buf)
        curated_dataset_size += len(buf)
        main_generated_dataset_size += len(buf)


def mycopyfile_with_metadata(src, dst, *, follow_symlinks=True):
    """
    Copy file src to dst with metadata (timestamp, permission, etc.) conserved

    Args:
        src: source file (string)
        dst: destination file (string)
    Returns:
        dst
    """
    if not follow_symlinks and os.path.islink(src):
        os.symlink(os.readlink(src), dst)
    else:
        with open(src, "rb") as fsrc:
            with open(dst, "wb") as fdst:
                mycopyfileobj(fsrc, fdst)
    shutil.copystat(src, dst)
    return dst


def check_empty_files_folders(soda):
    """
    Function to check for empty files and folders

    Args:
        soda: soda dict with information about all specified files and folders
    Output:
        error: error message with list of non valid local data files, if any
    """
    try:
        def recursive_empty_files_check(my_folder, my_relative_path, error_files):
            for folder_key, folder in my_folder["folders"].items():
                relative_path = my_relative_path + "/" + folder_key
                error_files = recursive_empty_files_check(
                    folder, relative_path, error_files
                )

            for file_key in list(my_folder["files"].keys()):
                file = my_folder["files"][file_key]
                file_type = file.get("location")
                if file_type == "local":
                    file_path = file["path"]
                    if isfile(file_path):
                        file_size = getsize(file_path)
                        if file_size == 0:
                            del my_folder["files"][file_key]
                            relative_path = my_relative_path + "/" + file_key
                            error_message = relative_path + " (path: " + file_path + ")"
                            error_files.append(error_message)

            return error_files

        def recursive_empty_local_folders_check(
                    my_folder,
                    my_folder_key,
                    my_folders_content,
                    my_relative_path,
                    error_folders,
                ):
            folders_content = my_folder["folders"]
            for folder_key in list(my_folder["folders"].keys()):
                folder = my_folder["folders"][folder_key]
                relative_path = my_relative_path + "/" + folder_key
                error_folders = recursive_empty_local_folders_check(
                    folder, folder_key, folders_content, relative_path, error_folders
                )

            if not my_folder["folders"] and not my_folder["files"]:
                ignore = False
                if "location" in my_folder and my_folder.get("location") == "ps":
                    ignore = True
                if not ignore:
                    error_message = my_relative_path
                    error_folders.append(error_message)
                    del my_folders_content[my_folder_key]
            return error_folders

        error_files = []
        error_folders = []
        if "dataset-structure" in soda.keys():
            dataset_structure = soda["dataset-structure"]
            if "folders" in dataset_structure:
                for folder_key, folder in dataset_structure["folders"].items():
                    relative_path = folder_key
                    error_files = recursive_empty_files_check(
                        folder, relative_path, error_files
                    )

                folders_content = dataset_structure["folders"]
                for folder_key in list(dataset_structure["folders"].keys()):
                    folder = dataset_structure["folders"][folder_key]
                    relative_path = folder_key
                    error_folders = recursive_empty_local_folders_check(
                        folder,
                        folder_key,
                        folders_content,
                        relative_path,
                        error_folders,
                    )

        if "metadata-files" in soda.keys():
            metadata_files = soda["metadata-files"]
            for file_key in list(metadata_files.keys()):
                file = metadata_files[file_key]
                file_type = file.get("location")
                if file_type == "local":
                    file_path = file["path"]
                    if isfile(file_path):
                        file_size = getsize(file_path)
                        if file_size == 0:
                            del metadata_files[file_key]
                            error_message = file_key + " (path: " + file_path + ")"
                            error_files.append(error_message)
            if not metadata_files:
                del soda["metadata-files"]

        if len(error_files) > 0:
            error_message = [
                "The following local file(s) is/are empty (0 kb) and will be ignored."
            ]
            error_files = error_message + [] + error_files

        if len(error_folders) > 0:
            error_message = [
                "The SPARC dataset structure does not allow empty folders. The following empty folders will be removed from your dataset:"
            ]
            error_folders = error_message + [] + error_folders

        return {
            "empty_files": error_files, 
            "empty_folders": error_folders, 
            "soda": soda
        }

    except Exception as e:
        raise e


def check_local_dataset_files_validity(soda):
    """
    Function to check that the local data files and folders specified in the dataset are valid

    Args:
        soda: soda dict with information about all specified files and folders
    Output:
        error: error message with list of non valid local data files, if any
    """

    def recursive_local_file_check(my_folder, my_relative_path, error):
        for folder_key, folder in my_folder["folders"].items():
            relative_path = my_relative_path + "/" + folder_key
            error = recursive_local_file_check(folder, relative_path, error)

        for file_key in list(my_folder["files"].keys()):
            file = my_folder["files"][file_key]
            if file_key in ["manifest.xlsx", "manifest.csv"]:
                continue
            file_type = file.get("location")
            if file_type == "local":
                file_path = file["path"]
                if file.get("location") == "ps":
                    continue
                if not isfile(file_path):
                    relative_path = my_relative_path + "/" + file_key
                    error_message = relative_path + " (path: " + file_path + ")"
                    error.append(error_message)
                else:
                    file_size = getsize(file_path)
                    if file_size == 0:
                        del my_folder["files"][file_key]

        return error

    def recursive_empty_local_folder_remove(
        my_folder, my_folder_key, my_folders_content
    ):

        folders_content = my_folder["folders"]
        for folder_key in list(my_folder["folders"].keys()):
            folder = my_folder["folders"][folder_key]
            recursive_empty_local_folder_remove(folder, folder_key, folders_content)

        if not my_folder.get("folders") and not my_folder.get("files") and my_folder.get("location") != "ps":
            del my_folders_content[my_folder_key]

    error = []
    if "dataset-structure" in soda.keys():
        dataset_structure = soda["dataset-structure"]
        # Remove 0kb files, files that can't be found, and any empty folders from the dataset data files
        if "folders" in dataset_structure:
            for folder_key, folder in dataset_structure["folders"].items():
                relative_path = folder_key
                error = recursive_local_file_check(folder, relative_path, error)

            folders_content = dataset_structure["folders"]
            for folder_key in list(dataset_structure["folders"].keys()):
                folder = dataset_structure["folders"][folder_key]
                recursive_empty_local_folder_remove(folder, folder_key, folders_content)

    # Return list of all the files that were not found. 
    if len(error) > 0:
        error_message = [
            "Error: The following local files were not found. Specify them again or remove them."
        ]
        error = error_message + error

    return error


# path to local SODA folder for saving manifest files
manifest_sparc = ["manifest.xlsx", "manifest.csv"]
manifest_folder_path = join(userpath, ".pysoda", "manifest_file")



def check_json_size(jsonStructure):
    """
        This function is called to check size of files that will be created locally on a user's device.
    """
    global total_dataset_size
    total_dataset_size = 0

    try:
        def recursive_dataset_scan(folder):
            global total_dataset_size

            if "files" in folder.keys():
                for file_key, file in folder["files"].items():
                    if "deleted" not in file["action"]:
                        file_type = file.get("location")
                        if file_type == "local":
                            file_path = file["path"]
                            if isfile(file_path):
                                total_dataset_size += getsize(file_path)

            if "folders" in folder.keys():
                for folder_key, folder in folder["folders"].items():
                    recursive_dataset_scan(folder)

        # scan dataset structure
        dataset_structure = jsonStructure["dataset-structure"]
        folderSection = dataset_structure["folders"]
        # gets keys like code, primary, source and their content...
        for keys, contents in folderSection.items():
            recursive_dataset_scan(contents)

        if "metadata-files" in jsonStructure.keys():
            metadata_files = jsonStructure["metadata-files"]
            for file_key, file in metadata_files.items():
                if file.get("location") == "local":
                    metadata_path = file["path"]
                    if isfile(metadata_path) and "new" in file["action"]:
                        total_dataset_size += getsize(metadata_path)

        if "manifest-files" in jsonStructure.keys():
            manifest_files_structure = create_high_level_manifest_files(jsonStructure, manifest_folder_path)
            for key in manifest_files_structure.keys():
                manifestpath = manifest_files_structure[key]
                if isfile(manifestpath):
                    total_dataset_size += getsize(manifestpath)

        # returns in bytes
        return {"dataset_size": total_dataset_size}
    except Exception as e:
        raise e


def generate_dataset_locally(soda):
    global logger
    logger.info("starting generate_dataset_locally")

    # Vars used for tracking progress on the frontend 
    global main_curate_progress_message
    global progress_percentage
    global main_total_generate_dataset_size
    global start_generate
    global main_curation_uploaded_files

    main_curation_uploaded_files = 0

    def recursive_dataset_scan(
        my_folder, my_folderpath, list_copy_files, list_move_files
    ):
        global main_total_generate_dataset_size

        if "folders" in my_folder.keys():
            for folder_key, folder in my_folder["folders"].items():
                folderpath = join(my_folderpath, folder_key)
                if not isdir(folderpath):
                    mkdir(folderpath)
                list_copy_files, list_move_files = recursive_dataset_scan(
                    folder, folderpath, list_copy_files, list_move_files
                )

        if "files" in my_folder.keys():
            for file_key, file in my_folder["files"].items():
                if "deleted" not in file["action"]:
                    file_type = file.get("location")
                    if file_type == "local":
                        file_path = file["path"]
                        if isfile(file_path):
                            destination_path = abspath(
                                join(my_folderpath, file_key)
                            )
                            if not isfile(destination_path):
                                if (
                                    "existing" in file["action"]
                                    and soda["generate-dataset"][
                                        "if-existing"
                                    ]
                                    == "merge"
                                ):
                                    list_move_files.append(
                                        [file_path, destination_path]
                                    )
                                else:
                                    main_total_generate_dataset_size += getsize(
                                        file_path
                                    )
                                    list_copy_files.append(
                                        [file_path, destination_path]
                                    )
                        else:
                            logger.info(f"file_path {file_path} does not exist. Skipping.")
        return list_copy_files, list_move_files


    logger.info("generate_dataset_locally step 1")
    # 1. Create new folder for dataset or use existing merge with existing or create new dataset
    main_curate_progress_message = "Generating folder structure and list of files to be included in the dataset"
    dataset_absolute_path = soda["generate-dataset"]["path"]
    if_existing = soda["generate-dataset"]["if-existing"]
    dataset_name = soda["generate-dataset"]["dataset-name"]
    datasetpath = join(dataset_absolute_path, dataset_name)
    datasetpath = return_new_path(datasetpath)
    mkdir(datasetpath)

    logger.info("generate_dataset_locally step 2")
    # 2. Scan the dataset structure and:
    # 2.1. Create all folders (with new name if renamed)
    # 2.2. Compile a list of files to be copied and a list of files to be moved (with new name recorded if renamed)
    list_copy_files = []
    list_move_files = []
    dataset_structure = soda["dataset-structure"]

    for folder_key, folder in dataset_structure["folders"].items():
        folderpath = join(datasetpath, folder_key)
        mkdir(folderpath)
        list_copy_files, list_move_files = recursive_dataset_scan(
            folder, folderpath, list_copy_files, list_move_files
        )

    # 3. Add high-level metadata files in the list
    if "dataset_metadata" in soda.keys():
        logger.info("generate_dataset_locally (optional) step 3 handling metadata-files")
        metadata_files = soda["dataset_metadata"]
        # log the metadata files that will be created
        logger.info(f"Metadata files to be created: {list(metadata_files.keys())}")
        for file_key, _ in metadata_files.items():
            logger.info(f"Processing metadata file: {file_key}")
            if file_key == "subjects":
                logger.info("Creating subjects metadata file")
                subjects.create_excel(soda, False, join(datasetpath, "subjects.xlsx"))
            elif file_key == "samples":
                samples.create_excel(soda, False, join(datasetpath, "samples.xlsx"))
            elif file_key == "code_description":
                code_description.create_excel(soda, False, join(datasetpath, "code_description.xlsx"))
            elif file_key == "dataset_description": 
                dataset_description.create_excel(soda, False, join(datasetpath, "dataset_description.xlsx"))
            elif file_key == "performances":
                performances.create_excel(soda, False, join(datasetpath, "performances.xlsx"))
            elif file_key == "resources":
                resources.create_excel(soda, False, join(datasetpath, "resources.xlsx"))
            elif file_key == "sites":
                sites.create_excel(soda, False, join(datasetpath, "sites.xlsx"))
            elif file_key == "submission":
                submission.create_excel(soda, False, join(datasetpath, "submission.xlsx"))
            elif file_key == "README":
                readme_changes.create_text_file(soda, False, join(datasetpath, "README.TXT"), "README")
            elif file_key == "CHANGES":
                readme_changes.create_text_file(soda, False, join(datasetpath, "CHANGES.TXT"), "CHANGES")

    # 4. Add manifest files in the list
    if "manifest_file" in soda["dataset_metadata"].keys():
        logger.info("generate_dataset_locally (optional) step 4 handling manifest-files")
        main_curate_progress_message = "Preparing manifest files"
        manifest.create_excel(soda, False, join(datasetpath,  "manifest.xlsx"))


    logger.info("generate_dataset_locally step 5 moving files to new location")
    # 5. Move files to new location
    main_curate_progress_message = "Moving files to new location"
    for fileinfo in list_move_files:
        srcfile = fileinfo[0]
        distfile = fileinfo[1]
        main_curate_progress_message = f"Moving file {str(srcfile)} to {str(distfile)}"
        shutil.move(srcfile, distfile)

    logger.info("generate_dataset_locally step 6 copying files to new location")
    # 6. Copy files to new location
    main_curate_progress_message = "Copying files to new location"
    start_generate = 1
    for fileinfo in list_copy_files:
        srcfile = fileinfo[0]
        distfile = fileinfo[1]
        main_curate_progress_message = f"Copying file {str(srcfile)} to {str(distfile)}"
        # track amount of copied files for loggin purposes
        mycopyfile_with_metadata(srcfile, distfile)
        main_curation_uploaded_files += 1

    logger.info("generate_dataset_locally step 7")
    # 7. Delete manifest folder and original folder if merge requested and rename new folder
    shutil.rmtree(manifest_folder_path) if isdir(manifest_folder_path) else 0
    if if_existing == "merge":
        logger.info("generate_dataset_locally (optional) step 7.1 delete manifest folder if merge requested")
        main_curate_progress_message = "Finalizing dataset"
        original_dataset_path = join(dataset_absolute_path, dataset_name)
        shutil.rmtree(original_dataset_path)
        rename(datasetpath, original_dataset_path)
        open_file(join(dataset_absolute_path, original_dataset_path))
    else:
        open_file(join(dataset_absolute_path, datasetpath))
    return datasetpath, main_total_generate_dataset_size


    


def ps_create_new_dataset(datasetname, ps):
    """
    Args:
        datasetname: name of the dataset to be created (string)
        bf: Pennsieve account object
    Action:
        Creates dataset for the account specified
    """
    try:
        error, count = "", 0
        datasetname = datasetname.strip()

        if check_forbidden_characters_ps(datasetname):
            error = (
                f"{error}Error: A Pennsieve dataset name cannot contain any of the following characters: "
                + forbidden_characters_bf
                + "<br>"
            )
            count += 1

        if not datasetname:
            error = f"{error}Error: Please enter valid dataset name<br>"
            count += 1

        if datasetname.isspace():
            error = error + "Error: Please enter valid dataset name" + "<br>"
            count += 1

        if count > 0:
            raise PennsieveDatasetNameInvalid(datasetname)

        try:
            dataset_list = get_users_dataset_list()
        except Exception as e:
            raise Exception("Failed to retrieve datasets from Pennsieve. Please try again later.")

        for dataset in dataset_list:
            if datasetname == dataset["content"]["name"]:
                raise PennsieveDatasetNameTaken("Dataset name already exists")
            
        
        # Create the dataset on Pennsieve
        r = requests.post(f"{PENNSIEVE_URL}/datasets", headers=create_request_headers(ps), json={"name": datasetname})
        r.raise_for_status()


        return r.json()

    # TODO: Remove unnecessary raise
    except Exception as e:
        raise e

double_extensions = [
    ".ome.tiff",
    ".ome.tif",
    ".ome.tf2,",
    ".ome.tf8",
    ".ome.btf",
    ".ome.xml",
    ".brukertiff.gz",
    ".mefd.gz",
    ".moberg.gz",
    ".nii.gz",
    ".mgh.gz",
    ".tar.gz",
    ".bcl.gz",
]


def create_high_lvl_manifest_files_existing_ps(
    soda, ps, my_tracking_folder
):
    """
    Function to create manifest files for each high-level SPARC folder.

    Args:
        soda: soda dict with information about the dataset to be generated/modified
    Action:
        manifest_files_structure: dict including the local path of the manifest files
    """
    def get_name_extension(file_name):
        double_ext = False
        for ext in double_extensions:
            if file_name.find(ext) != -1:
                double_ext = True
                break
        ext = ""
        name = ""
        if double_ext == False:
            name = os.path.splitext(file_name)[0]
            ext = os.path.splitext(file_name)[1]
        else:
            ext = (
                os.path.splitext(os.path.splitext(file_name)[0])[1]
                + os.path.splitext(file_name)[1]
            )
            name = os.path.splitext(os.path.splitext(file_name)[0])[0]
        return name, ext

    def recursive_import_ps_manifest_info(
        folder, my_relative_path, dict_folder_manifest, manifest_df
    ):
        """
        Import manifest information from the Pennsieve dataset for the given folder and its children.
        """

        if len(folder['children']) == 0:
            limit = 100
            offset = 0 
            ps_folder = {"children": []}
            while True: 
                r = requests.get(f"{PENNSIEVE_URL}/packages/{folder['content']['id']}?limit={limit}&offset={offset}", headers=create_request_headers(ps), json={"include": "files"})
                r.raise_for_status()
                page = r.json()
                normalize_tracking_folder(page)
                ps_folder["children"].extend(page)

                if len(page) < limit:
                    break
                offset += limit

            folder['children'] = ps_folder['children']

        for _, folder_item in folder["children"]["folders"].items():
            folder_name = folder_item['content']['name']
            relative_path = generate_relative_path(
                my_relative_path, folder_name
            )
            dict_folder_manifest = recursive_import_ps_manifest_info(
                folder_item, relative_path, dict_folder_manifest, manifest_df
            )
        for _, file in folder["children"]["files"].items():
            if file['content']['name'] != "manifest":
                file_id = file['content']['id']
                r = requests.get(f"{PENNSIEVE_URL}/packages/{file_id}/view", headers=create_request_headers(ps))
                r.raise_for_status()
                file_details = r.json()
                file_name = file_details[0]["content"]["name"]
                file_extension = splitext(file_name)[1]
                file_name_with_extension = (
                    splitext(file['content']['name'])[0] + file_extension
                )
                relative_path = generate_relative_path(
                    my_relative_path, file_name_with_extension
                )
                dict_folder_manifest["filename"].append(relative_path)
                # file type
                file_extension = get_name_extension(file_name)
                if file_extension == "":
                    file_extension = "None"
                dict_folder_manifest["file type"].append(file_extension)
                # timestamp, description, Additional Metadata
                if not manifest_df.empty:
                    if relative_path in manifest_df["filename"].values:
                        timestamp = manifest_df[
                            manifest_df["filename"] == relative_path
                        ]["timestamp"].iloc[0]
                        description = manifest_df[
                            manifest_df["filename"] == relative_path
                        ]["description"].iloc[0]
                        additional_metadata = manifest_df[
                            manifest_df["filename"] == relative_path
                        ]["Additional Metadata"].iloc[0]
                    else:
                        timestamp = ""
                        description = ""
                        additional_metadata = ""
                    dict_folder_manifest["timestamp"].append(timestamp)
                    dict_folder_manifest["description"].append(description)
                    dict_folder_manifest["Additional Metadata"].append(
                        additional_metadata
                    )
                else:
                    dict_folder_manifest["timestamp"].append("")
                    dict_folder_manifest["description"].append("")
                    dict_folder_manifest["Additional Metadata"].append("")
        return dict_folder_manifest

    # Merge existing folders
    def recursive_manifest_builder_existing_ps(
        my_folder,
        my_bf_folder,
        my_bf_folder_exists,
        my_relative_path,
        dict_folder_manifest,
    ):
        if "folders" in my_folder.keys():
            if my_bf_folder_exists:
                (
                    my_bf_existing_folders_name,
                ) = ps_get_existing_folders_details(my_bf_folder['children']['folders'])
            else:
                my_bf_existing_folders_name = []
            for folder_key, folder in my_folder["folders"].items():
                relative_path = generate_relative_path(my_relative_path, folder_key)
                if folder_key in my_bf_existing_folders_name:
                    bf_folder = my_bf_folder["children"]["folders"][folder_key]
                    bf_folder_exists = True
                else:
                    bf_folder = ""
                    bf_folder_exists = False
                dict_folder_manifest = recursive_manifest_builder_existing_ps(
                    folder,
                    bf_folder,
                    bf_folder_exists,
                    relative_path,
                    dict_folder_manifest,
                )
        if "files" in my_folder.keys():
            if my_bf_folder_exists:
                (
                    my_bf_existing_files_name,
                    my_bf_existing_files_name_with_extension,
                ) = ps_get_existing_files_details(my_bf_folder)
            else:
                my_bf_existing_files = []
                my_bf_existing_files_name = []
                my_bf_existing_files_name_with_extension = []
            for file_key, file in my_folder["files"].items():
                if file.get("location") == "local":
                    file_path = file["path"]
                    if isfile(file_path):
                        desired_name = splitext(file_key)[0]
                        file_extension = splitext(file_key)[1]
                        # manage existing file request
                        if existing_file_option == "skip" and file_key in my_bf_existing_files_name_with_extension:
                            continue
                        if existing_file_option == "replace" and file_key in my_bf_existing_files_name_with_extension:
                            # remove existing from manifest
                            filename = generate_relative_path(
                                my_relative_path, file_key
                            )
                            filename_list = dict_folder_manifest["filename"]
                            index_file = filename_list.index(filename)
                            del dict_folder_manifest["filename"][index_file]
                            del dict_folder_manifest["timestamp"][index_file]
                            del dict_folder_manifest["description"][index_file]
                            del dict_folder_manifest["file type"][index_file]
                            del dict_folder_manifest["Additional Metadata"][
                                index_file
                            ]
                            index_name = (
                                my_bf_existing_files_name_with_extension.index(
                                    file_key
                                )
                            )
                            del my_bf_existing_files[index_name]
                            del my_bf_existing_files_name[index_name]
                            del my_bf_existing_files_name_with_extension[
                                index_name
                            ]
                        if desired_name not in my_bf_existing_files_name:
                            final_name = file_key
                        else:
                            # expected final name
                            count_done = 0
                            final_name = desired_name
                            output = get_base_file_name(desired_name)
                            if output:
                                base_name = output[0]
                                count_exist = output[1]
                                while count_done == 0:
                                    if final_name in my_bf_existing_files_name:
                                        count_exist += 1
                                        final_name = (
                                            base_name + "(" + str(count_exist) + ")"
                                        )
                                    else:
                                        count_done = 1
                            else:
                                count_exist = 0
                                while count_done == 0:
                                    if final_name in my_bf_existing_files_name:
                                        count_exist += 1
                                        final_name = (
                                            desired_name
                                            + " ("
                                            + str(count_exist)
                                            + ")"
                                        )
                                    else:
                                        count_done = 1
                            final_name = final_name + file_extension
                            my_bf_existing_files_name.append(
                                splitext(final_name)[0]
                            )
                        # filename
                        filename = generate_relative_path(
                            my_relative_path, final_name
                        )
                        dict_folder_manifest["filename"].append(filename)
                        # timestamp
                        file_path = file["path"]
                        filepath = pathlib.Path(file_path)
                        mtime = filepath.stat().st_mtime
                        lastmodtime = datetime.fromtimestamp(mtime).astimezone(
                            local_timezone
                        )
                        dict_folder_manifest["timestamp"].append(
                            lastmodtime.isoformat()
                            .replace(".", ",")
                            .replace("+00:00", "Z")
                        )
                        # description
                        if "description" in file.keys():
                            dict_folder_manifest["description"].append(
                                file["description"]
                            )
                        else:
                            dict_folder_manifest["description"].append("")
                        # file type
                        if file_extension == "":
                            file_extension = "None"
                        dict_folder_manifest["file type"].append(file_extension)
                        # addtional metadata
                        if "additional-metadata" in file.keys():
                            dict_folder_manifest["Additional Metadata"].append(
                                file["additional-metadata"]
                            )
                        else:
                            dict_folder_manifest["Additional Metadata"].append("")
        return dict_folder_manifest

    double_extensions = [
        ".ome.tiff",
        ".ome.tif",
        ".ome.tf2,",
        ".ome.tf8",
        ".ome.btf",
        ".ome.xml",
        ".brukertiff.gz",
        ".mefd.gz",
        ".moberg.gz",
        ".nii.gz",
        ".mgh.gz",
        ".tar.gz",
        ".bcl.gz",
    ]

    try:
        # create local folder to save manifest files temporarly (delete any existing one first)
        shutil.rmtree(manifest_folder_path) if isdir(manifest_folder_path) else 0
        makedirs(manifest_folder_path)

        # import info about files already on ps
        dataset_structure = soda["dataset-structure"]
        manifest_dict_save = {}
        for high_level_folder_key, high_level_folder in my_tracking_folder["children"]["folders"].items():
            if (
                high_level_folder_key in dataset_structure["folders"].keys()
            ):

                relative_path = ""
                dict_folder_manifest = {}
                # Initialize dict where manifest info will be stored
                dict_folder_manifest["filename"] = []
                dict_folder_manifest["timestamp"] = []
                dict_folder_manifest["description"] = []
                dict_folder_manifest["file type"] = []
                dict_folder_manifest["Additional Metadata"] = []

                # pull manifest file into if exists 
                manifest_df = pd.DataFrame()
                for file_key, file in high_level_folder['children']['files'].items():
                    file_id = file['content']['id']
                    r = requests.get(f"{PENNSIEVE_URL}/packages/{file_id}/view", headers=create_request_headers(ps))
                    r.raise_for_status()
                    file_details = r.json()
                    file_name_with_extension = file_details[0]["content"]["name"]
                    if file_name_with_extension in manifest_sparc:
                        file_id_2 = file_details[0]["content"]["id"]
                        r = requests.get(f"{PENNSIEVE_URL}/packages/{file_id}/files/{file_id_2}", headers=create_request_headers(ps))
                        r.raise_for_status()
                        file_url_info = r.json()
                        file_url = file_url_info["url"]
                        manifest_df = pd.read_excel(file_url, engine="openpyxl")
                        manifest_df = manifest_df.fillna("")
                        if (
                            "filename" not in manifest_df.columns
                            or "description" not in manifest_df.columns
                            or "Additional Metadata" not in manifest_df.columns
                        ):
                            manifest_df = pd.DataFrame()
                        break

                # store the data frame pulled from Pennsieve into a dictionary
                dict_folder_manifest =  recursive_import_ps_manifest_info(
                    high_level_folder, relative_path, dict_folder_manifest, manifest_df
                )

                manifest_dict_save[high_level_folder_key] = {
                    "manifest": dict_folder_manifest,
                    "bf_folder": high_level_folder,
                }

        # import info from local files to be uploaded
        local_timezone = TZLOCAL()
        manifest_files_structure = {}
        existing_folder_option = soda["generate-dataset"]["if-existing"]
        existing_file_option = soda["generate-dataset"][
            "if-existing-files"
        ]
        for folder_key, folder in dataset_structure["folders"].items():
            relative_path = ""

            if (
                folder_key in manifest_dict_save
                and existing_folder_option == "merge"
            ):
                bf_folder = manifest_dict_save[folder_key]["bf_folder"]
                bf_folder_exists = True
                dict_folder_manifest = manifest_dict_save[folder_key]["manifest"]

            elif (
                folder_key in manifest_dict_save
                and folder_key
                not in my_tracking_folder["children"]["folders"].keys()
                and existing_folder_option == "skip"
            ):
                continue

            else:
                bf_folder = ""
                bf_folder_exists = False
                dict_folder_manifest = {}
                dict_folder_manifest["filename"] = []
                dict_folder_manifest["timestamp"] = []
                dict_folder_manifest["description"] = []
                dict_folder_manifest["file type"] = []
                dict_folder_manifest["Additional Metadata"] = []

            dict_folder_manifest = recursive_manifest_builder_existing_ps(
                folder, bf_folder, bf_folder_exists, relative_path, dict_folder_manifest
            )

            # create high-level folder at the temporary location
            folderpath = join(manifest_folder_path, folder_key)
            makedirs(folderpath)

            # save manifest file
            manifestfilepath = join(folderpath, "manifest.xlsx")
            df = pd.DataFrame.from_dict(dict_folder_manifest)
            df.to_excel(manifestfilepath, index=None, header=True)
            wb = load_workbook(manifestfilepath)
            ws = wb.active

            blueFill = PatternFill(
                start_color="9DC3E6", fill_type="solid"
            )
            greenFill = PatternFill(
                start_color="A8D08D", fill_type="solid"
            )
            yellowFill = PatternFill(
                start_color="FFD965", fill_type="solid"
            )
            ws['A1'].fill = blueFill
            ws['B1'].fill = greenFill
            ws['C1'].fill = greenFill
            ws['D1'].fill = greenFill
            ws['E1'].fill = yellowFill
            wb.save(manifestfilepath)

            manifest_files_structure[folder_key] = manifestfilepath

        return manifest_files_structure

    except Exception as e:
        raise e





def generate_relative_path(x, y):
    return x + "/" + y if x else y


def ps_get_existing_folders_details(ps_folders):
    ps_existing_folders = [ps_folders[folder] for folder in ps_folders if ps_folders[folder]["content"]["packageType"] == "Collection"]
    ps_existing_folders_name = [folder['content']["name"] for folder in ps_existing_folders]

    return ps_existing_folders, ps_existing_folders_name


def ps_get_existing_files_details(ps_folder):
    # TODO: Dorian -> ["extensions doesn't seem to be returned anymore by the endpoint"]
    def verify_file_name(file_name, extension):
        if extension == "":
            return file_name

        double_ext = False
        for ext in double_extensions:
            if file_name.find(ext) != -1:
                double_ext = True
                break

        extension_from_name = ""

        if double_ext == False:
            extension_from_name = os.path.splitext(file_name)[1]
        else:
            extension_from_name = (
                os.path.splitext(os.path.splitext(file_name)[0])[1]
                + os.path.splitext(file_name)[1]
            )

        if extension_from_name == ("." + extension):
            return file_name
        else:
            return file_name + ("." + extension)

    files = ps_folder["children"]["files"]
    double_extensions = [
        ".ome.tiff",
        ".ome.tif",
        ".ome.tf2,",
        ".ome.tf8",
        ".ome.btf",
        ".ome.xml",
        ".brukertiff.gz",
        ".mefd.gz",
        ".moberg.gz",
        ".nii.gz",
        ".mgh.gz",
        ".tar.gz",
        ".bcl.gz",
    ]


    bf_existing_files_name = [splitext(files[file]['content']["name"])[0] for file in files]
    bf_existing_files_name_with_extension = []

    # determine if we are at the root of the dataset
    content = ps_folder["content"]
    if (str(content['id'])[2:9]) == "dataset":
        r = requests.get(f"{PENNSIEVE_URL}/datasets/{content['id']}", headers=create_request_headers(get_access_token())) 
        r.raise_for_status()
        root_folder = r.json()
        root_children = root_folder["children"]
        for item in root_children:
            file_name_with_extension = ""
            item_id = item["content"]["id"]
            item_name = item["content"]["name"]
            if item_id[2:9] == "package":
                if("extension" not in root_children):
                    file_name_with_extension = verify_file_name(item_name,"")
                else:
                    file_name_with_extension = verify_file_name(item_name, root_children["extension"])

            if file_name_with_extension == "":
                continue
            bf_existing_files_name_with_extension.append(file_name_with_extension)
    else:
        #is collection - aka a folder in the dataset
        for file_key, file in files.items():
            file_name_with_extension = ""
            file_name = file["content"]["name"]
            file_id = file["content"]["id"]
            if file_id[2:9] == "package":
                if "extension" not in file:
                    file_name_with_extension = verify_file_name(file_name,"")
                else:
                    file_name_with_extension = verify_file_name(file_name, file["extension"])
            if file_name_with_extension == "":
                continue
            bf_existing_files_name_with_extension.append(file_name_with_extension)


    return (
        bf_existing_files_name,
        bf_existing_files_name_with_extension,
    )


def check_if_int(s):
    try:
        int(s)
        return True
    except ValueError:
        return False


def get_base_file_name(file_name):
    output = []
    if file_name[-1] == ")":
        string_length = len(file_name)
        count_start = string_length
        character = file_name[count_start - 1]
        while character != "(" and count_start >= 0:
            count_start -= 1
            character = file_name[count_start - 1]
        if character == "(":
            base_name = file_name[:count_start - 1]
            num = file_name[count_start : string_length - 1]
            if check_if_int(num):
                output = [base_name, int(num)]
    return output


def ps_update_existing_dataset(soda, ds, ps, resume):
    global logger

    logger.info("Starting ps_update_existing_dataset")

    global main_curate_progress_message
    global main_total_generate_dataset_size
    global start_generate
    global main_initial_bfdataset_size

    # Delete any files on Pennsieve that have been marked as deleted
    def recursive_file_delete(folder):
        if "files" in folder.keys():
            for item in list(folder["files"]):
                if "deleted" in folder["files"][item]["action"]:
                    file_path = folder["files"][item]["path"]
                    # remove the file from the dataset
                    r = requests.post(f"{PENNSIEVE_URL}/data/delete", headers=create_request_headers(ps), json={"things": [file_path]})
                    r.raise_for_status()
                    # remove the file from the soda json structure
                    del folder["files"][item]

        for item in list(folder["folders"]):
            recursive_file_delete(folder["folders"][item])

    # Delete any files on Pennsieve that have been marked as deleted
    def metadata_file_delete(soda):
        if "dataset_metadata" in soda.keys():
            folder = soda["dataset_metadata"]
            for item in list(folder):
                if "deleted" in folder[item]["action"]:
                    r = requests.post(f"{PENNSIEVE_URL}/data/delete", headers=create_request_headers(ps), json={"things": [folder[item]["path"]]})
                    r.raise_for_status()
                    del folder[item]


    def recursive_item_path_create(folder, path):
        """
        Recursively create the path for the item    # Add a new key containing the path to all the files and folders on the
        local data structure.
        Allows us to see if the folder path of a specfic file already
        exists on Pennsieve.
        """
        
        if "files" in folder.keys():
            for item in list(folder["files"]):
                if item in ["manifest.xslx", "manifest.csv"]:
                    continue
                if "folderpath" not in folder["files"][item]:
                    folder["files"][item]["folderpath"] = path[:]

        if "folders" in folder.keys():
            for item in list(folder["folders"]):
                if "folderpath" not in folder["folders"][item]:
                    folder["folders"][item]["folderpath"] = path[:]
                    folder["folders"][item]["folderpath"].append(item)
                recursive_item_path_create(
                    folder["folders"][item], folder["folders"][item]["folderpath"][:]
                )

        return

    # Check and create any non existing folders for the file move process (Used in the recursive_check_moved_files function)
    def recursive_check_and_create_ps_file_path(
        folderpath, index, current_folder_structure
    ):
        folder = folderpath[index]

        if folder not in current_folder_structure["folders"]:
            if index == 0:
                r = requests.post(f"{PENNSIEVE_URL}/packages", json={"name": folder, "parent": f"{current_folder_structure['path']}", "packageType": "collection", "dataset": ds['content']['id']},  headers=create_request_headers(ps))
                r.raise_for_status()
                new_folder = r.json()
            else:
                r = requests.post(f"{PENNSIEVE_URL}/packages", json={"name": folder, "parent": f"{current_folder_structure['path']}", "packageType": "collection", "dataset": ds['content']['id']},  headers=create_request_headers(ps))
                r.raise_for_status()
                new_folder = r.json()
            
            current_folder_structure["folders"][folder] = {
                "location": "ps",
                "action": ["existing"],
                "path": new_folder['content']['id'],
                "folders": {},
                "files": {},
            }

        index += 1
        # check if path exists for folder, if not then folder has not been created on Pennsieve yet, so create it and add it to the path key
        if "path" not in current_folder_structure["folders"][folder].keys() or current_folder_structure["folders"][folder]["location"] != "ps":
            r = requests.post(f"{PENNSIEVE_URL}/packages", headers=create_request_headers(ps), json=build_create_folder_request(folder, current_folder_structure["path"], ds['content']['id']))
            r.raise_for_status()
            new_folder_id = r.json()["content"]["id"]
            current_folder_structure["folders"][folder]["path"] = new_folder_id

        if index < len(folderpath):
            return recursive_check_and_create_ps_file_path(
                folderpath, index, current_folder_structure["folders"][folder]
            )
        else:
            return current_folder_structure["folders"][folder]["path"]

    # Check for any files that have been moved and verify paths before moving
    def recursive_check_moved_files(folder):
        if "files" in folder.keys():
            for item in list(folder["files"]):
                if (
                    "moved" in folder["files"][item]["action"]
                    and folder["files"][item]["location"] == "ps"
                ):
                    # create the folders if they do not exist
                    new_folder_id = ""
                    new_folder_id = recursive_check_and_create_ps_file_path(
                        folder["files"][item]["folderpath"].copy(), 0, dataset_structure
                    )
                    # move the file into the target folder on Pennsieve
                    r = requests.post(f"{PENNSIEVE_URL}/data/move",  json={"things": [folder["files"][item]["path"]], "destination": new_folder_id}, headers=create_request_headers(ps))
                    r.raise_for_status()

        for item in list(folder["folders"]):
            recursive_check_moved_files(folder["folders"][item])


    # Rename any files that exist on Pennsieve
    def recursive_file_rename(folder):
        if "files" in folder.keys():
            for item in list(folder["files"]):
                if (
                    "renamed" in folder["files"][item]["action"]
                    and folder["files"][item]["location"] == "ps"
                ):
                    # rename the file on Pennsieve
                    r = requests.put(f"{PENNSIEVE_URL}/packages/{folder['files'][item]['path']}?updateStorage=true", json={"name": item}, headers=create_request_headers(ps))
                    r.raise_for_status()

        for item in list(folder["folders"]):
            recursive_file_rename(folder["folders"][item])


    def recursive_folder_delete(folder):
        """
        Delete any stray folders that exist on Pennsieve
        Only top level files are deleted since the api deletes any
        files and folders that exist inside.
        """
        for item in list(folder["folders"]):
            if folder["folders"][item]["location"] == "ps":
                if "moved" in folder["folders"][item]["action"]:
                    file_path = folder["folders"][item]["path"]
                    # remove the file from the dataset
                    r = requests.post(f"{PENNSIEVE_URL}/data/delete", headers=create_request_headers(ps), json={"things": [file_path]})
                    r.raise_for_status()
                if "deleted" in folder["folders"][item]["action"]:
                    file_path = folder["folders"][item]["path"]
                    # remove the file from the dataset
                    r = requests.post(f"{PENNSIEVE_URL}/data/delete", headers=create_request_headers(ps), json={"things": [file_path]})
                    r.raise_for_status()
                    del folder["folders"][item]
                else:
                    recursive_folder_delete(folder["folders"][item])
            else:
                recursive_folder_delete(folder["folders"][item])


    # Rename any folders that still exist.
    def recursive_folder_rename(folder, mode):
        for item in list(folder["folders"]):
            if (
                folder["folders"][item]["location"] == "ps"
                and "action" in folder["folders"][item].keys()
                and mode in folder["folders"][item]["action"]
            ):
                folder_id = folder["folders"][item]["path"]
                r = requests.put(f"{PENNSIEVE_URL}/packages/{folder_id}?updateStorage=true", headers=create_request_headers(ps), json={"name": item})
                r.raise_for_status()
            recursive_folder_rename(folder["folders"][item], mode)


    ps_dataset = ""
    start = timer()
    # 1. Remove all existing files on Pennsieve, that the user deleted.
    logger.info("ps_update_existing_dataset step 1 remove existing files on Pennsieve the user deleted")
    main_curate_progress_message = "Checking Pennsieve for deleted files"
    dataset_structure = soda["dataset-structure"]
    recursive_file_delete(dataset_structure)
    main_curate_progress_message = (
        "Files on Pennsieve marked for deletion have been deleted"
    )

    # 2. Rename any deleted folders on Pennsieve to allow for replacements.
    logger.info("ps_update_existing_dataset step 2 rename deleted folders on Pennsieve to allow for replacements")
    main_curate_progress_message = "Checking Pennsieve for deleted folders"
    dataset_structure = soda["dataset-structure"]
    recursive_folder_rename(dataset_structure, "deleted")
    main_curate_progress_message = "Folders on Pennsieve have been marked for deletion"

    # 2.5 Rename folders that need to be in the final destination.
    logger.info("ps_update_existing_dataset step 2.5 rename folders that need to be in the final destination")
    main_curate_progress_message = "Renaming any folders requested by the user"
    recursive_folder_rename(dataset_structure, "renamed")
    main_curate_progress_message = "Renamed all folders requested by the user"

    # 3. Get the status of all files currently on Pennsieve and create
    # the folderpath for all items in both dataset structures.
    logger.info("ps_update_existing_dataset step 3 get the status of all files currently on Pennsieve and create the folderpath for all items in both dataset structures")
    main_curate_progress_message = "Fetching files and folders from Pennsieve"
    current_bf_dataset_files_folders = import_pennsieve_dataset(
        soda.copy()
    )["soda_object"]
    ps_dataset = current_bf_dataset_files_folders["dataset-structure"]
    main_curate_progress_message = "Creating file paths for all files on Pennsieve"
    recursive_item_path_create(dataset_structure, [])
    recursive_item_path_create(ps_dataset, [])
    main_curate_progress_message = "File paths created"

    # 4. Move any files that are marked as moved on Pennsieve.
    # Create any additional folders if required
    logger.info("ps_update_existing_dataset step 4 move any files that are marked as moved on Pennsieve")
    main_curate_progress_message = "Moving any files requested by the user"
    recursive_check_moved_files(dataset_structure)
    main_curate_progress_message = "Moved all files requested by the user"

    # 5. Rename any Pennsieve files that are marked as renamed.
    logger.info("ps_update_existing_dataset step 5 rename any Pennsieve files that are marked as renamed")
    main_curate_progress_message = "Renaming any files requested by the user"
    recursive_file_rename(dataset_structure)
    main_curate_progress_message = "Renamed all files requested by the user"

    # 6. Delete any Pennsieve folders that are marked as deleted.
    logger.info("ps_update_existing_dataset step 6 delete any Pennsieve folders that are marked as deleted")
    main_curate_progress_message = (
        "Deleting any additional folders present on Pennsieve"
    )
    recursive_folder_delete(dataset_structure)
    main_curate_progress_message = "Deletion of additional folders complete"

    # 7. Delete any metadata files that are marked as deleted.
    logger.info("ps_update_existing_dataset step 8 delete any metadata files that are marked as deleted")
    main_curate_progress_message = "Removing any metadata files marked for deletion"
    metadata_file_delete(soda)
    main_curate_progress_message = "Removed metadata files marked for deletion"

    # 8. Run the original code to upload any new files added to the dataset.
    logger.info("ps_update_existing_dataset step 9 run the ps_create_new_dataset code to upload any new files added to the dataset")
    if "dataset_metadata" in soda.keys() and "manifest_files" in soda["dataset_metadata"].keys():
        if "auto-generated" in soda["manifest-files"].keys():
            soda["manifest-files"] = {"destination": "ps", "auto-generated": True}
        else:
            soda["manifest-files"] = {"destination": "ps"}

    soda["generate-dataset"] = {
        "destination": "ps",
        "if-existing": "merge",
        "if-existing-files": "replace",
        "generate-option": "existing-ps"
    }

    end = timer()
    logger.info(f"Time for ps_update_existing_dataset function: {timedelta(seconds=end - start)}")
    ps_upload_to_dataset(soda, ps, ds, resume)


def get_origin_manifest_id(dataset_id):
    global logger
    max_attempts = 3
    for _ in range(max_attempts):
        manifests = get_upload_manifests(dataset_id)
        if manifests and "manifests" in manifests and manifests["manifests"]:
            # sort the manifests list by date_created timestamp field in descending order
            manifests["manifests"].sort(key=lambda x: x["date_created"], reverse=True)
            return manifests["manifests"][0]["id"]
        time.sleep(5)  # Wait for 5 seconds before the next attempt

    raise Exception("Did not get the origin manifest id in an expected amount of time.")



def normalize_tracking_folder(tracking_folder):
    """
    Normalize the tracking folder object to be a dictonary with the shape: {files: {}, folders: {}}. 
    This shape matches our dataset structure object. Recall, the tracking folder receives information about what folders and 
    files are stored on Pennsieve. We update this as we update Pennsieve's state. 
    """
    if tracking_folder == "":
        return {"folders": {}, "files": {} }
    
    temp_children = {"folders": {}, "files": {}}


    # add the files and folders to the temp_children structure 
    for child in tracking_folder["children"]:
        if child["content"]["packageType"] == "Collection":
            # add the folders ( designated collection on Pennsieve ) to the temp_children structure under folders
            temp_children["folders"][child["content"]["name"]] = child
        else:
            # add the files (anything not designated a collection) to the temp_children structure under files
            temp_children["files"][child["content"]["name"]] = child

    # replace the non-normalized children structure with the normalized children structure
    tracking_folder["children"] = temp_children


def build_create_folder_request(folder_name, folder_parent_id, dataset_id):
    """
    Create a folder on Pennsieve. 
    """
    body = {}

    # if creating a folder at the root of the dataset the api does not require a parent key
    if folder_parent_id.find("N:dataset") == -1:
        body["parent"] = folder_parent_id
    
    body["name"] = folder_name
    body["dataset"] = dataset_id
    body["packageType"] = "collection"

    return body


bytes_uploaded_per_file = {}
total_bytes_uploaded = {"value": 0}
current_files_in_subscriber_session = 0


bytes_file_path_dict = {}

# retry variables instantiated outside function
list_of_files_to_rename = {}
renamed_files_counter = 0 


def ps_upload_to_dataset(soda, ps, ds, resume=False):
    global logger

    # Progress tracking variables that are used for the frontend progress bar.
    global main_curate_progress_message
    global main_total_generate_dataset_size
    global main_generated_dataset_size
    global start_generate
    global main_initial_bfdataset_size
    global main_curation_uploaded_files
    global uploaded_folder_counter
    global current_size_of_uploaded_files
    global total_files
    global total_bytes_uploaded # current number of bytes uploaded to Pennsieve in the current session
    global client
    global files_uploaded
    global total_dataset_files
    global current_files_in_subscriber_session
    global renaming_files_flow
    global bytes_uploaded_per_file
    global total_bytes_uploaded_per_file
    global bytes_file_path_dict
    global elapsed_time
    global manifest_id
    global origin_manifest_id
    global main_curate_status
    global list_of_files_to_rename
    global renamed_files_counter


    total_files = 0
    total_dataset_files = 0
    total_metadata_files = 0
    total_manifest_files = 0
    main_curation_uploaded_files = 0
    total_bytes_uploaded = {"value": 0}
    total_bytes_uploaded_per_file = {}
    files_uploaded = 0
    renamed_files_counter = 0
    

    uploaded_folder_counter = 0
    current_size_of_uploaded_files = 0
    start = timer()
    try:


        def recursive_dataset_scan_for_new_upload(dataset_structure, list_upload_files, my_relative_path):
            """
            This function recursively gathers the files and folders in the dataset that will be uploaded to Pennsieve.
            It assumes the dataset is new based on the generate_option value and will spend less time comparing what is on Pennsieve.
            It will gather all the relative paths for the files and folders to pass along to the Pennsieve agent.
            Input:
            dataset_structure,
            my_relative_path

            Output:
            two lists in one tuple, the first list will have all the local file paths that will be uploaded to Pennsieve
            The second list will have the relative files paths according to the dataset structure.
            If the folder does not existing yet on Pennsieve the agent will create it.
            """
            global main_total_generate_dataset_size
            global bytes_file_path_dict
            # First loop will take place in the root of the dataset
            if "folders" in dataset_structure.keys():
                for folder_key, folder in dataset_structure["folders"].items():
                    relative_path = generate_relative_path(my_relative_path, folder_key)
                    list_upload_files = recursive_dataset_scan_for_new_upload(folder, list_upload_files, relative_path)
            if "files" in dataset_structure.keys():
                list_local_files = []
                list_projected_names = []
                list_desired_names = []
                list_final_names = []

                list_initial_names = []
                for file_key, file in dataset_structure["files"].items():
                    # relative_path = generate_relative_path(my_relative_path, file_key)
                    file_path = file["path"]
                    if isfile(file_path) and file.get("location") == "local":
                        projected_name = splitext(basename(file_path))[0]
                        projected_name_w_extension = basename(file_path)
                        desired_name = splitext(file_key)[0]
                        desired_name_with_extension = file_key


                        if projected_name != desired_name:
                            list_initial_names.append(projected_name)
                            list_local_files.append(file_path)
                            list_projected_names.append(projected_name_w_extension)
                            list_desired_names.append(desired_name_with_extension)
                            list_final_names.append(desired_name)
                        else:
                            list_local_files.append(file_path)
                            list_projected_names.append(projected_name_w_extension)
                            list_desired_names.append(desired_name_with_extension)
                            list_final_names.append(desired_name)
                            list_initial_names.append(projected_name)

                        file_size = getsize(file_path)
                        main_total_generate_dataset_size += file_size
                        bytes_file_path_dict[file_path] = file_size

                if list_local_files:
                    list_upload_files.append([
                        list_local_files,
                        list_projected_names,
                        list_desired_names,
                        list_final_names,
                        "/" if my_relative_path == soda["generate-dataset"]["dataset-name"] else my_relative_path,
                    ])


            return list_upload_files

        # See how to create folders with the Pennsieve agent
        def recursive_create_folder_for_ps(
            my_folder, my_tracking_folder, existing_folder_option
        ):
            """
            Creates a folder on Pennsieve for each folder in the dataset structure if they aren't already present in the dataset.
            Input:
                my_folder: The dataset structure to be created on Pennsieve. Pass in the soda json object to start. 
                my_tracking_folder: Tracks what folders have been created on Pennsieve thus far. Starts as an empty dictionary.
                existing_folder_option: Dictates whether to merge, duplicate, replace, or skip existing folders.
            """
            # Check if the current folder has any subfolders that already exist on Pennsieve. Important step to appropriately handle replacing and merging folders.
            if len(my_tracking_folder["children"]["folders"]) == 0 and my_tracking_folder["content"]["id"].find("N:dataset") == -1:
                limit = 100
                offset = 0
                ps_folder = {}
                ps_folder_children = []
                while True: 
                    r = requests.get(f"{PENNSIEVE_URL}/packages/{my_tracking_folder['content']['id']}?limit={limit}&offset={offset}", headers=create_request_headers(ps), json={"include": "files"})
                    r.raise_for_status()
                    ps_folder = r.json()
                    page = ps_folder["children"]
                    ps_folder_children.extend(page)
                    if len(page) < limit:
                        break
                    offset += limit
                    time.sleep(1)
                    
                ps_folder["children"] = ps_folder_children
                normalize_tracking_folder(ps_folder)
                my_tracking_folder["children"] = ps_folder["children"]

            # create/replace/skip folder
            if "folders" in my_folder.keys():
                for folder_key, folder in my_folder["folders"].items():
                    if existing_folder_option == "skip":
                        if folder_key not in my_tracking_folder["children"]["folders"]:
                            r = requests.post(f"{PENNSIEVE_URL}/packages", headers=create_request_headers(ps), json=build_create_folder_request(folder_key, my_tracking_folder['content']['id'], ds['content']['id']))
                            r.raise_for_status()
                            ps_folder = r.json()
                            normalize_tracking_folder(ps_folder)
                        else:
                            ps_folder = my_tracking_folder["children"]["folders"][folder_key]
                            normalize_tracking_folder(ps_folder)

                    elif existing_folder_option == "create-duplicate":
                        r = requests.post(f"{PENNSIEVE_URL}/packages", headers=create_request_headers(ps), json=build_create_folder_request(folder_key, my_tracking_folder['content']['id'], ds['content']['id']))
                        r.raise_for_status()
                        ps_folder = r.json()
                        normalize_tracking_folder(ps_folder)

                    elif existing_folder_option == "replace":
                        # if the folder exists on Pennsieve remove it
                        if folder_key in my_tracking_folder["children"]["folders"]:
                            ps_folder = my_tracking_folder["children"]["folders"][folder_key]

                            r = requests.post(f"{PENNSIEVE_URL}/data/delete", headers=create_request_headers(ps), json={"things": [ps_folder["content"]["id"]]})
                            r.raise_for_status()

                            # remove from ps_folder 
                            del my_tracking_folder["children"]["folders"][folder_key]

                        r = requests.post(f"{PENNSIEVE_URL}/packages", headers=create_request_headers(ps), json=build_create_folder_request(folder_key, my_tracking_folder['content']['id'], ds['content']['id']))
                        r.raise_for_status()
                        ps_folder = r.json()
                        normalize_tracking_folder(ps_folder)

                    elif existing_folder_option == "merge":
                        if folder_key in my_tracking_folder["children"]["folders"]:
                            ps_folder = my_tracking_folder["children"]["folders"][folder_key]
                            normalize_tracking_folder(ps_folder)
                        else:
                            # We are merging but this is a new folder - not one that already exists in the current dataset - so we create it.
                            r = requests.post(f"{PENNSIEVE_URL}/packages", headers=create_request_headers(ps), json=build_create_folder_request(folder_key, my_tracking_folder['content']['id'], ds['content']['id']))
                            r.raise_for_status()
                            ps_folder = r.json()
                            normalize_tracking_folder(ps_folder)


                    my_tracking_folder["children"]["folders"][folder_key] = ps_folder
                    tracking_folder = my_tracking_folder["children"]["folders"][folder_key] # get the folder we just added to the tracking folder
                    recursive_create_folder_for_ps(
                        folder, tracking_folder, existing_folder_option
                    )

        def recursive_dataset_scan_for_ps(
            my_folder,
            my_tracking_folder,
            existing_file_option,
            list_upload_files,
            my_relative_path,
        ):
            """
                Delete files that are marked to be replaced in the dataset. Create a list of files to upload to Pennsieve.
            """

            global main_total_generate_dataset_size


            # folder children are packages such as collections and files stored on the Pennsieve dataset
            ps_folder_children = my_tracking_folder["children"] #ds (dataset)



            if "folders" in my_folder.keys():
                for folder_key, folder in my_folder["folders"].items():
                    relative_path = generate_relative_path(my_relative_path, folder_key)

                    if existing_folder_option == "skip" and folder_key in my_tracking_folder["children"]["folders"]:
                        continue

                    tracking_folder = ps_folder_children["folders"][folder_key]
                    list_upload_files = recursive_dataset_scan_for_ps(
                        folder,
                        tracking_folder,
                        existing_file_option,
                        list_upload_files,
                        relative_path,
                    )

            if "files" in my_folder.keys(): 

                # delete files to be deleted
                (
                    my_bf_existing_files_name,
                    my_bf_existing_files_name_with_extension,
                ) = ps_get_existing_files_details(my_tracking_folder)

                for file_key, file in my_folder["files"].items():
                    # if local then we are either adding a new file to an existing/new dataset or replacing a file in an existing dataset
                    if file.get("location") == "local":
                        file_path = file["path"]
                        if isfile(file_path) and existing_file_option == "replace" and file_key in ps_folder_children["files"]:
                            my_file = ps_folder_children["files"][file_key]
                            # delete the package ( aka file ) from the dataset 
                            r = requests.post(f"{PENNSIEVE_URL}/data/delete", headers=create_request_headers(ps), json={"things": [f"{my_file['content']['id']}"]})
                            r.raise_for_status()
                            del ps_folder_children["files"][file_key]


                # create list of files to be uploaded with projected and desired names saved
                (
                    my_bf_existing_files_name,
                    my_bf_existing_files_name_with_extension,
                ) = ps_get_existing_files_details(my_tracking_folder)

                list_local_files = []
                list_projected_names = []
                list_desired_names = []
                list_final_names = []
                additional_upload_lists = []

                list_initial_names = []

                # add the files that are set to be uploaded to Pennsieve to a list 
                # handle renaming files and creating duplicates
                for file_key, file in my_folder["files"].items():
                    if file.get("location") == "local":
                        file_path = file["path"]
                        if isfile(file_path):
                            initial_name = splitext(basename(file_path))[0]
                            initial_extension = splitext(basename(file_path))[1]
                            initial_name_with_extension = basename(file_path)
                            desired_name = splitext(file_key)[0]
                            desired_name_extension = splitext(file_key)[1]
                            desired_name_with_extension = file_key
                            if existing_file_option == "skip" and desired_name_with_extension in my_bf_existing_files_name_with_extension:
                                continue

                            # check if initial filename exists on Pennsieve dataset and get the projected name of the file after upload
                            # used when a local file has a name that matches an existing name on Pennsieve
                            count_done = 0
                            count_exist = 0
                            projected_name = initial_name_with_extension
                            while count_done == 0:
                                if (
                                    projected_name
                                    in my_bf_existing_files_name_with_extension
                                ):
                                    count_exist += 1
                                    projected_name = (
                                        initial_name
                                        + " ("
                                        + str(count_exist)
                                        + ")"
                                        + initial_extension
                                    )
                                else:
                                    count_done = 1

                            # expected final name
                            count_done = 0
                            final_name = desired_name_with_extension
                            if output := get_base_file_name(desired_name):
                                base_name = output[0]
                                count_exist = output[1]
                                while count_done == 0:
                                    if final_name in my_bf_existing_files_name:
                                        count_exist += 1
                                        final_name = (
                                            base_name
                                            + "("
                                            + str(count_exist)
                                            + ")"
                                            + desired_name_extension
                                        )
                                    else:
                                        count_done = 1
                            else:
                                count_exist = 0
                                while count_done == 0:
                                    if final_name in my_bf_existing_files_name:
                                        count_exist += 1
                                        final_name = (
                                            desired_name
                                            + " ("
                                            + str(count_exist)
                                            + ")"
                                            + desired_name_extension
                                        )
                                    else:
                                        count_done = 1

                            # save in list accordingly
                            if (
                                initial_name in list_initial_names
                                or initial_name in list_final_names
                                or projected_name in list_final_names
                                or final_name in list_projected_names
                            ):
                                additional_upload_lists.append(
                                    [
                                        [file_path],
                                        ps_folder_children,
                                        [projected_name],
                                        [desired_name],
                                        [final_name],
                                        my_tracking_folder,
                                        my_relative_path,
                                    ]
                                )
                            else:
                                list_local_files.append(file_path)
                                list_projected_names.append(projected_name)
                                list_desired_names.append(desired_name_with_extension)
                                list_final_names.append(final_name)
                                list_initial_names.append(initial_name)

                            my_bf_existing_files_name.append(final_name)
                            if initial_extension in ps_recognized_file_extensions:
                                my_bf_existing_files_name_with_extension.append(
                                    final_name
                                )
                            else:
                                my_bf_existing_files_name_with_extension.append(
                                    final_name + initial_extension
                                )

                            # add to projected dataset size to be generated
                            main_total_generate_dataset_size += getsize(file_path)

                if list_local_files:
                    ds_name = soda["ps-dataset-selected"]["dataset-name"]
                    list_upload_files.append(
                        [
                            list_local_files,
                            ps_folder_children,
                            list_projected_names,
                            list_desired_names,
                            list_final_names,
                            my_tracking_folder,
                            "/" if my_relative_path == ds_name else my_relative_path,
                        ]
                    )

                for item in additional_upload_lists:
                    list_upload_files.append(item)

            return list_upload_files

        def monitor_subscriber_progress(events_dict):
            """
            Monitors the progress of a subscriber and unsubscribes once the upload finishes. 
            """
            global files_uploaded
            global total_bytes_uploaded
            global bytes_uploaded_per_file
            global main_curation_uploaded_files
            global main_total_generate_dataset_size


            if events_dict["type"] == 1:  # upload status: file_id, total, current, worker_id
                file_id = events_dict["upload_status"].file_id
                total_bytes_to_upload = events_dict["upload_status"].total
                current_bytes_uploaded = events_dict["upload_status"].current

                status = events_dict["upload_status"].status
                if status == "2" or status == 2:
                    ps.unsubscribe(10)
                    logger.info("[UPLOAD COMPLETE EVENT RECEIVED]")
                    logger.info(f"Amount of bytes uploaded via sum: {sum(bytes_uploaded_per_file.values())} vs total bytes uploaded via difference: {total_bytes_uploaded['value']}")
                    logger.info(f"Amount of bytes Pennsieve Agent says via sum: {sum(bytes_uploaded_per_file.values())} vs amount of bytes we calculated before hand: {main_total_generate_dataset_size}")


                # only update the byte count if the current bytes uploaded is greater than the previous bytes uploaded
                # if current_bytes_uploaded > previous_bytes_uploaded:
                # update the file id's current total bytes uploaded value 
                bytes_uploaded_per_file[file_id] = current_bytes_uploaded
                total_bytes_uploaded["value"] = sum(bytes_uploaded_per_file.values())

                # check if the given file has finished uploading
                if current_bytes_uploaded == total_bytes_to_upload and  file_id != "":
                    files_uploaded += 1
                    main_curation_uploaded_files += 1



        # Set the Pennsieve Python Client's dataset to the Pennsieve dataset that will be uploaded to.
        selected_id = ds["content"]["id"]
        ps.use_dataset(selected_id)

        # Set variables needed throughout generation flow
        list_upload_files = []
        list_upload_metadata_files = []
        list_upload_manifest_files = []
        list_of_files_to_rename = {}
        brand_new_dataset = False
        dataset_structure = soda["dataset-structure"]
        generate_option = soda["generate-dataset"]["generate-option"]
        starting_point = soda["starting-point"]["origin"]
        relative_path = ds["content"]["name"]

 

        # 1. Scan the dataset structure and create a list of files/folders to be uploaded with the desired renaming
        if generate_option == "new" and starting_point == "new":
            vs = ums.df_mid_has_progress()
            if resume == False or resume == True and not vs:
                logger.info("NO progress found so we will start from scratch and construct the manifest")
                main_curate_progress_message = "Preparing a list of files to upload"
                # we can assume no files/folders exist in the dataset since the generate option is new and starting point is also new
                # therefore, we can assume the dataset structure is the same as the tracking structure
                brand_new_dataset = True
                list_upload_files = recursive_dataset_scan_for_new_upload(dataset_structure, list_upload_files, relative_path)





            if "dataset_metadata" in soda.keys():
                for key, _ in soda["dataset_metadata"].items():
                    if key == "submission":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "submission.xlsx")
                        submission.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "subjects":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "subjects.xlsx")
                        subjects.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "samples":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "samples.xlsx")
                        samples.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "performances":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "performances.xlsx")
                        performances.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "resources":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "resources.xlsx")
                        resources.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "sites":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "sites.xlsx")
                        sites.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "dataset_description":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "dataset_description.xlsx")
                        dataset_description.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "code_description":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "code_description.xlsx")
                        code_description.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "manifest_file":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "manifest.xlsx")
                        manifest.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1

                    if key == "README":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "README.txt")
                        readme_changes.create_text_file(soda, False, metadata_path, "README")
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "CHANGES":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "CHANGES.txt")
                        readme_changes.create_text_file(soda, False, metadata_path, "CHANGES")
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1



        else:

            main_curate_progress_message = "Preparing a list of files to upload"

            existing_folder_option = soda["generate-dataset"]["if-existing"]
            existing_file_option = soda["generate-dataset"][
                "if-existing-files"
            ]

            # we will need a tracking structure to compare against
            tracking_json_structure = ds
            normalize_tracking_folder(tracking_json_structure)
            recursive_create_folder_for_ps(dataset_structure, tracking_json_structure, existing_folder_option)
            list_upload_files = recursive_dataset_scan_for_ps(
                dataset_structure,
                tracking_json_structure,
                existing_file_option,
                list_upload_files,
                relative_path,
            )



            # 3. Add high-level metadata files to a list
            if "dataset_metadata" in soda.keys():
                logger.info("ps_create_new_dataset (optional) step 3 create high level metadata list")
                # TODO: Add enahnced merge support post SDS3 launch
                # (
                #     my_bf_existing_files_name,
                #     _,
                # ) = ps_get_existing_files_details(ds)
                for key, _ in soda["dataset_metadata"].items():
                    if key == "submission":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "submission.xlsx")
                        submission.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "subjects":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "subjects.xlsx")
                        subjects.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "samples":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "samples.xlsx")
                        samples.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "performances":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "performances.xlsx")
                        performances.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "resources":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "resources.xlsx")
                        resources.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "sites":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "sites.xlsx")
                        sites.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "dataset_description":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "dataset_description.xlsx")
                        dataset_description.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "code_description":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "code_description.xlsx")
                        code_description.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1
                    if key == "manifest_file":
                        metadata_path = os.path.join(METADATA_UPLOAD_PS_PATH, "manifest.xlsx")
                        manifest.create_excel(soda, False, metadata_path)
                        list_upload_metadata_files.append(metadata_path)
                        main_total_generate_dataset_size += getsize(metadata_path)
                        total_files += 1
                        total_metadata_files += 1


                    # TODO: Post SDS3 release add enhanced support for merging into existing datasets with more than 0 files
                    # if file["location"] == "local":
                    #     # TODO: SDS 3 Determine if can be done without needing the path key as that isn't in my updated schema for SDS3. Probably is possible. Probably just create it if not set to skip. And delete. Voila.
                    #     metadata_path = file["path"]
                    #     if isfile(metadata_path):
                    #         initial_name = splitext(basename(metadata_path))[0]
                    #         if (
                    #             existing_file_option == "replace"
                    #             and initial_name in my_bf_existing_files_name
                    #         ):
                    #             my_file = ds['children']['files'][file_key]
                    #             # delete the file from Pennsieve
                    #             r = requests.post(f"{PENNSIEVE_URL}/data/delete", json={"things": [my_file['content']['id']]}, headers=create_request_headers(get_access_token()))
                    #             r.raise_for_status()
                    #         if (
                    #             existing_file_option == "skip"
                    #             and initial_name in my_bf_existing_files_name
                    #         ):
                    #             continue

                    #         list_upload_metadata_files.append(metadata_path)
                    #         main_total_generate_dataset_size += getsize(metadata_path)
                    #         total_files += 1
                    #         total_metadata_files += 1

            # 4. Prepare and add manifest files to a list
            if "dataset_metadata" in soda.keys() and "manifest_files" in soda["dataset_metadata"].keys():
                logger.info("ps_create_new_dataset (optional) step 4 create manifest list")
                # create local folder to save manifest files temporarly (delete any existing one first)
                # TODO: SDS 3 create manifests if not skipping and delete file on Pennsieve if it exists
                if "auto-generated" in soda["manifest-files"]:
                    if soda["manifest-files"]["auto-generated"] == True:
                        manifest_files_structure = (
                            get_auto_generated_manifest_files(soda)
                        )

                    # add manifest files to list after deleting existing ones
                    for key in manifest_files_structure.keys():
                        manifestpath = manifest_files_structure[key]
                        folder = tracking_json_structure["children"]["folders"][key]

                        # delete existing manifest files
                        for child_key in folder["children"]["files"]:
                            file_name_no_ext = os.path.splitext(folder['children']['files'][child_key]['content']['name'])[0]
                            if file_name_no_ext.lower() == "manifest":
                                # delete the manifest file from the given folder 
                                r = requests.post(f"{PENNSIEVE_URL}/data/delete", json={"things": [folder['children']['files'][child_key]['content']['id']]}, headers=create_request_headers(get_access_token()))
                                r.raise_for_status()

                        # upload new manifest files
                        # the number of files to upload and the total also determines when the upload subscribers should stop listening to the dataset upload progress ( when files uploaded == total files stop listening )
                        list_upload_manifest_files.append([manifestpath, key])
                        total_files += 1
                        total_manifest_files += 1
                        main_total_generate_dataset_size += getsize(manifestpath)


        # 2. Count how many files will be uploaded to inform frontend - do not count if we are resuming a previous upload that has made progress
        if not resume or resume and not ums.df_mid_has_progress():
            for folderInformation in list_upload_files:
                file_paths_count = len(folderInformation[0])
                total_files += file_paths_count
                total_dataset_files += file_paths_count


        # 3. Upload files and add to tracking list
        start_generate = 1

        
        # resuming a dataset that had no files to rename or that failed before renaming any files
        if resume and ums.df_mid_has_progress() and not ums.get_renaming_files_flow():
            main_curate_progress_message = ("Preparing to retry upload. Progress on partially uploaded files will be reset.")
            # reset necessary variables that were used in the failed upload session and cannot be reliably cached
            bytes_uploaded_per_file = {}

            # get the current manifest id for data files
            manifest_id = ums.get_df_mid()
            # get the cached values of the previous upload session 
            main_total_generate_dataset_size = ums.get_main_total_generate_dataset_size()

            total_files = ums.get_total_files_to_upload()
            total_dataset_files = total_files         
            current_files_in_subscriber_session = total_dataset_files

            main_curation_uploaded_files = total_files - ums.get_remaining_file_count(manifest_id, total_files)
            files_uploaded = main_curation_uploaded_files
            total_bytes_uploaded["value"] = ums.calculate_completed_upload_size(manifest_id, bytes_file_path_dict, total_files )

            # rename file information 
            list_of_files_to_rename = ums.get_list_of_files_to_rename()
            renamed_files_counter = ums.get_rename_total_files()
            
            time.sleep(5)


            # upload the manifest files
            try: 
                ps.manifest.upload(manifest_id)
                main_curate_progress_message = ("Uploading data files...")
                # subscribe to the manifest upload so we wait until it has finished uploading before moving on
                ps.subscribe(10, False, monitor_subscriber_progress)
            except Exception as e:
                logger.error("Error uploading dataset files")
                logger.error(e)
                raise PennsieveUploadException("The Pennsieve Agent has encountered an issue while uploading. Please retry the upload. If this issue persists please follow this <a target='_blank' rel='noopener noreferrer' href='https://docs.sodaforsparc.io/docs/how-to/how-to-reinstall-the-pennsieve-agent'> guide</a> on performing a full reinstallation of the Pennsieve Agent then click the retry button.")
        elif resume and ums.df_mid_has_progress() and ums.get_renaming_files_flow():
            # setup for rename files flow
            list_of_files_to_rename = ums.get_list_of_files_to_rename()
            renamed_files_counter = ums.get_rename_total_files()
        # create a manifest for files - IMP: We use a single file to start with since creating a manifest requires a file path.  We need to remove this at the end. 
        elif len(list_upload_files) > 0:
            main_curate_progress_message = ("Queuing dataset files for upload with the Pennsieve Agent..." + "<br>" + "This may take some time.")

            first_file_local_path = list_upload_files[0][0][0]

            if brand_new_dataset:
                first_relative_path = list_upload_files[0][4]
                first_final_name = list_upload_files[0][2][0]
            else:
                first_relative_path = list_upload_files[0][6]
                first_final_name = list_upload_files[0][4][0]

            folder_name = first_relative_path[first_relative_path.index("/")+1:]

            if first_final_name != basename(first_file_local_path):
                # if file name is not the same as local path, then it has been renamed in SODA
                if folder_name not in list_of_files_to_rename:
                    list_of_files_to_rename[folder_name] = {}
                if basename(first_file_local_path) not in list_of_files_to_rename[folder_name]:
                    list_of_files_to_rename[folder_name][basename(first_file_local_path)] = {
                        "final_file_name": first_final_name,
                        "id": "",
                    }
                    renamed_files_counter += 1

            manifest_data = ps.manifest.create(first_file_local_path, folder_name)
            manifest_id = manifest_data.manifest_id


            ums.set_df_mid(manifest_id)

            # remove the item just added to the manifest 
            list_upload_files[0][0].pop(0)

            # reset global variables used in the subscriber monitoring function
            bytes_uploaded_per_file = {}
            total_bytes_uploaded = {"value": 0}
            current_files_in_subscriber_session = total_dataset_files

            # there are files to add to the manifest if there are more than one file in the first folder or more than one folder
            if len(list_upload_files[0][0]) > 1 or len(list_upload_files) > 1:
                index_skip = True
                for folderInformation in list_upload_files:
                    list_file_paths = folderInformation[0]
                    if brand_new_dataset:
                        relative_path = folderInformation[4]
                        final_file_name_list = folderInformation[2]
                    else:
                        relative_path = folderInformation[6]
                        final_file_name_list = folderInformation[4]
                    # get the substring from the string relative_path that starts at the index of the / and contains the rest of the string
                    try:
                        folder_name = relative_path[relative_path.index("/")+1:]
                    except ValueError as e:
                        folder_name = relative_path

                    # Add files to manfiest"
                    final_files_index = 1 if index_skip else 0
                    index_skip = False
                    for file_path in list_file_paths:
                        file_file_name = final_file_name_list[final_files_index]
                        if file_file_name != basename(file_path):
                            # save the relative path, final name and local path of the file to be renamed
                            if folder_name not in list_of_files_to_rename:
                                list_of_files_to_rename[folder_name] = {}
                            if basename(file_path) not in list_of_files_to_rename[folder_name]:
                                renamed_files_counter += 1
                                list_of_files_to_rename[folder_name][basename(file_path)] = {
                                    "final_file_name": file_file_name,
                                    "id": "",
                                }
                        ps.manifest.add(file_path, folder_name, manifest_id)
                        final_files_index += 1


            # add metadata files to the manifest
            if list_upload_metadata_files:
                current_files_in_subscriber_session += total_metadata_files
                # add the files to the manifest
                for manifest_path in list_upload_metadata_files:
                    # subprocess call to the pennsieve agent to add the files to the manifest
                    ps.manifest.add(manifest_path, target_base_path="", manifest_id=manifest_id)


            # add manifest files to the upload manifest
            if list_upload_manifest_files:
                current_files_in_subscriber_session += total_manifest_files
                for manifest_file_path in list_upload_manifest_files:
                    # add the file to the manifest
                    ps.manifest.add(manifest_file_path, "/", manifest_id)

            
            # set rename files to ums for upload resuming if this upload fails
            if renamed_files_counter > 0:
                ums.set_list_of_files_to_rename(list_of_files_to_rename)
                ums.set_rename_total_files(renamed_files_counter)

            # upload the manifest files
            try: 
                ps.manifest.upload(manifest_id)

                main_curate_progress_message = ("Uploading data files...")

                # subscribe to the manifest upload so we wait until it has finished uploading before moving on
                ps.subscribe(10, False, monitor_subscriber_progress)

            except Exception as e:
                logger.error(e)
                raise PennsieveUploadException("The Pennsieve Agent has encountered an issue while uploading. Please retry the upload. If this issue persists please follow this <a target='_blank' rel='noopener noreferrer' href='https://docs.sodaforsparc.io/docs/how-to/how-to-reinstall-the-pennsieve-agent'> guide</a> on performing a full reinstallation of the Pennsieve Agent then click the retry button.")


        # wait for all of the Agent's processes to finish to avoid errors when deleting files on Windows
        time.sleep(1)

        # 6. Rename files
        if list_of_files_to_rename:
            renaming_files_flow = True
            logger.info("ps_create_new_dataset (optional) step 8 rename files")
            main_curate_progress_message = ("Preparing files to be renamed...")
            dataset_id = ds["content"]["id"]
            collection_ids = {}
            # gets the high level folders in the dataset
            r = requests.get(f"{PENNSIEVE_URL}/datasets/{dataset_id}", headers=create_request_headers(ps))
            r.raise_for_status()
            dataset_content = r.json()["children"]

            if dataset_content == []:
                while dataset_content == []:
                    time.sleep(3)
                    r = requests.get(f"{PENNSIEVE_URL}/datasets/{dataset_id}", headers=create_request_headers(ps))
                    r.raise_for_status()
                    dataset_content = r.json()["children"]

            collections_found = False
            while not collections_found:
                for item in dataset_content:
                    # high lvl folders' ids are stored to be used to find the file IDS
                    if item["content"]["packageType"] == "Collection":
                        collections_found = True
                        collection_ids[item["content"]["name"]] = {"id": item["content"]["nodeId"]}


                if not collections_found:
                    # No collections were found, metadata files were processed but not the high level folders
                    time.sleep(3)
                    r = requests.get(f"{PENNSIEVE_URL}/datasets/{dataset_id}", headers=create_request_headers(ps))
                    r.raise_for_status()
                    dataset_content = r.json()["children"]

            for key in list_of_files_to_rename:
                # split the key up if there are multiple folders in the relative path
                relative_path = key.split("/")
                high_lvl_folder_name = relative_path[0]
                subfolder_level = 0
                subfolder_amount = len(relative_path) - 1

                if high_lvl_folder_name in collection_ids:
                    # subfolder_amount will be the amount of subfolders we need to call until we can get the file ID to rename

                    high_lvl_folder_id = collection_ids[high_lvl_folder_name]["id"]
                    limit = 100
                    offset = 0
                    dataset_content = []
                    while True:
                        r = requests.get(f"{PENNSIEVE_URL}/packages/{high_lvl_folder_id}?limit={limit}&offset={offset}", headers=create_request_headers(ps))
                        r.raise_for_status()
                        page = r.json()["children"]
                        dataset_content.extend(page)

                        if len(page) < limit:
                            break
                        offset += limit

                    if dataset_content == []:
                        # request until there is no children content, (folder is empty so files have not been processed yet)
                        while dataset_content == []:
                            time.sleep(3)
                            limit = 100 
                            offset = 0 

                            while True:
                                r = requests.get(f"{PENNSIEVE_URL}/packages/{high_lvl_folder_id}?limit={limit}&offset={offset}", headers=create_request_headers(ps))
                                r.raise_for_status()
                                page = r.json()["children"]
                                dataset_content.extend(page)
                                if len(page) < limit:
                                    break
                                offset += limit
                            

                    if subfolder_amount == 0:
                        # the file is in the high level folder
                        if "id" not in list_of_files_to_rename[key]:
                            # store the id of the folder to be used again in case the file id is not found (happens when not all files have been processed yet)
                            list_of_files_to_rename[key]["id"] = high_lvl_folder_id


                        for item in dataset_content:
                            if item["content"]["packageType"] != "Collection":
                                file_name = item["content"]["name"]
                                file_id = item["content"]["nodeId"]

                                if file_name in list_of_files_to_rename[key]:
                                    # name
                                    # store the package id for now
                                    list_of_files_to_rename[key][file_name]["id"] = file_id
                    else:
                        # file is within a subfolder and we recursively iterate until we get to the last subfolder needed
                        subfolder_id = collection_ids[high_lvl_folder_name]["id"]
                        while subfolder_level != subfolder_amount:
                            if dataset_content == []:
                                # subfolder has no content so request again
                                while dataset_content == []:
                                    time.sleep(3)
                                    limit = 100 
                                    offset = 0
                                    while True: 
                                        r = requests.get(f"{PENNSIEVE_URL}/packages/{subfolder_id}", headers=create_request_headers(ps))
                                        r.raise_for_status()
                                        page = r.json()["children"]
                                        dataset_content.extend(page)
                                        if len(page) < limit:
                                            break
                                        offset += limit
                        

                            for item in dataset_content:
                                if item["content"]["packageType"] == "Collection":
                                    folder_name = item["content"]["name"]
                                    folder_id = item["content"]["nodeId"]

                                    if folder_name in relative_path:
                                        # we have found the folder we need to iterate through
                                        subfolder_level += 1

                                        limit = 100
                                        offset = 0 
                                        children = []
                                        while True: 
                                            r = requests.get(f"{PENNSIEVE_URL}/packages/{folder_id}?limit={limit}&offset={offset}", headers=create_request_headers(ps))
                                            r.raise_for_status()
                                            page = r.json()["children"]
                                            children.extend(page)
                                            if len(page) < limit:
                                                break
                                            offset += limit

                                        if subfolder_level != subfolder_amount:
                                            dataset_content = children
                                            if dataset_content == []:
                                                while dataset_content == []:
                                                    # subfolder has no content so request again
                                                    time.sleep(3)
                                                    limit = 100
                                                    offset = 0 
                                                    while True: 
                                                        r = requests.get(f"{PENNSIEVE_URL}/packages/{folder_id}", headers=create_request_headers(ps))
                                                        r.raise_for_status()
                                                        page = r.json()["children"]
                                                        dataset_content.extend(page)
                                                        if len(page) < limit:
                                                            break
                                                        offset += limit
                                                    
                                            subfolder_id = folder_id
                                            break
                                        else:
                                            # we are at the last folder in the relative path, we can get the file id
                                            if "id" not in list_of_files_to_rename[key]:
                                                # store the id of the last folder to directly call later in case not all files get an id
                                                list_of_files_to_rename[key]["id"] = folder_id
                                            for item in children:
                                                if item["content"]["packageType"] != "Collection":
                                                    file_name = item["content"]["name"]
                                                    file_id = item["content"]["nodeId"]

                                                    if file_name in list_of_files_to_rename[key]:
                                                        # store the package id for renaming
                                                        list_of_files_to_rename[key][file_name]["id"] = file_id
                                    else:
                                        continue

            # 8.5 Rename files - All or most ids have been fetched now rename the files or gather the ids again if not all files have been processed at this time
            main_curate_progress_message = "Renaming files..."
            main_generated_dataset_size = 0
            main_total_generate_dataset_size = renamed_files_counter
            for relative_path in list_of_files_to_rename:
                for file in list_of_files_to_rename[relative_path].keys():
                    collection_id = list_of_files_to_rename[relative_path]["id"]
                    if file == "id":
                        continue
                    new_name = list_of_files_to_rename[relative_path][file]["final_file_name"]
                    file_id = list_of_files_to_rename[relative_path][file]["id"]

                    if file_id != "":
                        # id was found so make api call to rename with final file name
                        try:
                            r = requests.put(f"{PENNSIEVE_URL}/packages/{file_id}?updateStorage=true", json={"name": new_name}, headers=create_request_headers(ps))
                            r.raise_for_status()
                        except Exception as e:
                            if r.status_code == 500:
                                continue
                        main_generated_dataset_size += 1
                    else:
                        # id was not found so keep trying to get the id until it is found
                        all_ids_found = False
                        while not all_ids_found:
                            collection_id = list_of_files_to_rename[relative_path]["id"]
                            if file == "id":
                                continue

                            
                            limit = 100
                            offset = 0
                            dataset_content = []

                            while True: 
                                r = requests.put(f"{PENNSIEVE_URL}/packages/{collection_id}?updateStorage=true&limit={limit}&offset={offset}", headers=create_request_headers(ps))
                                r.raise_for_status()
                                page = r.json()["children"]
                                dataset_content.extend(page)
                                if len(dataset_content) < limit:
                                    break
                                offset += limit
                            
                            for item in dataset_content:
                                if item["content"]["packageType"] != "Collection":
                                    file_name = item["content"]["name"]
                                    file_id = item["content"]["nodeId"]

                                    if file_name == file:
                                        # id was found so make api call to rename with file file name
                                        try:
                                            r = requests.put(f"{PENNSIEVE_URL}/packages/{file_id}", json={"name": new_name}, headers=create_request_headers(ps))
                                            r.raise_for_status()
                                        except Exception as e:
                                            if r.status_code == 500:
                                                continue
                                        main_generated_dataset_size += 1
                                        all_ids_found = True
                                        break


        


                # get the manifest id of the Pennsieve upload manifest created when uploading
        
                
        origin_manifest_id = get_origin_manifest_id(selected_id)

        # if files were uploaded but later receive the 'Failed' status in the Pennsieve manifest we allow users to retry the upload; set the pre-requisite information for the upload to 
        # be retried in that case
        # NOTE: We do not need to store the rename information here. Rationale: If the upload for a file failed the rename could not succeed and we would not reach this point. 
        #       What would happen instead is as follows(in an optimistic case where the upload doesnt keep being marked as Failed): 
        #      1. The upload for a file fails
        #      2. The upload information gets (including rename information ) stored in the catchall error handling block 
        #      3. The user retries the upload
        #      4. The manifest counts the Failed file as a file to be retried 
        #      5. The manifest is uploaded again and the file is uploaded again
        #      6. The file is renamed successfully this time
        ums.set_main_total_generate_dataset_size(main_total_generate_dataset_size)
        ums.set_total_files_to_upload(total_files)
        ums.set_elapsed_time(elapsed_time)        
        

        main_curate_progress_message = "Success: COMPLETED!"
        main_curate_status = "Done"

        
        shutil.rmtree(manifest_folder_path) if isdir(manifest_folder_path) else 0
        end = timer()
        logger.info(f"Time for ps_upload_to_dataset function: {timedelta(seconds=end - start)}")
    except Exception as e:
        # reset the total bytes uploaded for any file that has not been fully uploaded
        ums.set_main_total_generate_dataset_size(main_total_generate_dataset_size)
        ums.set_total_files_to_upload(total_files)
        ums.set_elapsed_time(elapsed_time)
        # store the renaming files information in case the upload fails and we need to rename files during the retry
        ums.set_renaming_files_flow(renaming_files_flow) # this determines if we failed while renaming files after the upload is complete
        ums.set_rename_total_files(renamed_files_counter)
        ums.set_list_of_files_to_rename(list_of_files_to_rename)
        raise e

main_curate_status = ""
main_curate_print_status = ""
main_curate_progress_message = ""
main_total_generate_dataset_size = 1
main_generated_dataset_size = 0
start_generate = 0
generate_start_time = 0
main_generate_destination = ""
main_initial_bfdataset_size = 0
myds = ""
renaming_files_flow = False
elapsed_time = None
manifest_id = None 
origin_manifest_id = None



def ps_check_dataset_files_validity(soda):
    """
    Function to check that the bf data files and folders specified in the dataset are valid

    Args:
        dataset_structure: soda dict with information about all specified files and folders
    Output:
        error: error message with list of non valid local data files, if any
    """
    def check_folder_validity(folder_id, folder_dict, folder_path, error):
        """
        Function to verify that the subfolders and files specified in the dataset are valid

        Args:
            folder_id: id of the folder in the dataset
            folder_dict: dict with information about the folder
            folder_path: path of the folder in the dataset
            error: error message with list of non valid files/folders, if any
        Output:
            error: error message with list of non valid files/folders, if any
        """
        # get the folder content through Pennsieve api
        limit = 100
        offset = 0
        folder_content = []
        while True: 
            r = requests.get(f"{PENNSIEVE_URL}/packages/{folder_id}?offset={offset}&limit={limit}", headers=create_request_headers(get_access_token()))
            r.raise_for_status()
            page = r.json()["children"]
            folder_content.extend(page)
            if len(page) < limit:
                break
            offset += limit

        # check that the subfolders and files specified in the dataset are valid
        if "files" in folder_dict.keys():
            for file_key, file in folder_dict["files"].items():
                file_type = file.get("location")
                relative_path = (f"{folder_path}/{file_key}")
                # If file is from Pennsieve we verify if file exists on Pennsieve
                if file_type == "ps":
                    file_actions = file["action"]
                    file_id = file["path"]
                    if "moved" in file_actions:
                        try:
                            r = requests.get(f"{PENNSIEVE_URL}/packages/{file_id}/view", headers=create_request_headers(get_access_token()))
                            r.raise_for_status()
                        except Exception as e:
                            error.append(f"{relative_path} id: {file_id}")
                        continue
                    if next((item for item in folder_content if item["content"]["id"] == file_id), None) is None:
                        error.append(f"{relative_path} id: {file_id}")
        
        if "folders" in folder_dict.keys():
            for folder_key, folder in folder_dict["folders"].items():
                folder_type = folder.get("location")
                relative_path = (f"{folder_path}/{folder_key}")
                if folder_type == "ps":
                    folder_id = folder["path"]
                    folder_action = folder["action"]
                    if "moved" in folder_action:
                        try:
                            r = requests.get(f"{PENNSIEVE_URL}/packages/{folder_id}", headers=create_request_headers(get_access_token()))
                            r.raise_for_status()
                        except Exception as e:
                            error.append(f"{relative_path} id: {folder_id}")
                        continue
                    if next((item for item in folder_content if item["content"]["id"] == folder_id), None) is None:
                        error.append(f"{relative_path} id: {folder_id}")
                    else:
                        check_folder_validity(folder_id, folder, relative_path, error)

        return error

    error = []
    # check that the files and folders specified in the dataset are valid
    dataset_name = soda["ps-dataset-selected"]["dataset-name"]
    dataset_id = get_dataset_id(dataset_name)
    r = requests.get(f"{PENNSIEVE_URL}/datasets/{dataset_id}", headers=create_request_headers(get_access_token()))
    r.raise_for_status()
    root_folder = r.json()["children"]

    if len(root_folder) == 0:
        return error

    if "dataset-structure" in soda.keys():
        dataset_structure = soda["dataset-structure"]
        if "folders" in dataset_structure:
            for folder_key, folder in dataset_structure["folders"].items():
                folder_type = folder.get("location")
                relative_path = folder_key
                if folder_type == "ps":
                    collection_id = folder["path"]
                    collection_actions = folder["action"]
                    if "moved" in collection_actions:
                        try:
                            r = requests.get(f"{PENNSIEVE_URL}/packages/{collection_id}/view", headers=create_request_headers(get_access_token()))
                            r.raise_for_status()
                        except Exception:
                            error.append(f"{relative_path} id: {collection_id}")
                        continue
                    if next((item for item in root_folder if item["content"]["id"] == collection_id), None) is None:
                        error.append(f"{relative_path} id: {collection_id}")
                    else:
                        # recursively check all files + subfolders of collection_id
                        error = check_folder_validity(collection_id, folder, relative_path, error)

    # if there are items in the error list, check if they have been "moved"
    if len(error) > 0:
        error_message = [
            "Error: The following Pennsieve files/folders are invalid. Specify them again or remove them."
        ]
        error = error_message + error

    return error


def check_server_access_to_files(file_list):
    # Return two lists, one that the server can open, and one that it can not.
    # This is to avoid the server trying to open files that it does not have access to.cf
    accessible_files = []
    inaccessible_files = []
    for file in file_list:
        if os.path.isfile(file) or os.path.isdir(file):
            accessible_files.append(file)
        else:
            inaccessible_files.append(file)

    return {"accessible_files": accessible_files, "inaccessible_files": inaccessible_files}


# TODO: Update for SDS 3.0
def clean_json_structure(soda):
    global logger
    # Delete any files on Pennsieve that have been marked as deleted
    def recursive_file_delete(folder):
        if "files" in folder.keys():
            for item in list(folder["files"]):
                if item in ["manifest.xlsx", "manifest.csv"]:
                    continue
                if "deleted" in folder["files"][item]["action"]:
                    # remove the file from the soda json structure
                    del folder["files"][item]

        for item in list(folder["folders"]):
            recursive_file_delete(folder["folders"][item])


    # Rename any files that exist on Pennsieve
    def recursive_file_rename(folder):
        if "files" in folder.keys():
            for item in list(folder["files"]):
                if (
                    "renamed" in folder["files"][item]["action"]
                    and folder["files"][item]["location"] == "ps"
                ):
                    continue

        for item in list(folder["folders"]):
            recursive_file_rename(folder["folders"][item])


    def recursive_folder_delete(folder):
        """
        Delete any stray folders that exist on Pennsieve
        Only top level files are deleted since the api deletes any
        files and folders that exist inside.
        """

        for folder_item in list(folder["folders"]):
            if folder["folders"][folder_item]["location"] == "ps":
                if "deleted" in folder["folders"][folder_item]["action"]:
                    del folder["folders"][folder_item]
                else:
                    recursive_folder_delete(folder["folders"][folder_item])
            else:
                recursive_folder_delete(folder["folders"][folder_item])
        return

    main_keys = soda.keys()
    dataset_structure = soda["dataset-structure"]

    if ("dataset-structure" not in main_keys and "dataset_metadata" not in main_keys):
        if "ps-dataset-selected" in main_keys:
            dataset_name = soda["ps-dataset-selected"]["dataset-name"]
        elif "generate-dataset" in main_keys:
            dataset_name = soda["generate-dataset"]["dataset-name"]
        else:
            dataset_name = "Unset Name"
        raise EmptyDatasetError(dataset_name)

    if "generate-dataset" in main_keys:
        # Check that local files/folders exist
        try:
            if error := check_local_dataset_files_validity(soda):
                raise LocalDatasetMissingSpecifiedFiles(error)
            # check that dataset is not empty after removing all the empty files and folders
            if not soda["dataset-structure"]["folders"] and "dataset_metadata" not in soda:
                if "ps-dataset-selected" in main_keys:
                    dataset_name = soda["ps-dataset-selected"]["dataset-name"]
                elif "generate-dataset" in main_keys:
                    dataset_name = soda["generate-dataset"]["dataset-name"]
                else:
                    dataset_name = "Unset Name"
                raise EmptyDatasetError(dataset_name)
        except Exception as e:
            raise e

    if "starting-point" in main_keys and soda["starting-point"][
        "origin"
    ] in ["ps", "local"]:
        recursive_file_delete(dataset_structure)
        recursive_folder_delete(dataset_structure)
        soda["dataset-structure"] = dataset_structure

    logger.info("clean_json_structure step 1")
    logger.info(soda)
    # here will be clean up the soda json object before creating the manifest file cards
    return {"soda": soda}



def validate_local_dataset_generate_path(soda):
    generate_dataset = soda["generate-dataset"]
    local_dataset_path = generate_dataset["path"]
    if not isdir(local_dataset_path):
        error_message = (
            "Error: The Path "
            + local_dataset_path
            + " is not found. Please select a valid destination folder for the new dataset"
        )
        raise FileNotFoundError(error_message)




def generating_on_ps(soda):
    return soda["generate-dataset"]["destination"] == "ps"

def uploading_with_ps_account(soda):
    return "ps-account-selected" in soda

def uploading_to_existing_ps_dataset(soda):
    return "ps-dataset-selected" in soda

def can_resume_prior_upload(resume_status):
    global ums 
    return resume_status and ums.df_mid_has_progress()

def virtual_dataset_empty(soda):
    return (
        "dataset-structure" not in soda
        and "metadata-files" not in soda
        )

def generate_options_set(soda):
    return "generate-dataset" in soda.keys()


def get_dataset_with_backoff(selected_dataset_id):
    # check that dataset was created with a limited retry (for some users the dataset isn't automatically accessible)
    attempts = 0
    while(attempts < 3):
        try: 
            # whether we are generating a new dataset or merging, we want the dataset information for later steps
            r = requests.get(f"{PENNSIEVE_URL}/datasets/{selected_dataset_id}", headers=create_request_headers(get_access_token()))
            r.raise_for_status()
            return r.json()
        except Exception as e:
            attempts += 1 
            # check if final attempt
            if attempts >= 2:
                # raise the error to the user
                raise e
            time.sleep(10)


def generate_new_ds_ps_resume(soda, dataset_name, ps):
    # get the dataset id by the name 
    try: 
        selected_dataset_id = get_dataset_id(dataset_name)
    except Exception as e:
        if e.code == 404:
            # dataset does not exist - create it 
            ds = ps_create_new_dataset(dataset_name, ps)
            selected_dataset_id = ds["content"]["id"]
    
    myds = get_dataset_with_backoff(selected_dataset_id)
    ps_upload_to_dataset(soda, ps, myds, True)

def generate_new_ds_ps(soda, dataset_name, ps):
    ds = ps_create_new_dataset(dataset_name, ps)
    selected_dataset_id = ds["content"]["id"]    
    myds = get_dataset_with_backoff(selected_dataset_id)
    ps_upload_to_dataset(soda, ps, myds, False)


def generate_dataset(soda, resume, ps):
    global main_generate_destination
    global main_total_generate_dataset_size

 
    # Generate dataset locally
    if generating_locally(soda):
        logger.info("generate_dataset generating_locally")
        main_generate_destination = soda["generate-dataset"][
            "destination"
        ]
        _, main_total_generate_dataset_size = generate_dataset_locally(
            soda
        )

    # Generate dataset to Pennsieve
    if generating_on_ps(soda):
        main_generate_destination = soda["generate-dataset"][
            "destination"
        ]
        generate_option = soda["generate-dataset"]["generate-option"]

        logger.info("generate_dataset generating_on_ps")
        logger.info(soda)

        if uploading_to_existing_ps_dataset(soda)  and soda["starting-point"]["origin"] != "new":
            
            selected_dataset_id = get_dataset_id(
                soda["ps-dataset-selected"]["dataset-name"]
            )
            # make an api request to pennsieve to get the dataset details
            r = requests.get(f"{PENNSIEVE_URL}/datasets/{selected_dataset_id}", headers=create_request_headers(get_access_token()))
            r.raise_for_status()
            myds = r.json()

            if can_resume_prior_upload(resume): 
                ps_upload_to_dataset(soda, ps, myds, resume)
            else:
                logger.info("We are updating an existing dataset")
                ps_update_existing_dataset(soda, myds, ps, resume)

        elif generate_option == "new" or generate_option == "existing-ps" and soda["starting-point"]["origin"] == "new":
            logger.info("We are generating into an existing but not updating an existing lol")
            # if dataset name is in the generate-dataset section, we are generating a new dataset
            if "dataset-name" in soda["generate-dataset"]:
                dataset_name = soda["generate-dataset"][
                    "dataset-name"
                ]
            elif "digital-metadata" in soda and "name" in soda["digital-metadata"]:
                dataset_name = soda["digital-metadata"]["name"]
            elif "ps-dataset-selected" in soda and "dataset-name" in soda["ps-dataset-selected"]:
                dataset_name = soda["ps-dataset-selected"]["dataset-name"]
            
            if resume: 
                generate_new_ds_ps_resume(soda, dataset_name, ps)
            else: 
                try: 
                    selected_dataset_id = get_dataset_id(dataset_name)
                except Exception as e:
                    if isinstance(e, PennsieveDatasetCannotBeFound):
                        generate_new_ds_ps(soda, dataset_name, ps)
                        return
                    else:
                        raise Exception(f"{e.status_code}, {e.message}")
                myds = get_dataset_with_backoff(selected_dataset_id)
            
                ps_upload_to_dataset(soda, ps, myds, resume)

                        
                


def validate_dataset_structure(soda, resume):

    global main_curate_status
    global main_curate_progress_message
    global logger

    # 1] Check for potential errors
    logger.info("main_curate_function step 1")

    if not generate_options_set(soda):
        main_curate_status = "Done"
        raise GenerateOptionsNotSet()

    # 1.1. If the dataset is being generated locally then check that the local destination is valid
    if generating_locally(soda): 
        main_curate_progress_message = "Checking that the local destination selected for generating your dataset is valid"
        try: 
            validate_local_dataset_generate_path(soda)
        except Exception as e:
            main_curate_status = "Done"
            raise e
        

    logger.info("main_curate_function step 1.2")

    # 1.2. If generating dataset to Pennsieve or any other Pennsieve actions are requested check that the destination is valid
    if uploading_with_ps_account(soda):
        # check that the Pennsieve account is valid
        try: 
            main_curate_progress_message = (
                "Checking that the selected Pennsieve account is valid"
            )
            accountname = soda["ps-account-selected"]["account-name"]
            connect_pennsieve_client(accountname)
        except Exception as e:
            main_curate_status = "Done"
            if isinstance(e, AttributeError):
                raise Exception("The Pennsieve Agent cannot access datasets but needs to in order to work. Please try again. If the issue persists, please contact the SODA team. The SODA team will contact Pennsieve to help resolve this issue.")
            else:
                raise PennsieveAccountInvalid("Please select a valid Pennsieve account.")

    if uploading_to_existing_ps_dataset(soda):
        # check that the Pennsieve dataset is valid
        try:
            main_curate_progress_message = (
                "Checking that the selected Pennsieve dataset is valid"
            )
            bfdataset = soda["ps-dataset-selected"]["dataset-name"]
            selected_dataset_id = get_dataset_id(bfdataset)

        except Exception as e:
            main_curate_status = "Done"
            bfdataset = soda["ps-dataset-selected"]["dataset-name"]
            raise PennsieveDatasetCannotBeFound(bfdataset)

        # check that the user has permissions for uploading and modifying the dataset
        main_curate_progress_message = "Checking that you have required permissions for modifying the selected dataset"
        role = pennsieve_get_current_user_permissions(selected_dataset_id, get_access_token())["role"]
        if role not in ["owner", "manager", "editor"]:
            main_curate_status = "Done"
            raise PennsieveActionNoPermission("uploading to Pennsieve dataset")

    logger.info("main_curate_function step 1.3")


    # 1.3. Check that specified dataset files and folders are valid (existing path) if generate dataset is requested
    # Note: Empty folders and 0 kb files will be removed without warning (a warning will be provided on the front end before starting the curate process)
    # Check at least one file or folder are added to the dataset
    main_curate_progress_message = "Checking that the dataset is not empty"
    if virtual_dataset_empty(soda):
        main_curate_status = "Done" 
        if "generate-options" in soda.keys():
            dataset_name = soda["generate-options"]["dataset-name"]
        elif "ps-dataset-selected" in soda.keys():
            dataset_name = soda["ps-dataset-selected"]["dataset-name"]
        else:
            dataset_name = "Name not set"
        raise EmptyDatasetError(dataset_name)


    logger.info("main_curate_function step 1.3.1")

    # Check that local files/folders exist
    if error := check_local_dataset_files_validity(soda):
        main_curate_status = "Done"
        raise LocalDatasetMissingSpecifiedFiles(error)


    # check that dataset is not empty after removing all the empty files and folders
    if virtual_dataset_empty(soda):
        main_curate_status = "Done"
        if "generate-options" in soda.keys():
            dataset_name = soda["generate-options"]["dataset-name"]
        elif "ps-dataset-selected" in soda.keys():
            dataset_name = soda["ps-dataset-selected"]["dataset-name"]
        else:
            dataset_name = "Name not set"
        raise EmptyDatasetError(dataset_name, "The dataset is empty after removing all the empty files and folders.")


    logger.info("main_curate_function step 1.3.2")
    # Check that bf files/folders exist (Only used for when generating from an existing Pennsieve dataset)
    if uploading_to_existing_ps_dataset(soda) and can_resume_prior_upload(resume) == False:                     
        try:
            main_curate_progress_message = (
                "Checking that the Pennsieve files and folders are valid"
            )
            if soda["generate-dataset"]["destination"] == "ps":
                if error := ps_check_dataset_files_validity(soda):
                    logger.info("Failed to validate dataset files")
                    logger.info(error)
                    main_curate_status = "Done"
                    raise PennsieveDatasetFilesInvalid(error)
        except Exception as e:
            main_curate_status = "Done"
            raise e



def reset_upload_session_environment(resume):
    global main_curate_status
    global main_curate_progress_message
    global main_total_generate_dataset_size
    global main_generated_dataset_size
    global start_generate
    global generate_start_time
    global main_generate_destination
    global main_initial_bfdataset_size
    global main_curation_uploaded_files
    global uploaded_folder_counter
    global ums

    global myds
    global generated_dataset_id
    global bytes_file_path_dict
    global renaming_files_flow

    start_generate = 0
    myds = ""

    generate_start_time = time.time()

    # variables for tracking the progress of the curate process on the frontend 
    main_curate_status = ""
    main_curate_progress_message = "Starting..."
    main_total_generate_dataset_size = 0
    main_generated_dataset_size = 0
    main_curation_uploaded_files = 0
    uploaded_folder_counter = 0
    generated_dataset_id = None

    main_curate_status = "Curating"
    main_curate_progress_message = "Starting dataset curation"
    main_generate_destination = ""
    main_initial_bfdataset_size = 0

    if not resume:
        ums.set_df_mid(None)
        ums.set_elapsed_time(None)
        ums.set_total_files_to_upload(0)
        ums.set_main_total_generate_dataset_size(0)
        # reset the rename information back to default
        ums.set_renaming_files_flow(False) # this determines if we failed while renaming files after the upload is complete
        ums.set_rename_total_files(None)
        ums.set_list_of_files_to_rename(None)
        renaming_files_flow = False
        # reset the calculated values for the upload session
        bytes_file_path_dict = {}




def main_curate_function(soda, resume):
    global logger
    global main_curate_status
    global manifest_id 
    global origin_manifest_id
    global total_files

    logger.info("Starting generating selected dataset")
    logger.info(f"Generating dataset metadata generate-options={soda['generate-dataset']}")


    reset_upload_session_environment(resume)


    validate_dataset_structure(soda, resume)

    logger.info("Generating dataset step 3")


    # 2] Generate
    main_curate_progress_message = "Generating dataset"
    try:
        if (soda["generate-dataset"]["destination"] == "local"):
            logger.info("main_curate_function generating locally")
            generate_dataset(soda, resume, ps=None)
        else:
            logger.info("main_curate_function generating on Pennsieve")
            accountname = soda["ps-account-selected"]["account-name"]
            ps = connect_pennsieve_client(accountname)
            generate_dataset(soda, resume, ps)
    except Exception as e:
        main_curate_status = "Done"
        raise e

    main_curate_status = "Done"
    main_curate_progress_message = "Success: COMPLETED!"


    logger.info(f"Finished generating dataset")
    return {
        "main_curate_progress_message": main_curate_progress_message,
        "main_total_generate_dataset_size": main_total_generate_dataset_size,
        "main_curation_uploaded_files": main_curation_uploaded_files,
        "local_manifest_id": manifest_id,
        "origin_manifest_id": origin_manifest_id,
        "main_curation_total_files": total_files,
    }



def main_curate_function_progress():
    """
    Function frequently called by front end to help keep track of the dataset generation progress
    """

    global main_curate_status  # empty if curate on going, "Done" when main curate function stopped (error or completed)
    global main_curate_progress_message
    global main_total_generate_dataset_size
    global main_generated_dataset_size
    global start_generate
    global generate_start_time
    global main_generate_destination
    global main_initial_bfdataset_size
    global main_curation_uploaded_files
    global total_bytes_uploaded # current number of bytes uploaded to Pennsieve in the upload session
    global myds
    global renaming_files_flow
    global ums 
    global elapsed_time


    prior_elapsed_time = ums.get_elapsed_time()
    if prior_elapsed_time is not None: 
        elapsed_time =  ( time.time() - generate_start_time ) + prior_elapsed_time
    else:
        elapsed_time = time.time() - generate_start_time

    elapsed_time_formatted = time_format(elapsed_time)


    if renaming_files_flow:
        testing_variable = main_generated_dataset_size
    else:
        testing_variable = total_bytes_uploaded["value"]

    return {
        "main_curate_status": main_curate_status,
        "start_generate": start_generate,
        "main_curate_progress_message": main_curate_progress_message,
        "main_total_generate_dataset_size": main_total_generate_dataset_size,
        "main_generated_dataset_size": testing_variable,
        "elapsed_time_formatted": elapsed_time_formatted,
        "total_files_uploaded": main_curation_uploaded_files,
        "generated_dataset_id": myds["content"]["id"] if myds != "" else None, # when a new dataset gets generated log its id to our analytics
        "generated_dataset_int_id": myds["content"]["intId"] if myds != "" else None,
    }


def preview_dataset(soda):
    """
    Associated with 'Preview' button in the SODA interface
    Creates a folder for preview and adds mock files based on the files specified in the UI by the user (same name as origin but 0 kb in size)
    Opens the dialog box to showcase the files / folders added

    Args:
        soda: soda dict with information about all specified files and folders
    Action:
        Opens the dialog box at preview_path
    Returns:
        preview_path: path of the folder where the preview files are located
    """

    preview_path = join(userpath, "SODA", "Preview_dataset")

    # remove empty files and folders from dataset
    try:
        check_empty_files_folders(soda)
    except Exception as e:
        raise e

    # create Preview_dataset folder
    try:
        if isdir(preview_path):
            shutil.rmtree(preview_path, ignore_errors=True)
        makedirs(preview_path)
    except Exception as e:
        raise e

    try:

        if "dataset-structure" in soda.keys():
            # create folder structure
            def recursive_create_mock_folder_structure(my_folder, my_folderpath):
                if "folders" in my_folder.keys():
                    for folder_key, folder in my_folder["folders"].items():
                        folderpath = join(my_folderpath, folder_key)
                        if not isdir(folderpath):
                            mkdir(folderpath)
                        recursive_create_mock_folder_structure(folder, folderpath)

                if "files" in my_folder.keys():
                    for file_key, file in my_folder["files"].items():
                        if "deleted" not in file["action"]:
                            open(join(my_folderpath, file_key), "a").close()

            dataset_structure = soda["dataset-structure"]
            folderpath = preview_path
            recursive_create_mock_folder_structure(dataset_structure, folderpath)

            if "manifest-files" in soda.keys() and "folders" in dataset_structure.keys():
                for folder_key, folder in dataset_structure["folders"].items():
                    manifest_path = join(preview_path, folder_key, "manifest.xlsx")
                    if not isfile(manifest_path):
                        open(manifest_path, "a").close()

        if "metadata-files" in soda.keys():
            for metadata_key in soda["metadata-files"].keys():
                open(join(preview_path, metadata_key), "a").close()

        if len(listdir(preview_path)) > 0:
            folder_in_preview = listdir(preview_path)[0]
            open_file(join(preview_path, folder_in_preview))
        else:
            open_file(preview_path)

        return preview_path

    except Exception as e:
        raise e


def generate_manifest_file_locally(generate_purpose, soda):
    """
    Function to generate manifest files locally
    """


    global manifest_folder_path

    def recursive_item_path_create(folder, path):
        if "files" in folder.keys():
            for item in list(folder["files"]):
                if "folderpath" not in folder["files"][item]:
                    folder["files"][item]["folderpath"] = path[:]

        if "folders" in folder.keys():
            for item in list(folder["folders"]):
                if "folderpath" not in folder["folders"][item]:
                    folder["folders"][item]["folderpath"] = path[:]
                    folder["folders"][item]["folderpath"].append(item)
                recursive_item_path_create(
                    folder["folders"][item], folder["folders"][item]["folderpath"][:]
                )

        return

    def copytree(src, dst, symlinks=False, ignore=None):
        for item in os.listdir(src):
            s = os.path.join(src, item)
            d = os.path.join(dst, item)
            if os.path.isdir(s):
                if os.path.exists(d):
                    shutil.rmtree(d)
                shutil.copytree(s, d, symlinks, ignore)
            else:
                shutil.copy2(s, d)

    dataset_structure = soda["dataset-structure"]
    manifest_destination = soda["manifest-files"]["local-destination"]

    recursive_item_path_create(dataset_structure, [])
    create_high_lvl_manifest_files_existing_ps_starting_point(soda, manifest_folder_path)

    if generate_purpose == "edit-manifest":
        manifest_destination = os.path.join(manifest_destination, "manifest_file")

    else:
        manifest_destination = return_new_path(
            os.path.join(manifest_destination, "manifest_file")
        )

        copytree(manifest_folder_path, manifest_destination)



    if generate_purpose == "edit-manifest":
        return {"success_message_or_manifest_destination": manifest_destination}

    open_file(manifest_destination)
    return {"success_message_or_manifest_destination": "success"}



def generate_manifest_file_data(dataset_structure):
    # Define common file extensions with special handling
    double_extensions = {
        ".ome.tiff", ".ome.tif", ".ome.tf2", ".ome.tf8", ".ome.btf", ".ome.xml",
        ".brukertiff.gz", ".mefd.gz", ".moberg.gz", ".nii.gz", ".mgh.gz", ".tar.gz", ".bcl.gz"
    }

    # Helper function: Get the complete file extension
    def get_file_extension(filename):
        for ext in double_extensions:
            if filename.endswith(ext):
                base_ext = os.path.splitext(os.path.splitext(filename)[0])[1]
                return base_ext + ext
        return os.path.splitext(filename)[1]
    
    def create_folder_entry(folder_name, path_parts):
        full_path = "/".join(path_parts + [folder_name]) + "/"
        entry = [
            full_path.lstrip("/"),  # Remove leading slash for consistency
            "", # Timestamp
            "", # Description
            "folder", # File type
            "",  # Entity (empty)
            "",  # Data modality (empty)
            "",  # Also in dataset (empty)
            "",  # Data dictionary path (empty)
            "",  # Entity is transitive (empty)
            "", # Additional Metadata
        ]
        return entry

        

    # Helper function: Build a single manifest entry
    def create_file_entry(item, folder, path_parts, timestamp, filename):
        full_path = "/".join(path_parts + [filename])
        file_info = folder["files"][item]

        entry = [
            full_path.lstrip("/"),  # Remove leading slash for consistency
            timestamp, # Timestamp
            file_info["description"], # Description
            get_file_extension(filename), # File type
            "",  # Entity (empty)
            "",  # Data modality (empty)
            "",  # Also in dataset (empty)
            "",  # Data dictionary path (empty)
            "",  # Entity is transitive (empty)
            file_info.get("additional-metadata", "") # Additional Metadata
        ]

        # Add any extra columns dynamically
        if "extra_columns" in file_info:
            for key, value in file_info["extra_columns"].items():
                entry.append(value)
                if key not in header_row:
                    header_row.append(key)

        return entry

    # Recursive function: Traverse dataset and collect file data
    def traverse_folders(folder, path_parts):
        # Add header row if processing files for the first time
        if not manifest_data:
            manifest_data.append(header_row)
        
        if "files" in folder:
            for item, file_info in folder["files"].items():

                if "path" in file_info:
                    file_path = file_info["path"]
                elif "pspath" in file_info:
                    file_path = file_info["pspath"]
                else: 
                    continue 

                # If the file is a manifest file, skip it
                if item in {"manifest.xlsx", "manifest.csv"}:
                    continue

                # Determine timestamp 
                filename = os.path.basename(file_path.replace("\\", "/"))
                logger.info(f"Processing file: {filename}")
                if file_info["location"] == "ps":
                    timestamp = file_info["timestamp"]
                else:
                    local_path = pathlib.Path(file_info["path"])
                    timestamp = datetime.fromtimestamp(
                        local_path.stat().st_mtime, tz=local_timezone
                    ).isoformat().replace(".", ",").replace("+00:00", "Z")

                # Add file entry
                manifest_data.append(create_file_entry(item, folder, path_parts, timestamp, filename))

        if "folders" in folder:
            for subfolder_name, subfolder in folder["folders"].items():
                # Add folder entry
                manifest_data.append(create_folder_entry(subfolder_name, path_parts))
                traverse_folders(subfolder, path_parts + [subfolder_name])

    # Initialize variables
    manifest_data = []  # Collects all rows for the manifest
    # TODO: Update to SDS 3.0
    header_row = [
        "filename", "timestamp", "description", "file type", "entity",
        "data modality", "also in dataset", "data dictionary path",
        "entity is transitive", "Additional Metadata"
    ]
    local_timezone = TZLOCAL()

    # Log the dataset structure
    logger.info("Generating manifest file data")
    logger.info(dataset_structure)

    # Start recursive traversal from the root
    traverse_folders(dataset_structure, [])

    return manifest_data