def read_json_file(file_path):
    """
    Reads JSON data from a file and returns it as a dictionary.
    """
    import json
    import os
    if os.path.exists(file_path):
        with open(file_path, 'r') as file:
            data = json.load(file)
    else:
        with open(file_path, 'w') as f:
            f.write('{}')
        data = {}
    return data


def write_json_file(file_path, data):
    """
    Writes a dictionary to a JSON file.
    """
    import json
    with open(file_path, 'w') as file:
        json.dump(data, file, indent=4)


def create_folder(folder_name):
    import os
    if not os.path.exists(folder_name):
        os.makedirs(folder_name)


def remove_folder(folder_name):
    import os
    import shutil
    if os.path.exists(folder_name):
        shutil.rmtree(folder_name)


def base_name_no_ext(file_path):
    import os
    base_name = os.path.basename(file_path)
    base_name_without_extension, _ = os.path.splitext(base_name)
    return base_name_without_extension


def convert_name_to_filename(name):
    import zhmiscellany.string
    return zhmiscellany.string.multi_replace(name, [("/","["), (":","]"), (".","+")])


def convert_filename_to_name(filename):
    import zhmiscellany.string
    return zhmiscellany.string.multi_replace(filename, [("[","/"), ("]",":"), ("+",".")])


def recursive_copy_files(source_dir, destination_dir, prints=False):
    import os
    import shutil
    if prints:
        print('Validating matching directory structure')
    for root, dirs, files in os.walk(source_dir):
        for dir in dirs:
            dir_path = os.path.join(root, dir)
            dest_dir_path = os.path.join(destination_dir, os.path.relpath(dir_path, source_dir))
            if not os.path.exists(dest_dir_path):
                print(f'Creating missing directory {dest_dir_path}')
                os.makedirs(dest_dir_path)
    if prints:
        print('Getting a list of files in the source directory')
    source_files = []
    for root, _, files in os.walk(source_dir):
        for file in files:
            source_files.append(os.path.join(root, file))
    if prints:
        print('Getting a list of files in the destination directory')
    dest_files = []
    for root, _, files in os.walk(destination_dir):
        for file in files:
            dest_files.append(os.path.join(root, file))
    if prints:
        print('Copying files from source to destination, skipping duplicates')
    for root, dirs, files in os.walk(source_dir):
        for file in files:
            source_file = os.path.join(root, file)
            rel_path = os.path.relpath(source_file, source_dir)
            dest_file = os.path.join(destination_dir, rel_path)
            if not os.path.exists(dest_file):
                if prints:
                    print(f'Copying {source_file}')
                shutil.copy2(source_file, dest_file)
            elif os.path.getmtime(source_file) != os.path.getmtime(dest_file):
                if prints:
                    print(f'Copying {source_file}')
                shutil.copy2(source_file, dest_file)


def empty_directory(directory_path):
    import os
    import shutil
    # Iterate over all items in the directory
    for item in os.listdir(directory_path):
        item_path = os.path.join(directory_path, item)
        if os.path.isfile(item_path):
            # If it's a file, delete it
            os.unlink(item_path)
        elif os.path.isdir(item_path):
            # If it's a directory, delete it recursively
            shutil.rmtree(item_path)


def abs_listdir(path):
    import os
    abs_directory_path = os.path.abspath(path)

    # Verify the path exists and is a directory
    if not os.path.exists(abs_directory_path):
        raise FileNotFoundError(f"Directory not found: {abs_directory_path}")

    if not os.path.isdir(abs_directory_path):
        raise NotADirectoryError(f"Path is not a directory: {abs_directory_path}")

    # Get all items in the directory
    items = os.listdir(abs_directory_path)

    # Create absolute paths by joining the directory path with each item
    absolute_paths = []
    for item in items:
        absolute_path = os.path.join(abs_directory_path, item)
        absolute_paths.append(absolute_path)

    return absolute_paths


def delete_ends_with(directory, string_endswith, avoid=[]):
    import os
    files = abs_listdir(directory)
    for file in files:
        if file.endswith(string_endswith):
            if not file in avoid:
                os.remove(file)


def read_bytes_section(file_path, section_start, section_end):
    with open(file_path, 'rb') as file:
        file.seek(section_start)  # Move the file pointer to the 'start' position
        bytes_to_read = section_end - section_start
        data = file.read(bytes_to_read)  # Read 'bytes_to_read' number of bytes
    return data


def copy_file_with_overwrite(src, dst):
    import os
    import shutil
    if os.path.exists(dst):
        os.remove(dst)
    shutil.copy2(src, dst)


def fast_dill_dumps(object):
    import pickle
    import dill
    try:
        data = pickle.dumps(object, protocol=5)  # pickle is much faster so at least attempt to use it at first
    except:
        data = dill.dumps(object, protocol=5)
    return data


def fast_dill_loads(data):
    import pickle
    import dill
    try:
        object = pickle.loads(data)  # pickle is much faster so at least attempt to use it at first
    except:
        object = dill.loads(data)
    return object


zstd_comp = None
zstd_decomp = None

def _get_std_objects():
    global zstd_comp, zstd_decomp
    if zstd_comp is None or zstd_decomp is None:
        import zstandard as zstd
        zstd_comp = zstd.ZstdCompressor(level=4)
        zstd_decomp = zstd.ZstdDecompressor()
    return zstd_comp, zstd_decomp


def save_object_to_file(object, file_name, compressed=False):
    zstd_comp, zstd_decomp = _get_std_objects()
    with open(file_name, 'wb') as f:
        if compressed:
            f.write(zstd_comp.compress(fast_dill_dumps(object)))
        else:
            f.write(fast_dill_dumps(object))


def load_object_from_file(file_name, compressed=False):
    zstd_comp, zstd_decomp = _get_std_objects()
    with open(file_name, 'rb') as f:
        if compressed:
            return fast_dill_loads(zstd_decomp.decompress(f.read()))
        else:
            return fast_dill_loads(f.read())


def pickle_and_encode(obj):
    zstd_comp, zstd_decomp = _get_std_objects()
    """Pickles an object and URL-safe encodes it."""
    import base64
    pickled_data = zstd_comp.compress(fast_dill_dumps(obj))  # Serialize the object
    encoded_data = base64.urlsafe_b64encode(pickled_data).decode()  # Base64 encode
    return encoded_data


def decode_and_unpickle(encoded_str):
    zstd_comp, zstd_decomp = _get_std_objects()
    """Decodes a URL-safe encoded string and unpickles the object."""
    import base64
    pickled_data = base64.urlsafe_b64decode(encoded_str)  # Decode from Base64
    obj = fast_dill_loads(zstd_decomp.decompress(pickled_data))  # Deserialize
    return obj


def list_files_by_modified_time(directory):
    import os
    files_with_times = [(file, os.path.getmtime(file)) for file in abs_listdir(directory) if os.path.isfile(os.path.join(directory, file))]
    sorted_files = sorted(files_with_times, key=lambda x: x[1], reverse=True)
    sorted_file_names = [file for file, _ in sorted_files]
    return sorted_file_names


def get_script_path():
    """Returns the path to the current script or executable."""
    import sys
    if getattr(sys, 'frozen', False):
        # Running as a standalone executable
        return sys.executable
    else:
        # Running as a Python script
        return sys.argv[0]


def chdir_to_script_dir():
    import os
    os.chdir(os.path.dirname(get_script_path()))


def cache(function, *args, _cache_compressed=False, **kwargs):
    """
    Caches the result of a function call to disk.
    """
    import os
    import inspect
    import orjson
    import hashlib
    from datetime import datetime
    import zhmiscellany.fileio

    cache_folder = 'zhmiscellany_cache'

    def normalize_for_json(obj):
        """Recursively normalize objects to be JSON-serializable."""
        if callable(obj):
            try:
                return ('__callable__', inspect.getsource(obj))
            except (OSError, TypeError):
                return ('__callable__', str(obj))

        # Handle dict-like objects (including bidict, defaultdict, etc.)
        if isinstance(obj, dict):
            # Convert non-string keys to strings
            return {str(k): normalize_for_json(v) for k, v in obj.items()}

        # Handle lists and tuples
        if isinstance(obj, (list, tuple)):
            return type(obj)(normalize_for_json(item) for item in obj)

        # Handle sets
        if isinstance(obj, set):
            return sorted([normalize_for_json(item) for item in obj])

        # Handle pandas DataFrames - OPTIMIZED FOR HASHING
        if hasattr(obj, '__class__') and obj.__class__.__name__ == 'DataFrame':
            try:
                import pandas as pd
                if isinstance(obj, pd.DataFrame):
                    # For hashing purposes, use a much faster representation
                    # We use the pandas internal hash which is very fast
                    import pandas.util as pd_util

                    # Fast hash based on shape, columns, dtypes (as a single string), and a sample of data
                    return {
                        '__type__': 'DataFrame',
                        'shape': obj.shape,
                        'columns': list(obj.columns),
                        'dtypes_str': str(obj.dtypes.to_dict()),  # Single conversion instead of dict comp
                        'index_name': obj.index.name,
                        # Use pandas' built-in hash on a sample for speed
                        'hash': str(pd_util.hash_pandas_object(obj.iloc[:min(100, len(obj))]).sum())
                    }
            except ImportError:
                pass

        # Handle pandas Series - OPTIMIZED
        if hasattr(obj, '__class__') and obj.__class__.__name__ == 'Series':
            try:
                import pandas as pd
                if isinstance(obj, pd.Series):
                    import pandas.util as pd_util
                    return {
                        '__type__': 'Series',
                        'dtype': str(obj.dtype),
                        'name': obj.name,
                        'shape': obj.shape,
                        'hash': str(pd_util.hash_pandas_object(obj.iloc[:min(100, len(obj))]).sum())
                    }
            except ImportError:
                pass

        # Handle bytes
        if isinstance(obj, bytes):
            return ('__bytes__', obj.hex())

        # Handle datetime
        if isinstance(obj, datetime):
            return ('__datetime__', obj.isoformat())

        # Handle custom objects with __dict__
        if hasattr(obj, '__dict__') and not isinstance(obj, type):
            return {f'__{obj.__class__.__name__}__': normalize_for_json(obj.__dict__)}

        # Return primitives as-is
        return obj

    def get_hash_orjson(data):
        # Pre-process the data to handle non-string keys and other issues
        normalized_data = normalize_for_json(data)

        json_bytes = orjson.dumps(
            normalized_data,
            option=orjson.OPT_SORT_KEYS
        )
        return hashlib.md5(json_bytes).hexdigest()

    seed = {
        'function': function,
        'args': args,
        'kwargs': kwargs,
        'compressed': _cache_compressed
    }

    seed_hash = get_hash_orjson(seed)

    cache_file = f'{cache_folder}/cache_{function.__name__}_{seed_hash}.pkl'

    if os.path.exists(cache_file):
        return load_object_from_file(cache_file, compressed=_cache_compressed)
    else:
        result = function(*args, **kwargs)
        zhmiscellany.fileio.create_folder(cache_folder)
        save_object_to_file(result, cache_file, compressed=_cache_compressed)
        return result


def load_all_cached():
    """
    Loads all cached objects from the cache folder.
    """
    import os
    cache_folder = 'zhmiscellany_cache'
    if os.path.exists(cache_folder):
        files = abs_listdir(cache_folder)
        files = [file for file in files if 'fn_cache_' in file]
        if files:
            return [load_object_from_file(file) for file in files]
        else:
            raise Exception('Nothing has been cached yet')
    else:
        raise Exception('Nothing has been cached yet')


def list_files_recursive(folder):
    """
    Recursively lists all files in a directory, excluding symlinks and junctions.
    """
    import os
    from ._fileio_supportfuncs import is_junction
    files = []
    try:
        for entry in os.scandir(folder):
            if entry.is_file():
                files.append(entry.path)
            elif entry.is_symlink() or is_junction(entry):
                continue
            elif entry.is_dir():
                files.extend(list_files_recursive(entry.path))
    except (PermissionError, FileNotFoundError):
        pass
    return files


def list_files_recursive_multiprocessed(dir_path, return_folders=False):
    import os
    import zhmiscellany.processing

    def is_junction(entry):
        try:
            st = entry.stat(follow_symlinks=False)
            # On Windows, st_file_attributes is available.
            # FILE_ATTRIBUTE_REPARSE_POINT (0x400) indicates a reparse point (e.g. junction).
            return hasattr(st, "st_file_attributes") and bool(st.st_file_attributes & 0x400)
        except Exception:
            return False
    
    def traversal(dir_path, depth):
        depth += 1
        files = []
        folders = []
        tasks = []
        try:
            for entry in os.scandir(dir_path):
                if entry.is_file():
                    files.append(entry.path)
                elif entry.is_symlink() or is_junction(entry):
                    continue
                elif entry.is_dir():
                    folders.append(entry.path)
                    if depth > max_python_depth:
                        tasks.append((traversal, (entry.path, -99999)))
                    else:
                        new_files, new_folders, new_tasks = traversal(entry.path, depth)
                        files.extend(new_files)
                        folders.extend(new_folders)
                        tasks.extend(new_tasks)
        except (PermissionError, FileNotFoundError):
            pass
        return (files, folders, tasks)
    
    max_python_depth = 1
    files, folders, tasks = traversal(dir_path, 0)
    file_groups = zhmiscellany.processing.batch_multiprocess(tasks)
    for group in file_groups:
        files.extend(group[0])
        folders.extend(group[1])
    if return_folders:
        return files, folders
    else:
        return files


def encode_safe_filename(s, max_length=16):
    """Encodes a string into a short, URL-safe, and file name-safe string."""
    import base64
    import hashlib
    encoded = base64.urlsafe_b64encode(s.encode()).decode().rstrip("=")  # URL-safe encoding
    if len(encoded) > max_length:  # Truncate if too long
        encoded = hashlib.md5(s.encode()).hexdigest()[:max_length]  # Use a hash
    return encoded


def list_files_recursive_cache_optimised_multiprocessed(dir_path, show_timings=False, cache_in_temp=True):
    import os
    import zhmiscellany.processing
    import zhmiscellany.fileio
    import tempfile
    from collections import defaultdict
    import random
    from itertools import chain
    import zhmiscellany.misc

    def is_junction(entry):
        try:
            st = entry.stat(follow_symlinks=False)
            # On Windows, st_file_attributes is available.
            # FILE_ATTRIBUTE_REPARSE_POINT (0x400) indicates a reparse point (e.g. junction).
            return hasattr(st, "st_file_attributes") and bool(st.st_file_attributes & 0x400)
        except Exception:
            return False
    
    def traversal(dir_path, depth=0):
        depth += 1
        files = defaultdict(list)
        folders = []
        tasks = []
        try:
            for entry in os.scandir(dir_path):
                if entry.is_file():
                    files[dir_path].append(entry.path)
                elif entry.is_symlink() or is_junction(entry):
                    continue
                elif entry.is_dir():
                    folders.append(entry.path)
                    if depth > max_python_depth:
                        tasks.append((traversal, (entry.path, -99999)))
                    else:
                        new_files, new_folders, new_tasks = traversal(entry.path, depth)
                        files.update(new_files)
                        folders.extend(new_folders)
                        tasks.extend(new_tasks)
        except (PermissionError, FileNotFoundError):
            pass
        return (files, folders, tasks)
    
    def list_folder(folder):
        files, folders = defaultdict(list), []
        try:
            for entry in os.scandir(folder):
                if entry.is_file():
                    files[folder].append(entry.path)
                elif entry.is_symlink() or is_junction(entry):
                    continue
                elif entry.is_dir():
                    folders.append(entry.path)
        except (PermissionError, FileNotFoundError):
            pass
        return files, folders
    
    def split_into_n_groups(lst, n):
        avg_size = len(lst) // n
        remainder = len(lst) % n
        sublists = []
        
        start = 0
        for i in range(n):
            end = start + avg_size + (1 if i < remainder else 0)  # Distribute remainder
            sublists.append(lst[start:end])
            start = end
        sublists = [sublist for sublist in sublists if sublist]
        return sublists
    
    def get_m_times(folders):
        groups = split_into_n_groups(folders, scan_mtime_worker_count)
        
        def atom(folders):
            mtimes = {}
            for folder in folders:
                try:
                    mtimes[folder] = os.path.getmtime(folder)
                except:
                    pass
            return mtimes
        
        tasks = [(atom, (group,)) for group in groups]
        results = zhmiscellany.processing.batch_multiprocess(tasks)
        mtimes = {}
        for i in results:
            mtimes.update(i)
        return mtimes
    
    # parameters
    scan_mtime_worker_count = 8
    scan_changed_folders_thread_group_count = 64
    fully_update_cache_threshold = 2**10
    cache_compression = False
    # end parameters
    
    if cache_in_temp:
        cache_folder = tempfile.gettempdir()
    else:
        cache_folder = 'zhmiscellany_cache'
        zhmiscellany.fileio.create_folder(cache_folder)
    
    cache_id = encode_safe_filename(dir_path)
    cache_file = f'GFI_{cache_id}.pkl'
    cache_file = os.path.join(cache_folder, cache_file)
    
    if show_timings: zhmiscellany.misc.time_it(None, 'lfrcom')
    if show_timings: zhmiscellany.misc.time_it(None, 'lfrcomt')
    
    max_python_depth = 1
    if not os.path.exists(cache_file):
        files, folders, tasks = traversal(dir_path)
        if show_timings: zhmiscellany.misc.time_it('initial traversal', 'lfrcom')
        
        file_groups = zhmiscellany.processing.batch_multiprocess(tasks)
        if show_timings: zhmiscellany.misc.time_it('multiprocessed deep traversal', 'lfrcom')
        for group in file_groups:
            files.update(group[0])
            folders.extend(group[1])
        if show_timings: zhmiscellany.misc.time_it('extending data', 'lfrcom')
        
        folders = get_m_times(folders)
        zhmiscellany.fileio.save_object_to_file((files, folders), cache_file, compressed=cache_compression)
        if show_timings: zhmiscellany.misc.time_it('creating cache', 'lfrcom')
        return list(chain.from_iterable(files.values()))
    else:
        files, folders = zhmiscellany.fileio.load_object_from_file(cache_file, compressed=cache_compression)
        if show_timings: zhmiscellany.misc.time_it('loading cache', 'lfrcom')
    
    fl_list = list(folders.keys())
    new_folders = get_m_times(fl_list)
    if show_timings: zhmiscellany.misc.time_it(f'getting m times of {len(fl_list)} folders', 'lfrcom')
    changed_folders = []
    for folder, mtime in new_folders.items():
        if folders[folder] != mtime:
            changed_folders.append(folder)
    random.shuffle(changed_folders)
    if show_timings: zhmiscellany.misc.time_it(f'creating {len(changed_folders)} changed folders', 'lfrcom')
    
    for i in changed_folders:  # clear files that might not exist
        try:
            del files[i]
        except KeyError:  # it is possible that the only thing that changed in a folder is another folder, so in that case it would not be inside the file dict
            pass
    
    if show_timings: zhmiscellany.misc.time_it(f'filtering files for changes', 'lfrcom')
    
    def atom(_folders):
        atom_files, atom_folders = defaultdict(list), []
        for _folder in _folders:
            fil, fol = list_folder(_folder)
            atom_files.update(fil)
            for fold in fol:
                if fold not in new_folders:
                    atom_folders.append(fold)
                    fil, fo, _ = traversal(fold, -99999)
                    atom_files.update(fil)
                    atom_folders.extend(fo)
        return atom_files, atom_folders
    
    groups = split_into_n_groups(changed_folders, scan_changed_folders_thread_group_count)
    tasks = [(atom, (group,)) for group in groups]
    if not tasks:
        results = []
    else:
        results = zhmiscellany.processing.batch_threading(tasks)
    if show_timings: zhmiscellany.misc.time_it('multithreading processing changed folders', 'lfrcom')
    
    new_new_folders = []
    for fi, fo in results:
        files.update(fi)
        new_new_folders.extend(fo)
    
    if len(changed_folders) > fully_update_cache_threshold:
        new_folders.update(get_m_times(new_new_folders))
        if show_timings: zhmiscellany.misc.time_it(f'get m times of {len(new_new_folders)} new folders')
        zhmiscellany.fileio.save_object_to_file((files, new_folders), cache_file)
        if show_timings: zhmiscellany.misc.time_it(f'writing to cache')
    
    ret = list(chain.from_iterable(files.values()))
    if show_timings: zhmiscellany.misc.time_it('Everything together', 'lfrcomt')
    return ret


def save_chunk(name, data):
    import zhmiscellany.string
    create_folder(name)
    chunk_path = f'{name}/chunk_{zhmiscellany.string.get_universally_unique_string()}.pkl'
    save_object_to_file(data, chunk_path)


def load_chunks(name):
    create_folder(name)
    chunks = abs_listdir(name)
    datas = []
    for chunk_file in chunks:
        datas.append(load_object_from_file(chunk_file))
    return datas


def clear_chunks(name):
    import os
    if os.path.exists(name):
        empty_directory(name)


def list_drives():
    import os
    import string
    return [f"{d}:\\" for d in string.ascii_uppercase if os.path.exists(f"{d}:\\")]