"""
Implements the :class:`FileSource` class and its essential helper classes.

The FileSource class lets you easily iterate through large amounts of files
stored locally, on disk, in zip archives, in cloud storages and even zip
archives in cloud storages.
"""

from __future__ import annotations

import io
import json
import os
from abc import abstractmethod
from collections import Counter
from datetime import datetime
from fnmatch import fnmatch
from hashlib import md5
from typing import Callable, Any, TYPE_CHECKING, Union

from pydantic import BaseModel, SecretStr

from filestag.file_source_iterator import (
    FileSourceIterator,
    FileIterationData,
    FilterCallback,
)
from filestag.protocols import is_azure_storage_source
from filestag.file_stag import FileStag

CACHE_VERSION = "cache_version"

if TYPE_CHECKING:
    from filestag.file_sink import FileSink


class FileSourceElement:
    """
    Provides the data of a single file returned from a :class:`FileSource`.
    """

    def __init__(self, data: bytes, name: str):
        """
        :param data: The file's content
        """
        self.data = data
        "Holds the file's data"
        self.filename = name
        "The file's name. Usually relative to its search path"


class FileSourcePathOptions:
    """
    Defines the options for accessing a file's full path.
    """

    def __init__(self, for_file_stag: bool = False):
        """
        :param for_file_stag: Defines if also return values are valid which
            can just be used internally, such as azure:// for Azure resources
            or zip:// for content in zip files. False by default.
        """
        self.for_file_stag = for_file_stag
        """
        Defines if also FileStag-only compatible return paths are allowed
        which can just be used within FileStag via e.g. :meth:`FileStag.load`.
        """


class FileListEntry(BaseModel):
    """
    Defines a single entry in the file list.
    """

    filename: str
    "The file's name"
    file_size: int = 0
    "The file's size in bytes"
    created: datetime = datetime.now()
    "The element's creation time"
    modified: datetime = datetime.now()
    "The element's last modification time"


class FileListModel(BaseModel):
    """
    Defines a list of files storable in a database.
    """

    user_version: int = 1
    """
    The user definable version number. If the version of the stored data does
    not match the one passed, the list is considered being invalid
    """
    format_version: int = 1
    """
    The format version
    """
    files: list[FileListEntry]
    """
    The file data
    """


FileList = list[FileListEntry]


class FileSource:
    """
    Base class for an iterable file source to batch process file lists such as
    directories, zip archives or cloud storages at small and large scale using
    a unified interface.
    """

    def __init__(
        self,
        search_mask: str = "*",
        search_path: str = "",
        recursive: bool = True,
        filter_callback: FilterCallback | None = None,
        index_filter: tuple[int, int] | None = None,
        fetch_file_list: bool = True,
        max_file_count: int = -1,
        file_list_name: str | tuple[str, int] | None = None,
        max_web_cache_age: float = 0.0,
        dont_load: bool = False,
        sorting_callback: Callable[[FileListEntry], Any] | None = None,
        validate_cache: bool = False,
    ):
        """
        For a detailed parameter description see :meth:`from_path`
        """
        self.search_mask = search_mask
        """
        The search mask to match the filenames against before they are returned
        """
        self.search_path = search_path
        "The path to search within, e.g. a file path"
        self.recursive = recursive
        "Defines if the search shall be executed recursive"
        self.filter_callback = filter_callback
        """
        The filter function which will be called for each file to verify if it
        shall be processed
        """
        self.user_data: dict = {}
        "The user data for further customization, e.g. of the filter callback"
        self.index_filter: tuple[int, int] | None = index_filter
        """
        The index filter helps splitting a processing task to multiple,
        threads nodes and/or processes.

        See initializer parameter.
        """
        self._file_list: FileList | None = None
        """
        A sorted list of all to files (if available e.g. by setting
         fetch_file_list=True).

        Note that settings such as :attr:`index_filter` and
        :attr:`max_file_count` have no effect on the file_list by default.
        You can though explicitly call the method :meth:`reduce_file_list` which
        will execute all filters in advance to provide you the final file_list
        and will disable these variable afterwards.
        """
        self._file_set: set | None = None
        """
        A set containing all known files. Only valid if file_list is available
        too
        """
        self.output_filename_list: list[str] | None = None
        """
        If defined it provides the output filenames for every file in
        self.file_list.
        """
        self.max_file_count = max_file_count
        """
        The maximum number of files to process. (excluding the impact of
        :attr:`index_filter`
        """
        self.is_closed = False
        "Defines if this file source was closed"
        self._statistics: dict | None = None
        "The statistics, only available when all files were iterated"
        self.dont_load = dont_load
        """
        If set to true the iterator ``for element in FileSource`` will not
        fetch the file's content but just iterate through its filenames
        """
        self._file_list_name: str | None = None
        "The name of the file from which the file list shall be loaded"
        self._file_list_version: int = -1
        """
        The version of the file list to assume. If it mismatches the
        stored version it will be replaced
        """
        self.sorting_callback = sorting_callback
        if sorting_callback is not None and not fetch_file_list:
            raise ValueError(
                "Sorting is only supported in combination w/ fetch_file_list=True"
            )
        """
        A function to be called (and pass into sorted) to sort the file list
        before it's stored.
        """
        if file_list_name is not None:
            if isinstance(file_list_name, tuple):
                self._file_list_name, self._file_list_version = file_list_name
            else:
                self._file_list_name = file_list_name
        self.max_web_cache_age = max_web_cache_age
        self.validate_cache = validate_cache
        """
        If True and file_list_name is set, validates the cached file list
        by checking if any files have been modified since the cache was created.
        Only effective for sources that support this (e.g., Azure Blob Storage).
        """
        self._cached_latest_modified: str | None = None
        """
        The latest modification timestamp from the cached file list.
        Used for cache validation.
        """
        self._cached_file_count: int | None = None
        """
        The file count from the cached file list.
        Used for cache validation.
        """
        self.download = self.copy
        self.download_all = self.copy_to

    @staticmethod
    def from_source(
        source: str | bytes | SecretStr,
        search_mask: str = "*",
        search_path: str = "",
        recursive: bool = True,
        filter_callback: FilterCallback | None = None,
        sorting_callback: Callable[[FileListEntry], Any] | None = None,
        index_filter: tuple[int, int] | None = None,
        fetch_file_list: bool = True,
        max_file_count: int = -1,
        file_list_name: str | tuple[str, int] | None = None,
        max_web_cache_age: float = 0.0,
        dont_load: bool = False,
        validate_cache: bool = False,
    ) -> "FileSource | None":
        """
        Auto-detects the required FileSource implementation for a given source
        path.

        :param source: The path you would like to iterate. The following path
            types are currently supported:
            * /home/aDirectory: Will return a FileSourceDisk object to iterate
                through a directory's content
            * /home/myZipArchive.zip: Will return a FileSourceZip object to
                iterate through a zip archive
            * azure://DefaultEndpointsProtocol=https;AccountName=...;AccountKey=.../container/path:
                will iterate to an Azure Blob Storage.
            * https://my_storage_name.blob.core.windows.net/my_container_name
                will iterate an Azure Blob Storage provided via SAS url.
            * A bytes object: Detects the source type and opens it. At the
                moment only zip archive data is supported.

            For more secure handling of connection strings with key you can
            also pass a SecretStr.
        :param search_mask: The file name filter mask
        :param search_path: The search path, e.g. directory name or relative
            path to the zip root, storage root etc.
        :param recursive: Defines if the search shall be executed recursive.
            True by default.
        :param filter_callback: A callback function to call for each file to
            verify if it shall be processed or ignored.
            See :const:`FilterCallback`
        :param sorting_callback:
            A function to be called (and pass into sorted) to sort the file list
            before it is stored.

            Is called for every element and has to return the sorting value,
            either a string, float or another size comparable data type.

            Does only work in combination with fetch_file_list.
        :param index_filter: The index filter helps splitting a processing task
            to multiple, threads nodes and/or processes.

            The first tuple element defines the total worker count, the second
            tuple element the current worker index (0 .. worker_count-1).
            If you want to for example process a zip archive by 4 threads in
            parallel just spawn 4 threads and pass (4,0) to the first, (4,1)
            to the second (4,2) to the third and (4,3) to the third.

            All four threads can then work in parallel and store their processed
            data parallel into one or multiple FileSinks which are (at least in
             most cases) multi-thread safe.
        :param fetch_file_list: If set to true the FileSource will try to
            iterate all filenames in advance. True by default.

            This is recommended especially if you are using sources where it's
            not guaranteed that the file names will always be provided in the
            same order and you intend to share a task among multiple threads to
            guarantee a consistent behavior.
        :param file_list_name: If provided the file list be cached in the
            file name passed. This makes working with large directories or
            cloud storages much faster.

            You can either pass a string, just containing the file name or a
            tuple of (filename, version) so you can enforce replacing the list
            when ever you pass a new version number.
        :param max_file_count: The maximum number of files to process (excluding
            the index filter's impact)
        :param max_web_cache_age: The count of seconds for how long files from
            this source may be stored and received from the cache if this
            source is remote, e.g. Azure, AWS.
        :param dont_load: If set to true the iterator will not provide the
            file's content but just iterate the filenames. Helpful if the
            consumer for example requires a path to files stored on disk.
        :param validate_cache: If True and file_list_name is set, validates
            the cached file list by checking if any files have been modified
            since the cache was created. If changes are detected, the cache
            is automatically refreshed. Only effective for sources that
            support this feature (e.g., Azure Blob Storage).
        :return: The FileSource implementation for your path. None if the path
            can not be identified.
        """
        params = {
            "search_mask": search_mask,
            "search_path": search_path,
            "recursive": recursive,
            "filter_callback": filter_callback,
            "index_filter": index_filter,
            "fetch_file_list": fetch_file_list,
            "file_list_name": file_list_name,
            "max_web_cache_age": max_web_cache_age,
            "max_file_count": max_file_count,
            "sorting_callback": sorting_callback,
            "dont_load": dont_load,
            "validate_cache": validate_cache,
        }
        if isinstance(source, SecretStr):
            source = source.get_secret_value()
        if isinstance(source, bytes):
            from filestag.sources.zip import FileSourceZip

            return FileSourceZip(source=source, **params)
        if is_azure_storage_source(source):
            from filestag.azure.source import AzureStorageFileSource

            return AzureStorageFileSource(source=source, **params)
        if not (source.endswith("/") or source.endswith("\\")) and source.endswith(
            ".zip"
        ):
            from filestag.sources.zip import FileSourceZip

            return FileSourceZip(source=source, **params)
        if "://" in source:
            raise NotImplementedError("Unknown protocol for FileSource")
        from filestag.sources.disk import FileSourceDisk

        return FileSourceDisk(path=source, **params)

    def refresh(self) -> None:
        """
        Refreshes the file list.
        """
        if self._file_list is not None:
            del self._file_list
        self._file_list = None
        if self._file_set is not None:
            del self._file_set
        self._file_set = None
        self._create_file_list_int(no_cache=True)

    def get_hash(self, max_content_size: int = 0) -> str:
        """
        Creates a hash of the current file list to detect potential
        modifications of the files. By default this hash is created using
        the file sizes and file time stamps. If desired the hash can
        also be created using the file's content.

        :param max_content_size: The maximum size of a file to compute up to
            which its hash is directly computed from its content.
        :return: The MD5 hash
        """
        stream = io.BytesIO()
        for cur_file in self.file_list:
            stream.write(hash(cur_file.filename).to_bytes(8, "little", signed=True))
            stream.write(cur_file.file_size.to_bytes(8, "little", signed=True))
            stream.write(hash(cur_file.created).to_bytes(8, "little", signed=True))
            stream.write(hash(cur_file.modified).to_bytes(8, "little", signed=True))
            if max_content_size > 0 and cur_file.file_size <= max_content_size:
                file_data = self.fetch(cur_file.filename)
                if file_data:
                    stream.write(md5(file_data).hexdigest().encode("utf-8"))
        return md5(stream.getvalue()).hexdigest()

    def __hash__(self) -> int:
        return hash(self.get_hash(max_content_size=0))

    @abstractmethod
    def _get_source_identifier(self) -> str:
        """
        Has to return a unique identifier for this file source which
        identifies the name of this source in the cache database.

        Can for example be the search path and the search mask or parts of the
        connection string.

        :return: The unique identifier
        """
        return ""

    @property
    def file_list(self) -> FileList | None:
        """
        Returns the file list (if available).

        Note that the file list is not available for all file sources.
        Pass fetch_file_list = true to the initializer of all supported
        FileSources to fetch the list in advance.

        :return: The list of filenames and their size (so far known).
        """
        return self._file_list

    def encode_file_list(self, version: int = -1) -> bytes:
        """
        Encodes the file list so it can be stored on disk.

        :param version: The user defined version number. It can be passed
            to enforce updating the list when ever this number is changed.

            If -1 is passed the version is ignored.
        :return: The encoded file list
        """
        file_list_data = [entry.model_dump(mode="json") for entry in self._file_list]
        # Find latest modification timestamp for cache validation
        latest_modified: str | None = None
        for entry in self._file_list:
            if entry.modified is not None:
                mod_str = entry.modified.isoformat()
                if latest_modified is None or mod_str > latest_modified:
                    latest_modified = mod_str
        data = {
            "format_version": 1,
            CACHE_VERSION: version,
            "latest_modified": latest_modified,
            "file_count": len(self._file_list),
            "files": file_list_data,
        }
        return json.dumps(data).encode("utf-8")

    def load_file_list(self, source: bytes | str, version: int = -1) -> bool:
        """
        Tries to load the file list from file.

        :param source: The file list source. Any FileStag compatible data
            source.
        :param version: The user defined version number. It can be passed
            to enforce updating the list when ever this number is changed.

            If -1 is passed the version is ignored.
        :return: True if a valid list could be loaded.
        """
        if not isinstance(source, bytes):
            source = FileStag.load(source)
        if source is None:
            return False
        try:
            data = json.loads(source.decode("utf-8"))
        except (json.JSONDecodeError, UnicodeDecodeError):
            return False
        if not isinstance(data, dict) or data.get("format_version") != 1:
            return False
        if version != -1 and data.get(CACHE_VERSION, -1) != version:
            return False
        # Store cache metadata for validation
        self._cached_latest_modified = data.get("latest_modified")
        self._cached_file_count = data.get("file_count")
        files = data.get("files", [])
        self.update_file_list(
            [FileListEntry.model_validate(entry) for entry in files],
            may_sort=False,
        )
        return True

    def save_file_list(self, target: str, version: int = -1) -> None:
        """
        Saves the file list to a file so it can be quickly restored after
        a restart of the application.

        :param target: The FileStag compatible file target, e.g. a local file
            name
        :param version: The user defined version number. It can be passed
            to enforce updating the list when ever this number is changed.

            If -1 is passed the version is ignored.
        """
        FileStag.save(target, self.encode_file_list(version=version))

    def set_file_list(self, new_list: list[str] | list[FileListEntry]) -> None:
        """
        Sets a custom file list provided by the user.

        Helpful for large jobs where the total file list is split into
        several working packages in advance and the shares need to be
        customized.

        :param new_list: The new list to be assigned. Either a list of
            "FileListEntry"s with all details or a list of filenames
        """
        if len(new_list) and isinstance(new_list[0], str):
            lst = [
                FileListEntry(filename=element, file_size=-1) for element in new_list
            ]
        else:
            lst = new_list
        self.update_file_list(lst)

    def get_absolute(
        self, filename: str, options: FileSourcePathOptions | None = None
    ) -> str | None:
        """
        Returns the full path to the resource if possible, e.g. the absolute
        path to a file on disk in case of an FileSourceDisk element.

        :param filename: The name of the file of which the full path shall be
            retrieved.
        :param options: The sharing options such as expiration date in case
            of cloud shares.
        :return: The absolute path to the file from which it can be downloaded
            via http or accessed on the disk.
        """
        _ = filename
        __ = options
        ___ = self
        return None

    def get_statistics(self) -> dict | None:
        """
        Returns statistics about the file source if available.

        Requires a valid file list, see :meth:`get_file_list`.

        :return: Dictionary with statistics about file types, total size etc.
        """
        if self._file_list is None:
            return None
        if self._statistics is not None:
            return self._statistics
        file_type_counter: Counter = Counter()
        size_by_filetype: Counter = Counter()
        dir_names: set = set()
        total_size = 0
        for cur_file in self._file_list:
            cur_name = cur_file.filename
            dir_name = os.path.dirname(cur_name)
            extension = os.path.splitext(cur_name)[1]
            dir_names.add(dir_name)
            file_type_counter[extension] += 1
            size_by_filetype[extension] += cur_file.file_size
            total_size += cur_file.file_size
        file_extension_list = sorted(
            list(file_type_counter.keys()),
            reverse=True,
            key=lambda x: file_type_counter[x],
        )
        sorted_keys = sorted(
            file_type_counter.keys(), reverse=True, key=lambda x: file_type_counter[x]
        )
        ext_details = {
            key: {
                "totalFileSizeMb": size_by_filetype[key] / 1000000.0,
                "totalFileCount": file_type_counter[key],
            }
            for key in sorted_keys
        }
        self._statistics = {
            "totalFileCount": len(self._file_list),
            "totalFileSizeMb": total_size / 1000000,
            "totalDirs": len(dir_names),
            "fileExtensions": file_extension_list,
            "extensionDetails": ext_details,
        }
        return self._statistics

    def __str__(self) -> str:
        result = self.__class__.__name__ + "\n"
        statistics = self.get_statistics()
        if statistics is not None:
            result += f"  Total files: {statistics.get('totalFileCount', 0)}\n"
            result += f"  Total size: {statistics.get('totalFileSizeMb', 0):.2f} MB\n"
            result += f"  Directories: {statistics.get('totalDirs', 0)}\n"
        result += f"* search-mask: {self.search_mask}"
        return result

    def __iter__(self) -> FileSourceIterator:
        """
        Returns an iterator object for this file source.
        :return: The iterator
        """
        return FileSourceIterator(self)

    def __enter__(self) -> "FileSource":
        """
        Provides the FileSource context. Allows automated clean closing of
        a file source via

        ``with FileSource.from_path("./my_folder") as source``

        as once the with with-block is left it will automatically call the
        source's :meth:`close` function.

        :return: The FileSource object
        """
        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        """
        Closes the context (and in consequence open connections, archives etc.)
        """
        if not self.is_closed:
            self.close()

    def __len__(self) -> int:
        """
        Returns the count of files of the file source (if fetch_file_list is
        True).

        :return: The count of elements
        """
        return len(self._file_list) if self._file_list is not None else 0

    def __contains__(self, item: str) -> bool:
        """
        Returns true if the file with given name exists.

        :param item: The name of the file
        :return: True if the file exists
        """
        return item in self._file_set if self._file_set else False

    @abstractmethod
    def _read_file_int(self, filename: str) -> bytes | None:
        """
        Reads a file from this file source, identified by name.

        Note: Not all FileSources support direct file access by name, so you
        should always prefer to just iterate through a FileSource object rather
        than accessing single files if your FileSource can be freely configured.
        For example an :class:`ImageFileSource` pointing to a camera can only
        provide its data frame by frame - and thus image file by image file -
        and not by name.

        :param filename: The name of the file to read
        :return: The file's content on success, None otherwise
        """
        return None

    def fetch(self, filename: str) -> bytes | None:
        """
        Reads a file from this file source, identified by name.

        Note: Not all FileSources support direct file access by name, so you
        should always prefer to just iterate through a FileSource object rather
        than accessing single files if your FileSource can be freely configured.
        For example an :class:`ImageFileSource` pointing to a camera can only
        provide its data frame by frame - and thus image file by image file -
        and not by name.

        :param filename: The name of the file to read
        :return: The file's content on success, None otherwise
        """
        from filestag.web import WebCache

        if self.max_web_cache_age != 0:  # try to fetch data if cache is on
            unique_name = self._get_source_identifier() + "/" + filename
            data = WebCache.fetch(unique_name, max_age=self.max_web_cache_age)
            if data is not None:
                return data
        result = self._read_file_int(filename)
        if self.max_web_cache_age != 0 and result is not None:  # store new data if cache is on
            unique_name = self._get_source_identifier() + "/" + filename
            WebCache.store(unique_name, result)
        return result

    def exists(self, filename: str) -> bool:
        """
        Verifies if a file exists.

        Note: This function may not be supported by all sources (such as
        streaming sources)

        :param filename: The file to look for
        :return: True if the file exists
        """
        if self._file_set is not None:
            return filename in self._file_set
        raise NotImplementedError("Missing implementation for exists method")

    def copy(
        self,
        filename: str,
        target_name: str,
        overwrite: bool = True,
        sink: Union["FileSink", None] = None,
        on_fetch: Callable[[str], None] | None = None,
        on_fetch_done: Callable[[str, int], None] | None = None,
        on_stored: Callable[[str, int], None] | None = None,
        on_error: Callable[[str, str], None] | None = None,
        on_skip: Callable[[str], None] | None = None,
    ) -> bool:
        """
        Copies a file from this FileSource to a FileStag compatible target
        path.

        :param filename: The name of the file to be copied
        :param target_name: The target path
        :param overwrite: Defines if the file shall be overwritten if it does
            already exist.
        :param sink: If defined the file will be copied to the specified sink
        :param on_skip: Is called if a file exists and will be skipped
        :param on_stored: Is called when ever a file was successfully uploaded
        :param on_fetch: Is called before a file is downloaded
        :param on_fetch_done: Is called after a file was
            successfully downloaded and will now be uploaded or stored.
        :param on_error: Is called when an error occurred
        :return: True if the file was copied
        """
        if sink is None and not overwrite and FileStag.exists(target_name):
            if on_skip is not None:
                on_skip(target_name)
            return False
        if on_fetch is not None:
            on_fetch(filename)
        data = self.fetch(filename)
        if data is None:
            if on_error is not None:
                on_error(filename, f"Could not load {filename}")
            return False
        if on_fetch_done is not None:
            on_fetch_done(filename, len(data))
        if sink is not None:
            result = sink.store(target_name, data, overwrite=overwrite)
        else:
            result = FileStag.save(target_name, data)
        if result:
            if on_stored is not None:
                on_stored(filename, len(data))
        else:
            if on_error is not None:
                on_error(filename, f"Could not store {filename}")
        return result

    def copy_to(
        self,
        target: Union["FileSink", str],
        error_log: list[str] | None = None,
        overwrite: bool = True,
    ) -> bool:
        """
        Copies the whole content of this FileSource to a defined target which
        can either be a FileSink or a local directory.

        :param target: The target, either a FileSink or a local file path
        :param error_log: If provided all errors will be added to this log
        :param overwrite: Defines if files in the target may be overwritten,
            True by default.
        :return: True if no errors occurred
        """
        if error_log is None:
            error_log = []
        from filestag.file_sink import FileSink

        if isinstance(target, FileSink):
            for cur_file in self:
                if cur_file.data is None:
                    error_log.append(f"Could not load file {cur_file.filename}")
                    continue
                if not target.store(
                    cur_file.filename, cur_file.data, overwrite=overwrite
                ):
                    error_log.append(f"Could not store file {cur_file.filename}")
        else:
            if overwrite or self._file_list is None:
                self._copy_to_local_iterator(target, overwrite, error_log)
            else:
                self._copy_to_local_file_list(target, overwrite, error_log)
        return len(error_log) == 0

    def _copy_to_local_file_list(
        self, target: str, overwrite: bool, error_log: list[str]
    ) -> None:
        """
        Copies to a local directory known the files in advance, so we can
        skip already existing files.

        :param target: The target directory
        :param overwrite: Defines if files may be overwritten
        :param error_log: The error log target list
        """
        for cur_file in self.file_list:
            target_name = target + "/" + cur_file.filename
            rel_path = os.path.dirname(target_name)
            os.makedirs(rel_path, exist_ok=True)
            if FileStag.exists(target_name):
                continue
            cur_file_data = self.fetch(cur_file.filename)
            if cur_file_data is None:
                error_log.append(f"Could not load file {cur_file.filename}")
                continue
            if not FileStag.save(target_name, cur_file_data):
                error_log.append(f"Could not store file {cur_file.filename}")

    def _copy_to_local_iterator(
        self, target: str, overwrite: bool, error_log: list[str]
    ) -> None:
        """
        Copies to a local directory NOT knowing the files, so we have to
        iterate them.

        :param target: The target directory
        :param overwrite: Defines if files may be overwritten
        :param error_log: The error log target list
        """
        for cur_file in self:
            if cur_file.data is None:
                error_log.append(f"Could not load file {cur_file.filename}")
                continue
            target_name = target + "/" + cur_file.filename
            rel_path = os.path.dirname(target_name)
            os.makedirs(rel_path, exist_ok=True)
            if not FileStag.save(target_name, cur_file.data):
                error_log.append(f"Could not store file {cur_file.filename}")

    def update_file_list(
        self, new_list: list[FileListEntry], may_sort: bool = True
    ) -> None:
        """
        Call this function if you want to manually update the file list.

        Updates the internal search index and other helper variables.

        :param new_list: The new list
        :param may_sort: Defines if the list may be sorted
        """
        self._file_list = new_list
        if self.sorting_callback is not None and may_sort:  # apply sorting
            self._file_list = sorted(self._file_list, key=self.sorting_callback)
        self._file_set = {element.filename for element in new_list}
        self._statistics = None

    def reduce_file_list(self) -> list[FileListEntry] | None:
        """
        Reduces the :attr:`file_list` by applying all filters (index_range,
        max_file_count, filter_callback) in advance. Requires the source being
        initialized with fetch_file_list in advance and thus requires a
        non-streaming file source where the full file list is known in advance.

        This way you know in advance which files (after all the filters) are
        really getting processed with your current filtering settings. So the
        filters are not applied twice this function also disables all
        callbacks and filter variables after its execution.

        :return: Returns the reduced file list
        """
        if self._file_list is None:
            return None
        output_filenames = []
        cleaned_list = []
        for index, element in enumerate(self._file_list):
            file_info = FileIterationData(self, index, element=element)
            new_filename = self.handle_skip_check(file_info)
            if new_filename is None:
                continue
            cleaned_list.append(element)
            output_filenames.append(new_filename)
        if self.max_file_count != -1 and len(cleaned_list) > self.max_file_count:
            cleaned_list = cleaned_list[0 : self.max_file_count]
            output_filenames = output_filenames[0 : self.max_file_count]
        self.output_filename_list = output_filenames
        self.update_file_list(cleaned_list)
        self.max_file_count = -1
        self.index_filter = None
        self.filter_callback = None
        return self._file_list

    def handle_next(self, iterator: FileSourceIterator) -> FileSourceElement | None:
        """
        Returns the next available element.

        :param iterator: The iterator object which keeps track of the current
            processing
        :return: The next file object if available
        """
        if (
            self.max_file_count != -1
            and iterator.processed_file_count >= self.max_file_count
        ):
            raise StopIteration
        while True:
            next_entry = self.handle_get_next_entry(iterator)
            if next_entry is None:  # stop if no files are available anymore
                return None
            # was already filtered using reduce_file_list
            if self.output_filename_list is not None:
                target_name = self.output_filename_list[iterator.file_index - 1]
            else:
                target_name = self.handle_skip_check(
                    FileIterationData(self, iterator.file_index - 1, element=next_entry)
                )
            # continue if just the current file is skipped
            if target_name is not None:
                break
        data = self.fetch(next_entry.filename) if not self.dont_load else None
        return self.handle_provide_result(iterator, target_name, data)

    def handle_get_next_entry(
        self, iterator: "FileSourceIterator"
    ) -> FileListEntry | None:
        """
        Returns the FileListEntry containing all information about the next file
        to be processed.

        Overwrite this method for your own, custom File iterator.

        :param iterator: The file iterator object
        :return: All details about the next file to be processed. None if no
            file needs to be processed anymore.
        """
        index = iterator.file_index
        iterator.file_index += 1
        if self._file_list is None:
            return None
        if index >= len(self._file_list):
            return None
        return self._file_list[index]

    def handle_fetch_file_list(self, force: bool = False) -> None:
        """
        Called when the file list shall be pre-fetched.

        If your custom FileSource is able to do so populate the self.file_list
        with a sorted list of all files available and instead of iterating the
        files live always access the matching file list entry using
        self.file_list[file_index] appropriately.

        :param force: Enforce an update of the file list, even if it was created
            before already
        """

    def get_latest_modified_timestamp(self) -> str | None:
        """
        Returns the latest modification timestamp from the source.

        This method can be overridden by subclasses to provide an efficient
        way to check if any files have been modified since the cache was
        created, without fetching the full file list.

        :return: ISO format timestamp of the most recently modified file,
            or None if not supported by this source type.
        """
        return None

    def _create_file_list_int(self, no_cache: bool = False) -> None:
        """
        Creates the file list by either scanning the source directory or
        loading a cache file list from disk.

        If caching is enabled the new list is stored to disk.

        :param no_cache: If defined the file list may not be loaded form disk,
            e.g. when this function is called from :meth:`refresh`.
        """
        loaded = False
        cache_valid = True
        if self._file_list_name is not None and not no_cache:
            loaded = self.load_file_list(
                self._file_list_name, version=self._file_list_version
            )
            # Validate cache freshness if requested
            if loaded and self.validate_cache and self._cached_latest_modified:
                current_latest = self.get_latest_modified_timestamp()
                if current_latest is not None:
                    if current_latest > self._cached_latest_modified:
                        # Source has newer files, invalidate cache
                        loaded = False
                        cache_valid = False
                        self._file_list = None
                        self._file_set = None
        if not loaded:
            self.handle_fetch_file_list()
        if not loaded and self._file_list_name is not None:
            self.save_file_list(self._file_list_name, version=self._file_list_version)

    def handle_file_list_filter(self, entry: FileListEntry) -> bool:
        """
        Verifies if the file is valid and shall be processed by comparing it to
        the file mask, the index_filter etc.

        Increases the file_index upon failure. Does NOT increase it upon success
        (as :meth:`provide_result` will do so).

        :param entry: The file entry to verify
        :return: A valid filename if the file shall be processed,
            None otherwise.
        """
        if not fnmatch(os.path.basename(entry.filename), self.search_mask):
            return False
        rest = entry.filename.lstrip("/").lstrip("\\")
        if not self.recursive:
            if "/" in rest or "\\" in rest:
                return False
        return True

    def handle_skip_check(self, file_info: FileIterationData) -> str | None:
        """
        Verifies if the file is valid and shall be processed by comparing it to
        the file mask, the index_filter etc.

        Increases the file_index upon failure. Does NOT increase it upon success
        (as :meth:`provide_result` will do so).

        :param file_info: Information about the current file
        :return: A valid filename if the file shall be processed, None otherwise.
        """
        filename = file_info.element.filename
        if self.index_filter is not None:
            if file_info.file_index % self.index_filter[0] != self.index_filter[1]:
                return None
        if self.filter_callback is not None:
            result = self.filter_callback(file_info)
            if result is None:
                return None
            if isinstance(result, bool):
                if not result:
                    return None
            else:
                filename = result
        return filename

    def handle_provide_result(
        self, iterator: FileSourceIterator, filename: str, data: bytes | None
    ) -> FileSourceElement:
        """
        Provides the file result for the current iterator index.

        :param iterator: The iterator handle
        :param filename: The name of the file to be stored
        :param data: The file data
        """
        iterator.processed_file_count += 1
        return FileSourceElement(data=data, name=filename)

    def close(self) -> None:
        """
        Closes the current file source, e.g. zip archive, streaming connection
        etc. if applicable.
        """
        self.is_closed = True

    # Async variants

    async def _read_file_int_async(self, filename: str) -> bytes | None:
        """
        Asynchronously reads a file from this file source, identified by name.

        Default implementation falls back to sync version in a thread pool.
        Subclasses can override for native async I/O.

        :param filename: The name of the file to read
        :return: The file's content on success, None otherwise
        """
        import asyncio

        return await asyncio.to_thread(self._read_file_int, filename)

    async def fetch_async(self, filename: str) -> bytes | None:
        """
        Asynchronously reads a file from this file source, identified by name.

        Note: Not all FileSources support direct file access by name, so you
        should always prefer to just iterate through a FileSource object rather
        than accessing single files if your FileSource can be freely configured.

        :param filename: The name of the file to read
        :return: The file's content on success, None otherwise
        """
        from filestag.web import WebCache

        if self.max_web_cache_age != 0:  # try to fetch data if cache is on
            unique_name = self._get_source_identifier() + "/" + filename
            data = await WebCache.fetch_async(unique_name, max_age=self.max_web_cache_age)
            if data is not None:
                return data
        result = await self._read_file_int_async(filename)
        if self.max_web_cache_age != 0 and result is not None:  # store new data if cache is on
            unique_name = self._get_source_identifier() + "/" + filename
            await WebCache.store_async(unique_name, result)
        return result

    async def copy_async(
        self,
        filename: str,
        target_name: str,
        overwrite: bool = True,
        sink: Union["FileSink", None] = None,
        on_fetch: Callable[[str], None] | None = None,
        on_fetch_done: Callable[[str, int], None] | None = None,
        on_stored: Callable[[str, int], None] | None = None,
        on_error: Callable[[str, str], None] | None = None,
        on_skip: Callable[[str], None] | None = None,
    ) -> bool:
        """
        Asynchronously copies a file from this FileSource to a FileStag
        compatible target path.

        :param filename: The name of the file to be copied
        :param target_name: The target path
        :param overwrite: Defines if the file shall be overwritten if it does
            already exist.
        :param sink: If defined the file will be copied to the specified sink
        :param on_skip: Is called if a file exists and will be skipped
        :param on_stored: Is called when ever a file was successfully uploaded
        :param on_fetch: Is called before a file is downloaded
        :param on_fetch_done: Is called after a file was
            successfully downloaded and will now be uploaded or stored.
        :param on_error: Is called when an error occurred
        :return: True if the file was copied
        """
        if sink is None and not overwrite and await FileStag.exists_async(target_name):
            if on_skip is not None:
                on_skip(target_name)
            return False
        if on_fetch is not None:
            on_fetch(filename)
        data = await self.fetch_async(filename)
        if data is None:
            if on_error is not None:
                on_error(filename, f"Could not load {filename}")
            return False
        if on_fetch_done is not None:
            on_fetch_done(filename, len(data))
        if sink is not None:
            # FileSink.store is sync, wrap in thread for now
            import asyncio

            result = await asyncio.to_thread(
                sink.store, target_name, data, overwrite=overwrite
            )
        else:
            result = await FileStag.save_async(target_name, data)
        if result:
            if on_stored is not None:
                on_stored(filename, len(data))
        else:
            if on_error is not None:
                on_error(filename, f"Could not store {filename}")
        return result

    async def load_file_list_async(self, source: bytes | str, version: int = -1) -> bool:
        """
        Asynchronously tries to load the file list from file.

        :param source: The file list source. Any FileStag compatible data
            source.
        :param version: The user defined version number. It can be passed
            to enforce updating the list when ever this number is changed.

            If -1 is passed the version is ignored.
        :return: True if a valid list could be loaded.
        """
        if not isinstance(source, bytes):
            source = await FileStag.load_async(source)
        if source is None:
            return False
        try:
            data = json.loads(source.decode("utf-8"))
        except (json.JSONDecodeError, UnicodeDecodeError):
            return False
        if not isinstance(data, dict) or data.get("format_version") != 1:
            return False
        if version != -1 and data.get(CACHE_VERSION, -1) != version:
            return False
        files = data.get("files", [])
        self.update_file_list(
            [FileListEntry.model_validate(entry) for entry in files],
            may_sort=False,
        )
        return True

    async def save_file_list_async(self, target: str, version: int = -1) -> None:
        """
        Asynchronously saves the file list to a file so it can be quickly
        restored after a restart of the application.

        :param target: The FileStag compatible file target, e.g. a local file
            name
        :param version: The user defined version number. It can be passed
            to enforce updating the list when ever this number is changed.

            If -1 is passed the version is ignored.
        """
        await FileStag.save_async(target, self.encode_file_list(version=version))
