from __future__ import annotations

import configparser
import fnmatch
import logging
import re
import time
import shutil
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable, List, Optional, Union, Tuple, Literal, Dict, Any, overload, Sequence
from urllib.parse import quote, urlparse

import boto3
from botocore.config import Config as BotoCoreConfig
from botocore.exceptions import ClientError, EndpointConnectionError

from .tool import (
    find_file_in_archives,
    read_raster_info,
    get_raster_metadata,
    archive_results as tool_archive_results,
)
from .logging_config import get_console_handler
# --------------------------- Logging ---------------------------

log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
log.addHandler(get_console_handler())
log.propagate = False

for noisy in ["boto3", "botocore", "s3transfer", "urllib3"]:
    logging.getLogger(noisy).setLevel(logging.WARNING)

# --------------------------- Constants ---------------------------

DEFAULT_S3_URL_BASE = "https://s3.waw3-1.cloudferro.com/swift/v1"

# --------------------------- Small utils ---------------------------

def _retry(fn, *, retries: int = 2, base_delay: float = 1.0):
    """Retry transient S3 errors with exponential backoff."""
    attempt = 0
    while True:
        try:
            return fn()
        except (EndpointConnectionError, ClientError) as e:
            attempt += 1
            if attempt > retries:
                raise
            delay = base_delay * (2 ** (attempt - 1))
            log.warning(
                "S3 op failed (attempt %d/%d): %s; retrying in %.1fs",
                attempt,
                retries + 1,
                e,
                delay,
            )
            time.sleep(delay)


def _norm_bool(v: str | None) -> bool:
    return str(v).strip().lower() in {"1", "true", "yes", "on"}


def _alias_from_remote_name(remote_name: str) -> str:
    """Return the rclone alias part from '<alias>:<bucket>' (or the whole string if no ':')."""
    alias, sep, _bucket = remote_name.partition(":")
    return alias if sep else remote_name


def _norm_posix_key(s: str) -> str:
    """
    Normalize to a safe S3 key:
    - use forward slashes
    - collapse duplicate slashes
    - remove any leading slash
    """
    s = s.replace("\\", "/")
    s = re.sub(r"/+", "/", s)
    return s.lstrip("/")


# --------------------------- rclone→boto3 helpers ---------------------------

def load_rclone_credentials(config_file_path: str | Path, remote_alias: str) -> dict:
    """
    Read S3 creds from an rclone config section.
    Expected keys (common rclone fields):
      - access_key_id, secret_access_key
      - endpoint (or endpoint_url)
      - region (optional)
      - session_token (optional)
      - v2_auth (bool) or signature_version ('s3' for SigV2, 's3v4' default)
    """
    cfg = configparser.ConfigParser()
    cfg.read(str(config_file_path))

    if remote_alias not in cfg.sections():
        raise KeyError(f"Remote '{remote_alias}' not in rclone config: {config_file_path}")

    section = cfg[remote_alias]
    endpoint_url = section.get("endpoint_url", fallback=section.get("endpoint"))
    v2_auth = _norm_bool(section.get("v2_auth"))
    signature_version = section.get("signature_version", fallback=("s3" if v2_auth else "s3v4"))

    # Normalize endpoint_url: ensure it has a scheme, but don't double-prefix
    if endpoint_url:
        endpoint_url = endpoint_url.strip()
        if not endpoint_url.startswith(("http://", "https://")):
            endpoint_url = "https://" + endpoint_url

    return {
        "aws_access_key_id": section.get("access_key_id"),
        "aws_secret_access_key": section.get("secret_access_key"),
        "aws_session_token": section.get("session_token"),
        "endpoint_url": endpoint_url,
        "region_name": section.get("region"),
        "signature_version": signature_version,
    }


def create_boto3_client_from_rclone(
    rclone_config_path: str | Path,
    remote_name: str,
    *,
    addressing_style: str = "path",   # Swift/CloudFerro usually needs 'path'
    retries_max_attempts: int = 3,
    default_region: str = "us-east-1",
) -> "boto3.client":
    """
    Build a boto3 S3 client using creds in an rclone config and a '<alias>:<bucket>' remote.
    """
    alias = _alias_from_remote_name(remote_name)
    creds = load_rclone_credentials(rclone_config_path, alias)

    boto_cfg = BotoCoreConfig(
        s3={"addressing_style": addressing_style},
        retries={"max_attempts": retries_max_attempts, "mode": "standard"},
        signature_version=creds.get("signature_version") or "s3v4",
    )

    s3 = boto3.client(
        "s3",
        endpoint_url=creds.get("endpoint_url"),
        region_name=creds.get("region_name") or default_region,
        aws_access_key_id=creds.get("aws_access_key_id"),
        aws_secret_access_key=creds.get("aws_secret_access_key"),
        aws_session_token=creds.get("aws_session_token"),
        config=boto_cfg,
    )
    return s3


# Optional aliases (mirroring screenshot names)
def create_boto3_write_client(rclone_config_path: str | Path, remote_name: str):
    return create_boto3_client_from_rclone(rclone_config_path, remote_name)


def create_boto3_read_client(rclone_config_path: str | Path, remote_name: str):
    return create_boto3_client_from_rclone(rclone_config_path, remote_name)


# --------------------------- ArchiveConfig ---------------------------

@dataclass
class ArchiveConfig:
    localarchive: bool
    remotearchive: bool
    remote_name: str | None
    config_path: Path | None
    input_archive_roots: list[Path]
    output_archive_root: Path
    output_archive_tmp: Path
    s3client: object | None
    # http url builder base
    s3_url_base: str = DEFAULT_S3_URL_BASE
    remote_baseoutputdir: str = "output"
    
    @classmethod
    def from_runtime(
        cls,
        *,
        support_data: Path,
        input_archive_roots: list[Path],
        output_archive_root: Path,
        output_archive_tmp: Path,
        remote_name: str | None = None,
        remote_baseoutputdir: str | None = None,
        addressing_style: str = "path",
        verify_remote_access: bool = True,
        rclone_conf_rel: Path = None,
        rclone_tmp_path: Path = Path("/tmp/rclone.conf"),
    ) -> "ArchiveConfig":
        """
        Create an ArchiveConfig instance from runtime parameters.

        This factory supports both local-only and remote (rclone/S3) archive setups.

        Remote Configuration
        --------------------
        If `remote_name` is provided, remote archive access is configured using an
        rclone configuration file. The location of `rclone.conf` is resolved using
        the following rules:

        Resolution order for `rclone_conf_rel`:

        1. If `rclone_conf_rel` is None:
           Uses the default location:
               <support_data>/project_info/rclone.conf

        2. If `rclone_conf_rel` points to a file ending with ".conf":
           That file is used directly (absolute or relative path supported).

        3. If `rclone_conf_rel` points to a directory:
           The config file is expected at:
               <rclone_conf_rel>/project_info/rclone.conf

        4. If `rclone_conf_rel` is a relative path:
           It is resolved relative to `support_data`, and the same
           file/directory logic applies.

        Parameters
        ----------
        support_data : Path
            Base project support directory containing configuration
            and auxiliary project files.

        input_archive_roots : list[Path]
            One or more archive roots used for input data lookup.

        output_archive_root : Path
            Root directory for final archive outputs.

        output_archive_tmp : Path
            Temporary output location before archiving.

        remote_name : str | None
            Name of the rclone remote (as defined in rclone.conf).
            If None, archive operates in local-only mode.

        remote_baseoutputdir : str | None
            Base directory on the remote under which outputs are stored.
            Defaults to "output" if not provided.

        addressing_style : str
            S3 addressing style ("path" or "virtual").

        verify_remote_access : bool
            If True, remote connectivity is verified during initialization.

        rclone_conf_rel : Path | None
            Custom rclone configuration location (file or directory).
            See resolution rules above.

        rclone_tmp_path : Path
            Temporary location where rclone.conf may be copied before use.

        Returns
        -------
        ArchiveConfig
            Fully initialized archive configuration instance.
        """
        
        remote_name = (remote_name or "").strip() or None
        
        remote_baseoutputdir = remote_baseoutputdir or "output"

        if remote_name:
            log.info("Configuring remote archive access...")

            # pick rclone.conf source
            if rclone_conf_rel is None:
                # Default behaviour
                rclone_conf = Path(support_data) / "project_info" / "rclone.conf"

            else:
                rclone_conf_rel = Path(rclone_conf_rel)

                # Case 1: user passed a direct .conf file
                if rclone_conf_rel.suffix == ".conf" and rclone_conf_rel.is_file():
                    rclone_conf = rclone_conf_rel

                # Case 2: user passed a folder
                elif rclone_conf_rel.is_dir():
                    rclone_conf = rclone_conf_rel / "project_info" / "rclone.conf"

                # Case 3: relative path inside support_data
                else:
                    candidate = Path(support_data) / rclone_conf_rel
                    if candidate.suffix == ".conf":
                        rclone_conf = candidate
                    else:
                        rclone_conf = candidate / "project_info" / "rclone.conf"

            config_path: Path | None = None
            if rclone_conf.exists():
                config_path = Path(rclone_tmp_path)
                config_path.parent.mkdir(parents=True, exist_ok=True)
                shutil.copy(rclone_conf, config_path)
            else:
                raise FileNotFoundError(f"Missing rclone config: {rclone_conf}")

            s3client = create_boto3_client_from_rclone(
                rclone_config_path=config_path,
                remote_name=remote_name,
                addressing_style=addressing_style,
            )

            cfg = cls(
                localarchive=False,
                remotearchive=True,
                remote_name=remote_name,
                remote_baseoutputdir=remote_baseoutputdir,
                config_path=config_path,
                input_archive_roots=input_archive_roots,
                output_archive_root=output_archive_root,
                output_archive_tmp=output_archive_tmp,
                s3client=s3client,
            )
            
            if verify_remote_access:
                cfg.check_remote_access()
                
            return cfg

        log.info("Configuring local archive access...")
        return cls(
            localarchive=True,
            remotearchive=False,
            remote_name=None,
            remote_baseoutputdir=remote_baseoutputdir,
            config_path=None,
            input_archive_roots=input_archive_roots,
            output_archive_root=output_archive_root,
            output_archive_tmp=output_archive_tmp,
            s3client=None,
        )

    def __post_init__(self) -> None:
        # Normalize incoming paths / lists; allow None safely
        if self.config_path is not None and not isinstance(self.config_path, Path):
            self.config_path = Path(self.config_path)

        self.input_archive_roots = [p if isinstance(p, Path) else Path(p) for p in self.input_archive_roots]
        if not isinstance(self.output_archive_root, Path):
            self.output_archive_root = Path(self.output_archive_root)
        if not isinstance(self.output_archive_tmp, Path):
            self.output_archive_tmp = Path(self.output_archive_tmp)

        # Sanity checks
        if not (self.localarchive or self.remotearchive):
            raise ValueError("At least one of localarchive or remotearchive must be True.")

        if self.remotearchive:
            if self.remote_name is None or ":" not in self.remote_name:
                raise ValueError("remote_name must be '<alias>:<bucket>' when remotearchive=True.")
            if self.s3client is None:
                # Allow late wiring via helpers, but your current code expects an eager client
                raise ValueError("remotearchive=True but s3client=None. Provide a boto3 S3 client.")

    # ---------- Remote checks ----------

    def check_remote_access(self) -> str:
        if self.localarchive and not self.remotearchive:
            msg = "Local archive is enabled; skipping remote access check."
            log.info(msg)
            return msg

        bucket = self.bucketname
        prefix = self._prefix_with_trailing_slash(self.remote_baseoutputdir)

        def _do():
            try:
                # Fast bucket permission probe
                self.s3client.head_bucket(Bucket=bucket)
            except ClientError:
                # Minimal list to validate access/prefix
                self.s3client.list_objects_v2(Bucket=bucket, Prefix=prefix, MaxKeys=1)
            return True

        _retry(_do)
        log.info("Access to remote bucket '%s' OK (prefix='%s').", bucket, prefix)
        return "Access to remote is successful."

    # ---------- URL / key helpers ----------

    @property
    def bucketname(self) -> str:
        _, _, bucket = (self.remote_name or "").partition(":")
        if not bucket:
            raise ValueError(f"remote_name '{self.remote_name}' has no bucket part.")
        return bucket

    def getremoteurlpath(self, filerelpath: Union[str, Path]) -> str:
        """
        Build a public HTTPS path using s3_url_base:
          {s3_url_base}/{bucket}/{key}
        """
        rel = Path(filerelpath).as_posix()
        key = self._join_key(self.remote_baseoutputdir, rel)
        return f"{self.s3_url_base}/{quote(self.bucketname)}/{quote(key, safe='/')}"

    def _extract_bucket_and_key(self, url_or_path: str) -> Tuple[str, str]:
        """
        From a remote reference (https://... or s3://...), return (bucket, key) with a normalized key.
        - For s3://bucket/key → ('bucket', 'key')
        - For https://base/<bucket>/key or https://.../swift/v1/<bucket>/key → (self.bucketname, 'key')
          (if detected bucket matches configured one, it is used; otherwise configured bucket is kept
           and we strip at the first occurrence of '/<bucket>/' in the URL path.)
        Never falls back to leaf name; preserves nested key path.
        """
        s = url_or_path.strip().replace("\\", "/")
        p = urlparse(s)
        configured_bucket = (getattr(self, "bucketname", "") or "").strip("/")
        base = str(self.s3_url_base).rstrip("/")

        # s3:// bucket/key
        if p.scheme == "s3":
            bucket = p.netloc or configured_bucket
            key = _norm_posix_key(p.path)
            return bucket, key

        # http(s)://...
        if p.scheme in {"http", "https"}:
            bucket = configured_bucket
            path = _norm_posix_key(p.path)  # e.g. swift/v1/<bucket>/key  OR  <bucket>/key

            needle = f"{bucket}/" if bucket else ""
            if needle and needle in path:
                key = path.split(needle, 1)[1]
                return bucket, _norm_posix_key(key)

            if needle and path.startswith(needle):
                return bucket, _norm_posix_key(path[len(needle) :])

            # last resort: treat entire path (minus leading slash) as key
            return bucket, path


        # If it's not a URL, treat as a key/path (already relative-ish)
        # Also handle the case where caller passes a full https base string without scheme checks
        if self.s3_url_base and s.startswith(base):
            # Strip base from the left and re-run as path
            stripped = s[len(base) :].lstrip("/")
            # stripped is now "<bucket>/key" or "swift/v1/<bucket>/key"
            # Reuse same logic by faking it as a path
            path = _norm_posix_key(stripped)
            bucket = configured_bucket
            needle = f"{bucket}/" if bucket else ""
            if needle and needle in path:
                key = path.split(needle, 1)[1]
                return bucket, _norm_posix_key(key)
            if needle and path.startswith(needle):
                return bucket, _norm_posix_key(path[len(needle) :])
            return bucket, path

        # If it's not a URL, treat as a key (already relative)
        return configured_bucket, _norm_posix_key(s)

    # ---------- Copy helpers ----------

    def copy_to_tmp(
        self,
        files: Union[str, Path, list[Union[str, Path]]],
        *,
        dest: Optional[Union[str, Path]] = None,
        flatten: bool = False,
        overwrite: bool = False,
    ) -> Union[Path, list[Path]]:
        """
        Copy one or more files (local or remote) to `dest`.

        Args:
            files: One path/URL or a list of paths/URLs.
                   - Local paths are copied.
                   - Remote URLs (self.s3_url_base/BUCKET/KEY... or s3://BUCKET/KEY) are downloaded via self.s3client.
            dest:  Destination folder OR (for a single source) a full file path.
            flatten: If True, don't preserve subfolders; place items directly inside `dest`.
            overwrite: If True, replace existing files at destination.

        Returns:
            Path (for single input) or list[Path] (for multiple).
        """
        # ---------- normalize inputs ----------
        if not isinstance(files, (list, tuple, set)):
            single = True
            src_list = [files]
        else:
            single = False
            src_list = list(files)

        if dest is None:
            raise ValueError("`dest` is required.")
        dest = Path(dest)

        # ---------- helper: should dest be treated as a directory ----------
        def _dest_is_dir_hint(p: Path) -> bool:
            if p.exists() and p.is_dir():
                return True
            if not single:
                return True
            return (p.suffix == "")

        dest_is_dir = _dest_is_dir_hint(dest)

        # ---------- validations ----------
        if not single and dest.exists() and dest.is_file():
            raise ValueError("When copying multiple sources, `dest` must be a folder (got a file path).")

        # Ensure destination directory exists (or its parent, if file path)
        if dest_is_dir:
            dest.mkdir(parents=True, exist_ok=True)
        else:
            dest.parent.mkdir(parents=True, exist_ok=True)

        # ---------- helpers for relative structure ----------
        def _rel_for_local(p: Path) -> Path:
            # Try to preserve path relative to any of the archive roots
            for root in getattr(self, "input_archive_roots", []):
                try:
                    return p.relative_to(root)
                except Exception:
                    pass
            # fallback: just the filename
            return Path(p.name)

        # ---------- main copy loop ----------
        out_paths: list[Path] = []

        for src in src_list:
            src_str = str(src).strip()
            parsed = urlparse(src_str)

            # -----------------------------
            # Detect source type
            # -----------------------------
            is_remote_http = False
            is_remote_s3 = False
            is_remote_relkey = False  # <-- NEW: "looks local" but should be treated as remote key
            is_local = False

            if parsed.scheme in ("http", "https"):
                is_remote_http = True
            elif parsed.scheme == "s3":
                is_remote_s3 = True
            elif self.s3_url_base and src_str.startswith(str(self.s3_url_base)):
                is_remote_http = True
            else:
                # No scheme / not a full URL. Could be:
                # - a real local path
                # - OR a remote relative key returned by findfile()
                p = Path(src_str)

                if p.exists():
                    is_local = True
                else:
                    # If remote is configured, treat missing local as remote rel key
                    if getattr(self, "remotearchive", False) and getattr(self, "s3client", None):
                        is_remote_relkey = True
                    else:
                        is_local = True  # will fail later with "does not exist"
    
            # -----------------------------
            # Decide destination path
            # -----------------------------
            if dest_is_dir:
                if flatten:
                    if is_local:
                        leaf = Path(src_str).name
                    else:
                        # for URLs or rel keys, leaf = last path element
                        leaf = Path(urlparse(src_str).path).name if (is_remote_http or is_remote_s3) else Path(src_str).name
                    target = dest / leaf
                else:
                    if is_local:
                        rel = _rel_for_local(Path(src_str))
                        target = dest / rel
                    else:
                        if is_remote_relkey:
                            # preserve rel key path as-is
                            target = dest / Path(_norm_posix_key(src_str))
                        else:
                            _bucket, rel_key = self._extract_bucket_and_key(src_str)
                            target = dest / Path(rel_key)
            else:
                if not single:
                    raise ValueError("Internal: multiple sources cannot target a single file path.")
                target = dest
    
            target.parent.mkdir(parents=True, exist_ok=True)
    
            # Skip if exists and not overwriting
            if target.exists() and not overwrite:
                log.info("Already exists, skipping: %s", target)
                out_paths.append(target)
                continue

            try:
                if is_local:
                    if not Path(src_str).exists():
                        log.warning("Local source does not exist, skipping: %s", src_str)
                        continue
                    shutil.copy2(src_str, target)
                    log.info("Copied local: %s -> %s", src_str, target)
    
                else:
                    # -----------------------------
                    # Remote download
                    # -----------------------------
                    if not getattr(self, "s3client", None):
                        raise RuntimeError("No S3 client configured on self.s3client; cannot download remote source.")
    
                    if is_remote_relkey:
                        # Treat src_str as a KEY RELATIVE TO remote_baseoutputdir
                        bucket_for_dl = self.bucketname
                        rel_key = _norm_posix_key(self._join_key(self.remote_baseoutputdir, src_str))
                    else:
                        bucket_for_dl, rel_key = self._extract_bucket_and_key(src_str)
                        rel_key = _norm_posix_key(rel_key)
    
                    log.debug("Remote download resolved: Bucket=%s Key=%s → %s", bucket_for_dl, rel_key, target)
    
                    self.s3client.download_file(
                        Bucket=bucket_for_dl,
                        Key=rel_key,
                        Filename=str(target),
                    )
                    log.info("Downloaded remote: s3://%s/%s -> %s", bucket_for_dl, rel_key, target)
    
                out_paths.append(target)
    
            except Exception as e:
                log.error("Failed to copy/download %s -> %s: %s", src_str, target, e)

        # ---------- return type symmetry ----------
        return out_paths[0] if single and out_paths else out_paths

    # ---------- Finders ----------

    def _list_objects(self, prefix: str) -> List[dict]:
        """List all objects under a prefix (handles pagination)."""
        bucket = self.bucketname
        kwargs = dict(Bucket=bucket, Prefix=prefix, MaxKeys=1000)
        objects: List[dict] = []

        def _page():
            nonlocal kwargs
            resp = self.s3client.list_objects_v2(**kwargs)
            contents = resp.get("Contents", [])
            objects.extend(contents)
            if resp.get("IsTruncated"):
                kwargs["ContinuationToken"] = resp["NextContinuationToken"]
                return False  # continue
            return True       # done

        while True:
            done = _retry(_page)
            if done:
                break
        return objects

    def find_files_in_remotearchive(self, wildcard_relpath: Union[str, Path]) -> list[Path]:
        """
        Return a list of *relative* Paths matching a wildcard pattern under remote_baseoutputdir.
        """
        wildcard_relpath = Path(wildcard_relpath)
        parent = wildcard_relpath.parent.as_posix().strip("/")
        pattern = wildcard_relpath.name

        base_prefix = self._join_prefix(self.remote_baseoutputdir, parent)
        if base_prefix and not base_prefix.endswith("/"):
            base_prefix += "/"

        objects = self._list_objects(base_prefix)
        results: list[Path] = []

        for obj in objects:
            key = obj["Key"]  # full key: '<remote_baseoutputdir>/<parent>/<filename>'
            rel_key = self._strip_prefix(key, self._prefix_with_trailing_slash(self.remote_baseoutputdir))

            # Enforce parent
            if parent:
                if not rel_key.startswith(parent + "/"):
                    continue
                candidate = rel_key[len(parent) + 1:]  # filename/leaf
            else:
                candidate = rel_key

            if fnmatch.fnmatch(candidate, pattern):
                results.append(Path(parent) / candidate if parent else Path(candidate))

        log.info("Remote match %d file(s) under s3://%s/%s pattern=%s",
                  len(results), self.bucketname, base_prefix, pattern)
        return results

    def findfilepattern(self, filepattern: Union[str, Path], *, return_all: bool = False) -> Union[Path, list[Path], None]:
        """
        Search locally (and/or remotely) for a glob-style pattern relative to archive roots/baseoutputdir.
        """
        filepattern = Path(filepattern)

        if self.remotearchive:
            matches = self.find_files_in_remotearchive(filepattern)
            if matches:
                return matches if return_all else matches[0]

        if self.localarchive:
            import glob as _glob
            local_matches: list[Path] = []
            for root in self.input_archive_roots:
                globbed = [_p for p in _glob.glob(str((root / filepattern).as_posix()))
                           if (_p := Path(p)).is_file()]
                local_matches.extend(globbed)
            if local_matches:
                return local_matches if return_all else local_matches[0]

        return [] if return_all else None

    def findfile(self, filerelpath: Union[str, Path]) -> Optional[Path]:
        """
        Locate a single file by exact relative path: local first, then remote existence check.
        Returns a *relative* Path for remote hits (to be used with copy helpers).
        """
        rel = Path(filerelpath)

        if self.localarchive:
            local = find_file_in_archives(rel, self.input_archive_roots, error_on_missing=False)
            if local:
                return local

        if self.remotearchive:
            bucket = self.bucketname
            key = self._join_key(self.remote_baseoutputdir, rel.as_posix())

            log.debug(
                "Searching for remote file %s in s3://%s/%s",
                rel,
                bucket,
                key,
            )

            def _do():
                try:
                    self.s3client.head_object(Bucket=bucket, Key=key)

                    log.info(
                        "Found remote file %s at s3://%s/%s",
                        rel.name,
                        bucket,
                        key,
                    )
                    
                    return rel
                except ClientError as e:
                    code = (e.response.get("Error") or {}).get("Code")
                    status = (e.response.get("ResponseMetadata") or {}).get("HTTPStatusCode")
                    if code in {"404", "NoSuchKey"} or status == 404:
                        return None
                    raise

            return _retry(_do)

        return None

    # ---------- Raster helpers ----------

    def findarchivefile_rasterinfo(self, filerelpath: Union[str, Path]):
        rel = Path(filerelpath)
        if self.localarchive:
            local = find_file_in_archives(rel, self.input_archive_roots, error_on_missing=False)
            if local:
                return read_raster_info(local)

        if self.remotearchive:
            url = self.getremoteurlpath(rel)
            return get_raster_metadata(url)

        raise FileNotFoundError(f"Raster not found locally and no remote enabled: {rel}")

    # ---------- Public: list folder ----------

    def _list_objects_and_prefixes(self, prefix: str, delimiter: str = "/") -> tuple[list[dict], list[str]]:
        """
        List S3 objects and 'directory' prefixes under a prefix using a delimiter.
        Returns (objects, prefixes). Prefixes are full keys ending with delimiter.
        """
        bucket = self.bucketname
        kwargs = dict(Bucket=bucket, Prefix=prefix, Delimiter=delimiter, MaxKeys=1000)
        objects: list[dict] = []
        prefixes: list[str] = []

        def _page():
            nonlocal kwargs
            resp = self.s3client.list_objects_v2(**kwargs)
            objects.extend(resp.get("Contents", []))
            prefixes.extend([p["Prefix"] for p in resp.get("CommonPrefixes", [])])
            if resp.get("IsTruncated"):
                kwargs["ContinuationToken"] = resp["NextContinuationToken"]
                return False
            return True

        while True:
            done = _retry(_page)
            if done:
                break
        return objects, prefixes

    def _immediate_children_remote(
        self,
        base_prefix: str,
        *,
        include_dirs: bool,
        include_files: bool,
    ) -> Iterable[Path]:
        """
        Yield immediate children (files/dirs) under base_prefix on S3.
        """
        # Ensure trailing slash semantics for 'folder'
        if base_prefix and not base_prefix.endswith("/"):
            base_prefix = base_prefix + "/"

        objects, prefixes = self._list_objects_and_prefixes(base_prefix, delimiter="/")

        # Directories
        if include_dirs:
            for p in prefixes:
                rel = self._strip_prefix(p, base_prefix).rstrip("/")
                if rel:
                    yield Path(rel)

        # Files directly under the folder (no additional '/')
        if include_files:
            for obj in objects:
                rel_key = self._strip_prefix(obj["Key"], base_prefix)
                if rel_key and "/" not in rel_key and rel_key != "":
                    yield Path(rel_key)

    def findfiles_in_folder(
        self,
        folder_relpath: Union[str, Path] = ".",
        *,
        recursive: bool = False,
        pattern: Optional[str] = None,       # e.g., "*.tif"
        include_dirs: bool = False,
        include_files: bool = True,
        sort: bool = True,
    ) -> list[Union[Path, str]]:
        """
        List items inside a folder (local and/or remote depending on config).
        - Returns **absolute local paths** or **full remote URLs**, depending on where files are found.
        - If both local and remote are enabled, results are merged and de-duplicated.
        - `recursive=True` walks all descendants; otherwise only immediate children.
        - `pattern` is a glob applied to the *leaf* names (e.g., "*.tif").
        """
        folder_relpath = Path(folder_relpath)
        results: set[Union[Path, str]] = set()

        # ----- Remote side -----
        if self.remotearchive:
            base_prefix = self._join_prefix(self.remote_baseoutputdir, folder_relpath.as_posix())

            if recursive:
                prefix = self._prefix_with_trailing_slash(base_prefix)
                for obj in self._list_objects(prefix):
                    key = obj["Key"]
                    rel = self._strip_prefix(key, prefix)
                    if rel:
                        if include_files:
                            url = self.getremoteurlpath(folder_relpath / rel)
                            results.add(url)
                        if include_dirs:
                            parts = Path(rel).parts
                            for i in range(1, len(parts)):
                                url = self.getremoteurlpath(folder_relpath / Path(*parts[:i]))
                                results.add(url)
            else:
                for rel in self._immediate_children_remote(
                    self._join_prefix(self.remote_baseoutputdir, folder_relpath.as_posix()),
                    include_dirs=include_dirs,
                    include_files=include_files,
                ):
                    url = self.getremoteurlpath(folder_relpath / rel)
                    results.add(url)

        # ----- Local side -----
        if self.localarchive:
            for root in self.input_archive_roots:
                base = (root / folder_relpath)
                if not base.exists():
                    continue

                it = base.rglob("*") if recursive else base.glob("*")

                for p in it:
                    if (p.is_file() and include_files) or (p.is_dir() and include_dirs):
                        results.add(p.resolve())

        # ----- Pattern filter (glob on leaf name) -----
        if pattern:
            results = {r for r in results if fnmatch.fnmatch(Path(r).name, pattern)}

        out = list(results)
        if sort:
            out.sort(key=lambda p: str(p))
        return out

    # ---------- Internal path/key utils ----------

    @staticmethod
    def _join_key(prefix: str, rel: str) -> str:
        prefix = (prefix or "").strip("/")
        rel = (rel or "").strip("/")
        return "/".join(p for p in [prefix, rel] if p)

    @staticmethod
    def _join_prefix(prefix: str, extra: str) -> str:
        prefix = (prefix or "").strip("/")
        extra = (extra or "").strip("/")
        return "/".join(p for p in [prefix, extra] if p)

    @staticmethod
    def _prefix_with_trailing_slash(prefix: str) -> str:
        p = (prefix or "").strip("/")
        return (p + "/") if p else ""

    @staticmethod
    def _strip_prefix(text: str, prefix: str) -> str:
        return text[len(prefix):] if prefix and text.startswith(prefix) else text



    # ---------------------------
    # ---- ArchiveResults -------
    # ---------------------------

    @overload
    def archive_results(
        self,
        src_dst_pairs: list[tuple[Union[str, Path], Union[str, Path]]],
        task_id: int,
        *,
        target: Literal["remote"],
    ) -> list[str]: ...
    @overload
    def archive_results(
        self,
        src_dst_pairs: list[tuple[Union[str, Path], Union[str, Path]]],
        task_id: int,
        *,
        target: Literal["local"],
    ) -> list[Path]: ...
    @overload
    def archive_results(
        self,
        src_dst_pairs: list[tuple[Union[str, Path], Union[str, Path]]],
        task_id: int,
        *,
        target: Literal["auto"],
    ) -> Union[list[str], list[Path]]: ...
    @overload
    def archive_results(
        self,
        src_dst_pairs: list[tuple[Union[str, Path], Union[str, Path]]],
        task_id: int,
        *,
        target: Literal["both"],
    ) -> Dict[str, Any]: ...

    def archive_results(
        self,
        tmp_dir: Path = None,
        src_dst_pairs: list[tuple[Union[str, Path], Union[str, Path]]] = [],
        *,
        target: Literal["auto", "remote", "local", "both"] = "auto",
    ):
        """
        Archive a list of (src_path, dst_path) pairs.

        Parameters
        ----------
        src_dst_pairs : list[tuple[src_path, dst_path]]
            Each dst_path represents the *intended* archive location under output_archive_root.
        task_id : int
            Used to build local task folder when target includes 'local'.
        target : {'auto','remote','local','both'}
            - 'auto'   : remote if configured, else local
            - 'remote' : force remote; raises if remote not configured
            - 'local'  : force local
            - 'both'   : do remote then local; returns {'remote': [...], 'local': [...]}

        Returns
        -------
        - 'remote' → list[str]    (HTTPS URLs)
        - 'local'  → list[Path]   (local paths)
        - 'auto'   → list[str] or list[Path] depending on the path chosen
        - 'both'   → dict(remote=list[str], local=list[Path])
        """
        def _remote_ready() -> bool:
            return bool(self.remotearchive and self.remote_name and self.s3client)

        if target == "remote":
            if not _remote_ready():
                raise RuntimeError("Remote archiving requested but remote is not configured.")
            out_remote = self._archive_results_remote(src_dst_pairs)
            log.info("Results have been archived (remote -> remote).")
            return out_remote

        if target == "local":
            out_local = self._archive_results_local(src_dst_pairs, tmp_dir)
            log.info("Results have been archived (local -> local).")
            return out_local

        if target == "both":
            if not _remote_ready():
                raise RuntimeError("Target 'both' requires a configured remote.")
            out_remote = self._archive_results_remote(src_dst_pairs)
            out_local  = self._archive_results_local(src_dst_pairs, tmp_dir)
            log.info("Results have been archived (remote & local).")
            return {"remote": out_remote, "local": out_local}

        # target == 'auto'
        if _remote_ready():
            out_remote = self._archive_results_remote(src_dst_pairs)
            log.info("Results have been archived (auto→remote).")
            return out_remote
        else:
            out_local = self._archive_results_local(src_dst_pairs, tmp_dir)
            log.info("Results have been archived (auto→local).")
            return out_local


    def _archive_results_remote(
        self,
        src_dst_pairs: list[tuple[Union[str, Path], Union[str, Path]]],
    ) -> list[str]:
        """
        Upload results to S3 using boto3, building keys:
          <remote_baseoutputdir>/<output_archive_root.name>/<relative_to_output_archive_root>

        If dst_path is not under output_archive_root, we replace its first folder
        with output_archive_root.name and keep the rest, so the remote layout
        is consistent.
        """
        bucket = self.bucketname
        urls: list[str] = []

        base_folder_name = self.output_archive_root.name  # the "tree_first_folder" you referenced
        remote_prefix = (self.remote_baseoutputdir or "").strip("/")
        
        for src_path, dst_path in src_dst_pairs:
            src_path = Path(src_path)
            dst_path = Path(dst_path)

            # --- compute 'rel' (the part that goes under base_folder_name) ---
            try:
                # Case A: already under output_archive_root → keep exact relative path
                rel = dst_path.relative_to(self.output_archive_root)
            except ValueError:
                # Case B: dst_path is elsewhere → "replace" its first folder with base_folder_name
                # Strip drive/anchor and split
                parts = list(Path(dst_path.as_posix()).parts)

                # If it already starts with base_folder_name, drop that head; else drop the first head anyway
                if parts and parts[0] == base_folder_name:
                    remainder = parts[1:]  # drop the duplicate base name
                else:
                    remainder = parts[1:] if len(parts) > 1 else [Path(dst_path).name]

                # This is the path portion that will live under base_folder_name on remote
                rel = Path(*remainder) if remainder else Path(Path(dst_path).name)

            if remote_prefix == base_folder_name:
                key_rel = rel
            else:
                key_rel = Path(base_folder_name) / rel
                
            # --- final key: remote_baseoutputdir / output_archive_root.name / rel ---
            key = _norm_posix_key(self._join_key(remote_prefix, key_rel.as_posix()))

            # ensure file exists locally before upload
            if not src_path.exists():
                log.warning("Source file missing, skipping upload: %s", src_path)
                continue

            def _do():
                # IMPORTANT: use the fully normalized 'key'
                self.s3client.upload_file(str(src_path), bucket, key)
                return True

            _retry(_do)

            url = f"{self.s3_url_base}/{quote(bucket)}/{quote(key, safe='/')}"
            urls.append(url)
            log.info("Uploaded: %s -> s3://%s/%s", src_path, bucket, key)

        log.info("Results have been archived (remote).")
        return urls

    def _archive_results_local(
        self,
        src_dst_pairs: list[tuple[Union[str, Path], Union[str, Path]]],
        tmp_dir
    ) -> list[Path]:
        """
        Copy results locally into:
        <output_archive_tmp>/task_<task_id>/<dst relative to output_archive_root>
        Returns local destination Paths.
        """
        out_paths: list[Path] = []

        # Copy results to archive.
        tool_archive_results(tmp_dir, src_dst_pairs)

        log.info("Results have been archived (local).")
        return out_paths

# -------------------------------------------------------------------------
# SUMMARY OF PUBLIC INTERFACE
# -------------------------------------------------------------------------
#
# Module for unified access to local and remote (CloudFerro S3-compatible) archives.
#
# PUBLIC FUNCTIONS
#   - load_rclone_credentials(config_file_path, remote_alias)
#   - create_boto3_client_from_rclone(rclone_config_path, remote_name, …)
#   - create_boto3_write_client(rclone_config_path, remote_name)
#   - create_boto3_read_client(rclone_config_path, remote_name)
#
# PUBLIC METHODS (ArchiveConfig)
#   - check_remote_access()
#   - copy_to_tmp(files, dest=..., flatten=False, overwrite=False)
#   - findfile(filerelpath)
#   - findfilepattern(filepattern, return_all=False)
#   - find_files_in_remotearchive(wildcard_relpath)
#   - getremoteurlpath(filerelpath)
#   - findarchivefile_rasterinfo(filerelpath)
#   - findfiles_in_folder(folder_relpath='.', recursive=False, pattern=None, …)
#
# -------------------------------------------------------------------------
# End of file
# -------------------------------------------------------------------------

