# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
"""Registry compile operations.

This module provides the core logic for the `registry compile` command,
which scans a registry bucket, determines the "latest" GA version for each
connector, ensures `latest/` directories are up-to-date, and writes
global registry JSON index files plus per-connector version indexes.

High-level algorithm
--------------------
1. **Scan** -- glob for `metadata.yaml` files to discover all
   (connector, version) tuples.  No file *contents* are downloaded.
2. **Yank set** -- glob for `version-yank.yml` markers to build an
   exclusion set.
3. **Compute latest** -- for each connector, pick the highest GA semver
   that is not yanked and not a pre-release.
4. **Fast-check latest/** -- glob for `version=*` marker files inside
   `latest/` directories.  Compare with the computed latest.  Only
   connectors whose marker disagrees (or is missing) need a resync.
5. **Resync stale latest/** -- `fsspec.generic.rsync` from the
   versioned directory to `latest/`, then write/update the
   `version=<x.y.z>` marker.
6. **Write index files**:
   a. Global `registries/v0/cloud_registry.json` and
      `registries/v0/oss_registry.json` (backwards-compatible).
   b. Per-connector `versions.json` (new).

When `with_secrets_mask=True`, the compile step also regenerates
`registries/v0/specs_secrets_mask.yaml` by scanning all connector specs
for properties marked `airbyte_secret: true`.
"""

from __future__ import annotations

import copy
import json
import logging
import os
import re
import sys
from dataclasses import dataclass, field
from typing import Any

import dpath.util
import gcsfs
import yaml
from packaging.version import InvalidVersion, Version

from airbyte_ops_mcp.registry._constants import (
    METADATA_FOLDER,
)

logger = logging.getLogger(__name__)

# Regex for the `version=<semver>` marker filename inside `latest/`
_VERSION_MARKER_RE = re.compile(r"^version=(.+)$")

# Registry index output path (relative to bucket root)
_REGISTRIES_PREFIX = "registries/v0"

# Specs secrets mask filename
_SPECS_SECRETS_MASK_FILENAME = "specs_secrets_mask.yaml"


# ---------------------------------------------------------------------------
# Result model
# ---------------------------------------------------------------------------


@dataclass
class CompileResult:
    """Result of a registry compile operation."""

    target: str
    connectors_scanned: int = 0
    versions_found: int = 0
    yanked_versions: int = 0
    latest_updated: int = 0
    latest_already_current: int = 0
    cloud_registry_entries: int = 0
    oss_registry_entries: int = 0
    version_indexes_written: int = 0
    specs_secrets_mask_properties: int = 0
    errors: list[str] = field(default_factory=list)
    dry_run: bool = False

    @property
    def status(self) -> str:
        if self.dry_run:
            return "dry-run"
        if self.errors:
            return "completed-with-errors"
        return "success"

    def summary(self) -> str:
        return (
            f"[{self.status}] Scanned {self.connectors_scanned} connectors, "
            f"{self.versions_found} versions ({self.yanked_versions} yanked). "
            f"Latest updated: {self.latest_updated}, "
            f"already current: {self.latest_already_current}. "
            f"Registry entries: cloud={self.cloud_registry_entries}, "
            f"oss={self.oss_registry_entries}. "
            f"Version indexes: {self.version_indexes_written}. "
            f"Specs secrets mask: {self.specs_secrets_mask_properties} properties. "
            f"Errors: {len(self.errors)}."
        )


@dataclass
class ConnectorVersionInfo:
    """Lightweight info about a single published version."""

    version: str
    yanked: bool = False
    is_latest: bool = False
    release_stage: str | None = None
    support_level: str | None = None


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _log_progress(msg: str, *args: object) -> None:
    """Log a progress message to both the logger and stderr."""
    logger.info(msg, *args)
    formatted = msg % args if args else msg
    print(formatted, file=sys.stderr, flush=True)


def _get_gcs_credentials_token() -> dict[str, str]:
    """Get GCS credentials as a token dict for gcsfs."""
    gcs_creds = os.environ.get("GCS_CREDENTIALS")
    if not gcs_creds:
        raise ValueError(
            "GCS_CREDENTIALS environment variable is required for GCS operations."
        )
    return json.loads(gcs_creds)


def _is_ga_version(version_str: str) -> bool:
    """Return True if the version string is a GA (non-prerelease) semver."""
    try:
        v = Version(version_str)
    except InvalidVersion:
        return False
    return not v.is_prerelease and not v.is_devrelease


def _parse_version(version_str: str) -> Version | None:
    """Parse a version string, returning None if it is not valid semver."""
    try:
        return Version(version_str)
    except InvalidVersion:
        return None


def _extract_connector_version(
    path: str, bucket: str, prefix: str = ""
) -> tuple[str, str] | None:
    """Extract (connector_name, version) from a GCS path.

    Expected path format:
        `<bucket>/[<prefix>/]metadata/airbyte/<connector>/<version>/metadata.yaml`

    Args:
        path: Full GCS path returned by glob.
        bucket: Bucket name.
        prefix: Optional path prefix (e.g. `"devin-test/"` or `""`).  Must
            include trailing `/` if non-empty.

    Returns None if the path does not match.
    """
    expected_prefix = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte/"
    if not path.startswith(expected_prefix):
        return None
    remainder = path[len(expected_prefix) :]
    parts = remainder.split("/")
    if len(parts) < 2:
        return None
    connector = parts[0]
    version = parts[1]
    # Skip non-version dirs like "latest", "release_candidate"
    if version in ("latest", "release_candidate"):
        return None
    return connector, version


# ---------------------------------------------------------------------------
# Step 1 + 2: Scan versions and yank markers via glob
# ---------------------------------------------------------------------------


def _scan_versions_and_yanks(
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
    connector_name: list[str] | None = None,
) -> tuple[dict[str, list[str]], set[tuple[str, str]]]:
    """Scan the registry for all (connector, version) tuples and yank markers.

    Uses efficient glob patterns -- no file contents are downloaded.

    Args:
        fs: Authenticated GCSFileSystem.
        bucket: Bucket name (with optional prefix already applied).
        prefix: Path prefix within the bucket (e.g. "" or "aj-test100/").
        connector_name: If provided, only scan these connector names.

    Returns:
        Tuple of:
        - dict mapping connector_name -> list of version strings
        - set of (connector_name, version) tuples that are yanked
    """
    base = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte"

    # Step 1: glob for metadata.yaml to discover all versions
    if connector_name:
        metadata_paths: list[str] = []
        for name in connector_name:
            pattern = f"{base}/{name}/*/metadata.yaml"
            metadata_paths.extend(fs.glob(pattern))
    else:
        pattern = f"{base}/*/*/metadata.yaml"
        metadata_paths = fs.glob(pattern)

    _log_progress("Glob found %d metadata.yaml files", len(metadata_paths))

    connector_versions: dict[str, list[str]] = {}
    for path in metadata_paths:
        parsed = _extract_connector_version(path, bucket, prefix)
        if parsed is None:
            continue
        connector, version = parsed
        connector_versions.setdefault(connector, []).append(version)

    # Step 2: glob for version-yank.yml markers
    if connector_name:
        yank_paths: list[str] = []
        for name in connector_name:
            yank_pattern = f"{base}/{name}/*/version-yank.yml"
            yank_paths.extend(fs.glob(yank_pattern))
    else:
        yank_pattern = f"{base}/*/*/version-yank.yml"
        yank_paths = fs.glob(yank_pattern)

    _log_progress("Glob found %d version-yank.yml markers", len(yank_paths))

    yanked: set[tuple[str, str]] = set()
    for path in yank_paths:
        # Path: bucket/[prefix/]metadata/airbyte/<connector>/<version>/version-yank.yml
        parts = path.split("/")
        # Find "metadata" index, then airbyte/<connector>/<version>
        try:
            meta_idx = parts.index("metadata")
            connector = parts[meta_idx + 2]
            version = parts[meta_idx + 3]
            yanked.add((connector, version))
        except (ValueError, IndexError):
            logger.warning("Could not parse yank path: %s", path)

    return connector_versions, yanked


# ---------------------------------------------------------------------------
# Step 3: Compute latest GA version per connector
# ---------------------------------------------------------------------------


def _compute_latest_versions(
    connector_versions: dict[str, list[str]],
    yanked: set[tuple[str, str]],
) -> dict[str, str]:
    """For each connector, determine the highest GA semver that is not yanked.

    Returns:
        dict mapping connector_name -> latest version string.
        Connectors with no eligible GA versions are omitted.
    """
    latest: dict[str, str] = {}
    for connector, versions in connector_versions.items():
        candidates: list[tuple[Version, str]] = []
        for v_str in versions:
            if (connector, v_str) in yanked:
                continue
            if not _is_ga_version(v_str):
                continue
            parsed = _parse_version(v_str)
            if parsed is not None:
                candidates.append((parsed, v_str))
        if candidates:
            candidates.sort(reverse=True)
            latest[connector] = candidates[0][1]
        else:
            logger.warning(
                "No eligible GA version found for %s (%d total versions, all yanked or pre-release).",
                connector,
                len(versions),
            )
    return latest


# ---------------------------------------------------------------------------
# Step 4: Check version markers in latest/ directories
# ---------------------------------------------------------------------------


def _scan_latest_markers(
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
    connector_name: list[str] | None = None,
) -> dict[str, str]:
    """Glob for `version=*` marker files in `latest/` directories.

    Returns:
        dict mapping connector_name -> current marker version string.
    """
    base = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte"

    if connector_name:
        marker_paths: list[str] = []
        for name in connector_name:
            marker_pattern = f"{base}/{name}/latest/version=*"
            marker_paths.extend(fs.glob(marker_pattern))
    else:
        marker_pattern = f"{base}/*/latest/version=*"
        marker_paths = fs.glob(marker_pattern)

    _log_progress("Glob found %d version marker files in latest/", len(marker_paths))

    markers: dict[str, str] = {}
    for path in marker_paths:
        filename = path.split("/")[-1]
        match = _VERSION_MARKER_RE.match(filename)
        if not match:
            continue
        version_str = match.group(1)
        # Extract connector name from path
        parts = path.split("/")
        try:
            meta_idx = parts.index("metadata")
            connector = parts[meta_idx + 2]
            markers[connector] = version_str
        except (ValueError, IndexError):
            logger.warning("Could not parse marker path: %s", path)

    return markers


# ---------------------------------------------------------------------------
# Step 5: Resync stale latest/ directories
# ---------------------------------------------------------------------------


def _sync_latest_dir(
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
    connector: str,
    version: str,
    old_marker_version: str | None,
    dry_run: bool = False,
) -> None:
    """Copy files from the versioned dir to `latest/` and update the marker.

    Uses the already-authenticated `fs` instance to copy files individually,
    avoiding credential-propagation issues with `fsspec.generic.rsync`.
    """
    base = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte/{connector}"
    source_dir = f"{base}/{version}"
    dest_dir = f"{base}/latest"

    if dry_run:
        _log_progress(
            "  [DRY RUN] Would sync %s/ → %s/ and write marker version=%s",
            source_dir,
            dest_dir,
            version,
        )
        return

    _log_progress("  Syncing %s/ → %s/", source_dir, dest_dir)

    # List source files and copy each to latest/
    source_files = fs.ls(source_dir, detail=False)
    source_filenames = {src_path.split("/")[-1] for src_path in source_files}

    for src_path in source_files:
        filename = src_path.split("/")[-1]
        dst_path = f"{dest_dir}/{filename}"
        fs.copy(src_path, dst_path)

    # Delete stale files from latest/ that no longer exist in the source version
    try:
        dest_files = fs.ls(dest_dir, detail=False)
    except FileNotFoundError:
        dest_files = []

    for dst_path in dest_files:
        filename = dst_path.split("/")[-1]
        # Preserve version marker files — they are managed separately below
        if filename.startswith("version="):
            continue
        if filename not in source_filenames:
            try:
                fs.rm(dst_path)
                _log_progress("    Deleted stale file from latest/: %s", filename)
            except Exception as exc:
                logger.warning("Failed to delete stale file %s: %s", dst_path, exc)

    # Delete old marker if present and different
    if old_marker_version and old_marker_version != version:
        old_marker_path = f"{base}/latest/version={old_marker_version}"
        try:
            if fs.exists(old_marker_path):
                fs.rm(old_marker_path)
        except Exception as exc:
            logger.warning("Failed to delete old marker %s: %s", old_marker_path, exc)

    # Write new marker (zero-byte file)
    marker_path = f"{base}/latest/version={version}"
    with fs.open(marker_path, "wb") as f:
        f.write(b"")

    _log_progress("  Wrote marker: version=%s", version)


def _apply_overrides_to_latest_entry(
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
    connector: str,
    version: str,
) -> None:
    """Post-process `latest/` registry entries after syncing from a versioned dir.

    When the compile step copies files from `{version}/` to `latest/`, the
    embedded `cloud.json` and `oss.json` still contain version-specific
    values.  This function:

    1. Applies `registryOverrides` from the metadata so that the `latest/`
       entry reflects the overridden values (e.g. a pinned `dockerImageTag`).
    2. When an override pins `dockerImageTag` to a different version, copies
       version-sensitive fields (`spec`, `packageInfo`) from the pinned
       version's entry so that `latest/` is fully consistent.
    3. Updates `generated.source_file_info.metadata_file_path` to reference
       the `latest/` path instead of the versioned path.
    """
    base = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte/{connector}"
    latest_dir = f"{base}/latest"

    # Read metadata.yaml from latest/ to get registry overrides.
    metadata_path = f"{latest_dir}/metadata.yaml"
    try:
        with fs.open(metadata_path, "r") as f:
            raw_metadata = yaml.safe_load(f)
    except FileNotFoundError:
        logger.warning(
            "No metadata.yaml in latest/ for %s, skipping override application",
            connector,
        )
        return
    except Exception as exc:
        logger.warning("Failed to read metadata for %s: %s", connector, exc)
        return

    metadata_data: dict[str, Any] = raw_metadata.get("data", {})
    registry_overrides = metadata_data.get("registryOverrides", {})

    latest_metadata_file_path = (
        f"{METADATA_FOLDER}/airbyte/{connector}/latest/metadata.yaml"
    )

    for registry_type in ("cloud", "oss"):
        entry_path = f"{latest_dir}/{registry_type}.json"
        try:
            if not fs.exists(entry_path):
                continue
            with fs.open(entry_path, "r") as f:
                entry = json.load(f)
        except Exception as exc:
            logger.warning(
                "Failed to read %s.json for %s: %s", registry_type, connector, exc
            )
            continue

        modified = False

        # --- Apply registry overrides (skip_docker_image_tag=False for latest) ---
        overrides = copy.deepcopy(registry_overrides.get(registry_type, {}))
        overrides.pop("enabled", None)
        overrides = {k: v for k, v in overrides.items() if v is not None}

        if overrides:
            entry.update(overrides)
            modified = True

        # --- When dockerImageTag is overridden to a different version, copy
        #     version-sensitive fields from the pinned version's entry. ---
        pinned_tag = overrides.get("dockerImageTag")
        if pinned_tag and pinned_tag != version:
            pinned_entry_path = f"{base}/{pinned_tag}/{registry_type}.json"
            try:
                if fs.exists(pinned_entry_path):
                    with fs.open(pinned_entry_path, "r") as fp:
                        pinned_entry = json.load(fp)
                    for field_name in ("spec", "packageInfo"):
                        if field_name in pinned_entry:
                            entry[field_name] = pinned_entry[field_name]
                            modified = True
                else:
                    logger.warning(
                        "Pinned version %s/%s.json not found for %s",
                        pinned_tag,
                        registry_type,
                        connector,
                    )
            except Exception as exc:
                logger.warning(
                    "Failed to read pinned entry %s/%s.json for %s: %s",
                    pinned_tag,
                    registry_type,
                    connector,
                    exc,
                )

        # --- Update metadata_file_path to point to latest/ ---
        generated = entry.get("generated")
        if isinstance(generated, dict):
            source_file_info = generated.get("source_file_info")
            if isinstance(source_file_info, dict):
                current_path = source_file_info.get("metadata_file_path")
                if current_path != latest_metadata_file_path:
                    source_file_info["metadata_file_path"] = latest_metadata_file_path
                    modified = True

        if modified:
            content = json.dumps(entry, indent=2, sort_keys=True) + "\n"
            with fs.open(entry_path, "w") as fout:
                fout.write(content)


# ---------------------------------------------------------------------------
# Step 6a: Compile global registry JSONs
# ---------------------------------------------------------------------------


def _compile_global_registry(
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
    latest_versions: dict[str, str],
    registry_type: str,
) -> list[dict[str, Any]]:
    """Read cloud.json or oss.json from each connector's latest/ dir.

    Args:
        registry_type: "cloud" or "oss".

    Returns:
        List of registry entry dicts.
    """
    base = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte"
    entries: list[dict[str, Any]] = []

    for connector in sorted(latest_versions):
        json_path = f"{base}/{connector}/latest/{registry_type}.json"
        try:
            if not fs.exists(json_path):
                continue
            with fs.open(json_path, "r") as f:
                entry = json.load(f)
            entries.append(entry)
        except Exception as exc:
            logger.warning(
                "Failed to read %s for %s: %s", registry_type, connector, exc
            )

    return entries


def _build_global_registry_json(entries: list[dict[str, Any]]) -> dict[str, Any]:
    """Assemble the global registry JSON from a list of connector entries.

    Splits entries into sources and destinations based on the presence of
    `sourceDefinitionId` or `destinationDefinitionId`.
    """
    sources = []
    destinations = []
    for entry in entries:
        if "sourceDefinitionId" in entry:
            sources.append(entry)
        elif "destinationDefinitionId" in entry:
            destinations.append(entry)
        else:
            # Fallback: guess from dockerRepository
            docker_repo = entry.get("dockerRepository", "")
            if "destination" in docker_repo:
                destinations.append(entry)
            else:
                sources.append(entry)

    return {"sources": sources, "destinations": destinations}


# ---------------------------------------------------------------------------
# Step 6c: Specs secrets mask
# ---------------------------------------------------------------------------


def _extract_secret_property_names(
    entries: list[dict[str, Any]],
) -> set[str]:
    """Extract property names marked as `airbyte_secret` from connector specs.

    Walks the `spec.connectionSpecification.properties` tree of each entry
    and collects the leaf property name for every node that has
    `airbyte_secret: true`.

    Args:
        entries: List of registry entry dicts (from cloud or oss registries).

    Returns:
        A set of property names that are marked as secrets.
    """
    secret_names: set[str] = set()
    for entry in entries:
        spec = entry.get("spec", {})
        conn_spec = spec.get("connectionSpecification", {})
        properties = conn_spec.get("properties")
        if not properties:
            continue
        for type_path, _ in dpath.util.search(properties, "**/type", yielded=True):
            absolute_path = f"/{type_path}"
            if "/" in type_path:
                property_path, _ = absolute_path.rsplit(sep="/", maxsplit=1)
            else:
                property_path = absolute_path
            try:
                property_definition = dpath.util.get(properties, property_path)
            except KeyError:
                continue
            if isinstance(property_definition, dict) and property_definition.get(
                "airbyte_secret", False
            ):
                secret_names.add(property_path.split("/")[-1])
    return secret_names


# ---------------------------------------------------------------------------
# Step 6b: Per-connector version index
# ---------------------------------------------------------------------------


def _build_version_index(
    connector: str,
    versions: list[str],
    yanked: set[tuple[str, str]],
    latest_version: str | None,
    fs: gcsfs.GCSFileSystem,
    bucket: str,
    prefix: str,
) -> dict[str, Any]:
    """Build the per-connector versions.json content.

    For each version, reads `metadata.yaml` to extract `releaseStage`
    and `supportLevel` (only for the latest version to keep scanning fast).
    """
    base = f"{bucket}/{prefix}{METADATA_FOLDER}/airbyte/{connector}"

    version_entries: list[dict[str, Any]] = []
    for v_str in sorted(
        versions, key=lambda s: _parse_version(s) or Version("0"), reverse=True
    ):
        is_yanked = (connector, v_str) in yanked
        is_latest = v_str == latest_version

        entry: dict[str, Any] = {
            "version": v_str,
            "yanked": is_yanked,
            "is_latest": is_latest,
        }

        # For the latest version, enrich with metadata fields
        if is_latest:
            try:
                meta_path = f"{base}/{v_str}/metadata.yaml"
                with fs.open(meta_path, "r") as f:
                    raw = yaml.safe_load(f)
                data = raw.get("data", {})
                entry["release_stage"] = data.get("releaseStage")
                entry["support_level"] = data.get("supportLevel")
            except Exception as exc:
                logger.warning(
                    "Failed to read metadata for %s@%s: %s", connector, v_str, exc
                )

        version_entries.append(entry)

    definition_id = None
    if latest_version:
        # Try to get the definition ID from the latest version's metadata
        try:
            meta_path = f"{base}/{latest_version}/metadata.yaml"
            with fs.open(meta_path, "r") as f:
                raw = yaml.safe_load(f)
            data = raw.get("data", {})
            definition_id = data.get("definitionId")
        except Exception:
            pass

    result: dict[str, Any] = {
        "connector": connector,
        "versions": version_entries,
    }
    if definition_id:
        result["definition_id"] = definition_id

    return result


# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------


def compile_registry(
    bucket_name: str,
    prefix: str = "",
    connector_name: list[str] | None = None,
    dry_run: bool = False,
    with_secrets_mask: bool = False,
) -> CompileResult:
    """Compile the registry: sync latest/ dirs and write index files.

    Steps:
        1. Glob for all `metadata.yaml` to discover (connector, version) pairs.
        2. Glob for `version-yank.yml` to build the yanked set.
        3. Compute the latest GA semver per connector.
        4. Glob for `version=*` markers in `latest/` dirs for a fast check.
        5. Rsync stale `latest/` dirs from the calculated version.
        6. Write global registry JSONs and per-connector `versions.json`.
        6c. (Optional) Regenerate `specs_secrets_mask.yaml`.

    Args:
        bucket_name: GCS bucket name to compile from/to.
        prefix: Optional path prefix within the bucket.
        connector_name: If provided, only compile these connectors.
        dry_run: If True, report what would be done without writing.
        with_secrets_mask: If True, regenerate `specs_secrets_mask.yaml`.

    Returns:
        A `CompileResult` describing what was done.
    """
    prefix = f"{prefix}/" if prefix else ""

    target_label = f"{bucket_name}/{prefix}" if prefix else bucket_name
    result = CompileResult(target=target_label, dry_run=dry_run)

    token = _get_gcs_credentials_token()
    fs = gcsfs.GCSFileSystem(token=token)

    # --- Step 1 + 2: Scan versions and yanks ---
    _log_progress("Step 1-2: Scanning versions and yank markers...")
    connector_versions, yanked = _scan_versions_and_yanks(
        fs, bucket_name, prefix, connector_name
    )
    result.connectors_scanned = len(connector_versions)
    result.versions_found = sum(len(v) for v in connector_versions.values())
    result.yanked_versions = len(yanked)
    _log_progress(
        "  Found %d connectors, %d versions, %d yanked",
        result.connectors_scanned,
        result.versions_found,
        result.yanked_versions,
    )

    # --- Step 3: Compute latest ---
    _log_progress("Step 3: Computing latest GA version per connector...")
    latest_versions = _compute_latest_versions(connector_versions, yanked)
    _log_progress("  Computed latest for %d connectors", len(latest_versions))

    # --- Step 4: Check existing latest markers ---
    _log_progress("Step 4: Checking existing latest/ markers...")
    existing_markers = _scan_latest_markers(fs, bucket_name, prefix, connector_name)
    _log_progress("  Found %d existing markers", len(existing_markers))

    stale_connectors: list[str] = []
    for connector, expected_version in latest_versions.items():
        current_marker = existing_markers.get(connector)
        if current_marker == expected_version:
            result.latest_already_current += 1
        else:
            stale_connectors.append(connector)

    _log_progress(
        "  %d connectors need latest/ update, %d already current",
        len(stale_connectors),
        result.latest_already_current,
    )

    # --- Step 5: Resync stale latest/ dirs ---
    if stale_connectors:
        _log_progress(
            "Step 5: Syncing %d stale latest/ directories...", len(stale_connectors)
        )
        for connector in sorted(stale_connectors):
            version = latest_versions[connector]
            old_marker = existing_markers.get(connector)
            try:
                _sync_latest_dir(
                    fs, bucket_name, prefix, connector, version, old_marker, dry_run
                )
                if not dry_run:
                    _apply_overrides_to_latest_entry(
                        fs, bucket_name, prefix, connector, version
                    )
                result.latest_updated += 1
            except Exception as exc:
                error_msg = f"Failed to sync latest/ for {connector}: {exc}"
                logger.error(error_msg)
                result.errors.append(error_msg)
                # Remove the version marker so the next compile retries this
                # connector instead of treating it as "already current".
                marker_path = (
                    f"{bucket_name}/{prefix}{METADATA_FOLDER}/airbyte"
                    f"/{connector}/latest/version={version}"
                )
                try:
                    if fs.exists(marker_path):
                        fs.rm(marker_path)
                        logger.info(
                            "Deleted stale marker %s after failure", marker_path
                        )
                except Exception as marker_exc:
                    logger.warning(
                        "Could not delete stale marker %s: %s",
                        marker_path,
                        marker_exc,
                    )
    else:
        _log_progress("Step 5: All latest/ directories are current, nothing to sync.")

    # --- Step 6a: Compile global registry JSONs ---
    _log_progress("Step 6a: Compiling global registry JSON files...")
    all_registry_entries: list[dict[str, Any]] = []  # collected for Step 6c
    for registry_type in ("cloud", "oss"):
        entries = _compile_global_registry(
            fs, bucket_name, prefix, latest_versions, registry_type
        )
        all_registry_entries.extend(entries)
        registry_json = _build_global_registry_json(entries)

        output_path = (
            f"{bucket_name}/{prefix}{_REGISTRIES_PREFIX}/{registry_type}_registry.json"
        )
        entry_count = len(registry_json["sources"]) + len(registry_json["destinations"])

        if registry_type == "cloud":
            result.cloud_registry_entries = entry_count
        else:
            result.oss_registry_entries = entry_count

        if dry_run:
            _log_progress(
                "  [DRY RUN] Would write %s_registry.json (%d entries)",
                registry_type,
                entry_count,
            )
        else:
            content = json.dumps(registry_json, indent=2, sort_keys=True) + "\n"
            with fs.open(output_path, "w") as f:
                f.write(content)
            _log_progress(
                "  Wrote %s_registry.json (%d entries)",
                registry_type,
                entry_count,
            )

    # --- Step 6b: Per-connector version indexes ---
    _log_progress("Step 6b: Writing per-connector version indexes...")
    base = f"{bucket_name}/{prefix}{METADATA_FOLDER}/airbyte"
    for connector in sorted(connector_versions):
        versions = connector_versions[connector]
        latest_v = latest_versions.get(connector)
        index = _build_version_index(
            connector, versions, yanked, latest_v, fs, bucket_name, prefix
        )
        index_path = f"{base}/{connector}/versions.json"

        if dry_run:
            _log_progress(
                "  [DRY RUN] Would write %s/versions.json (%d versions)",
                connector,
                len(versions),
            )
        else:
            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
            with fs.open(index_path, "w") as f:
                f.write(content)

        result.version_indexes_written += 1

    # --- Step 6c: Specs secrets mask (optional) ---
    if with_secrets_mask:
        if connector_name:
            _log_progress(
                "Step 6c: SKIPPED — --with-secrets-mask requires all connectors "
                "but --connector-name is set. Run without --connector-name "
                "to regenerate the full secrets mask."
            )
        else:
            _log_progress("Step 6c: Generating specs secrets mask...")
            # Reuse entries collected during Step 6a to avoid redundant GCS reads.
            secret_names = _extract_secret_property_names(all_registry_entries)
            sorted_names = sorted(secret_names)
            result.specs_secrets_mask_properties = len(sorted_names)
            mask_content = yaml.dump(
                {"properties": sorted_names}, default_flow_style=False
            )
            mask_path = f"{bucket_name}/{prefix}{_REGISTRIES_PREFIX}/{_SPECS_SECRETS_MASK_FILENAME}"

            _log_progress(
                "  Found %d secret properties: %s",
                len(sorted_names),
                ", ".join(sorted_names),
            )

            if dry_run:
                _log_progress(
                    "  [DRY RUN] Would write %s",
                    _SPECS_SECRETS_MASK_FILENAME,
                )
            else:
                with fs.open(mask_path, "w") as f:
                    f.write(mask_content)
                _log_progress(
                    "  Wrote %s",
                    _SPECS_SECRETS_MASK_FILENAME,
                )

    _log_progress(result.summary())
    return result
