"""Catalog workspace creation with cross-server and selective asset support.

This module provides `create_ml_workspace` for creating a partial catalog clone
containing data reachable from a root RID. It handles the common case of
incoherent row-level policies in source catalogs.

The solution uses a three-stage approach:
1. Create schema WITHOUT foreign keys
2. Copy all data (using async operations for performance)
3. Apply foreign keys, handling violations by either:
   - Deleting orphan rows (rows with dangling FK references)
   - Nullifying references (setting dangling FK values to NULL)
   - Failing the operation (reporting violations)
"""

from __future__ import annotations

import asyncio
import json
import logging
import time
from collections.abc import Callable
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from enum import Enum
from typing import Any
from urllib.parse import quote as urlquote

from deriva.core import DerivaServer, ErmrestCatalog, get_credential
from deriva.core.asyncio import AsyncCatalogWrapper, AsyncErmrestCatalog
from deriva.core.asyncio.async_datapath import copy_tables_concurrent_async
from deriva.core.hatrac_store import HatracStore

from deriva_ml.model.catalog import VOCAB_COLUMNS
from deriva_ml.schema import create_ml_schema

logger = logging.getLogger("deriva_ml")


# =============================================================================
# Enums and Dataclasses
# =============================================================================


class CloneIssueSeverity(Enum):
    """Severity level of clone issues."""

    INFO = "info"
    WARNING = "warning"
    ERROR = "error"
    CRITICAL = "critical"


class CloneIssueCategory(Enum):
    """Category of clone issues."""

    ACCESS_DENIED = "access_denied"
    ORPHAN_ROWS = "orphan_rows"
    DATA_INTEGRITY = "data_integrity"
    SCHEMA_ISSUE = "schema_issue"
    RESTORE_FAILURE = "restore_failure"
    FK_VIOLATION = "fk_violation"
    FK_PRUNED = "fk_pruned"  # FK was intentionally not applied
    POLICY_INCOHERENCE = "policy_incoherence"
    INDEX_REBUILT = "index_rebuilt"  # Index was dropped and rebuilt due to size limits


class OrphanStrategy(Enum):
    """Strategy for handling orphan rows (rows with dangling FK references).

    When cloning a catalog with incoherent row-level policies, some rows may
    reference parent rows that are hidden from the cloning user. These orphan
    rows would violate FK constraints.
    """

    FAIL = "fail"  # Fail the clone if FK violations occur
    DELETE = "delete"  # Delete rows with dangling references
    NULLIFY = "nullify"  # Set dangling FK values to NULL (requires nullok)


class AssetCopyMode(Enum):
    """How to handle assets during catalog cloning."""

    NONE = "none"  # Don't copy assets
    REFERENCES = "refs"  # Keep URLs pointing to source
    FULL = "full"  # Download and re-upload assets


@dataclass
class CloneIssue:
    """A single issue encountered during catalog cloning."""

    severity: CloneIssueSeverity
    category: CloneIssueCategory
    message: str
    table: str | None = None
    details: str | None = None
    action: str | None = None
    row_count: int = 0
    skipped_rids: list[str] | None = None  # RIDs of rows that were skipped

    def to_dict(self) -> dict[str, Any]:
        result = {
            "severity": self.severity.value,
            "category": self.category.value,
            "message": self.message,
            "table": self.table,
            "details": self.details,
            "action": self.action,
            "row_count": self.row_count,
        }
        if self.skipped_rids:
            result["skipped_rids"] = self.skipped_rids
        return result

    def __str__(self) -> str:
        parts = [f"[{self.severity.value.upper()}]"]
        if self.table:
            parts.append(f"{self.table}:")
        parts.append(self.message)
        if self.row_count > 0:
            parts.append(f"({self.row_count} rows)")
        result = " ".join(parts)
        if self.skipped_rids:
            if len(self.skipped_rids) <= 5:
                result += f"\n    Skipped RIDs: {', '.join(self.skipped_rids)}"
            else:
                result += f"\n    Skipped RIDs: {len(self.skipped_rids)} rows (see JSON for full list)"
        return result


@dataclass
class CloneReportSummary:
    """Summary statistics for a clone operation."""

    total_issues: int
    errors: int
    warnings: int
    tables_restored: int
    tables_failed: int
    tables_skipped: int
    total_rows_restored: int
    orphan_rows_removed: int
    orphan_rows_nullified: int
    fkeys_applied: int
    fkeys_failed: int
    fkeys_pruned: int


@dataclass
class CloneReport:
    """Comprehensive report of catalog clone operation.

    Tracks all issues encountered during cloning, including:
    - Policy incoherence issues (FK violations due to hidden data)
    - Orphan rows that were deleted or nullified
    - FKs that were pruned or failed
    - Tables that were restored, failed, or skipped

    Provides both JSON and text output formats for reporting.
    """

    issues: list[CloneIssue] = field(default_factory=list)
    tables_restored: dict[str, int] = field(default_factory=dict)
    tables_failed: list[str] = field(default_factory=list)
    tables_skipped: list[str] = field(default_factory=list)
    orphan_details: dict[str, dict] = field(default_factory=dict)
    fkeys_applied: int = 0
    fkeys_failed: int = 0
    fkeys_pruned: int = 0

    def add_issue(self, issue: CloneIssue) -> None:
        self.issues.append(issue)

    @property
    def summary(self) -> CloneReportSummary:
        """Return summary statistics as a dataclass."""
        return CloneReportSummary(
            total_issues=len(self.issues),
            errors=len([i for i in self.issues if i.severity == CloneIssueSeverity.ERROR]),
            warnings=len([i for i in self.issues if i.severity == CloneIssueSeverity.WARNING]),
            tables_restored=len(self.tables_restored),
            tables_failed=len(self.tables_failed),
            tables_skipped=len(self.tables_skipped),
            total_rows_restored=sum(self.tables_restored.values()),
            orphan_rows_removed=sum(
                d.get("rows_removed", 0) for d in self.orphan_details.values()
            ),
            orphan_rows_nullified=sum(
                d.get("rows_nullified", 0) for d in self.orphan_details.values()
            ),
            fkeys_applied=self.fkeys_applied,
            fkeys_failed=self.fkeys_failed,
            fkeys_pruned=self.fkeys_pruned,
        )

    def to_dict(self) -> dict[str, Any]:
        """Return the report as a JSON-serializable dictionary."""
        return {
            "summary": asdict(self.summary),
            "issues": [i.to_dict() for i in self.issues],
            "tables_restored": self.tables_restored,
            "tables_failed": self.tables_failed,
            "tables_skipped": self.tables_skipped,
            "orphan_details": self.orphan_details,
        }

    def to_json(self, indent: int = 2) -> str:
        """Return the report as a formatted JSON string."""
        return json.dumps(self.to_dict(), indent=indent)

    def to_text(self) -> str:
        """Return the report as human-readable text."""
        lines = []
        lines.append("=" * 70)
        lines.append("CATALOG CLONE REPORT")
        lines.append("=" * 70)
        lines.append("")

        # Summary
        summary = self.to_dict()["summary"]
        lines.append("SUMMARY")
        lines.append("-" * 40)
        lines.append(f"Tables restored:       {summary['tables_restored']}")
        lines.append(f"Tables failed:         {summary['tables_failed']}")
        lines.append(f"Tables skipped:        {summary['tables_skipped']}")
        lines.append(f"Total rows restored:   {summary['total_rows_restored']}")
        lines.append(f"Orphan rows removed:   {summary['orphan_rows_removed']}")
        lines.append(f"Orphan rows nullified: {summary['orphan_rows_nullified']}")
        lines.append(f"FKs applied:           {summary['fkeys_applied']}")
        lines.append(f"FKs failed:            {summary['fkeys_failed']}")
        lines.append(f"FKs pruned:            {summary['fkeys_pruned']}")
        lines.append(f"Errors:                {summary['errors']}")
        lines.append(f"Warnings:              {summary['warnings']}")
        lines.append("")

        # Issues by severity
        if self.issues:
            lines.append("ISSUES")
            lines.append("-" * 40)

            for severity in [CloneIssueSeverity.CRITICAL, CloneIssueSeverity.ERROR,
                           CloneIssueSeverity.WARNING, CloneIssueSeverity.INFO]:
                severity_issues = [i for i in self.issues if i.severity == severity]
                if severity_issues:
                    lines.append(f"\n{severity.value.upper()} ({len(severity_issues)}):")
                    for issue in severity_issues:
                        lines.append(f"  - {issue}")
                        if issue.details:
                            details = issue.details[:100] + "..." if len(issue.details) > 100 else issue.details
                            lines.append(f"    Details: {details}")
                        if issue.action:
                            lines.append(f"    Action: {issue.action}")
            lines.append("")

        # Orphan details
        if self.orphan_details:
            lines.append("ORPHAN ROW DETAILS")
            lines.append("-" * 40)
            for table, details in self.orphan_details.items():
                removed = details.get("rows_removed", 0)
                nullified = details.get("rows_nullified", 0)
                lines.append(f"  {table}:")
                if removed > 0:
                    lines.append(f"    Rows deleted: {removed}")
                if nullified > 0:
                    lines.append(f"    Rows nullified: {nullified}")
                missing = details.get("missing_references", {})
                for ref_table, count in missing.items():
                    lines.append(f"    -> missing references to {ref_table}: {count}")
            lines.append("")

        # Assessment
        lines.append("CLONE ASSESSMENT")
        lines.append("-" * 40)
        if summary['errors'] > 0:
            lines.append("Clone completed with ERRORS. Some FKs could not be applied.")
            lines.append("The catalog schema may be degraded.")
        elif summary['orphan_rows_removed'] > 0 or summary['orphan_rows_nullified'] > 0:
            lines.append("Clone completed with orphan handling.")
            lines.append("Source catalog may have incoherent row-level policies.")
        elif summary['fkeys_pruned'] > 0:
            lines.append("Clone completed with pruned FKs.")
            lines.append("Some FK constraints were skipped due to hidden reference data.")
        else:
            lines.append("Clone completed successfully.")
        lines.append("")
        lines.append("=" * 70)

        return "\n".join(lines)

    def __str__(self) -> str:
        """Return text representation of the report."""
        return self.to_text()


@dataclass
class AssetFilter:
    """Filter for selecting which assets to copy during cloning."""

    tables: list[str] | None = None
    rids: list[str] | None = None


@dataclass
class TruncatedValue:
    """Record of a value that was truncated during cloning."""

    table: str
    rid: str
    column: str
    original_bytes: int
    truncated_bytes: int

    def to_dict(self) -> dict[str, Any]:
        return {
            "table": self.table,
            "rid": self.rid,
            "column": self.column,
            "original_bytes": self.original_bytes,
            "truncated_bytes": self.truncated_bytes,
        }


@dataclass
class CloneCatalogResult:
    """Result of a catalog clone operation."""

    catalog_id: str
    hostname: str
    schema_only: bool
    asset_mode: AssetCopyMode
    source_hostname: str
    source_catalog_id: str
    source_snapshot: str | None = None
    alias: str | None = None
    ml_schema_added: bool = False
    datasets_reinitialized: int = 0
    orphan_rows_removed: int = 0
    orphan_rows_nullified: int = 0
    fkeys_pruned: int = 0
    rows_skipped: int = 0
    truncated_values: list[TruncatedValue] = field(default_factory=list)
    report: CloneReport | None = None


# Clone state annotation URL (same as deriva-py)
_clone_state_url = "tag:isrd.isi.edu,2018:clone-state"

# Catalog provenance annotation URL
_catalog_provenance_url = "tag:deriva-ml.org,2025:catalog-provenance"

# Pattern to detect btree index size errors
_BTREE_INDEX_ERROR_PATTERN = "index row size"
_BTREE_INDEX_NAME_PATTERN = r'for index "([^"]+)"'

# Maximum safe size for btree index values (with margin below 2704 limit)
_MAX_INDEX_VALUE_BYTES = 2600
_TRUNCATE_SUFFIX = "...[TRUNCATED]"


class CatalogCreationMethod(Enum):
    """How a catalog was created."""

    CLONE = "clone"  # Cloned from another catalog
    CREATE = "create"  # Created programmatically (e.g., create_catalog)
    SCHEMA = "schema"  # Created from schema definition
    UNKNOWN = "unknown"  # Unknown or pre-existing catalog


@dataclass
class CloneDetails:
    """Details specific to cloned catalogs."""

    source_hostname: str
    source_catalog_id: str
    source_snapshot: str | None = None
    source_schema_url: str | None = None  # Hatrac URL to source schema JSON
    # Clone parameters
    orphan_strategy: str = "fail"
    truncate_oversized: bool = False
    prune_hidden_fkeys: bool = False
    schema_only: bool = False
    asset_mode: str = "refs"
    exclude_schemas: list[str] = field(default_factory=list)
    exclude_objects: list[str] = field(default_factory=list)
    add_ml_schema: bool = False
    copy_annotations: bool = True
    copy_policy: bool = True
    reinitialize_dataset_versions: bool = True
    # Statistics
    rows_copied: int = 0
    rows_skipped: int = 0
    skipped_rids: list[str] = field(default_factory=list)
    truncated_count: int = 0
    orphan_rows_removed: int = 0
    orphan_rows_nullified: int = 0
    fkeys_pruned: int = 0

    def to_dict(self) -> dict[str, Any]:
        result = {
            "source_hostname": self.source_hostname,
            "source_catalog_id": self.source_catalog_id,
            "source_snapshot": self.source_snapshot,
            "source_schema_url": self.source_schema_url,
            "orphan_strategy": self.orphan_strategy,
            "truncate_oversized": self.truncate_oversized,
            "prune_hidden_fkeys": self.prune_hidden_fkeys,
            "schema_only": self.schema_only,
            "asset_mode": self.asset_mode,
            "exclude_schemas": self.exclude_schemas,
            "exclude_objects": self.exclude_objects,
            "add_ml_schema": self.add_ml_schema,
            "copy_annotations": self.copy_annotations,
            "copy_policy": self.copy_policy,
            "reinitialize_dataset_versions": self.reinitialize_dataset_versions,
            "rows_copied": self.rows_copied,
            "rows_skipped": self.rows_skipped,
            "truncated_count": self.truncated_count,
            "orphan_rows_removed": self.orphan_rows_removed,
            "orphan_rows_nullified": self.orphan_rows_nullified,
            "fkeys_pruned": self.fkeys_pruned,
        }
        if self.skipped_rids:
            result["skipped_rids"] = self.skipped_rids
        return result

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> "CloneDetails":
        return cls(
            source_hostname=data.get("source_hostname", ""),
            source_catalog_id=data.get("source_catalog_id", ""),
            source_snapshot=data.get("source_snapshot"),
            source_schema_url=data.get("source_schema_url"),
            orphan_strategy=data.get("orphan_strategy", "fail"),
            truncate_oversized=data.get("truncate_oversized", False),
            prune_hidden_fkeys=data.get("prune_hidden_fkeys", False),
            schema_only=data.get("schema_only", False),
            asset_mode=data.get("asset_mode", "refs"),
            exclude_schemas=data.get("exclude_schemas", []),
            exclude_objects=data.get("exclude_objects", []),
            add_ml_schema=data.get("add_ml_schema", False),
            copy_annotations=data.get("copy_annotations", True),
            copy_policy=data.get("copy_policy", True),
            reinitialize_dataset_versions=data.get("reinitialize_dataset_versions", True),
            rows_copied=data.get("rows_copied", 0),
            rows_skipped=data.get("rows_skipped", 0),
            skipped_rids=data.get("skipped_rids", []),
            truncated_count=data.get("truncated_count", 0),
            orphan_rows_removed=data.get("orphan_rows_removed", 0),
            orphan_rows_nullified=data.get("orphan_rows_nullified", 0),
            fkeys_pruned=data.get("fkeys_pruned", 0),
        )


@dataclass
class CatalogProvenance:
    """Provenance information for a catalog.

    This metadata is stored as a catalog-level annotation and tracks
    how the catalog was created, by whom, and with what parameters.
    """

    creation_method: CatalogCreationMethod
    created_at: str
    hostname: str
    catalog_id: str
    created_by: str | None = None
    name: str | None = None
    description: str | None = None
    workflow_url: str | None = None
    workflow_version: str | None = None
    clone_details: CloneDetails | None = None

    def to_dict(self) -> dict[str, Any]:
        result = {
            "creation_method": self.creation_method.value,
            "created_at": self.created_at,
            "hostname": self.hostname,
            "catalog_id": self.catalog_id,
            "created_by": self.created_by,
            "name": self.name,
            "description": self.description,
            "workflow_url": self.workflow_url,
            "workflow_version": self.workflow_version,
        }
        if self.clone_details:
            result["clone_details"] = self.clone_details.to_dict()
        return result

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> "CatalogProvenance":
        clone_details = None
        if data.get("clone_details"):
            clone_details = CloneDetails.from_dict(data["clone_details"])

        # Handle legacy format where creation_method might be missing
        method_str = data.get("creation_method", "unknown")
        try:
            creation_method = CatalogCreationMethod(method_str)
        except ValueError:
            creation_method = CatalogCreationMethod.UNKNOWN

        return cls(
            creation_method=creation_method,
            created_at=data.get("created_at", ""),
            hostname=data.get("hostname", ""),
            catalog_id=data.get("catalog_id", ""),
            created_by=data.get("created_by"),
            name=data.get("name"),
            description=data.get("description"),
            workflow_url=data.get("workflow_url"),
            workflow_version=data.get("workflow_version"),
            clone_details=clone_details,
        )

    @property
    def is_clone(self) -> bool:
        """Return True if this catalog was cloned from another catalog."""
        return self.creation_method == CatalogCreationMethod.CLONE and self.clone_details is not None


# =============================================================================
# Provenance Utilities
# =============================================================================


def _upload_source_schema(
    hostname: str,
    catalog_id: str,
    schema_json: dict[str, Any],
    credential: dict | None,
) -> str | None:
    """Upload source schema JSON to Hatrac."""
    try:
        cred = credential or get_credential(hostname)
        hatrac = HatracStore("https", hostname, credentials=cred)

        namespace = f"/hatrac/catalog/{catalog_id}/provenance"
        try:
            hatrac.create_namespace(namespace, parents=True)
        except Exception:
            pass  # Namespace may already exist

        schema_bytes = json.dumps(schema_json, indent=2).encode("utf-8")
        object_path = f"{namespace}/source-schema.json"

        url = hatrac.put_obj(
            object_path,
            schema_bytes,
            content_type="application/json",
        )

        logger.info(f"Uploaded source schema to {url}")
        return url

    except Exception as e:
        logger.warning(f"Failed to upload source schema to Hatrac: {e}")
        return None


def _set_catalog_provenance(
    dst_catalog: ErmrestCatalog,
    provenance: CatalogProvenance,
) -> None:
    """Set the catalog provenance annotation on a catalog."""
    try:
        dst_catalog.put(
            f"/annotation/{urlquote(_catalog_provenance_url)}",
            json=provenance.to_dict(),
        )
        logger.info("Set catalog provenance annotation")
    except Exception as e:
        logger.warning(f"Failed to set catalog provenance annotation: {e}")


def set_catalog_provenance(
    catalog: ErmrestCatalog,
    name: str | None = None,
    description: str | None = None,
    workflow_url: str | None = None,
    workflow_version: str | None = None,
    creation_method: CatalogCreationMethod = CatalogCreationMethod.CREATE,
) -> CatalogProvenance:
    """Set catalog provenance information for a newly created catalog.

    Args:
        catalog: The catalog to annotate.
        name: Human-readable name for the catalog.
        description: Description of the catalog's purpose.
        workflow_url: URL to the workflow/script that created the catalog.
        workflow_version: Version of the workflow.
        creation_method: How the catalog was created. Defaults to CREATE.

    Returns:
        The CatalogProvenance object that was set.
    """
    created_by = None
    try:
        session_info = catalog.get("/authn/session").json()
        if session_info and "client" in session_info:
            client = session_info["client"]
            created_by = client.get("display_name") or client.get("id")
    except Exception:
        pass

    try:
        catalog_info = catalog.get("/").json()
        hostname = catalog_info.get("meta", {}).get("host", "")
        catalog_id = str(catalog.catalog_id)
    except Exception:
        hostname = ""
        catalog_id = str(catalog.catalog_id)

    provenance = CatalogProvenance(
        creation_method=creation_method,
        created_at=datetime.now(timezone.utc).isoformat(),
        hostname=hostname,
        catalog_id=catalog_id,
        created_by=created_by,
        name=name,
        description=description,
        workflow_url=workflow_url,
        workflow_version=workflow_version,
    )

    _set_catalog_provenance(catalog, provenance)
    return provenance


def get_catalog_provenance(catalog: ErmrestCatalog) -> CatalogProvenance | None:
    """Get the catalog provenance information.

    Returns provenance information if the catalog has it set.

    Args:
        catalog: The catalog to check.

    Returns:
        CatalogProvenance if available, None otherwise.
    """
    try:
        model = catalog.getCatalogModel()
        provenance_data = model.annotations.get(_catalog_provenance_url)
        if provenance_data:
            return CatalogProvenance.from_dict(provenance_data)
    except Exception as e:
        logger.debug(f"Could not get catalog provenance: {e}")

    return None


# =============================================================================
# Index Error Handling
# =============================================================================


def _parse_index_error(error_msg: str) -> tuple[str | None, str | None]:
    """Parse a btree index size error to extract index name and column."""
    import re

    if _BTREE_INDEX_ERROR_PATTERN not in error_msg:
        return None, None

    match = re.search(_BTREE_INDEX_NAME_PATTERN, error_msg)
    if not match:
        return None, None

    index_name = match.group(1)

    if "__" in index_name and index_name.endswith("_idx"):
        parts = index_name.rsplit("__", 1)
        if len(parts) == 2:
            column_name = parts[1].replace("_idx", "")
            return index_name, column_name

    return index_name, None


def _rebuild_deferred_indexes(
    dst_catalog: ErmrestCatalog,
    deferred_indexes: dict[str, list[dict]],
    report: CloneReport,
) -> None:
    """Report on indexes that had issues during data copy."""
    if not deferred_indexes:
        return

    logger.info(f"Reporting {sum(len(v) for v in deferred_indexes.values())} index issues...")


# =============================================================================
# Export Annotation Parsing
# =============================================================================


# Export annotation tag
_export_tag = "tag:isrd.isi.edu,2019:export"


def _parse_export_annotation_tables(
    table: Any,
    paths_discovered: list[list[str]] | None = None,
) -> tuple[list[str], list[list[str]]]:
    """Parse export annotation from a table to extract tables and paths.

    The export annotation (tag:isrd.isi.edu,2019:export) defines which tables
    should be exported when downloading a row as a BDBag.

    Args:
        table: ERMrest Table object with annotations.
        paths_discovered: Optional list to append discovered paths to.

    Returns:
        Tuple of (tables_list, paths_list) where:
        - tables_list: List of table names in "schema:table" format
        - paths_list: List of paths, each path is a list of "schema:table" strings
    """
    if paths_discovered is None:
        paths_discovered = []

    tables: set[str] = set()

    root_table_spec = f"{table.schema.name}:{table.name}"
    tables.add(root_table_spec)

    export_annotation = table.annotations.get(_export_tag, {})

    for context_key, context_value in export_annotation.items():
        templates = context_value.get("templates", [])
        for template in templates:
            outputs = template.get("outputs", [])
            for output in outputs:
                source = output.get("source", {})
                path_str = source.get("path", "")

                if not path_str:
                    continue

                path_segments = path_str.split("/")
                current_path: list[str] = [root_table_spec]

                for segment in path_segments:
                    if not segment:
                        continue

                    if "=" in segment:
                        continue

                    if ":" in segment:
                        parts = segment.split(":")
                        if len(parts) == 2 and not any(c in segment for c in ["(", ")", "!", "@"]):
                            schema, tname = parts
                            table_spec = f"{schema}:{tname}"
                            tables.add(table_spec)
                            current_path.append(table_spec)

                if len(current_path) > 1:
                    paths_discovered.append(current_path)

    return sorted(tables), paths_discovered


# =============================================================================
# Table Discovery
# =============================================================================


def _discover_reachable_tables(
    model: Any,
    start_tables: list[str],
    exclude_tables: set[tuple[str, str]] | None = None,
    exclude_schemas: set[str] | None = None,
) -> list[str]:
    """Discover all tables reachable from start tables via FK relationships.

    Traverses FK graph in both directions (outbound and inbound FKs).

    Args:
        model: ERMrest Model object.
        start_tables: Starting tables in "schema:table" format.
        exclude_tables: Set of (schema, table) tuples to exclude.
        exclude_schemas: Set of schema names to exclude.

    Returns:
        List of reachable table names in "schema:table" format.
    """
    exclude_tables = exclude_tables or set()
    exclude_schemas = exclude_schemas or set()

    system_schemas = {"public", "_acl_admin", "WWW"}
    all_excluded_schemas = system_schemas | exclude_schemas

    discovered: set[tuple[str, str]] = set()
    to_visit: list[tuple[str, str]] = []

    for table_spec in start_tables:
        if ":" not in table_spec:
            raise ValueError(f"Table must be specified as 'schema:table', got: {table_spec}")
        schema, table = table_spec.split(":", 1)
        key = (schema, table)
        if key not in exclude_tables and schema not in all_excluded_schemas:
            discovered.add(key)
            to_visit.append(key)

    while to_visit:
        current_key = to_visit.pop(0)
        schema_name, table_name = current_key

        try:
            table = model.schemas[schema_name].tables[table_name]
        except KeyError:
            continue

        for fk in table.foreign_keys:
            pk_table = fk.pk_table
            pk_key = (pk_table.schema.name, pk_table.name)

            if pk_key in discovered or pk_key in exclude_tables:
                continue
            if pk_table.schema.name in all_excluded_schemas:
                continue

            discovered.add(pk_key)
            to_visit.append(pk_key)

        for fk in table.referenced_by:
            ref_table = fk.table
            ref_key = (ref_table.schema.name, ref_table.name)

            if ref_key in discovered or ref_key in exclude_tables:
                continue
            if ref_table.schema.name in all_excluded_schemas:
                continue

            discovered.add(ref_key)
            to_visit.append(ref_key)

    return [f"{schema}:{table}" for schema, table in sorted(discovered)]


def _expand_tables_with_associations(
    model: Any,
    include_tables: list[str],
) -> tuple[list[str], list[str]]:
    """Expand table list to include association tables needed for FK integrity.

    Args:
        model: ERMrest Model object.
        include_tables: List of table names in "schema:table" format.

    Returns:
        Tuple of (all_tables, association_tables_added).
    """
    included_set: set[tuple[str, str]] = set()
    for table_spec in include_tables:
        if ":" in table_spec:
            schema, table = table_spec.split(":", 1)
            included_set.add((schema, table))
        else:
            raise ValueError(f"Table must be specified as 'schema:table', got: {table_spec}")

    associations_added: list[str] = []

    for schema_name, table_name in list(included_set):
        try:
            table = model.schemas[schema_name].tables[table_name]
        except KeyError:
            continue

        for assoc in table.find_associations(pure=False):
            assoc_table = assoc.table
            assoc_key = (assoc_table.schema.name, assoc_table.name)

            if assoc_key in included_set:
                continue

            for other_fk in assoc.other_fkeys:
                other_table = other_fk.pk_table
                other_key = (other_table.schema.name, other_table.name)

                if other_key in included_set:
                    included_set.add(assoc_key)
                    assoc_spec = f"{assoc_key[0]}:{assoc_key[1]}"
                    if assoc_spec not in associations_added:
                        associations_added.append(assoc_spec)
                    break

    all_tables = list(include_tables) + associations_added
    return all_tables, associations_added


def _expand_tables_with_vocabularies(
    model: Any,
    include_tables: list[str],
) -> tuple[list[str], list[str]]:
    """Expand table list to include vocabulary tables referenced by included tables.

    Args:
        model: ERMrest Model object.
        include_tables: List of table names in "schema:table" format.

    Returns:
        Tuple of (all_tables, vocabulary_tables_added).
    """
    def is_vocabulary(table) -> bool:
        return VOCAB_COLUMNS.issubset({c.name.upper() for c in table.columns})

    included_set: set[tuple[str, str]] = set()
    for table_spec in include_tables:
        if ":" in table_spec:
            schema, table = table_spec.split(":", 1)
            included_set.add((schema, table))

    vocabularies_added: list[str] = []

    for schema_name, table_name in list(included_set):
        try:
            table = model.schemas[schema_name].tables[table_name]
        except KeyError:
            continue

        for fk in table.foreign_keys:
            pk_table = fk.pk_table
            pk_key = (pk_table.schema.name, pk_table.name)

            if pk_key in included_set:
                continue

            if is_vocabulary(pk_table):
                included_set.add(pk_key)
                vocab_spec = f"{pk_key[0]}:{pk_key[1]}"
                if vocab_spec not in vocabularies_added:
                    vocabularies_added.append(vocab_spec)

    all_tables = list(include_tables) + vocabularies_added
    return all_tables, vocabularies_added


def _quote_table_spec(table_spec: str) -> str:
    """URL-quote a table specification for ERMrest queries."""
    schema, table = table_spec.split(":", 1)
    return f"{urlquote(schema)}:{urlquote(table)}"


# =============================================================================
# Schema Building (extracted from _clone_three_stage)
# =============================================================================


def _build_subset_schema(
    src_model: Any,
    included_tables: set[tuple[str, str]],
    copy_annotations: bool,
    copy_policy: bool,
    prune_hidden_fkeys: bool,
    report: CloneReport,
) -> tuple[list[dict], list[tuple[str, str, dict]], int]:
    """Build schema model JSON for subset of tables, deferring foreign keys.

    Args:
        src_model: Source catalog model.
        included_tables: Set of (schema, table) tuples to include.
        copy_annotations: Whether to copy annotations.
        copy_policy: Whether to copy ACL policies.
        prune_hidden_fkeys: Whether to prune FKs with hidden reference data.
        report: Clone report for recording issues.

    Returns:
        Tuple of (model_json, deferred_fkeys, fkeys_pruned) where:
        - model_json: List of schema/table definitions for POST /schema
        - deferred_fkeys: List of (schema, table, fk_def) tuples
        - fkeys_pruned: Number of FKs that were pruned
    """
    new_model = []
    fkeys_deferred = []
    fkeys_pruned = 0

    def prune_parts(d, *extra_victims):
        victims = set(extra_victims)
        if not copy_annotations:
            victims |= {'annotations'}
        if not copy_policy:
            victims |= {'acls', 'acl_bindings'}
        for k in victims:
            d.pop(k, None)
        return d

    def should_prune_fkey(fkdef):
        """Check if FK should be pruned due to hidden data."""
        if not prune_hidden_fkeys:
            return False

        for ref_col in fkdef.get('referenced_columns', []):
            ref_schema = ref_col.get('schema_name')
            ref_table = ref_col.get('table_name')
            ref_col_name = ref_col.get('column_name')

            if ref_schema and ref_table and ref_col_name:
                try:
                    ref_table_obj = src_model.schemas[ref_schema].tables[ref_table]
                    col_obj = ref_table_obj.column_definitions[ref_col_name]
                    rights = getattr(col_obj, 'rights', None)
                    if rights and rights.get('select') is None:
                        return True
                except (KeyError, AttributeError):
                    pass
        return False

    # Collect schemas that have included tables
    included_schemas: set[str] = {schema for schema, _ in included_tables}

    for sname in included_schemas:
        if sname not in src_model.schemas:
            continue

        schema = src_model.schemas[sname]
        schema_def = prune_parts(schema.prejson(), 'tables')
        new_model.append(schema_def)

        for tname, table in schema.tables.items():
            if (sname, tname) not in included_tables:
                continue

            if table.kind != 'table':
                continue

            if 'RID' not in table.column_definitions.elements:
                logger.warning(f"Table {sname}.{tname} lacks system columns, skipping")
                report.tables_skipped.append(f"{sname}:{tname}")
                continue

            # Create table definition without FKs
            table_def = prune_parts(table.prejson(), 'foreign_keys')
            table_def['column_definitions'] = [
                prune_parts(c) for c in table_def['column_definitions']
            ]
            table_def['keys'] = [prune_parts(k) for k in table_def.get('keys', [])]
            table_def.setdefault('annotations', {})[_clone_state_url] = 1

            new_model.append(table_def)

            # Collect FKs (only those between included tables)
            for fkdef in table.prejson().get('foreign_keys', []):
                # Skip FKs to system tables
                skip = False
                for ref_col in fkdef.get('referenced_columns', []):
                    if ref_col.get('schema_name') == 'public' \
                       and ref_col.get('table_name') in {'ERMrest_Client', 'ERMrest_Group', 'ERMrest_RID_Lease'}:
                        skip = True
                        break

                if skip:
                    continue

                # Check if FK references only included tables
                include_fk = True
                for ref_col in fkdef.get('referenced_columns', []):
                    ref_schema = ref_col.get('schema_name')
                    ref_table = ref_col.get('table_name')
                    if (ref_schema, ref_table) not in included_tables:
                        include_fk = False
                        break

                if not include_fk:
                    continue

                if should_prune_fkey(fkdef):
                    fkeys_pruned += 1
                    fk_name = fkdef.get('names', [[sname, 'unknown']])[0]
                    report.add_issue(CloneIssue(
                        severity=CloneIssueSeverity.WARNING,
                        category=CloneIssueCategory.FK_PRUNED,
                        message="FK pruned due to hidden reference data",
                        table=f"{sname}:{tname}",
                        details=f"FK {fk_name} references columns with 'select': null",
                        action="Source catalog may have incoherent policies",
                    ))
                    continue

                fkeys_deferred.append((sname, tname, prune_parts(fkdef.copy())))

    return new_model, fkeys_deferred, fkeys_pruned


# =============================================================================
# FK Application with Orphan Handling (extracted from _clone_three_stage)
# =============================================================================


def _identify_orphan_values(
    dst_catalog: ErmrestCatalog,
    sname: str,
    tname: str,
    fk_def: dict,
) -> set:
    """Identify orphan FK values without deleting them.

    Returns: Set of values that exist in the FK column but not in the referenced table.
    """
    fk_columns = fk_def.get('foreign_key_columns', [])
    ref_columns = fk_def.get('referenced_columns', [])

    if not fk_columns or not ref_columns:
        return set()

    src_col = fk_columns[0].get('column_name')
    ref_schema = ref_columns[0].get('schema_name')
    ref_table = ref_columns[0].get('table_name')
    ref_col = ref_columns[0].get('column_name')

    if not all([src_col, ref_schema, ref_table, ref_col]):
        return set()

    src_uri = f"{urlquote(sname)}:{urlquote(tname)}"
    ref_uri = f"{urlquote(ref_schema)}:{urlquote(ref_table)}"

    try:
        src_values = dst_catalog.get(
            f"/attributegroup/{src_uri}/{urlquote(src_col)}"
        ).json()
        src_value_set = {row[src_col] for row in src_values if row.get(src_col) is not None}
    except Exception as e:
        logger.error(f"Failed to get source values for {sname}:{tname}.{src_col}: {e}")
        return set()

    try:
        ref_values = dst_catalog.get(
            f"/attributegroup/{ref_uri}/{urlquote(ref_col)}"
        ).json()
        ref_value_set = {row[ref_col] for row in ref_values if row.get(ref_col) is not None}
    except Exception as e:
        logger.error(f"Failed to get reference values for {ref_schema}:{ref_table}.{ref_col}: {e}")
        return set()

    return src_value_set - ref_value_set


def _delete_orphan_rows(
    dst_catalog: ErmrestCatalog,
    sname: str,
    tname: str,
    fk_def: dict,
    orphan_values: set,
    report: CloneReport,
) -> tuple[int, int]:
    """Delete rows with orphan FK values.

    Returns: (rows_removed, rows_nullified)
    """
    fk_columns = fk_def.get('foreign_key_columns', [])
    ref_columns = fk_def.get('referenced_columns', [])

    if not fk_columns or not ref_columns:
        return 0, 0

    src_col = fk_columns[0].get('column_name')
    ref_schema = ref_columns[0].get('schema_name')
    ref_table = ref_columns[0].get('table_name')

    src_uri = f"{urlquote(sname)}:{urlquote(tname)}"

    rows_removed = 0
    for value in orphan_values:
        encoded_value = urlquote(str(value), safe='') if isinstance(value, str) else str(value)
        try:
            dst_catalog.delete(f"/entity/{src_uri}/{urlquote(src_col)}={encoded_value}")
            rows_removed += 1
        except Exception as e:
            logger.debug(f"Could not delete {sname}:{tname} where {src_col}={value}: {e}")

    if rows_removed > 0:
        table_key = f"{sname}:{tname}"
        if table_key not in report.orphan_details:
            report.orphan_details[table_key] = {
                "rows_removed": 0,
                "rows_nullified": 0,
                "missing_references": {},
            }

        report.orphan_details[table_key]["rows_removed"] += rows_removed
        ref_key = f"{ref_schema}:{ref_table}"
        report.orphan_details[table_key]["missing_references"][ref_key] = len(orphan_values)

        report.add_issue(CloneIssue(
            severity=CloneIssueSeverity.WARNING,
            category=CloneIssueCategory.ORPHAN_ROWS,
            message="Orphan rows deleted",
            table=table_key,
            details=f"Missing references to: {ref_key} ({len(orphan_values)})",
            action="Source catalog may have incoherent row-level policies",
            row_count=rows_removed,
        ))

    return rows_removed, 0


def _handle_fk_violation(
    dst_catalog: ErmrestCatalog,
    sname: str,
    tname: str,
    fk_def: dict,
    strategy: OrphanStrategy,
    report: CloneReport,
) -> tuple[int, int]:
    """Handle FK violation by deleting or nullifying orphan rows.

    Returns: (rows_removed, rows_nullified)
    """
    fk_columns = fk_def.get('foreign_key_columns', [])
    ref_columns = fk_def.get('referenced_columns', [])

    if not fk_columns or not ref_columns:
        return 0, 0

    src_col = fk_columns[0].get('column_name')
    ref_schema = ref_columns[0].get('schema_name')
    ref_table = ref_columns[0].get('table_name')
    ref_col = ref_columns[0].get('column_name')

    if not all([src_col, ref_schema, ref_table, ref_col]):
        return 0, 0

    src_uri = f"{urlquote(sname)}:{urlquote(tname)}"
    ref_uri = f"{urlquote(ref_schema)}:{urlquote(ref_table)}"

    try:
        src_values = dst_catalog.get(
            f"/attributegroup/{src_uri}/{urlquote(src_col)}"
        ).json()
        src_value_set = {row[src_col] for row in src_values if row.get(src_col) is not None}
    except Exception as e:
        logger.error(f"Failed to get source values: {e}")
        return 0, 0

    try:
        ref_values = dst_catalog.get(
            f"/attributegroup/{ref_uri}/{urlquote(ref_col)}"
        ).json()
        ref_value_set = {row[ref_col] for row in ref_values if row.get(ref_col) is not None}
    except Exception as e:
        logger.error(f"Failed to get reference values: {e}")
        return 0, 0

    orphan_values = src_value_set - ref_value_set

    if not orphan_values:
        return 0, 0

    logger.info(f"Found {len(orphan_values)} orphan values in {sname}:{tname}.{src_col}")

    rows_removed = 0
    rows_nullified = 0

    for value in orphan_values:
        encoded_value = urlquote(str(value), safe='') if isinstance(value, str) else str(value)

        if strategy == OrphanStrategy.DELETE:
            try:
                dst_catalog.delete(f"/entity/{src_uri}/{urlquote(src_col)}={encoded_value}")
                rows_removed += 1
            except Exception as e:
                logger.warning(f"Failed to delete orphans for {src_col}={value}: {e}")
        elif strategy == OrphanStrategy.NULLIFY:
            try:
                dst_catalog.put(
                    f"/attributegroup/{src_uri}/{urlquote(src_col)}={encoded_value}/{urlquote(src_col)}",
                    json=None
                )
                rows_nullified += 1
            except Exception as e:
                logger.warning(f"Failed to nullify {src_col}={value}: {e}")

    # Record in report
    table_key = f"{sname}:{tname}"
    if table_key not in report.orphan_details:
        report.orphan_details[table_key] = {
            "rows_removed": 0,
            "rows_nullified": 0,
            "missing_references": {},
        }

    report.orphan_details[table_key]["rows_removed"] += rows_removed
    report.orphan_details[table_key]["rows_nullified"] += rows_nullified
    ref_key = f"{ref_schema}:{ref_table}"
    report.orphan_details[table_key]["missing_references"][ref_key] = len(orphan_values)

    action_taken = "deleted" if strategy == OrphanStrategy.DELETE else "nullified"
    report.add_issue(CloneIssue(
        severity=CloneIssueSeverity.WARNING,
        category=CloneIssueCategory.ORPHAN_ROWS,
        message=f"Orphan rows {action_taken}",
        table=table_key,
        details=f"Missing references to: {ref_key} ({len(orphan_values)})",
        action="Source catalog may have incoherent row-level policies",
        row_count=rows_removed + rows_nullified,
    ))

    return rows_removed, rows_nullified


def _apply_fkeys_with_orphan_handling(
    dst_catalog: ErmrestCatalog,
    fkeys_deferred: list[tuple[str, str, dict]],
    orphan_strategy: OrphanStrategy,
    report: CloneReport,
) -> tuple[int, int]:
    """Apply deferred foreign keys with orphan handling.

    Args:
        dst_catalog: Destination catalog.
        fkeys_deferred: List of (schema, table, fk_def) tuples.
        orphan_strategy: How to handle orphan rows.
        report: Clone report for recording issues.

    Returns:
        Tuple of (orphan_rows_removed, orphan_rows_nullified).
    """
    orphan_rows_removed = 0
    orphan_rows_nullified = 0

    if orphan_strategy == OrphanStrategy.DELETE:
        # Three-phase approach: identify orphans, delete, then apply FKs

        # Phase 1: Identify orphan values for each FK
        logger.info("Phase 1: Identifying orphan values...")
        fk_orphans: list[tuple[str, str, dict, set]] = []

        for sname, tname, fk in fkeys_deferred:
            orphan_values = _identify_orphan_values(dst_catalog, sname, tname, fk)
            if orphan_values:
                fk_orphans.append((sname, tname, fk, orphan_values))
                logger.info(f"Found {len(orphan_values)} orphan values in {sname}:{tname}")

        # Phase 2: Delete orphan rows in cascading waves
        if fk_orphans:
            logger.info("Phase 2: Deleting orphan rows...")

            max_waves = 20
            all_processed_fks: set[tuple[str, str, str]] = set()

            for wave in range(max_waves):
                current_orphans: list[tuple[str, str, dict, set]] = []

                for sname, tname, fk in fkeys_deferred:
                    fk_names = fk.get('names', [])
                    fk_id = (sname, tname, str(fk_names))
                    if fk_id in all_processed_fks:
                        continue

                    orphan_values = _identify_orphan_values(dst_catalog, sname, tname, fk)
                    if orphan_values:
                        current_orphans.append((sname, tname, fk, orphan_values))

                if not current_orphans:
                    logger.info(f"Deletion wave {wave + 1}: no more orphans found")
                    break

                logger.info(f"Deletion wave {wave + 1}: processing {len(current_orphans)} FKs with orphans")

                wave_deleted = 0
                for sname, tname, fk, orphan_values in current_orphans:
                    removed, nullified = _delete_orphan_rows(
                        dst_catalog, sname, tname, fk, orphan_values, report
                    )
                    orphan_rows_removed += removed
                    orphan_rows_nullified += nullified
                    wave_deleted += removed

                    if removed == len(orphan_values):
                        fk_names = fk.get('names', [])
                        fk_id = (sname, tname, str(fk_names))
                        all_processed_fks.add(fk_id)

                if wave_deleted == 0:
                    logger.warning(f"Deletion wave {wave + 1}: no rows deleted, may have circular dependencies")
                    break

        # Phase 3: Apply all FKs
        logger.info("Phase 3: Applying foreign keys...")
        failed_fks = []

        for sname, tname, fk in fkeys_deferred:
            try:
                dst_catalog.post("/schema", json=[fk])
                report.fkeys_applied += 1
            except Exception as e:
                failed_fks.append((sname, tname, fk, str(e)))

        # Retry failed FKs with additional orphan cleanup
        for retry_round in range(10):
            if not failed_fks:
                break

            logger.info(f"FK retry round {retry_round + 1}: {len(failed_fks)} FKs still failing")

            for sname, tname, fk, last_error in failed_fks:
                orphan_values = _identify_orphan_values(dst_catalog, sname, tname, fk)
                if orphan_values:
                    removed, nullified = _delete_orphan_rows(
                        dst_catalog, sname, tname, fk, orphan_values, report
                    )
                    orphan_rows_removed += removed
                    orphan_rows_nullified += nullified

            still_failed = []
            for sname, tname, fk, last_error in failed_fks:
                try:
                    dst_catalog.post("/schema", json=[fk])
                    report.fkeys_applied += 1
                except Exception as e:
                    still_failed.append((sname, tname, fk, str(e)))

            if len(still_failed) == len(failed_fks):
                logger.warning(f"FK retry round {retry_round + 1}: no progress, stopping retries")
                break

            failed_fks = still_failed

        # Record final failures
        for sname, tname, fk, error_msg in failed_fks:
            report.fkeys_failed += 1
            report.add_issue(CloneIssue(
                severity=CloneIssueSeverity.ERROR,
                category=CloneIssueCategory.FK_VIOLATION,
                message="FK still failing after handling orphans",
                table=f"{sname}:{tname}",
                details=error_msg[:500],
            ))

    else:
        # For NULLIFY or FAIL strategies: single-pass approach
        for sname, tname, fk in fkeys_deferred:
            try:
                dst_catalog.post("/schema", json=[fk])
                report.fkeys_applied += 1
            except Exception as e:
                error_msg = str(e)

                if orphan_strategy == OrphanStrategy.FAIL:
                    report.fkeys_failed += 1
                    report.add_issue(CloneIssue(
                        severity=CloneIssueSeverity.ERROR,
                        category=CloneIssueCategory.FK_VIOLATION,
                        message="FK constraint failed",
                        table=f"{sname}:{tname}",
                        details=error_msg[:500],
                        action="Use orphan_strategy=DELETE or NULLIFY to handle",
                    ))
                    continue

                # NULLIFY strategy
                removed, nullified = _handle_fk_violation(
                    dst_catalog, sname, tname, fk, orphan_strategy, report
                )
                orphan_rows_removed += removed
                orphan_rows_nullified += nullified

                try:
                    dst_catalog.post("/schema", json=[fk])
                    report.fkeys_applied += 1
                except Exception as retry_error:
                    report.fkeys_failed += 1
                    report.add_issue(CloneIssue(
                        severity=CloneIssueSeverity.ERROR,
                        category=CloneIssueCategory.FK_VIOLATION,
                        message="FK still failing after nullifying orphans",
                        table=f"{sname}:{tname}",
                        details=str(retry_error)[:500],
                    ))

    return orphan_rows_removed, orphan_rows_nullified


# =============================================================================
# Configuration Copy
# =============================================================================


def _copy_configuration(
    src_model,
    dst_catalog: ErmrestCatalog,
    copy_annotations: bool,
    copy_policy: bool,
    exclude_schemas: list[str],
    excluded_tables: set[tuple[str, str]],
) -> None:
    """Copy annotations and policies after FK application."""
    dst_model = dst_catalog.getCatalogModel()

    for sname, src_schema in src_model.schemas.items():
        if sname in exclude_schemas or sname not in dst_model.schemas:
            continue

        dst_schema = dst_model.schemas[sname]

        if copy_annotations:
            for k, v in src_schema.annotations.items():
                if k != _clone_state_url:
                    dst_schema.annotations[k] = v

        if copy_policy:
            if hasattr(dst_schema, 'acls') and hasattr(src_schema, 'acls'):
                dst_schema.acls.update(src_schema.acls)
            if hasattr(dst_schema, 'acl_bindings') and hasattr(src_schema, 'acl_bindings'):
                dst_schema.acl_bindings.update(src_schema.acl_bindings)

        for tname, src_table in src_schema.tables.items():
            if (sname, tname) in excluded_tables or tname not in dst_schema.tables:
                continue

            dst_table = dst_schema.tables[tname]

            if copy_annotations:
                for k, v in src_table.annotations.items():
                    if k != _clone_state_url:
                        dst_table.annotations[k] = v

            if copy_policy:
                if hasattr(dst_table, 'acls') and hasattr(src_table, 'acls'):
                    dst_table.acls.update(src_table.acls)
                if hasattr(dst_table, 'acl_bindings') and hasattr(src_table, 'acl_bindings'):
                    dst_table.acl_bindings.update(src_table.acl_bindings)

    try:
        dst_model.apply()
    except Exception as e:
        logger.warning(f"Failed to apply some configuration: {e}")


# =============================================================================
# Post-Clone Operations
# =============================================================================


def _get_catalog_snapshot(
    hostname: str,
    catalog_id: str,
    credential: dict | None,
) -> str | None:
    """Get the current snapshot ID for a catalog."""
    try:
        cred = credential or get_credential(hostname)
        server = DerivaServer("https", hostname, credentials=cred)
        catalog = server.connect_ermrest(catalog_id)
        response = catalog.get("/")
        if response.status_code == 200:
            data = response.json()
            return data.get("snaptime")
    except Exception as e:
        logger.warning(f"Could not get catalog snapshot: {e}")
    return None


def _post_clone_operations(
    result: CloneCatalogResult,
    alias: str | None,
    add_ml_schema: bool,
    credential: dict | None,
) -> CloneCatalogResult:
    """Perform post-clone operations."""
    cred = credential or get_credential(result.hostname)
    server = DerivaServer("https", result.hostname, credentials=cred)

    if alias:
        try:
            server.create_ermrest_alias(id=alias, alias_target=str(result.catalog_id))
            result.alias = alias
        except Exception as e:
            logger.warning(f"Failed to create alias '{alias}': {e}")

    if add_ml_schema:
        try:
            catalog = server.connect_ermrest(result.catalog_id)
            create_ml_schema(catalog)
            result.ml_schema_added = True

            try:
                from deriva_ml import DerivaML
                ml = DerivaML(result.hostname, result.catalog_id, check_auth=False)
                ml.apply_catalog_annotations()
                logger.info("Applied catalog annotations (chaise-config, navbar)")
            except Exception as e:
                logger.warning(f"Failed to apply catalog annotations: {e}")
                if result.report:
                    result.report.add_issue(CloneIssue(
                        severity=CloneIssueSeverity.WARNING,
                        category=CloneIssueCategory.SCHEMA_ISSUE,
                        message="Failed to apply catalog annotations",
                        details=str(e),
                        action="Manually call apply_catalog_annotations() after clone",
                    ))
        except Exception as e:
            logger.warning(f"Failed to add ML schema: {e}")
            if result.report:
                result.report.add_issue(CloneIssue(
                    severity=CloneIssueSeverity.ERROR,
                    category=CloneIssueCategory.SCHEMA_ISSUE,
                    message="Failed to add DerivaML schema",
                    details=str(e),
                    action="ML schema was not added to the clone",
                ))

    return result


def _reinitialize_dataset_versions(
    result: CloneCatalogResult,
    credential: dict | None,
) -> CloneCatalogResult:
    """Reinitialize dataset versions after cloning."""
    try:
        cred = credential or get_credential(result.hostname)
        server = DerivaServer("https", result.hostname, credentials=cred)
        catalog = server.connect_ermrest(result.catalog_id)

        model = catalog.getCatalogModel()
        if "deriva-ml" not in model.schemas:
            return result

        datasets = catalog.get("/entity/deriva-ml:Dataset").json()

        for dataset in datasets:
            try:
                rid = dataset["RID"]
                catalog.post(
                    "/entity/deriva-ml:Dataset_Version",
                    json=[{
                        "Dataset": rid,
                        "Version": "0.0.1",
                        "Description": f"Cloned from {result.source_hostname}:{result.source_catalog_id}",
                    }]
                )
                result.datasets_reinitialized += 1
            except Exception as e:
                logger.warning(f"Failed to reinitialize version for dataset {rid}: {e}")

    except Exception as e:
        logger.warning(f"Failed to reinitialize dataset versions: {e}")

    return result


# =============================================================================
# Async Data Copy Helpers
# =============================================================================


async def _copy_path_query_async(
    src_async: AsyncErmrestCatalog,
    dst_async: AsyncErmrestCatalog,
    path_query: str,
    table_spec: str,
    page_size: int = 1000,
    truncate_oversized: bool = False,
) -> int:
    """Copy data using an async path query with pagination.

    Fetches data from the source via a path query and inserts into the destination.
    Handles duplicate rows (via onconflict=skip) and btree index errors (via
    row-by-row fallback).

    Args:
        src_async: Source async catalog.
        dst_async: Destination async catalog.
        path_query: ERMrest path query (e.g., /entity/Schema:Table/RID=X/Schema:Next).
        table_spec: Target table in "schema:table" format.
        page_size: Number of rows per page.
        truncate_oversized: Whether to truncate oversized values.

    Returns:
        Number of rows copied.
    """
    rows_copied = 0
    last_rid = ""

    while True:
        # Add pagination
        if last_rid:
            query = f"{path_query}@sort(RID)@after({urlquote(last_rid)})?limit={page_size}"
        else:
            query = f"{path_query}@sort(RID)?limit={page_size}"

        try:
            response = await src_async.get_async(query)
            page = response.json()
        except Exception as e:
            logger.warning(f"Path query failed {path_query}: {e}")
            break

        if not page:
            break

        # Optionally truncate oversized values
        if truncate_oversized:
            page = [_truncate_row_values(row) for row in page]

        # Insert into destination with onconflict=skip to handle overlapping paths
        dst_path = f"/entity/{_quote_table_spec(table_spec)}?nondefaults=RID,RCT,RCB&onconflict=skip"
        try:
            await dst_async.post_async(dst_path, json_data=page)
            rows_copied += len(page)
        except Exception as e:
            error_msg = str(e)
            if _BTREE_INDEX_ERROR_PATTERN in error_msg:
                # Row-by-row fallback for index size errors
                for row in page:
                    try:
                        await dst_async.post_async(dst_path, json_data=[row])
                        rows_copied += 1
                    except Exception:
                        logger.debug(f"Skipping row due to index error")
            else:
                logger.warning(f"Failed to insert into {table_spec}: {e}")

        if len(page) < page_size:
            break
        last_rid = page[-1].get("RID", "")
        if not last_rid:
            break

    return rows_copied


def _truncate_row_values(row: dict) -> dict:
    """Truncate oversized text values in a row for btree index compatibility."""
    modified = row.copy()
    for col, value in row.items():
        if isinstance(value, str):
            value_bytes = len(value.encode('utf-8'))
            if value_bytes > _MAX_INDEX_VALUE_BYTES:
                max_chars = _MAX_INDEX_VALUE_BYTES - len(_TRUNCATE_SUFFIX.encode('utf-8'))
                truncated = value[:max_chars] + _TRUNCATE_SUFFIX
                while len(truncated.encode('utf-8')) > _MAX_INDEX_VALUE_BYTES:
                    max_chars -= 100
                    truncated = value[:max_chars] + _TRUNCATE_SUFFIX
                modified[col] = truncated
    return modified


async def _copy_data_via_export_paths_async(
    src_async: AsyncErmrestCatalog,
    dst_async: AsyncErmrestCatalog,
    root_table: str,
    root_rid: str,
    export_paths: list[list[str]],
    all_tables: list[str],
    report: CloneReport,
    truncate_oversized: bool = False,
    page_size: int = 1000,
) -> dict[str, int]:
    """Copy data by following export paths asynchronously.

    Copies the root row, then follows each export path to copy related data.
    Tables not covered by export paths (e.g., vocabulary tables) are noted
    as uncovered for later fill-in.

    Args:
        src_async: Source async catalog.
        dst_async: Destination async catalog.
        root_table: Root table in "schema:table" format.
        root_rid: Root RID to start from.
        export_paths: Paths from export annotation.
        all_tables: All tables to copy.
        report: Clone report.
        truncate_oversized: Whether to truncate oversized values.
        page_size: Number of rows per batch.

    Returns:
        Dict mapping table spec -> rows copied.
    """
    rows_by_table: dict[str, int] = {}

    # Copy root row
    root_query = f"/entity/{_quote_table_spec(root_table)}/RID={urlquote(root_rid)}"
    try:
        response = await src_async.get_async(root_query)
        root_row = response.json()
        if root_row:
            dst_path = f"/entity/{_quote_table_spec(root_table)}?nondefaults=RID,RCT,RCB"
            await dst_async.post_async(dst_path, json_data=root_row)
            rows_by_table[root_table] = 1
            logger.info(f"Copied 1 row for {root_table}")
    except Exception as e:
        logger.warning(f"Failed to copy root row: {e}")

    # Copy data for each export path
    tables_copied = {root_table}
    for path in export_paths:
        if len(path) < 2:
            continue

        # Build the path query starting from root
        query = f"/entity/{_quote_table_spec(root_table)}/RID={urlquote(root_rid)}"
        for table_spec in path[1:]:
            query += f"/{_quote_table_spec(table_spec)}"

        target_table = path[-1]
        if target_table in tables_copied:
            continue

        rows = await _copy_path_query_async(
            src_async, dst_async, query, target_table,
            page_size=page_size, truncate_oversized=truncate_oversized,
        )
        rows_by_table[target_table] = rows_by_table.get(target_table, 0) + rows
        tables_copied.add(target_table)
        logger.info(f"Copied {rows} rows for {target_table}")

    return rows_by_table


# =============================================================================
# Main Async Implementation
# =============================================================================


async def _create_ml_workspace_async(
    src_catalog: ErmrestCatalog,
    dst_catalog: ErmrestCatalog,
    src_model: Any,
    source_hostname: str,
    source_catalog_id: str,
    dest_hostname: str,
    root_table_key: str,
    root_rid: str,
    all_tables: list[str],
    export_paths: list[list[str]],
    included_tables: set[tuple[str, str]],
    report: CloneReport,
    src_cred: dict | None,
    dst_cred: dict | None,
    truncate_oversized: bool,
    page_size: int = 1000,
    table_concurrency: int = 1,
    progress_callback: Callable[[str, float], None] | None = None,
) -> dict[str, int]:
    """Async implementation of data copying phase.

    Creates async catalog connections, copies data via export paths,
    then fills in uncovered tables using concurrent async table copy.

    Returns:
        Dict mapping table spec -> rows copied.
    """
    src_async = AsyncErmrestCatalog(
        "https", source_hostname, source_catalog_id, src_cred
    )
    dst_async = AsyncErmrestCatalog(
        "https", dest_hostname, str(dst_catalog.catalog_id), dst_cred
    )
    src_wrapper = AsyncCatalogWrapper(src_catalog, src_async)

    try:
        rows_by_table: dict[str, int] = {}

        if export_paths:
            # Copy data via export paths (respects row-level ACLs)
            if progress_callback:
                progress_callback("Copying data via export paths", 30.0)

            rows_by_table = await _copy_data_via_export_paths_async(
                src_async, dst_async,
                root_table=root_table_key,
                root_rid=root_rid,
                export_paths=export_paths,
                all_tables=all_tables,
                report=report,
                truncate_oversized=truncate_oversized,
                page_size=page_size,
            )

        # Fill in tables not covered by export paths
        # (vocabulary tables, association tables, etc.)
        fill_tables = [t for t in all_tables if t not in rows_by_table]
        if fill_tables:
            if progress_callback:
                progress_callback("Filling uncovered tables", 50.0)

            logger.info(f"Filling {len(fill_tables)} uncovered tables (concurrency={table_concurrency})")
            fill_results = await copy_tables_concurrent_async(
                src_wrapper, dst_async, fill_tables,
                table_concurrency=table_concurrency,
                page_size=10000,
            )
            rows_by_table.update(fill_results)

        return rows_by_table

    finally:
        await src_async.close()
        await dst_async.close()


# =============================================================================
# Public API
# =============================================================================


def create_ml_workspace(
    source_hostname: str,
    source_catalog_id: str,
    root_rid: str,
    *,
    include_tables: list[str] | None = None,
    exclude_objects: list[str] | None = None,
    exclude_schemas: list[str] | None = None,
    include_associations: bool = True,
    include_vocabularies: bool = True,
    dest_hostname: str | None = None,
    alias: str | None = None,
    add_ml_schema: bool = True,
    asset_mode: AssetCopyMode = AssetCopyMode.REFERENCES,
    copy_annotations: bool = True,
    copy_policy: bool = True,
    source_credential: dict | None = None,
    dest_credential: dict | None = None,
    orphan_strategy: OrphanStrategy = OrphanStrategy.FAIL,
    prune_hidden_fkeys: bool = False,
    truncate_oversized: bool = False,
    reinitialize_dataset_versions: bool = True,
    table_concurrency: int = 1,
    progress_callback: Callable[[str, float], None] | None = None,
) -> CloneCatalogResult:
    """Create an ML workspace by cloning data reachable from a root RID.

    This is the primary function for creating partial catalog clones. It uses the
    root table's export annotation (if available) to determine which tables and
    paths to follow, then fills in any uncovered tables (vocabularies, associations).

    The function uses a three-stage approach:
    1. Create schema WITHOUT foreign keys (only for included tables)
    2. Copy data asynchronously (export paths + fill-in tables)
    3. Apply foreign keys, handling violations based on orphan_strategy

    Args:
        source_hostname: Hostname of the source catalog server.
        source_catalog_id: ID of the catalog to clone from.
        root_rid: The starting RID from which to trace reachability.
        include_tables: Optional additional tables ("schema:table" format)
            to include beyond what's discovered from the root table.
        exclude_objects: Tables ("schema:table" format) to exclude.
        exclude_schemas: Schema names to exclude entirely.
        include_associations: If True, auto-include association tables.
        include_vocabularies: If True, auto-include vocabulary tables.
        dest_hostname: Destination hostname. If None, uses source_hostname.
        alias: Optional alias for the new catalog.
        add_ml_schema: If True, add DerivaML schema to clone.
        asset_mode: How to handle assets (NONE, REFERENCES, or FULL).
        copy_annotations: If True, copy annotations to clone.
        copy_policy: If True, copy ACLs/ACL bindings to clone.
        source_credential: Credentials for source catalog.
        dest_credential: Credentials for destination catalog.
        orphan_strategy: How to handle orphan rows (FAIL, DELETE, or NULLIFY).
        prune_hidden_fkeys: If True, prune FKs with hidden reference data.
        truncate_oversized: If True, truncate values exceeding index size limits.
        reinitialize_dataset_versions: If True, reinitialize dataset versions.
        table_concurrency: Maximum number of tables to copy concurrently
            during the fill phase. Lower values reduce server load. Default: 2.
        progress_callback: Optional callback(message, percent_complete) for
            progress reporting (e.g., from MCP tools).

    Returns:
        CloneCatalogResult with details of the cloned catalog.

    Raises:
        ValueError: If root_rid is not found in any table.
        ValueError: If include_tables contains invalid table specifications.

    Example:
        >>> result = create_ml_workspace(
        ...     source_hostname="www.facebase.org",
        ...     source_catalog_id="1",
        ...     root_rid="3-HXMC",
        ...     alias="my-project-clone",
        ...     orphan_strategy=OrphanStrategy.DELETE,
        ... )
        >>> print(f"Created catalog {result.catalog_id}")
    """
    include_tables = include_tables or []
    exclude_objects = exclude_objects or []
    exclude_schemas_set = set(exclude_schemas) if exclude_schemas else set()

    # Validate table format
    for table_spec in include_tables:
        if ":" not in table_spec:
            raise ValueError(f"Table must be specified as 'schema:table', got: {table_spec}")

    # Parse exclude_objects into set of tuples
    excluded_tables: set[tuple[str, str]] = set()
    for table_spec in exclude_objects:
        if ":" not in table_spec:
            raise ValueError(f"exclude_objects entries must be 'schema:table', got: {table_spec}")
        schema, table = table_spec.split(":", 1)
        excluded_tables.add((schema, table))

    dest_hostname = dest_hostname or source_hostname

    if progress_callback:
        progress_callback("Connecting to source catalog", 0.0)

    # =========================================================================
    # Phase 0: Setup
    # =========================================================================

    start_time = time.time()
    src_cred = source_credential or get_credential(source_hostname)
    dst_cred = dest_credential or get_credential(dest_hostname)

    src_server = DerivaServer("https", source_hostname, credentials=src_cred)
    src_catalog = src_server.connect_ermrest(source_catalog_id)
    src_model = src_catalog.getCatalogModel()

    logger.info(
        f"Connected to source catalog {source_hostname}/{source_catalog_id} "
        f"in {time.time() - start_time:.2f}s"
    )

    # Get source snapshot for provenance
    source_snapshot = _get_catalog_snapshot(
        source_hostname, source_catalog_id, source_credential
    )

    # Capture source schema for provenance
    source_schema_json = src_catalog.get("/schema").json()

    # Find the table containing the root RID
    if progress_callback:
        progress_callback("Finding root RID", 5.0)

    root_table_key = None
    for sname, schema in src_model.schemas.items():
        if sname in {"public", "_acl_admin", "WWW"} or sname in exclude_schemas_set:
            continue
        for tname, table in schema.tables.items():
            if (sname, tname) in excluded_tables:
                continue
            if table.kind != 'table' or 'RID' not in table.column_definitions.elements:
                continue
            try:
                table_spec = f"{sname}:{tname}"
                uri = f"/entity/{_quote_table_spec(table_spec)}/RID={urlquote(root_rid)}"
                result = src_catalog.get(uri).json()
                if result:
                    root_table_key = table_spec
                    break
            except Exception:
                continue
        if root_table_key:
            break

    if root_table_key is None:
        raise ValueError(f"Root RID {root_rid} not found in any accessible table")

    logger.info(f"Root RID {root_rid} found in table {root_table_key}")

    # =========================================================================
    # Phase 1: Table Discovery
    # =========================================================================

    if progress_callback:
        progress_callback("Discovering tables", 10.0)

    root_schema, root_tname = root_table_key.split(":", 1)
    root_table_obj = src_model.schemas[root_schema].tables[root_tname]

    # Try export annotation first, fall back to FK graph traversal
    export_paths: list[list[str]] = []
    discovered_tables, export_paths = _parse_export_annotation_tables(
        root_table_obj, []
    )

    if discovered_tables and len(discovered_tables) > 1:
        logger.info(f"Export annotation defines {len(discovered_tables)} tables and {len(export_paths)} paths")

        # Add explicitly included tables
        for table_spec in include_tables:
            if table_spec not in discovered_tables:
                discovered_tables.append(table_spec)

        # Filter out excluded tables
        discovered_tables = [
            t for t in discovered_tables
            if tuple(t.split(":", 1)) not in excluded_tables
        ]
    else:
        # Fall back to FK graph traversal
        logger.info("No export annotation found, using FK graph traversal")
        export_paths = []

        start_tables = [root_table_key]
        for table_spec in include_tables:
            if table_spec not in start_tables:
                start_tables.append(table_spec)

        discovered_tables = _discover_reachable_tables(
            model=src_model,
            start_tables=start_tables,
            exclude_tables=excluded_tables,
            exclude_schemas=exclude_schemas_set,
        )

    logger.info(f"Discovered {len(discovered_tables)} connected tables")

    # Expand with associations and vocabularies
    all_tables = list(discovered_tables)
    associations_added: list[str] = []
    vocabularies_added: list[str] = []

    if include_associations:
        all_tables, associations_added = _expand_tables_with_associations(src_model, all_tables)
        associations_added = [
            t for t in associations_added
            if tuple(t.split(":", 1)) not in excluded_tables
        ]
        all_tables = [
            t for t in all_tables
            if tuple(t.split(":", 1)) not in excluded_tables
        ]
        if associations_added:
            logger.info(f"Auto-added association tables: {associations_added}")

    if include_vocabularies:
        all_tables, vocabularies_added = _expand_tables_with_vocabularies(src_model, all_tables)
        vocabularies_added = [
            t for t in vocabularies_added
            if tuple(t.split(":", 1)) not in excluded_tables
        ]
        all_tables = [
            t for t in all_tables
            if tuple(t.split(":", 1)) not in excluded_tables
        ]
        if vocabularies_added:
            logger.info(f"Auto-added vocabulary tables: {vocabularies_added}")

    logger.info(f"Will clone {len(all_tables)} tables")

    # Parse tables into set for schema building
    included_tables: set[tuple[str, str]] = set()
    for table_spec in all_tables:
        schema, table = table_spec.split(":", 1)
        included_tables.add((schema, table))

    # =========================================================================
    # Phase 2: Schema Creation
    # =========================================================================

    if progress_callback:
        progress_callback("Creating destination catalog", 20.0)

    report = CloneReport()
    clone_timestamp = datetime.now(timezone.utc).isoformat()

    dst_server = DerivaServer("https", dest_hostname, credentials=dst_cred)
    dst_catalog = dst_server.create_ermrest_catalog()
    dst_catalog_id = str(dst_catalog.catalog_id)

    logger.info(f"Created destination catalog {dest_hostname}/{dst_catalog_id}")

    try:
        # Copy top-level ACLs
        if copy_policy and src_model.acls:
            try:
                dst_catalog.put('/acl', json=src_model.acls)
            except Exception as e:
                logger.warning(f"Could not copy ACLs (may not be owner): {e}")

        if copy_annotations:
            dst_catalog.put('/annotation', json=src_model.annotations)

        # Build schema without FKs
        new_model, fkeys_deferred, fkeys_pruned = _build_subset_schema(
            src_model=src_model,
            included_tables=included_tables,
            copy_annotations=copy_annotations,
            copy_policy=copy_policy,
            prune_hidden_fkeys=prune_hidden_fkeys,
            report=report,
        )

        logger.info("Stage 1: Creating schema without foreign keys...")
        if new_model:
            dst_catalog.post("/schema", json=new_model)

        # =====================================================================
        # Phase 3: Async Data Copy
        # =====================================================================

        if progress_callback:
            progress_callback("Copying data", 25.0)

        logger.info("Stage 2: Copying data asynchronously...")
        rows_by_table = asyncio.run(
            _create_ml_workspace_async(
                src_catalog=src_catalog,
                dst_catalog=dst_catalog,
                src_model=src_model,
                source_hostname=source_hostname,
                source_catalog_id=source_catalog_id,
                dest_hostname=dest_hostname,
                root_table_key=root_table_key,
                root_rid=root_rid,
                all_tables=all_tables,
                export_paths=export_paths,
                included_tables=included_tables,
                report=report,
                src_cred=src_cred,
                dst_cred=dst_cred,
                truncate_oversized=truncate_oversized,
                table_concurrency=table_concurrency,
                progress_callback=progress_callback,
            )
        )

        # Update report with copied rows
        total_rows_copied = 0
        for table_key, rows in rows_by_table.items():
            report.tables_restored[table_key] = rows
            total_rows_copied += rows

        logger.info(f"Stage 2 complete: {total_rows_copied} rows copied across {len(rows_by_table)} tables")

        # =====================================================================
        # Phase 4: FK Application
        # =====================================================================

        if progress_callback:
            progress_callback("Applying foreign keys", 70.0)

        logger.info(f"Stage 3: Applying {len(fkeys_deferred)} foreign keys...")
        orphan_rows_removed, orphan_rows_nullified = _apply_fkeys_with_orphan_handling(
            dst_catalog, fkeys_deferred, orphan_strategy, report,
        )
        report.fkeys_pruned = fkeys_pruned

        # Copy configuration (annotations, policies)
        if copy_annotations or copy_policy:
            _copy_configuration(
                src_model, dst_catalog, copy_annotations, copy_policy,
                exclude_schemas or [], excluded_tables,
            )

        # =====================================================================
        # Phase 5: Post-Clone Operations
        # =====================================================================

        if progress_callback:
            progress_callback("Post-clone operations", 85.0)

        # Build result
        result = CloneCatalogResult(
            catalog_id=dst_catalog_id,
            hostname=dest_hostname,
            schema_only=False,
            asset_mode=asset_mode,
            source_hostname=source_hostname,
            source_catalog_id=source_catalog_id,
            source_snapshot=source_snapshot,
            orphan_rows_removed=orphan_rows_removed,
            orphan_rows_nullified=orphan_rows_nullified,
            fkeys_pruned=fkeys_pruned,
            report=report,
        )

        # Upload source schema and set provenance
        source_schema_url = _upload_source_schema(
            hostname=dest_hostname,
            catalog_id=dst_catalog_id,
            schema_json=source_schema_json,
            credential=dst_cred,
        )

        created_by = None
        try:
            session_info = dst_catalog.get("/authn/session").json()
            if session_info and "client" in session_info:
                client = session_info["client"]
                created_by = client.get("display_name") or client.get("id")
        except Exception:
            pass

        clone_details = CloneDetails(
            source_hostname=source_hostname,
            source_catalog_id=source_catalog_id,
            source_snapshot=source_snapshot,
            source_schema_url=source_schema_url,
            orphan_strategy=orphan_strategy.value,
            truncate_oversized=truncate_oversized,
            prune_hidden_fkeys=prune_hidden_fkeys,
            schema_only=False,
            asset_mode=asset_mode.value,
            exclude_schemas=exclude_schemas or [],
            exclude_objects=exclude_objects or [],
            add_ml_schema=add_ml_schema,
            copy_annotations=copy_annotations,
            copy_policy=copy_policy,
            reinitialize_dataset_versions=reinitialize_dataset_versions,
            rows_copied=total_rows_copied,
            orphan_rows_removed=orphan_rows_removed,
            orphan_rows_nullified=orphan_rows_nullified,
            fkeys_pruned=fkeys_pruned,
        )

        provenance = CatalogProvenance(
            creation_method=CatalogCreationMethod.CLONE,
            created_at=clone_timestamp,
            hostname=dest_hostname,
            catalog_id=dst_catalog_id,
            created_by=created_by,
            name=alias or f"Clone of {source_catalog_id}",
            description=f"Partial clone from {source_hostname}:{source_catalog_id} (root: {root_rid})",
            clone_details=clone_details,
        )
        _set_catalog_provenance(dst_catalog, provenance)

        # Post-clone operations (alias, ML schema)
        result = _post_clone_operations(
            result=result,
            alias=alias,
            add_ml_schema=add_ml_schema,
            credential=dst_cred,
        )

        if reinitialize_dataset_versions:
            result = _reinitialize_dataset_versions(result, dst_cred)

        # Set defaultTable to the root table
        try:
            chaise_config_url = "tag:isrd.isi.edu,2019:chaise-config"
            dst_model = dst_catalog.getCatalogModel()
            dst_model.annotations[chaise_config_url] = dst_model.annotations.get(chaise_config_url, {})
            root_schema, root_tname = root_table_key.split(":", 1)
            dst_model.annotations[chaise_config_url]["defaultTable"] = {
                "schema": root_schema,
                "table": root_tname,
            }
            dst_model.apply()
            logger.info(f"Set defaultTable to {root_table_key}")
        except Exception as e:
            logger.warning(f"Failed to set defaultTable annotation: {e}")

        if progress_callback:
            progress_callback("Complete", 100.0)

        logger.info(
            f"Workspace creation complete: {dest_hostname}/{dst_catalog_id} "
            f"({total_rows_copied} rows in {len(rows_by_table)} tables)"
        )

        return result

    except Exception as e:
        # Clean up on failure
        logger.error(f"Workspace creation failed: {e}")
        try:
            dst_server.delete_ermrest_catalog(dst_catalog_id)
            logger.info(f"Cleaned up failed catalog {dst_catalog_id}")
        except Exception:
            pass
        raise
