"""
Manager-facing API
"""

from __future__ import annotations

import asyncio
import logging
import os
from contextlib import contextmanager as ctxmgr
from datetime import datetime
from http import HTTPStatus
from pathlib import Path, PurePosixPath
from typing import (
    TYPE_CHECKING,
    Any,
    AsyncContextManager,
    Awaitable,
    Callable,
    Iterator,
    NotRequired,
    Optional,
    TypedDict,
    cast,
)

import attr
import jwt
import trafaret as t
from aiohttp import hdrs, web

from ai.backend.common import validators as tx
from ai.backend.common.api_handlers import (
    APIResponse,
    BodyParam,
    api_handler,
)
from ai.backend.common.defs import DEFAULT_VFOLDER_PERMISSION_MODE
from ai.backend.common.dto.internal.health import HealthResponse, HealthStatus
from ai.backend.common.dto.storage.request import FileDeleteAsyncRequest
from ai.backend.common.dto.storage.response import (
    FileDeleteAsyncResponse,
    VFolderCloneResponse,
    VFolderDeleteResponse,
)
from ai.backend.common.events.event_types.volume.broadcast import (
    DoVolumeMountEvent,
    DoVolumeUnmountEvent,
    VolumeMounted,
    VolumeUnmounted,
)
from ai.backend.common.json import dump_json_str
from ai.backend.common.metrics.http import (
    build_api_metric_middleware,
    build_prometheus_metrics_handler,
)
from ai.backend.common.metrics.metric import CommonMetricRegistry
from ai.backend.common.middlewares.exception import general_exception_middleware
from ai.backend.common.types import (
    AgentId,
    BinarySize,
    ItemResult,
    QuotaScopeID,
    ResultSet,
    VolumeMountableNodeType,
)
from ai.backend.logging import BraceStyleAdapter
from ai.backend.storage.bgtask.types import StorageBgtaskName

from .. import __version__
from ..bgtask.tasks.clone import VFolderCloneManifest
from ..bgtask.tasks.delete import VFolderDeleteManifest
from ..bgtask.tasks.delete_files import FileDeleteManifest
from ..dto.context import StorageRootCtx
from ..errors import (
    ExternalStorageServiceError,
    InvalidAPIParameters,
    InvalidQuotaConfig,
    InvalidSubpathError,
    ProcessExecutionError,
    QuotaScopeAlreadyExists,
    QuotaScopeNotFoundError,
    StorageProxyError,
    VFolderNotFoundError,
)
from ..types import QuotaConfig, VFolderID
from ..utils import check_params, log_manager_api_entry
from ..watcher import ChownTask, MountTask, UmountTask
from .v1.registries.huggingface import create_app as create_huggingface_registries_app
from .v1.registries.reservoir import create_app as create_reservoir_registries_app
from .v1.storages.object_storage import create_app as create_object_storages_app
from .v1.storages.vfs_storage import create_app as create_vfs_storages_app
from .vfolder.handler import VFolderHandler

if TYPE_CHECKING:
    from ..context import RootContext, ServiceContext
    from ..volumes.abc import AbstractVolume

log = BraceStyleAdapter(logging.getLogger(__spec__.name))


@web.middleware
async def token_auth_middleware(
    request: web.Request,
    handler: Callable[[web.Request], Awaitable[web.StreamResponse]],
) -> web.StreamResponse:
    skip_token_auth = getattr(handler, "skip_token_auth", False)
    if not skip_token_auth:
        token = request.headers.get("X-BackendAI-Storage-Auth-Token", None)
        if not token:
            raise web.HTTPForbidden(
                text=dump_json_str(
                    {
                        "type": "https://api.backend.ai/probs/storage/forbidden",
                        "title": "Forbidden (missing auth token)",
                    },
                ),
                content_type="application/problem+json",
            )
        ctx: RootContext = request.app["ctx"]
        if token != ctx.local_config.api.manager.secret:
            raise web.HTTPForbidden(
                text=dump_json_str(
                    {
                        "type": "https://api.backend.ai/probs/storage/forbidden",
                        "title": "Forbidden (invalid auth token)",
                    },
                ),
                content_type="application/problem+json",
            )
    return await handler(request)


def skip_token_auth(
    handler: Callable[[web.Request], Awaitable[web.StreamResponse]],
) -> Callable[[web.Request], Awaitable[web.StreamResponse]]:
    setattr(handler, "skip_token_auth", True)
    return handler


@skip_token_auth
async def check_health(request: web.Request) -> web.Response:
    """Health check endpoint with dependency connectivity status"""

    request["do_not_print_access_log"] = True

    ctx: RootContext = request.app["ctx"]
    connectivity = await ctx.health_probe.get_connectivity_status()
    response = HealthResponse(
        status=HealthStatus.OK if connectivity.overall_healthy else HealthStatus.DEGRADED,
        version=__version__,
        component="storage-proxy",
        connectivity=connectivity,
    )

    return web.json_response(response.model_dump(mode="json"))


@skip_token_auth
async def check_status(request: web.Request) -> web.Response:
    async with check_params(request, None) as params:
        await log_manager_api_entry(log, "get_status", params)
        return web.json_response(
            {
                "status": "ok",
                "type": "manager-facing",
                "storage-proxy": __version__,
            },
        )


@ctxmgr
def handle_fs_errors(
    volume: AbstractVolume,
    vfid: VFolderID,
) -> Iterator[None]:
    try:
        yield
    except OSError as e:
        related_paths = []
        msg = str(e) if e.strerror is None else e.strerror

        def _append_fpath(fname: Any) -> None:
            try:
                related_paths.append(str(volume.strip_vfpath(vfid, Path(fname))))
            except (ValueError, OSError):
                related_paths.append(fname)

        if e.filename:
            _append_fpath(e.filename)
        if e.filename2:
            _append_fpath(e.filename2)
        raise web.HTTPBadRequest(
            text=dump_json_str(
                {
                    "msg": msg,
                    "errno": e.errno,
                    "paths": related_paths,
                },
            ),
            content_type="application/json",
        )


@ctxmgr
def handle_external_errors() -> Iterator[None]:
    try:
        yield
    except ExternalStorageServiceError as e:
        raise web.HTTPInternalServerError(
            text=dump_json_str({
                "msg": str(e),
            }),
            content_type="application/json",
        )


async def get_volumes(request: web.Request) -> web.Response:
    async def _get_caps(ctx: RootContext, volume_name: str) -> list[str]:
        async with ctx.get_volume(volume_name) as volume:
            return [*await volume.get_capabilities()]

    async with check_params(request, None) as params:
        await log_manager_api_entry(log, "get_volumes", params)
        ctx: RootContext = request.app["ctx"]
        volumes = ctx.list_volumes()
        return web.json_response(
            {
                "volumes": [
                    {
                        "name": name,
                        "backend": info.backend,
                        "path": str(info.path),
                        "fsprefix": str(info.fsprefix),
                        "capabilities": await _get_caps(ctx, name),
                    }
                    for name, info in volumes.items()
                ],
            },
        )


async def get_hwinfo(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "get_hwinfo", params)
        ctx: RootContext = request.app["ctx"]
        async with ctx.get_volume(params["volume"]) as volume:
            data = await volume.get_hwinfo()
            return web.json_response(data)


async def create_quota_scope(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        qsid: QuotaScopeID
        options: QuotaConfig | None
        extra_args: NotRequired[dict[str, Any]]

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("qsid"): tx.QuotaScopeID(),
                    t.Key("options", default=None): t.Null | QuotaConfig.as_trafaret(),
                    t.Key("extra_args", default=None): t.Null | t.Mapping(t.String, t.Any),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "create_quota_scope", params)
        ctx: RootContext = request.app["ctx"]
        async with ctx.get_volume(params["volume"]) as volume:
            try:
                with handle_external_errors():
                    await volume.quota_model.create_quota_scope(
                        params["qsid"], params["options"], params["extra_args"]
                    )
            except QuotaScopeAlreadyExists:
                return web.json_response(
                    {"msg": "Volume already exists with given quota scope."},
                    status=HTTPStatus.CONFLICT,
                )
            return web.Response(status=HTTPStatus.NO_CONTENT)


async def get_quota_scope(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        qsid: QuotaScopeID

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("qsid"): tx.QuotaScopeID(),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "get_quota_scope", params)
        ctx: RootContext = request.app["ctx"]
        async with ctx.get_volume(params["volume"]) as volume:
            with handle_external_errors():
                quota_usage = await volume.quota_model.describe_quota_scope(params["qsid"])
            if not quota_usage:
                raise QuotaScopeNotFoundError
            return web.json_response({
                "used_bytes": quota_usage.used_bytes if quota_usage.used_bytes >= 0 else None,
                "limit_bytes": quota_usage.limit_bytes if quota_usage.limit_bytes >= 0 else None,
            })


async def update_quota_scope(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        qsid: QuotaScopeID
        options: QuotaConfig

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("qsid"): tx.QuotaScopeID(),
                    t.Key("options"): QuotaConfig.as_trafaret(),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "update_quota_scope", params)
        ctx: RootContext = request.app["ctx"]
        async with ctx.get_volume(params["volume"]) as volume:
            with handle_external_errors():
                quota_usage = await volume.quota_model.describe_quota_scope(params["qsid"])
                if not quota_usage:
                    await volume.quota_model.create_quota_scope(params["qsid"], params["options"])
                else:
                    try:
                        await volume.quota_model.update_quota_scope(
                            params["qsid"], params["options"]
                        )
                    except InvalidQuotaConfig:
                        return web.json_response(
                            {"msg": "Invalid quota config option"},
                            status=HTTPStatus.BAD_REQUEST,
                        )
            return web.Response(status=HTTPStatus.NO_CONTENT)


async def unset_quota(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        qsid: QuotaScopeID

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("qsid"): tx.QuotaScopeID(),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "unset_quota", params)
        ctx: RootContext = request.app["ctx"]
        async with ctx.get_volume(params["volume"]) as volume:
            with handle_external_errors():
                quota_usage = await volume.quota_model.describe_quota_scope(params["qsid"])
            if not quota_usage:
                raise QuotaScopeNotFoundError
            await volume.quota_model.unset_quota(params["qsid"])
            return web.Response(status=HTTPStatus.NO_CONTENT)


async def create_vfolder(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID
        mode: Optional[int]
        options: dict[str, Any] | None  # deprecated

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                    # TODO: Change `mode` parameter to not-nullable
                    t.Key("mode", default=DEFAULT_VFOLDER_PERMISSION_MODE): t.Null | t.Int,
                    t.Key("options", default=None): t.Null | t.Dict().allow_extra("*"),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "create_vfolder", params)
        if params["vfid"].quota_scope_id is None:
            raise InvalidAPIParameters("quota_scope_id is required for vfid")
        ctx: RootContext = request.app["ctx"]
        perm_mode = cast(
            int, params["mode"] if params["mode"] is not None else DEFAULT_VFOLDER_PERMISSION_MODE
        )
        async with ctx.get_volume(params["volume"]) as volume:
            try:
                await volume.create_vfolder(params["vfid"], mode=perm_mode)
            except QuotaScopeNotFoundError:
                if not ctx.local_config.storage_proxy.auto_quota_scope_creation:
                    raise
                if not params["vfid"].quota_scope_id:
                    raise InvalidAPIParameters(
                        "quota_scope_id is required for auto quota scope creation"
                    )
                if initial_max_size_for_quota_scope := (params["options"] or {}).get(
                    "initial_max_size_for_quota_scope"
                ):
                    options = QuotaConfig(initial_max_size_for_quota_scope)
                else:
                    options = None
                await volume.quota_model.create_quota_scope(
                    params["vfid"].quota_scope_id, options=options
                )
                try:
                    await volume.create_vfolder(params["vfid"], mode=perm_mode)
                except QuotaScopeNotFoundError:
                    raise ExternalStorageServiceError(
                        "Failed to create vfolder due to quota scope not found."
                    )
            return web.Response(status=HTTPStatus.NO_CONTENT)


async def delete_vfolder(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "delete_vfolder", params)
        ctx: RootContext = request.app["ctx"]
        vfid: VFolderID = params["vfid"]
        delete_manifest = VFolderDeleteManifest(
            volume=params["volume"],
            vfolder_id=vfid,
        )
        task_id = await ctx.background_task_manager.start_retriable(
            StorageBgtaskName.DELETE_VFOLDER,
            delete_manifest,
        )
        data = VFolderDeleteResponse(bgtask_id=task_id).model_dump(mode="json")
        return web.json_response(data, status=HTTPStatus.ACCEPTED)


async def clone_vfolder(request: web.Request) -> web.Response:
    class Params(TypedDict):
        src_volume: str
        src_vfid: VFolderID
        dst_volume: str | None  # deprecated
        dst_vfid: VFolderID

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("src_volume"): t.String(),
                    t.Key("src_vfid"): tx.VFolderID(),
                    t.Key("dst_volume"): t.String() | t.Null,
                    t.Key("dst_vfid"): tx.VFolderID(),
                },
            ).allow_extra("*"),
        ),
    ) as params:
        if params["dst_volume"] is not None and params["dst_volume"] != params["src_volume"]:
            raise StorageProxyError("Cross-volume vfolder cloning is not implemented yet")
        await log_manager_api_entry(log, "clone_vfolder", params)
        ctx: RootContext = request.app["ctx"]
        if params["dst_volume"] is not None and params["dst_volume"] != params["src_volume"]:
            raise StorageProxyError("Cross-volume vfolder cloning is not implemented yet")
        clone_manifest = VFolderCloneManifest(
            volume=params["src_volume"],
            src_vfolder=params["src_vfid"],
            dst_vfolder=params["dst_vfid"],
        )
        task_id = await ctx.background_task_manager.start_retriable(
            StorageBgtaskName.CLONE_VFOLDER,
            clone_manifest,
        )
        data = VFolderCloneResponse(bgtask_id=task_id).model_dump(mode="json")
        return web.json_response(data)


async def get_vfolder_mount(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID
        subpath: str

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                    t.Key("subpath", default="."): t.String(),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "get_container_mount", params)
        ctx: RootContext = request.app["ctx"]
        async with ctx.get_volume(params["volume"]) as volume:
            try:
                mount_path = await volume.get_vfolder_mount(
                    params["vfid"],
                    params["subpath"],
                )
            except VFolderNotFoundError:
                raise web.HTTPBadRequest(
                    text=dump_json_str(
                        {
                            "msg": "VFolder not found",
                            "vfid": str(params["vfid"]),
                        },
                    ),
                    content_type="application/json",
                )
            except InvalidSubpathError as e:
                raise web.HTTPBadRequest(
                    text=dump_json_str(
                        {
                            "msg": "Invalid vfolder subpath",
                            "vfid": str(params["vfid"]),
                            "subpath": str(e.args[1]),
                        },
                    ),
                    content_type="application/json",
                )
            return web.json_response(
                {
                    "path": str(mount_path),
                },
            )


async def get_performance_metric(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "get_performance_metric", params)
        ctx: RootContext = request.app["ctx"]
        async with ctx.get_volume(params["volume"]) as volume:
            metric = await volume.get_performance_metric()
            return web.json_response(
                {
                    "metric": attr.asdict(cast(attr.AttrsInstance, metric)),
                },
            )


async def fetch_file(request: web.Request) -> web.StreamResponse:
    """
    Direct file streaming API for internal use, such as retrieving
    task logs from a user vfolder ".logs".
    """

    class Params(TypedDict):
        volume: str
        vfid: VFolderID
        relpath: PurePosixPath

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                    t.Key("relpath"): tx.PurePath(relative_only=True),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "fetch_file", params)
        ctx: RootContext = request.app["ctx"]
        response = web.StreamResponse(status=HTTPStatus.OK)
        response.headers[hdrs.CONTENT_TYPE] = "application/octet-stream"
        prepared = False
        try:
            async with ctx.get_volume(params["volume"]) as volume:
                with handle_fs_errors(volume, params["vfid"]):
                    async for chunk in volume.read_file(
                        params["vfid"],
                        params["relpath"],
                    ):
                        if not chunk:
                            return response
                        if not prepared:
                            await response.prepare(request)
                            prepared = True
                        await response.write(chunk)
        except FileNotFoundError:
            response = web.Response(status=HTTPStatus.NOT_FOUND, reason="Log data not found")
        finally:
            if prepared:
                await response.write_eof()
            return response


async def get_metadata(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "get_metadata", params)
        return web.json_response(
            {
                "status": "ok",
            },
        )


async def set_metadata(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID
        payload: bytes

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                    t.Key("payload"): t.Bytes(),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "set_metadata", params)
        return web.json_response(
            {
                "status": "ok",
            },
        )


async def get_vfolder_fs_usage(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "get_vfolder_fs_usage", params)
        ctx: RootContext = request.app["ctx"]
        async with ctx.get_volume(params["volume"]) as volume:
            fs_usage = await volume.get_fs_usage()
            return web.json_response(
                {
                    "capacity_bytes": fs_usage.capacity_bytes,
                    "used_bytes": fs_usage.used_bytes,
                },
            )


async def get_vfolder_usage(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                },
            ),
        ),
    ) as params:
        try:
            await log_manager_api_entry(log, "get_vfolder_usage", params)
            ctx: RootContext = request.app["ctx"]
            async with ctx.get_volume(params["volume"]) as volume:
                usage = await volume.get_usage(params["vfid"])
                return web.json_response(
                    {
                        "file_count": usage.file_count,
                        "used_bytes": usage.used_bytes,
                    },
                )
        except ProcessExecutionError:
            return web.Response(
                status=HTTPStatus.INTERNAL_SERVER_ERROR,
                reason="Storage server is busy. Please try again",
            )


async def get_vfolder_used_bytes(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                },
            ),
        ),
    ) as params:
        try:
            await log_manager_api_entry(log, "get_vfolder_used_bytes", params)
            ctx: RootContext = request.app["ctx"]
            async with ctx.get_volume(params["volume"]) as volume:
                usage = await volume.get_used_bytes(params["vfid"])
                return web.json_response(
                    {
                        "used_bytes": usage,
                    },
                )
        except ProcessExecutionError:
            return web.Response(
                status=HTTPStatus.INTERNAL_SERVER_ERROR,
                reason="Storage server is busy. Please try again",
            )


async def get_quota(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid", default=None): t.Null | tx.VFolderID,
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "get_quota", params)
        return web.Response(
            status=HTTPStatus.BAD_REQUEST,
            reason="get_quota (for individual vfolder) is a deprecated API",
        )


async def set_quota(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID
        size_bytes: BinarySize

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid", default=None): t.Null | tx.VFolderID,
                    t.Key("size_bytes"): tx.BinarySize,
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "update_quota", params)
        return web.Response(
            status=HTTPStatus.BAD_REQUEST,
            reason="set_quota (for individual vfolder) is a deprecated API",
        )


async def mkdir(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID
        relpath: PurePosixPath | list[PurePosixPath]
        parents: bool
        exist_ok: bool

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                    t.Key("relpath"): tx.PurePath(relative_only=True)
                    | t.List(tx.PurePath(relative_only=True), max_length=50),
                    t.Key("parents", default=True): t.ToBool,
                    t.Key("exist_ok", default=False): t.ToBool,
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "mkdir", params)
        ctx: RootContext = request.app["ctx"]
        vfid = params["vfid"]
        parents = params["parents"]
        exist_ok = params["exist_ok"]
        relpath = params["relpath"]
        relpaths = relpath if isinstance(relpath, list) else [relpath]
        failed_results: list[ItemResult] = []
        success_results: list[ItemResult] = []

        async with ctx.get_volume(params["volume"]) as volume:
            mkdir_tasks = [
                volume.mkdir(vfid, rpath, parents=parents, exist_ok=exist_ok) for rpath in relpaths
            ]
            result_group = await asyncio.gather(*mkdir_tasks, return_exceptions=True)
            failed_cases = [isinstance(res, BaseException) for res in result_group]

        for relpath, result_or_exception in zip(relpaths, result_group):
            if isinstance(result_or_exception, BaseException):
                log.error(
                    "Failed to create the directory {!r} in vol:{}/vfid:{}:",
                    relpath,
                    volume,
                    vfid,
                    exc_info=result_or_exception,
                )
                failed_results.append({
                    "msg": repr(result_or_exception),
                    "item": str(relpath),
                })
            else:
                success_results.append({
                    "msg": None,
                    "item": str(relpath),
                })
        results: ResultSet = {
            "success": success_results,
            "failed": failed_results,
        }
        if all(failed_cases):
            status_code = 422
        elif any(failed_cases):
            status_code = 207
        else:
            status_code = 200
        return web.json_response(
            {"results": results},
            status=status_code,
        )


async def list_files(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID
        relpath: PurePosixPath

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                    t.Key("relpath"): tx.PurePath(relative_only=True),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "list_files", params)
        ctx: RootContext = request.app["ctx"]
        async with ctx.get_volume(params["volume"]) as volume:
            with handle_fs_errors(volume, params["vfid"]):
                items = [
                    {
                        "name": item.name,
                        "type": item.type.name,
                        "stat": {
                            "mode": item.stat.mode,
                            "size": item.stat.size,
                            "created": item.stat.created.isoformat(),
                            "modified": item.stat.modified.isoformat(),
                        },
                        "symlink_target": item.symlink_target,
                    }
                    async for item in volume.scandir(
                        params["vfid"],
                        params["relpath"],
                        recursive=False,
                    )
                ]
        return web.json_response(
            {
                "items": items,
            },
        )


async def rename_file(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID
        relpath: PurePosixPath
        new_name: str
        is_dir: bool

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                    t.Key("relpath"): tx.PurePath(relative_only=True),
                    t.Key("new_name"): t.String(),
                    # ignored since 22.03
                    t.Key("is_dir", default=False): t.ToBool,
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "rename_file", params)
        ctx: RootContext = request.app["ctx"]
        async with ctx.get_volume(params["volume"]) as volume:
            with handle_fs_errors(volume, params["vfid"]):
                await volume.move_file(
                    params["vfid"],
                    params["relpath"],
                    params["relpath"].with_name(params["new_name"]),
                )
        return web.Response(status=HTTPStatus.NO_CONTENT)


async def move_file(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID
        src_relpath: PurePosixPath
        dst_relpath: PurePosixPath

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                    t.Key("src_relpath"): tx.PurePath(relative_only=True),
                    t.Key("dst_relpath"): tx.PurePath(relative_only=True),
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "move_file", params)
        ctx: RootContext = request.app["ctx"]
        async with ctx.get_volume(params["volume"]) as volume:
            with handle_fs_errors(volume, params["vfid"]):
                await volume.move_file(
                    params["vfid"],
                    params["src_relpath"],
                    params["dst_relpath"],
                )
        return web.Response(status=HTTPStatus.NO_CONTENT)


async def create_download_session(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID
        relpath: PurePosixPath
        archive: bool
        unmanaged_path: str | None

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                    t.Key("relpath"): tx.PurePath(relative_only=True),
                    t.Key("archive", default=False): t.ToBool,
                    t.Key("unmanaged_path", default=None): t.Null | t.String,
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "create_download_session", params)
        ctx: RootContext = request.app["ctx"]
        token_data = {
            "op": "download",
            "volume": params["volume"],
            "vfid": str(params["vfid"]),
            "relpath": str(params["relpath"]),
            "exp": datetime.utcnow() + ctx.local_config.storage_proxy.session_expire,
        }
        token = jwt.encode(
            token_data,
            ctx.local_config.storage_proxy.secret,
            algorithm="HS256",
        )
        return web.json_response(
            {
                "token": token,
            },
        )


async def create_upload_session(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID
        relpath: PurePosixPath
        size: int

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                    t.Key("relpath"): tx.PurePath(relative_only=True),
                    t.Key("size"): t.ToInt,
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "create_upload_session", params)
        ctx: RootContext = request.app["ctx"]
        async with ctx.get_volume(params["volume"]) as volume:
            session_id = await volume.prepare_upload(params["vfid"])
        token_data = {
            "op": "upload",
            "volume": params["volume"],
            "vfid": str(params["vfid"]),
            "relpath": str(params["relpath"]),
            "size": params["size"],
            "session": session_id,
            "exp": datetime.utcnow() + ctx.local_config.storage_proxy.session_expire,
        }
        token = jwt.encode(
            token_data,
            ctx.local_config.storage_proxy.secret,
            algorithm="HS256",
        )
        return web.json_response(
            {
                "token": token,
            },
        )


async def delete_files(request: web.Request) -> web.Response:
    class Params(TypedDict):
        volume: str
        vfid: VFolderID
        relpaths: list[PurePosixPath]
        recursive: bool

    async with cast(
        AsyncContextManager[Params],
        check_params(
            request,
            t.Dict(
                {
                    t.Key("volume"): t.String(),
                    t.Key("vfid"): tx.VFolderID(),
                    t.Key("relpaths"): t.List(tx.PurePath(relative_only=True)),
                    t.Key("recursive", default=False): t.ToBool,
                },
            ),
        ),
    ) as params:
        await log_manager_api_entry(log, "delete_files", params)
        ctx: RootContext = request.app["ctx"]
        async with ctx.get_volume(params["volume"]) as volume:
            with handle_fs_errors(volume, params["vfid"]):
                await volume.delete_files(
                    params["vfid"],
                    params["relpaths"],
                    recursive=params["recursive"],
                )
        return web.json_response(
            {
                "status": "ok",
            },
        )


class FileOperationHandler:
    """Handler for file operations within vfolders."""

    @api_handler
    async def delete_files_async(
        self,
        body: BodyParam[FileDeleteAsyncRequest],
        ctx: StorageRootCtx,
    ) -> APIResponse:
        """Delete files or directories asynchronously using background tasks."""
        # Create file deletion manifest
        delete_manifest = FileDeleteManifest(
            volume=body.parsed.volume,
            vfolder_id=body.parsed.vfid,
            relpaths=body.parsed.relpaths,
            recursive=body.parsed.recursive,
        )

        # Start background task
        task_id = await ctx.root_ctx.background_task_manager.start_retriable(
            StorageBgtaskName.DELETE_FILES,
            delete_manifest,
        )

        # Return task ID with HTTP 202 ACCEPTED
        return APIResponse.build(
            status_code=HTTPStatus.ACCEPTED,
            response_model=FileDeleteAsyncResponse(bgtask_id=task_id),
        )


async def _shutdown(app: web.Application) -> None:
    pass


def init_v2_volume_app(service_ctx: ServiceContext) -> web.Application:
    app = web.Application()
    handler = VFolderHandler(service_ctx.volume_service)
    app.router.add_route("GET", "/volumes", handler.get_volumes)
    app.router.add_route("GET", "/volumes/{volume_id}", handler.get_volume)
    app.router.add_route(
        "GET",
        "/volumes/{volume_id}/quota-scope/{quota_scope_type}/{quota_scope_uuid}",
        handler.get_quota_scope,
    )
    app.router.add_route(
        "POST",
        "/volumes/{volume_id}/quota-scope/{quota_scope_type}/{quota_scope_uuid}",
        handler.create_quota_scope,
    )
    app.router.add_route(
        "PUT",
        "/volumes/{volume_id}/quota-scope/{quota_scope_type}/{quota_scope_uuid}",
        handler.update_quota_scope,
    )
    app.router.add_route(
        "DELETE",
        "/volumes/{volume_id}/quota-scope/{quota_scope_type}/{quota_scope_uuid}",
        handler.delete_quota_scope,
    )
    app.router.add_route(
        "POST",
        "/volumes/{volume_id}/quota-scope/{quota_scope_type}/{quota_scope_uuid}/vfolder/{folder_uuid}",
        handler.create_vfolder,
    )
    app.router.add_route(
        "GET",
        "/volumes/{volume_id}/quota-scope/{quota_scope_type}/{quota_scope_uuid}/vfolder/{folder_uuid}",
        handler.get_vfolder_info,
    )
    app.router.add_route(
        "PUT",
        "/volumes/{volume_id}/quota-scope/{quota_scope_type}/{quota_scope_uuid}/vfolder/{folder_uuid}/clone",
        handler.clone_vfolder,
    )
    app.router.add_route(
        "DELETE",
        "/volumes/{volume_id}/quota-scope/{quota_scope_type}/{quota_scope_uuid}/vfolder/{folder_uuid}",
        handler.delete_vfolder,
    )

    return app


async def init_manager_app(ctx: RootContext) -> web.Application:
    app = web.Application(
        middlewares=[
            token_auth_middleware,
            general_exception_middleware,
            build_api_metric_middleware(ctx.metric_registry.api),
        ],
    )
    app["ctx"] = ctx
    app.on_shutdown.append(_shutdown)

    # Initialize handlers
    file_operation_handler = FileOperationHandler()

    app.router.add_route("GET", "/", check_status)
    app.router.add_route("GET", "/health", check_health)
    app.router.add_route("GET", "/status", check_status)
    app.router.add_route("GET", "/volumes", get_volumes)
    app.router.add_route("GET", "/volume/hwinfo", get_hwinfo)
    app.router.add_route("POST", "/quota-scope", create_quota_scope)
    app.router.add_route("GET", "/quota-scope", get_quota_scope)
    app.router.add_route("PATCH", "/quota-scope", update_quota_scope)
    app.router.add_route("DELETE", "/quota-scope/quota", unset_quota)
    app.router.add_route("POST", "/folder/create", create_vfolder)
    app.router.add_route("POST", "/folder/delete", delete_vfolder)
    app.router.add_route("POST", "/folder/clone", clone_vfolder)
    app.router.add_route("GET", "/folder/mount", get_vfolder_mount)
    app.router.add_route("GET", "/volume/performance-metric", get_performance_metric)
    app.router.add_route("GET", "/folder/metadata", get_metadata)
    app.router.add_route("POST", "/folder/metadata", set_metadata)
    app.router.add_route("GET", "/volume/quota", get_quota)
    app.router.add_route("PATCH", "/volume/quota", set_quota)
    app.router.add_route("GET", "/folder/usage", get_vfolder_usage)
    app.router.add_route("GET", "/folder/used-bytes", get_vfolder_used_bytes)
    app.router.add_route("GET", "/folder/fs-usage", get_vfolder_fs_usage)
    app.router.add_route("POST", "/folder/file/mkdir", mkdir)
    app.router.add_route("POST", "/folder/file/list", list_files)
    app.router.add_route("POST", "/folder/file/rename", rename_file)
    app.router.add_route("POST", "/folder/file/move", move_file)
    app.router.add_route("POST", "/folder/file/fetch", fetch_file)
    app.router.add_route("POST", "/folder/file/download", create_download_session)
    app.router.add_route("POST", "/folder/file/upload", create_upload_session)
    app.router.add_route("POST", "/folder/file/delete", delete_files)
    app.router.add_route(
        "POST", "/folder/file/delete-async", file_operation_handler.delete_files_async
    )

    app.add_subapp("/v1/storages/s3", create_object_storages_app(ctx))
    app.add_subapp("/v1/storages/vfs", create_vfs_storages_app(ctx))
    app.add_subapp("/v1/registries/huggingface", create_huggingface_registries_app(ctx))
    app.add_subapp("/v1/registries/reservoir", create_reservoir_registries_app(ctx))

    # passive events
    evd = ctx.event_dispatcher
    evd.subscribe(DoVolumeMountEvent, ctx, handle_volume_mount, name="storage.volume.mount")
    evd.subscribe(DoVolumeUnmountEvent, ctx, handle_volume_umount, name="storage.volume.umount")
    return app


def init_internal_app(ctx: RootContext) -> web.Application:
    app = web.Application()
    app["ctx"] = ctx
    metric_registry = CommonMetricRegistry.instance()
    app.router.add_route("GET", "/health", check_health)
    app.router.add_route("GET", "/metrics", build_prometheus_metrics_handler(metric_registry))
    return app


async def handle_volume_mount(
    context: RootContext,
    source: AgentId,
    event: DoVolumeMountEvent,
) -> None:
    if context.pidx != 0:
        return
    if context.watcher is None:
        log.debug(f"{context.node_id}: Watcher is disabled. Skip handling mount event.")
        return
    err_msg: str | None = None
    mount_prefix = await context.etcd.get("volumes/_mount")
    volume_mount_path = str(context.local_config.volume[event.volume_backend_name].path)
    mount_path = Path(volume_mount_path, event.dir_name)
    mount_task = MountTask.from_event(event, mount_path=mount_path, mount_prefix=mount_prefix)
    resp = await context.watcher.request_task(mount_task)
    if not resp.succeeded:
        # Produce volume mounted event with error message.
        # And skip chown.
        err_msg = resp.body
        await context.event_producer.broadcast_event(
            VolumeMounted(
                str(context.node_id),
                VolumeMountableNodeType.STORAGE_PROXY,
                str(mount_path),
                event.quota_scope_id,
                err_msg,
            )
        )
        return

    # change owner of mounted directory
    chown_task = ChownTask(str(mount_path), os.getuid(), os.getgid())
    resp = await context.watcher.request_task(chown_task)
    if not resp.succeeded:
        err_msg = resp.body
    await context.event_producer.broadcast_event(
        VolumeMounted(
            str(context.node_id),
            VolumeMountableNodeType.STORAGE_PROXY,
            str(mount_path),
            event.quota_scope_id,
            err_msg,
        )
    )


async def handle_volume_umount(
    context: RootContext,
    source: AgentId,
    event: DoVolumeUnmountEvent,
) -> None:
    if context.pidx != 0:
        return
    if context.watcher is None:
        log.debug(f"{context.node_id}: Watcher is disabled. Skip handling umount event.")
        return
    mount_prefix = await context.etcd.get("volumes/_mount")
    timeout = await context.etcd.get("config/watcher/file-io-timeout")
    volume_mount_path = str(context.local_config.volume[event.volume_backend_name].path)
    mount_path = Path(volume_mount_path, event.dir_name)
    umount_task = UmountTask.from_event(
        event,
        mount_path=mount_path,
        mount_prefix=mount_prefix,
        timeout=float(timeout) if timeout is not None else None,
    )
    resp = await context.watcher.request_task(umount_task)
    err_msg = resp.body if not resp.succeeded else None
    if resp.body:
        log.warning(resp.body)
    await context.event_producer.broadcast_event(
        VolumeUnmounted(
            str(context.node_id),
            VolumeMountableNodeType.STORAGE_PROXY,
            str(mount_path),
            event.quota_scope_id,
            err_msg,
        )
    )
