import faulthandler
import io
import json
import hashlib
import logging
import os
import re
import signal
import socket
import ssl
import sys
import tempfile
import traceback
from functools import wraps
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Iterable, Union
from urllib.parse import urlparse

faulthandler.enable(file=sys.stderr)
faulthandler.register(signal.SIGUSR1, file=sys.stderr, all_threads=True)

import tracemalloc as _tracemalloc

_TRACEMALLOC_ACTIVE = _tracemalloc.is_tracing()


def _tracemalloc_snapshot_text(top_n: int = 30) -> str:
    """Return a human-readable tracemalloc snapshot, or a status message if not tracing."""
    if not _tracemalloc.is_tracing():
        return "tracemalloc is not active. Set PYTHONTRACEMALLOC=<nframes> to enable.\n"
    snap = _tracemalloc.take_snapshot()
    snap = snap.filter_traces([
        _tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
        _tracemalloc.Filter(False, "<frozen importlib._bootstrap_external>"),
        _tracemalloc.Filter(False, "<unknown>"),
    ])
    buf = io.StringIO()
    current, peak = _tracemalloc.get_traced_memory()
    buf.write(f"tracemalloc: current={current / 1024 / 1024:.1f} MiB, peak={peak / 1024 / 1024:.1f} MiB\n\n")
    stats = snap.statistics("lineno")
    buf.write(f"Top {top_n} allocations by line:\n")
    for i, stat in enumerate(stats[:top_n], 1):
        buf.write(f"  #{i}: {stat}\n")
    buf.write(f"\nTop {top_n} allocations by file:\n")
    stats_file = snap.statistics("filename")
    for i, stat in enumerate(stats_file[:top_n], 1):
        buf.write(f"  #{i}: {stat}\n")
    return buf.getvalue()


def _memory_profile_text() -> str:
    """Lightweight memory profile: RSS, gc object type counts, tracemalloc summary."""
    import gc
    import resource
    buf = io.StringIO()
    rss_kb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
    buf.write(f"Peak RSS: {rss_kb / 1024:.0f} MiB\n")
    try:
        with open("/proc/self/status") as f:
            for line in f:
                if line.startswith(("VmRSS", "RssAnon", "VmData")):
                    buf.write(f"  {line.strip()}\n")
    except OSError:
        pass
    buf.write("\n")
    if _tracemalloc.is_tracing():
        current, peak = _tracemalloc.get_traced_memory()
        buf.write(f"tracemalloc: current={current / 1024 / 1024:.1f} MiB, peak={peak / 1024 / 1024:.1f} MiB\n\n")
    type_counts: dict[str, list[int]] = {}
    for obj in gc.get_objects():
        tp = type(obj).__name__
        sz = sys.getsizeof(obj, 0)
        entry = type_counts.get(tp)
        if entry:
            entry[0] += 1
            entry[1] += sz
        else:
            type_counts[tp] = [1, sz]
    sorted_by_size = sorted(type_counts.items(), key=lambda x: -x[1][1])
    buf.write("Top 30 types by total shallow size:\n")
    for tp, (count, total_sz) in sorted_by_size[:30]:
        buf.write(f"  {tp:40s} count={count:>10,}  size={total_sz / 1024 / 1024:>10.1f} MiB\n")
    buf.write(f"\nTop 30 types by count:\n")
    sorted_by_count = sorted(type_counts.items(), key=lambda x: -x[1][0])
    for tp, (count, total_sz) in sorted_by_count[:30]:
        buf.write(f"  {tp:40s} count={count:>10,}  size={total_sz / 1024 / 1024:>10.1f} MiB\n")
    return buf.getvalue()


def _dump_tracemalloc_background():
    """Run the expensive tracemalloc snapshot in a daemon thread to avoid blocking."""
    text = _memory_profile_text()
    sys.stderr.write(f"\n{'='*60}\n")
    sys.stderr.write(f"SIGUSR2 memory profile (gc objects)\n")
    sys.stderr.write(f"{'='*60}\n")
    sys.stderr.write(text)
    if _tracemalloc.is_tracing():
        sys.stderr.write(f"\n--- tracemalloc snapshot (may take a while) ---\n")
        sys.stderr.flush()
        tm_text = _tracemalloc_snapshot_text(top_n=50)
        sys.stderr.write(tm_text)
    sys.stderr.write(f"{'='*60}\n\n")
    sys.stderr.flush()


def _sigusr2_handler(signum, frame):
    threading.Thread(target=_dump_tracemalloc_background, daemon=True).start()


signal.signal(signal.SIGUSR2, _sigusr2_handler)

import boto3
import pytz
import requests
from requests.exceptions import ConnectionError as RequestsConnectionError
from mitmproxy import ctx, http
from mitmproxy.script import concurrent
from botocore.config import Config as BotoConfig
from botocore.exceptions import ClientError
from werkzeug.http import parse_date

from passsage.range_utils import (
    parse_single_range as _parse_single_range_impl,
    is_unsatisfiable_range as _is_unsatisfiable_range_impl,
    if_range_matches as _if_range_matches_impl,
)

from passsage.cache_key import (
    CacheKeyResolver,
    Context as CacheKeyContext,
    get_default_cache_key_resolver,
    set_default_cache_key_resolver,
)
from passsage.policy import (
    POLICY_BY_NAME,
    Context as PolicyContext,
    NoCache,
    NoRefresh,
    PolicyResolver,
    ResolvedPolicy,
    StaleIfError,
    Standard,
    TimeoutConfig,
    get_default_resolver,
    policy_from_name,
    set_default_resolver,
)
from passsage.cache_utils import (
    url_ext as _url_ext,
    hashed_s3_key as _hashed_s3_key,
    get_cache_key,
    get_vary_index_key,
    es_doc_id,
    es_vary_index_id,
)
from passsage.es_client import (
    configure as _es_configure,
    es_get_doc,
    es_mget_docs,
    es_index_doc,
    es_create_index,
    LastAccessBatcher,
)
from passsage.log_storage import AccessLogWriter, ErrorLogWriter, build_access_log_config, build_error_log_config

try:
    from passsage import __version__
except ImportError:
    __version__ = "dev"

LOG = logging.getLogger("passsage.proxy")

SERVER_NAME = "passsage"
SERVER_VERSION = __version__
_VIA_HOSTNAME = (socket.gethostname() or SERVER_NAME).strip() or SERVER_NAME
VIA_HEADER_VALUE = f"1.1 {_VIA_HOSTNAME} ({SERVER_NAME}/{SERVER_VERSION})"
POLICY_HEADER = "X-Passsage-Policy"

_S3_ENDPOINT = os.environ.get("S3_ENDPOINT_URL")
S3_REGION = os.environ.get("AWS_DEFAULT_REGION", os.environ.get("AWS_REGION", "us-west-2"))
S3_BUCKET = os.environ.get("S3_BUCKET", "proxy-cache")
if _S3_ENDPOINT:
    p = urlparse(_S3_ENDPOINT)
    S3_HOST = p.hostname or "localhost"
    S3_PORT = p.port or (443 if p.scheme == "https" else 80)
    S3_SCHEME = p.scheme or "http"
    S3_URL = f"{S3_SCHEME}://{S3_HOST}:{S3_PORT}/{S3_BUCKET}"
    S3_PATH_STYLE = True
else:
    S3_HOST = f"{S3_BUCKET}.s3.{S3_REGION}.amazonaws.com"
    S3_PORT = 80
    S3_SCHEME = "http"
    S3_URL = f"{S3_SCHEME}://{S3_HOST}"
    S3_PATH_STYLE = False
CACHE_TIMEOUT = 10

_TRUTHY = ("1", "true", "yes")
_FALSY = ("0", "false", "no")


def _env_bool(name, default=""):
    return os.environ.get(name, default).strip().lower() in _TRUTHY


def _env_bool_default_true(name):
    return os.environ.get(name, "1").strip().lower() not in _FALSY


HEALTH_PORT = int(os.environ.get("PASSSAGE_HEALTH_PORT", "8082"))
HEALTH_HOST = os.environ.get("PASSSAGE_HEALTH_HOST", "0.0.0.0")
HEALTH_CHECK_S3 = _env_bool_default_true("PASSSAGE_HEALTH_CHECK_S3")
PUBLIC_PROXY_URL = os.environ.get("PASSSAGE_PUBLIC_PROXY_URL", "").strip()
ACCESS_LOGS = _env_bool("PASSSAGE_ACCESS_LOGS")
ACCESS_LOG_PREFIX = os.environ.get("PASSSAGE_ACCESS_LOG_PREFIX", "__passsage_logs__").strip()
ACCESS_LOG_DIR = os.environ.get("PASSSAGE_ACCESS_LOG_DIR", "/tmp/passsage-logs").strip()
ACCESS_LOG_FLUSH_SECONDS = os.environ.get("PASSSAGE_ACCESS_LOG_FLUSH_SECONDS", "30").strip()
ACCESS_LOG_FLUSH_BYTES = os.environ.get("PASSSAGE_ACCESS_LOG_FLUSH_BYTES", "1G").strip()
ACCESS_LOG_HEADERS = os.environ.get(
    "PASSSAGE_ACCESS_LOG_HEADERS",
    "accept,accept-encoding,cache-control,content-type,content-encoding,"
    "etag,last-modified,range,user-agent,via,x-cache,x-cache-lookup,x-amz-request-id",
).strip()
ERROR_LOGS = _env_bool("PASSSAGE_ERROR_LOGS")
ERROR_LOG_PREFIX = os.environ.get("PASSSAGE_ERROR_LOG_PREFIX", "__passsage_error_logs__").strip()
ERROR_LOG_DIR = os.environ.get("PASSSAGE_ERROR_LOG_DIR", "/tmp/passsage-errors").strip()
ERROR_LOG_FLUSH_SECONDS = os.environ.get("PASSSAGE_ERROR_LOG_FLUSH_SECONDS", "30").strip()
ERROR_LOG_FLUSH_BYTES = os.environ.get("PASSSAGE_ERROR_LOG_FLUSH_BYTES", "256M").strip()



def _s3_hash_prefix_depth() -> int:
    """Get the S3 hash prefix depth from mitmproxy options, with fallback."""
    try:
        return getattr(ctx.options, "s3_hash_prefix_depth", 4)
    except Exception:
        return int(os.environ.get("PASSSAGE_S3_HASH_PREFIX_DEPTH", "4"))


class _HealthHandler(BaseHTTPRequestHandler):
    protocol_version = "HTTP/1.1"

    def log_message(self, format: str, *args) -> None:
        return

    def do_GET(self) -> None:
        if self.path == "/health":
            ok, message = _health_check()
            body = (message or "ok").encode("utf-8")
            self.send_response(200 if ok else 503)
            self.send_header("Content-Type", "text/plain")
            self.send_header("Content-Length", str(len(body)))
            self.end_headers()
            self.wfile.write(body)
        elif self.path == "/debug/threads":
            body = _dump_all_threads().encode("utf-8")
            self.send_response(200)
            self.send_header("Content-Type", "text/plain")
            self.send_header("Content-Length", str(len(body)))
            self.end_headers()
            self.wfile.write(body)
        elif self.path == "/debug/connections":
            body = _dump_connections().encode("utf-8")
            self.send_response(200)
            self.send_header("Content-Type", "text/plain")
            self.send_header("Content-Length", str(len(body)))
            self.end_headers()
            self.wfile.write(body)
        elif self.path == "/debug/memory" or self.path.startswith("/debug/memory?"):
            if "tracemalloc" in self.path:
                body = _tracemalloc_snapshot_text().encode("utf-8")
            else:
                body = _memory_profile_text().encode("utf-8")
            self.send_response(200)
            self.send_header("Content-Type", "text/plain")
            self.send_header("Content-Length", str(len(body)))
            self.end_headers()
            self.wfile.write(body)
        else:
            self.send_response(404)
            self.end_headers()


def _health_check() -> tuple[bool, str]:
    if HEALTH_CHECK_S3:
        try:
            get_s3_client().head_bucket(Bucket=S3_BUCKET)
        except Exception as exc:
            return False, f"s3_unhealthy: {exc}"
    return True, "ok"


def _dump_all_threads() -> str:
    """Dump Python tracebacks for all threads."""
    buf = io.StringIO()
    buf.write(f"Thread dump at {datetime.now(timezone.utc).isoformat()}\n")
    buf.write(f"Total threads: {threading.active_count()}\n\n")
    frames = sys._current_frames()
    for thread in threading.enumerate():
        frame = frames.get(thread.ident)
        buf.write(f"--- Thread {thread.ident} ({thread.name}) daemon={thread.daemon} ---\n")
        if frame:
            for line in traceback.format_stack(frame):
                buf.write(line)
        else:
            buf.write("  (no frame)\n")
        buf.write("\n")
    return buf.getvalue()


_TCP_STATES = {
    "01": "ESTABLISHED", "02": "SYN_SENT", "03": "SYN_RECV", "04": "FIN_WAIT1",
    "05": "FIN_WAIT2", "06": "TIME_WAIT", "07": "CLOSE", "08": "CLOSE_WAIT",
    "09": "LAST_ACK", "0A": "LISTEN", "0B": "CLOSING",
}


def _dump_connections() -> str:
    """Dump TCP connection info from /proc/net/tcp."""
    buf = io.StringIO()
    buf.write(f"TCP connections at {datetime.now(timezone.utc).isoformat()}\n\n")
    try:
        lines = Path("/proc/net/tcp").read_text().splitlines()[1:]
    except Exception as exc:
        buf.write(f"Error reading /proc/net/tcp: {exc}\n")
        return buf.getvalue()

    state_counts: dict[str, int] = {}
    connections: list[str] = []
    for line in lines:
        parts = line.split()
        if len(parts) < 4:
            continue
        local_hex, local_port_hex = parts[1].split(":")
        remote_hex, remote_port_hex = parts[2].split(":")
        st = parts[3]
        state_name = _TCP_STATES.get(st, f"UNKNOWN({st})")
        state_counts[state_name] = state_counts.get(state_name, 0) + 1

        def _hex_to_ip(h: str) -> str:
            b = bytes.fromhex(h)
            return f"{b[3]}.{b[2]}.{b[1]}.{b[0]}"

        lip = _hex_to_ip(local_hex)
        lp = int(local_port_hex, 16)
        rip = _hex_to_ip(remote_hex)
        rp = int(remote_port_hex, 16)
        connections.append(f"  {lip}:{lp} -> {rip}:{rp}  {state_name}")

    buf.write("State summary:\n")
    for k, v in sorted(state_counts.items(), key=lambda x: -x[1]):
        buf.write(f"  {k}: {v}\n")
    buf.write(f"\nAll connections ({len(connections)}):\n")
    for c in connections:
        buf.write(c + "\n")
    return buf.getvalue()


def _start_health_server() -> None:
    if HEALTH_PORT <= 0:
        return
    if getattr(ctx, "_health_server", None):
        return
    try:
        server = ThreadingHTTPServer((HEALTH_HOST, HEALTH_PORT), _HealthHandler)
    except OSError as exc:
        LOG.warning("Health server failed to bind to %s:%s: %s", HEALTH_HOST, HEALTH_PORT, exc)
        return
    thread = threading.Thread(target=server.serve_forever, daemon=True)
    thread.start()
    ctx._health_server = server
    ctx._health_thread = thread


def _create_s3_client():
    logging.getLogger("botocore.credentials").setLevel(logging.WARNING)
    kwargs = {"region_name": S3_REGION}
    if _S3_ENDPOINT:
        kwargs["endpoint_url"] = _S3_ENDPOINT
        kwargs["config"] = BotoConfig(s3={"addressing_style": "path"})
    else:
        kwargs["config"] = BotoConfig(signature_version="s3v4")
    return boto3.session.Session().client("s3", **kwargs)


_s3_client = None
_s3_client_lock = threading.Lock()


def get_s3_client():
    """Return a process-wide shared S3 client (boto3 clients are thread-safe)."""
    global _s3_client
    if _s3_client is not None:
        return _s3_client
    with _s3_client_lock:
        if _s3_client is None:
            _s3_client = _create_s3_client()
        return _s3_client


@dataclass
class CacheMeta:
    """Result of a cache metadata lookup.

    *status_code* is 200 when metadata was found, 404 otherwise.
    *meta* is the parsed JSON metadata dict (empty on miss).
    *source* tracks where the metadata came from ("elasticsearch").
    """
    status_code: int
    meta: dict
    source: str = ""


def _xs3lerator_link_manifest(base_key: str, vary_key: str) -> None:
    """Ask xs3lerator to create a manifest alias for Vary-aware cache keys."""
    xs3_url = getattr(ctx.options, "xs3lerator_url", "")
    if not xs3_url:
        LOG.warning("xs3lerator_url not configured, cannot link manifest")
        return
    url = f"{xs3_url}/manifest-alias"
    headers = {
        "X-Xs3lerator-Link-Manifest-Source": base_key,
        "X-Xs3lerator-Link-Manifest-Target": vary_key,
    }
    try:
        resp = requests.post(url, headers=headers, timeout=30)
        resp.raise_for_status()
        LOG.debug("Manifest alias linked: %s -> %s", base_key, vary_key)
    except Exception as exc:
        LOG.warning("Manifest alias failed (%s -> %s): %s", base_key, vary_key, exc)


def load_proxy_env_script() -> bytes:
    try:
        from importlib import resources
        script = resources.files("passsage").joinpath("onboarding/proxy-env.sh").read_text()
        public_url = os.environ.get("PASSSAGE_PUBLIC_PROXY_URL", "").strip() or "http://localhost:8080"
        script = script.replace("__PASSSAGE_PUBLIC_PROXY_URL__", public_url)
        script = script.replace("__PASSSAGE_S3_HOST__", S3_HOST)
        no_proxy_extra = os.environ.get("PASSSAGE_NO_PROXY_EXTRA", "").strip()
        if no_proxy_extra:
            no_proxy_extra = "," + no_proxy_extra
        script = script.replace("__PASSSAGE_NO_PROXY_EXTRA__", no_proxy_extra)
        cert_pem = _read_ca_cert_pem()
        if cert_pem:
            script = script.replace("__PASSSAGE_MITM_CA_PEM__", cert_pem)
        return script.encode("utf-8")
    except Exception as exc:
        LOG.error("Failed to load proxy env script: %s", exc)
        return b""


def _read_ca_cert_pem() -> str:
    confdir = getattr(ctx.options, "confdir", None)
    if confdir:
        candidate = os.path.join(confdir, "mitmproxy-ca-cert.pem")
        if os.path.isfile(candidate):
            return Path(candidate).read_text()
    default_path = os.path.expanduser("~/.mitmproxy/mitmproxy-ca-cert.pem")
    if os.path.isfile(default_path):
        return Path(default_path).read_text()
    LOG.error("Mitmproxy CA certificate not found in confdir or ~/.mitmproxy")
    return ""


def _is_tls_verify_error(err: Exception | None) -> bool:
    if err is None:
        return False
    if isinstance(err, (requests.exceptions.SSLError, ssl.SSLError)):
        return True
    message = str(err).lower()
    return "certificate verify failed" in message or "tls" in message


def _load_policy_module(path: str):
    import importlib.util

    spec = importlib.util.spec_from_file_location("passsage_policy_overrides", path)
    if spec is None or spec.loader is None:
        raise ValueError(f"Unable to load policy file: {path}")
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def _load_policy_overrides(path: str) -> None:
    if not path:
        return
    try:
        module = _load_policy_module(path)
    except Exception as exc:
        LOG.error("Policy file load failed path=%s error=%s", path, exc)
        return
    policy_loaded = False
    cache_key_loaded = False

    if hasattr(module, "get_resolver"):
        resolver = module.get_resolver()
        if isinstance(resolver, PolicyResolver):
            set_default_resolver(resolver)
            LOG.info("Policy overrides loaded via get_resolver path=%s", path)
            policy_loaded = True
        else:
            LOG.error("Policy file get_resolver did not return PolicyResolver path=%s", path)
    elif hasattr(module, "get_rules"):
        rules = module.get_rules()
        if rules:
            set_default_resolver(PolicyResolver(rules=rules))
            LOG.info("Policy overrides loaded via get_rules path=%s", path)
            policy_loaded = True
        else:
            LOG.error("Policy file get_rules returned no rules path=%s", path)
    elif hasattr(module, "RULES"):
        rules = module.RULES
        if rules:
            set_default_resolver(PolicyResolver(rules=rules))
            LOG.info("Policy overrides loaded via RULES path=%s", path)
            policy_loaded = True
        else:
            LOG.error("Policy file RULES is empty path=%s", path)

    if hasattr(module, "get_cache_key_resolver"):
        resolver = module.get_cache_key_resolver()
        if isinstance(resolver, CacheKeyResolver):
            set_default_cache_key_resolver(resolver)
            LOG.info("Cache key overrides loaded via get_cache_key_resolver path=%s", path)
            cache_key_loaded = True
        else:
            LOG.error(
                "Policy file get_cache_key_resolver did not return CacheKeyResolver path=%s",
                path,
            )
    elif hasattr(module, "CACHE_KEY_RESOLVER"):
        resolver = module.CACHE_KEY_RESOLVER
        if isinstance(resolver, CacheKeyResolver):
            set_default_cache_key_resolver(resolver)
            LOG.info("Cache key overrides loaded via CACHE_KEY_RESOLVER path=%s", path)
            cache_key_loaded = True
        else:
            LOG.error(
                "Policy file CACHE_KEY_RESOLVER is not CacheKeyResolver path=%s",
                path,
            )
    elif hasattr(module, "get_cache_key_rules"):
        rules = module.get_cache_key_rules()
        if rules:
            set_default_cache_key_resolver(CacheKeyResolver(rules=rules))
            LOG.info("Cache key overrides loaded via get_cache_key_rules path=%s", path)
            cache_key_loaded = True
        else:
            LOG.error("Policy file get_cache_key_rules returned no rules path=%s", path)
    elif hasattr(module, "CACHE_KEY_RULES"):
        rules = module.CACHE_KEY_RULES
        if rules:
            set_default_cache_key_resolver(CacheKeyResolver(rules=rules))
            LOG.info("Cache key overrides loaded via CACHE_KEY_RULES path=%s", path)
            cache_key_loaded = True
        else:
            LOG.error("Policy file CACHE_KEY_RULES is empty path=%s", path)

    if not policy_loaded and not cache_key_loaded:
        LOG.error(
            "Policy file missing policy or cache-key hooks path=%s",
            path,
        )


def save_response(proxy, flow, data: bytes) -> Union[bytes, Iterable[bytes]]:
    """
    This function will be called for each chunk of request/response body data that arrives at the proxy,
    and once at the end of the message with an empty bytes argument (b"").

    It may either return bytes or an iterable of bytes (which would result in multiple HTTP/2 data frames).
    """
    if flow._counter not in proxy.files:
        proxy.files[flow._counter] = tempfile.TemporaryFile()
        proxy.hashes[flow._counter] = hashlib.sha224()

    proxy.files[flow._counter].write(data)
    proxy.hashes[flow._counter].update(data)
    return data


def _parse_single_range(header: str, total: int) -> tuple[int, int] | None:
    """Parse a single-range ``Range`` header.  Returns (start, end) inclusive or None."""
    return _parse_single_range_impl(header, total)


def _is_unsatisfiable_range(header: str, total: int) -> bool:
    """Return True when the Range spec is syntactically valid but wholly outside [0, total)."""
    return _is_unsatisfiable_range_impl(header, total)


def _if_range_matches(flow) -> bool:
    """Evaluate If-Range against cached metadata per RFC 9110 Section 14.5."""
    if_range = flow.request.headers.get("if-range")
    meta = flow._cache_head.meta if getattr(flow, "_cache_head", None) else None
    return _if_range_matches_impl(if_range, meta)


def _make_range_stream(proxy, flow, start: int, end: int):
    """Return a stream function that saves the full body but serves only [start, end] to the client."""
    pos = [0]

    def stream_fn(data: bytes):
        saved = save_response(proxy, flow, data)
        if not data:
            return saved  # end-of-stream marker
        chunk_start = pos[0]
        chunk_end = chunk_start + len(data) - 1
        pos[0] += len(data)
        if chunk_end < start or chunk_start > end:
            return b""
        slice_start = max(0, start - chunk_start)
        slice_end = min(len(data), end - chunk_start + 1)
        return data[slice_start:slice_end]

    return stream_fn


def get_policy(flow):
    if (
        flow.request.headers
        and hasattr(ctx, "options")
        and getattr(ctx.options, "allow_policy_header", False)
    ):
        policy_name = flow.request.headers.get(POLICY_HEADER)
        if policy_name:
            policy = POLICY_BY_NAME.get(policy_name.strip().lower())
            if policy:
                LOG.debug("Policy header override=%s", policy_name)
                flow._timeout_config = None
                flow._forced_swr_seconds = None
                return policy
            LOG.warning("Unknown policy header override=%s", policy_name)
    request_ctx = PolicyContext(
        url=flow.request.url,
        method=flow.request.method,
        headers=list(flow.request.headers.items()) if flow.request.headers else None,
    )
    resolver = get_default_resolver()
    if hasattr(ctx, "options") and hasattr(ctx.options, "default_policy"):
        resolver.set_default_policy(policy_from_name(ctx.options.default_policy))
    resolved = resolver.resolve(request_ctx)
    flow._timeout_config = resolved.timeouts
    flow._forced_swr_seconds = resolved.forced_swr_seconds
    return resolved.policy


def http_2xx(r):
    if r and 200 <= r.status_code <= 299:
        return True
    return False


def _is_connection_error(err: Exception | None) -> bool:
    if not err:
        return False
    if isinstance(err, RequestsConnectionError):
        return True
    return "Connection refused" in str(err) or "Failed to establish a new connection" in str(err)


def _build_upstream_error_response(status_code: int, title: str, detail: str) -> http.Response:
    body = f"""<html>
<head><title>{status_code} {title}</title></head>
<body>
<center><h1>{status_code} {title}</h1></center>
<p>Upstream request failed.</p>
<pre>{detail}</pre>
</body>
</html>""".encode("utf-8")
    return http.Response.make(status_code, body, {"Content-Type": "text/html"})


def _is_s3_cache_request(flow: http.HTTPFlow) -> bool:
    return flow.request.pretty_host == S3_HOST


def parse_cache_control(value: str) -> dict[str, str | bool]:
    directives: dict[str, str | bool] = {}
    if not value:
        return directives
    for part in value.split(","):
        part = part.strip()
        if not part:
            continue
        if "=" in part:
            k, v = part.split("=", 1)
            directives[k.strip().lower()] = v.strip().strip('"')
        else:
            directives[part.lower()] = True
    return directives


def parse_cache_control_seconds(cc: dict[str, str | bool], name: str) -> int | None:
    value = cc.get(name)
    if value is None or value is True:
        return None
    try:
        return int(value)
    except (TypeError, ValueError):
        return None


@dataclass(frozen=True)
class RefreshPattern:
    pattern: re.Pattern[str]
    min_seconds: int
    percent: int
    max_seconds: int
    options: set[str]

    def ttl_seconds(self, now: datetime, meta: dict) -> int:
        lastmod = (meta.get("headers") or {}).get("last-modified")
        if lastmod:
            lastmod_dt = parse_date(lastmod)
            age = max(0, int((now - lastmod_dt).total_seconds())) if lastmod_dt else 0
        else:
            age = 0
        ttl = self.min_seconds + int(age * (self.percent / 100))
        if self.max_seconds > 0:
            ttl = min(ttl, self.max_seconds)
        return max(0, ttl)


def parse_refresh_patterns(spec: str) -> list[RefreshPattern]:
    patterns: list[RefreshPattern] = []
    for line in (spec or "").splitlines():
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        parts = line.split()
        if len(parts) < 4:
            continue
        regex, min_s, percent, max_s, *opts = parts
        try:
            patterns.append(
                RefreshPattern(
                    pattern=re.compile(regex, re.IGNORECASE),
                    min_seconds=int(min_s),
                    percent=int(percent),
                    max_seconds=int(max_s),
                    options={o.lower() for o in opts},
                )
            )
        except Exception:
            continue
    return patterns


def cache_stored_at(meta: dict) -> datetime | None:
    stored = meta.get("stored_at")
    if stored is not None:
        try:
            if isinstance(stored, str):
                return datetime.fromisoformat(stored)
            return datetime.fromtimestamp(float(stored), tz=timezone.utc)
        except (ValueError, TypeError, OSError):
            return None
    return None


@dataclass(frozen=True)
class CacheFreshness:
    age_seconds: int
    ttl_seconds: int | None
    is_fresh: bool
    stale_while_revalidate_seconds: int | None
    stale_if_error_seconds: int | None
    allow_stale_while_revalidate: bool
    allow_stale_if_error: bool


def cache_ttl_seconds(
    url: str,
    meta: dict,
    refresh_patterns: list[RefreshPattern],
    now: datetime,
    stored_at: datetime,
) -> int | None:
    for pattern in refresh_patterns:
        if pattern.pattern.search(url):
            return pattern.ttl_seconds(now, meta)
    hdrs = meta.get("headers", {})
    cc = parse_cache_control(hdrs.get("cache-control", ""))
    if "no-store" in cc:
        return None
    if "no-cache" in cc:
        return 0
    if (s_maxage := cc.get("s-maxage")) is not None:
        try:
            return int(s_maxage)
        except ValueError:
            return None
    if (max_age := cc.get("max-age")) is not None:
        try:
            return int(max_age)
        except ValueError:
            return None
    if (exp := hdrs.get("expires")):
        exp_dt = parse_date(exp)
        if exp_dt:
            return max(0, int((exp_dt - stored_at).total_seconds()))
    if (lastmod := (meta.get("headers") or {}).get("last-modified")):
        lastmod_dt = parse_date(lastmod)
        if lastmod_dt:
            lifetime = max(0, int((now - lastmod_dt).total_seconds() * 0.1))
            return lifetime
    return None


def cache_freshness(
    url: str, meta: dict, refresh_patterns: list[RefreshPattern]
) -> CacheFreshness:
    now = datetime.now(tz=pytz.utc)
    stored_at = cache_stored_at(meta)
    if stored_at is None:
        return CacheFreshness(
            age_seconds=0,
            ttl_seconds=None,
            is_fresh=False,
            stale_while_revalidate_seconds=None,
            stale_if_error_seconds=None,
            allow_stale_while_revalidate=False,
            allow_stale_if_error=False,
        )
    age_seconds = max(0, int((now - stored_at).total_seconds()))
    hdrs = meta.get("headers", {})
    cc = parse_cache_control(hdrs.get("cache-control", ""))
    ttl_seconds = cache_ttl_seconds(url, meta, refresh_patterns, now, stored_at)
    is_fresh = ttl_seconds is not None and age_seconds < ttl_seconds
    stale_while_revalidate_seconds = parse_cache_control_seconds(cc, "stale-while-revalidate")
    stale_if_error_seconds = parse_cache_control_seconds(cc, "stale-if-error")
    allow_stale_while_revalidate = False
    if (
        not is_fresh
        and ttl_seconds is not None
        and stale_while_revalidate_seconds is not None
        and age_seconds <= ttl_seconds + stale_while_revalidate_seconds
        and "must-revalidate" not in cc
        and "proxy-revalidate" not in cc
    ):
        allow_stale_while_revalidate = True
    allow_stale_if_error = False
    if (
        not is_fresh
        and ttl_seconds is not None
        and stale_if_error_seconds is not None
        and age_seconds <= ttl_seconds + stale_if_error_seconds
    ):
        allow_stale_if_error = True
    if not allow_stale_if_error and "immutable" in cc and not is_fresh and ttl_seconds is not None:
        # Immutable implies the resource is versioned; if upstream fails after expiry,
        # allow serving the stale cached copy even without stale-if-error.
        allow_stale_if_error = True
    return CacheFreshness(
        age_seconds=age_seconds,
        ttl_seconds=ttl_seconds,
        is_fresh=is_fresh,
        stale_while_revalidate_seconds=stale_while_revalidate_seconds,
        stale_if_error_seconds=stale_if_error_seconds,
        allow_stale_while_revalidate=allow_stale_while_revalidate,
        allow_stale_if_error=allow_stale_if_error,
    )


def request_requires_revalidation(request_headers: dict[str, str], freshness: CacheFreshness) -> bool:
    cc = parse_cache_control(request_headers.get("cache-control", ""))
    if "no-cache" in cc:
        return True
    if (req_max_age := parse_cache_control_seconds(cc, "max-age")) is not None:
        return freshness.age_seconds >= req_max_age
    return False




def apply_cached_metadata(flow: http.HTTPFlow) -> None:
    """Apply cached metadata from flow._cache_head to the response."""
    meta = getattr(flow, "_cache_head", None)
    if not meta or not meta.meta:
        return
    m = meta.meta
    if (status_code := m.get("status_code")):
        if flow.response.status_code != 206:
            flow.response.status_code = int(status_code)
    for k, v in m.get("headers", {}).items():
        flow.response.headers[k] = v


def refresh_cache_metadata(cache_key: str, meta: dict) -> None:
    """Update stored_at timestamp in the ES doc."""
    meta = dict(meta)
    meta["stored_at"] = datetime.now(timezone.utc).isoformat()
    try:
        es_meta_index = os.environ.get("PASSSAGE_ELASTICSEARCH_META_INDEX", "passsage_meta")
        doc_id = cache_key
        es_index_doc(es_meta_index, doc_id, meta)
    except Exception as exc:
        LOG.warning("Cache metadata refresh failed for %s: %s", cache_key, exc)


def _es_meta_index() -> str:
    return os.environ.get("PASSSAGE_ELASTICSEARCH_META_INDEX", "passsage_meta")


def get_cache_metadata(cache_key: str) -> CacheMeta:
    data = es_get_doc(_es_meta_index(), cache_key)
    if data is None:
        return CacheMeta(404, {}, source="elasticsearch")
    batcher = getattr(ctx, "_last_access_batcher", None)
    if batcher:
        batcher.touch(cache_key)
    return CacheMeta(200, data, source="elasticsearch")


def get_vary_index(vary_index_key: str) -> CacheMeta:
    data = es_get_doc(_es_meta_index(), vary_index_key)
    if data is None:
        return CacheMeta(404, {}, source="elasticsearch")
    return CacheMeta(200, data, source="elasticsearch")


def get_refresh_patterns() -> list[RefreshPattern]:
    spec = getattr(ctx.options, "refresh_pattern", "") if hasattr(ctx, "options") else ""
    cached = getattr(ctx, "_refresh_patterns_cache", None)
    if cached and cached.get("spec") == spec:
        return cached["patterns"]
    patterns = parse_refresh_patterns(spec)
    ctx._refresh_patterns_cache = {"spec": spec, "patterns": patterns}
    return patterns


def _parse_header_names(raw: str) -> list[str]:
    return [name.strip().lower() for name in raw.split(",") if name.strip()]


def _select_headers(headers, names: list[str]) -> dict[str, str]:
    if not headers or not names:
        return {}
    selected: dict[str, str] = {}
    for name in names:
        value = headers.get(name)
        if value is not None:
            selected[name] = value
    return selected


def _safe_int(value) -> int | None:
    try:
        return int(value)
    except (TypeError, ValueError):
        return None


def _access_log_writer() -> AccessLogWriter | None:
    return getattr(ctx, "_access_log_writer", None)


def _error_log_writer() -> ErrorLogWriter | None:
    return getattr(ctx, "_error_log_writer", None)


def _log_exception(
    exc: Exception,
    *,
    flow=None,
    context: dict | None = None,
    error_type: str | None = None,
    request_url: str | None = None,
    method: str | None = None,
    request_headers: dict | None = None,
) -> None:
    writer = _error_log_writer()
    if not writer:
        LOG.exception("Unhandled error: %s", exc)
        return
    tb_text = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
    try:
        writer.add(
            _build_error_log_record(
                error_type=error_type or type(exc).__name__,
                error_message=str(exc),
                traceback_text=tb_text,
                context=context or {},
                flow=flow,
                request_url=request_url,
                method=method,
                request_headers=request_headers,
            )
        )
    except Exception as write_exc:
        LOG.debug("Error log write skipped: %s", write_exc)


def _log_hook_errors(hook_name: str):
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            flow = kwargs.get("flow")
            if flow is None:
                for arg in args:
                    if isinstance(arg, http.HTTPFlow):
                        flow = arg
                        break
            try:
                return func(*args, **kwargs)
            except Exception as exc:
                _log_exception(exc, flow=flow, context={"hook": hook_name})
                raise

        return wrapper

    return decorator


def _init_access_logs() -> None:
    writer = _access_log_writer()
    if not getattr(ctx.options, "access_logs", False):
        if writer:
            writer.stop()
        ctx._access_log_writer = None
        return
    config = build_access_log_config(
        S3_BUCKET,
        getattr(ctx.options, "access_log_prefix", ACCESS_LOG_PREFIX),
        getattr(ctx.options, "access_log_dir", ACCESS_LOG_DIR),
        getattr(ctx.options, "access_log_flush_seconds", ACCESS_LOG_FLUSH_SECONDS),
        getattr(ctx.options, "access_log_flush_bytes", ACCESS_LOG_FLUSH_BYTES),
    )
    if writer:
        writer.stop()
    ctx._access_log_writer = AccessLogWriter(get_s3_client(), config, LOG)
    ctx._access_log_header_names = _parse_header_names(
        getattr(ctx.options, "access_log_headers", ACCESS_LOG_HEADERS)
    )
    ctx._access_log_writer.start()


def _init_error_logs() -> None:
    writer = _error_log_writer()
    if not getattr(ctx.options, "error_logs", False):
        if writer:
            writer.stop()
        ctx._error_log_writer = None
        return
    config = build_error_log_config(
        S3_BUCKET,
        getattr(ctx.options, "error_log_prefix", ERROR_LOG_PREFIX),
        getattr(ctx.options, "error_log_dir", ERROR_LOG_DIR),
        getattr(ctx.options, "error_log_flush_seconds", ERROR_LOG_FLUSH_SECONDS),
        getattr(ctx.options, "error_log_flush_bytes", ERROR_LOG_FLUSH_BYTES),
    )
    if writer:
        writer.stop()
    ctx._error_log_writer = ErrorLogWriter(get_s3_client(), config, LOG)
    ctx._error_log_header_names = _parse_header_names(
        getattr(ctx.options, "access_log_headers", ACCESS_LOG_HEADERS)
    )
    ctx._error_log_writer.start()


_ES_MAPPING = {
    "settings": {
        "number_of_shards": 9,
        "number_of_replicas": 1,
        "refresh_interval": "60s",
    },
    "mappings": {
        "dynamic": False,
        "properties": {
            "status_code": {"type": "integer", "index": False},
            "url": {"type": "keyword"},
            "extension": {"type": "keyword"},
            "content_size": {"type": "long", "index": False},
            "stored_at": {"type": "date"},
            "last_access": {"type": "date"},
            "headers": {"type": "object", "enabled": False},
            "vary": {"type": "keyword", "index": False},
            "vary_request": {"type": "keyword", "index": False},
            "doc_type": {"type": "keyword"},
            "manifest_b64": {"type": "keyword", "index": False, "doc_values": False},
        },
    },
}


def _init_elasticsearch() -> None:
    if getattr(ctx, "_es_initialized", False):
        return
    es_url = os.environ.get("PASSSAGE_ELASTICSEARCH_URL", "")
    if not es_url:
        return
    meta_index = os.environ.get("PASSSAGE_ELASTICSEARCH_META_INDEX", "passsage_meta")
    flush_interval = float(os.environ.get("PASSSAGE_ELASTICSEARCH_FLUSH_INTERVAL", "30"))

    shards = int(os.environ.get("PASSSAGE_ELASTICSEARCH_SHARDS", "9"))
    replicas = int(os.environ.get("PASSSAGE_ELASTICSEARCH_REPLICAS", "1"))

    _es_configure(es_url)

    mapping = dict(_ES_MAPPING)
    mapping["settings"] = {
        "number_of_shards": shards,
        "number_of_replicas": replicas,
        "refresh_interval": "60s",
    }
    es_create_index(meta_index, mapping)

    ctx._last_access_batcher = LastAccessBatcher(meta_index, flush_interval)
    ctx._es_initialized = True
    LOG.info("Elasticsearch initialized: url=%s index=%s flush_interval=%ss", es_url, meta_index, flush_interval)


def _build_access_log_record(flow, error: str | None = None) -> dict:
    now = time.time()
    start_ts = getattr(flow, "_access_log_start", now)
    duration_ms = int((now - start_ts) * 1000)
    start_dt = datetime.fromtimestamp(start_ts, tz=pytz.utc)
    request_url = getattr(flow, "_req_orig_url", None) or flow.request.url
    parsed = urlparse(request_url)
    client_addr = getattr(flow.client_conn, "peername", None) or ()
    client_ip = client_addr[0] if isinstance(client_addr, tuple) and client_addr else None
    client_port = client_addr[1] if isinstance(client_addr, tuple) and len(client_addr) > 1 else None
    request_headers = _select_headers(
        flow.request.headers,
        getattr(ctx, "_access_log_header_names", []),
    )
    response = getattr(flow, "response", None)
    response_headers = (
        _select_headers(response.headers, getattr(ctx, "_access_log_header_names", []))
        if response
        else {}
    )
    content_length = None
    if response:
        content_length = _safe_int(response.headers.get("content-length"))
    if content_length is None and response and response.content is not None:
        content_length = len(response.content)
    cache_head = getattr(flow, "_cache_head", None)
    upstream_error = getattr(flow, "_upstream_error", None)
    return {
        "timestamp": start_dt,
        "request_id": getattr(flow, "id", None),
        "client_ip": client_ip,
        "client_port": _safe_int(client_port),
        "method": flow.request.method,
        "url": request_url,
        "host": parsed.hostname,
        "scheme": parsed.scheme,
        "port": _safe_int(parsed.port),
        "path": parsed.path,
        "query": parsed.query,
        "user_agent": flow.request.headers.get("User-Agent"),
        "request_headers": request_headers,
        "status_code": response.status_code if response else None,
        "reason": response.reason if response else None,
        "response_headers": response_headers,
        "content_length": content_length,
        "content_type": response.headers.get("content-type") if response else None,
        "content_encoding": response.headers.get("content-encoding") if response else None,
        "policy": getattr(flow, "_policy", None).__name__ if getattr(flow, "_policy", None) else None,
        "cached": getattr(flow, "_cached", None),
        "cache_key": getattr(flow, "_cache_key", None),
        "cache_vary": getattr(flow, "_cache_vary", None),
        "cache_hit": getattr(flow, "_cache_hit", None),
        "cache_fresh": getattr(flow, "_cache_fresh", None),
        "stale_while_revalidate": getattr(flow, "_allow_stale_while_revalidate", None),
        "stale_if_error": getattr(flow, "_allow_stale_if_error", None),
        "forced_swr": getattr(flow, "_forced_swr", False),
        "upstream_error": str(upstream_error) if upstream_error else None,
        "cache_head_status": cache_head.status_code if cache_head else None,
        "cache_head_etag": cache_head.meta.get("headers", {}).get("etag") if cache_head else None,
        "cache_head_last_modified": cache_head.meta.get("last_modified") if cache_head else None,
        "cache_head_method": cache_head.meta.get("method") if cache_head else None,
        "metadata_source": getattr(flow, "_metadata_source", None),
        "serve_reason": getattr(flow, "_serve_reason", None),
        "cached_method": getattr(flow, "_cached_method", None),
        "vary_index_key": getattr(flow, "_vary_index_key", None),
        "cache_save": getattr(flow, "_cache_saved", False),
        "range_stripped": getattr(flow, "_range_stripped", False),
        "object_store_rewrite": getattr(flow, "_object_store_rewrite", False),
        "error": error,
        "duration_ms": duration_ms,
        "bytes_sent": content_length,
    }


def _build_error_log_record(
    error_type: str,
    error_message: str,
    traceback_text: str,
    context: dict | None = None,
    flow=None,
    request_url: str | None = None,
    method: str | None = None,
    request_headers: dict | None = None,
) -> dict:
    now = time.time()
    start_ts = getattr(flow, "_access_log_start", now) if flow else now
    start_dt = datetime.fromtimestamp(start_ts, tz=pytz.utc)
    url_value = request_url or (flow.request.url if flow else None)
    parsed = urlparse(url_value) if url_value else None
    client_addr = getattr(flow.client_conn, "peername", None) if flow else None
    client_addr = client_addr or ()
    client_ip = client_addr[0] if isinstance(client_addr, tuple) and client_addr else None
    client_port = client_addr[1] if isinstance(client_addr, tuple) and len(client_addr) > 1 else None
    selected_request_headers = (
        _select_headers(
            flow.request.headers,
            getattr(ctx, "_error_log_header_names", []),
        )
        if flow
        else request_headers
        or {}
    )
    response = getattr(flow, "response", None) if flow else None
    response_headers = (
        _select_headers(response.headers, getattr(ctx, "_error_log_header_names", []))
        if response
        else {}
    )
    cache_head = getattr(flow, "_cache_head", None) if flow else None
    upstream_error = getattr(flow, "_upstream_error", None) if flow else None
    policy = getattr(flow, "_policy", None) if flow else None
    return {
        "timestamp": start_dt,
        "request_id": getattr(flow, "id", None) if flow else None,
        "client_ip": client_ip,
        "client_port": _safe_int(client_port),
        "method": method or (flow.request.method if flow else None),
        "url": url_value,
        "host": parsed.hostname if parsed else None,
        "scheme": parsed.scheme if parsed else None,
        "port": _safe_int(parsed.port) if parsed else None,
        "path": parsed.path if parsed else None,
        "query": parsed.query if parsed else None,
        "user_agent": flow.request.headers.get("User-Agent") if flow else None,
        "request_headers": selected_request_headers,
        "status_code": response.status_code if response else None,
        "response_headers": response_headers,
        "policy": policy.__name__ if policy else None,
        "cached": getattr(flow, "_cached", None) if flow else None,
        "cache_key": getattr(flow, "_cache_key", None) if flow else None,
        "cache_vary": getattr(flow, "_cache_vary", None) if flow else None,
        "serve_reason": getattr(flow, "_serve_reason", None) if flow else None,
        "cache_head_status": cache_head.status_code if cache_head else None,
        "upstream_error": str(upstream_error) if upstream_error else None,
        "error_type": error_type,
        "error_message": error_message,
        "traceback": traceback_text,
        "context": json.dumps(context or {}, default=str),
    }


def _normalize_url(flow) -> str:
    url = getattr(flow, "_original_url", None) or flow.request.url
    request_ctx = CacheKeyContext(
        url=url,
        method=flow.request.method,
        headers=list(flow.request.headers.items()) if flow.request.headers else None,
    )
    resolver = get_default_cache_key_resolver()
    return resolver.resolve(request_ctx)



def compute_vary_key(vary_header: str, request_headers) -> str | None:
    raw = build_vary_request(vary_header, request_headers)
    if raw is None:
        return None
    return hashlib.sha224(raw.encode("utf-8")).hexdigest()


def build_vary_request(vary_header: str, request_headers) -> str | None:
    if "*" in vary_header:
        return None
    names = [n.strip().lower() for n in vary_header.split(",") if n.strip()]
    parts = [f"{name}={request_headers.get(name, '')}" for name in names]
    return "|".join(parts)


# Request headers that are known to affect response bytes.  Used to
# proactively differentiate cache/dedup keys before the upstream Vary
# header is discovered — prevents xs3lerator from coalescing concurrent
# requests for the same URL but different encodings.
_CONTENT_AFFECTING_HEADERS = ("accept-encoding",)


def compute_proactive_vary_key(request_headers) -> str | None:
    """Hash content-affecting request headers into a vary key *before*
    seeing the upstream Vary response header.

    Returns None when no content-affecting headers are present (no need
    to differentiate the key).

    The output intentionally matches ``compute_vary_key()`` for the
    same set of headers so that the ES metadata key is identical
    whether computed proactively or reactively.
    """
    names_with_values = [
        name for name in _CONTENT_AFFECTING_HEADERS
        if request_headers.get(name)
    ]
    if not names_with_values:
        return None
    return compute_vary_key(",".join(names_with_values), request_headers)


def _rewrite_to_object_store(flow):
    """Rewrite a cache-hit GET to fetch via xs3lerator.

    xs3lerator handles S3 cache retrieval (with parallel downloads and
    automatic upstream fallback).  Non-GET methods never reach here — HEAD
    requests are handled separately and other verbs go straight to upstream.
    """
    _rewrite_to_xs3lerator(flow, cache_skip=False)


def _rewrite_to_xs3lerator(flow, cache_skip: bool):
    """Rewrite GET request to route through xs3lerator.

    xs3lerator handles parallel downloads, S3 caching, and range requests.
    The upstream URL is placed in the request path; the cache key is sent
    in the X-Xs3lerator-Cache-Key header.
    """
    xs3lerator_url = ctx.options.xs3lerator_url
    parsed = urlparse(xs3lerator_url)
    flow._original_url = flow.request.url

    flow.request.scheme = parsed.scheme or "http"
    flow.request.host = parsed.hostname
    flow.request.port = parsed.port or 8080

    s3_key = flow._cache_key
    flow._xs3lerator_stored_key = s3_key
    flow.request.path = f"/{flow._original_url}"

    flow.request.headers["X-Xs3lerator-Cache-Key"] = s3_key
    if cache_skip:
        flow.request.headers["X-Xs3lerator-Cache-Skip"] = "true"
    elif flow._cache_head and flow._cache_head.meta.get("headers", {}).get("content-length"):
        flow.request.headers["X-Xs3lerator-Object-Size"] = (
            flow._cache_head.meta["headers"]["content-length"]
        )

    if not cache_skip and flow._cache_head:
        manifest_b64 = flow._cache_head.meta.get("manifest_b64")
        if manifest_b64 and len(manifest_b64) <= 65536:
            flow.request.headers["X-Xs3lerator-Manifest"] = manifest_b64

    tc = getattr(flow, "_timeout_config", None)
    if tc is not None:
        if tc.connect_timeout is not None:
            flow.request.headers["X-Xs3lerator-Connect-Timeout"] = str(tc.connect_timeout)
        if tc.read_timeout is not None:
            flow.request.headers["X-Xs3lerator-Read-Timeout"] = str(tc.read_timeout)

    flow._xs3lerator_rewrite = True
    flow._object_store_rewrite = True
    flow._cached = not cache_skip
    flow._save_response = cache_skip
    flow.request.headers.pop("If-Range", None)


def _rewrite_to_xs3lerator_miss(flow):
    """Rewrite a cache miss GET to route through xs3lerator."""
    xs3lerator_url = getattr(ctx.options, "xs3lerator_url", "")
    if not xs3lerator_url:
        return
    _rewrite_to_xs3lerator(flow, cache_skip=True)


def _rewrite_to_xs3lerator_revalidate(flow):
    """Rewrite a stale-cache GET to do conditional revalidation via xs3lerator.

    Sends the cached ETag/Last-Modified as contract headers so xs3lerator
    can issue a conditional GET to upstream.  On 304, xs3lerator serves from
    its S3 cache.  On upstream error (when stale-if-error is eligible),
    xs3lerator serves cached content instead of passing the error through.
    """
    xs3lerator_url = getattr(ctx.options, "xs3lerator_url", "")
    if not xs3lerator_url:
        return
    _rewrite_to_xs3lerator(flow, cache_skip=False)

    cm = flow._cache_head.meta
    etag = cm.get("headers", {}).get("etag")
    if etag:
        flow.request.headers["X-Xs3lerator-If-None-Match"] = etag
    last_modified = (cm.get("headers") or {}).get("last-modified")
    if not last_modified:
        last_modified = cm.get("last_modified")
    if last_modified:
        flow.request.headers["X-Xs3lerator-If-Modified-Since"] = last_modified

    if flow._policy == StaleIfError or getattr(flow, "_allow_stale_if_error", False):
        flow.request.headers["X-Xs3lerator-Stale-If-Error"] = "true"

    flow._cached = False
    flow._save_response = False
    flow._conditional_revalidation = True


def _rewrite_to_object_store_with_bg_revalidation(flow):
    """Rewrite a stale-cache GET to serve from cache immediately while
    requesting xs3lerator to do a background conditional revalidation.

    xs3lerator returns the cached body right away and spawns an async
    task to revalidate with the upstream origin.
    """
    xs3lerator_url = getattr(ctx.options, "xs3lerator_url", "")
    if not xs3lerator_url:
        return
    _rewrite_to_xs3lerator(flow, cache_skip=False)

    cm = flow._cache_head.meta
    etag = cm.get("headers", {}).get("etag")
    if etag:
        flow.request.headers["X-Xs3lerator-If-None-Match"] = etag
    last_modified = (cm.get("headers") or {}).get("last-modified")
    if not last_modified:
        last_modified = cm.get("last_modified")
    if last_modified:
        flow.request.headers["X-Xs3lerator-If-Modified-Since"] = last_modified

    flow.request.headers["X-Xs3lerator-Background-Revalidate"] = "true"
    flow.request.headers["X-Xs3lerator-Stale-If-Error"] = "true"

    flow._cached = True
    flow._save_response = False


def _rewrite_head_to_xs3lerator(flow):
    """Rewrite a stale/miss HEAD to route through xs3lerator for connection pooling."""
    xs3lerator_url = getattr(ctx.options, "xs3lerator_url", "")
    if not xs3lerator_url:
        return
    parsed = urlparse(xs3lerator_url)
    flow._original_url = flow.request.url
    flow.request.scheme = parsed.scheme or "http"
    flow.request.host = parsed.hostname
    flow.request.port = parsed.port or 8080
    flow.request.path = f"/{flow._original_url}"
    flow._save_response = False
    flow._xs3lerator_rewrite = True
    flow._serve_reason = "head_via_xs3lerator"


def _fallback_fetch_from_object_store(flow) -> bool:
    """Fetch cached object via xs3lerator as stale-if-error fallback.

    Used when the upstream already returned an error response and we need to
    replace it with the cached object.  Returns True on success.
    """
    cache_key = flow._cache_key
    if not cache_key:
        return False
    xs3lerator_url = ctx.options.xs3lerator_url
    if not xs3lerator_url:
        return False
    original_url = flow._original_url if hasattr(flow, "_original_url") else flow.request.url
    parsed = urlparse(xs3lerator_url)
    url = f"{parsed.scheme}://{parsed.netloc}/{original_url}"
    try:
        resp = requests.get(url, timeout=30, headers={
            "X-Xs3lerator-Cache-Key": cache_key,
        })
        if resp.status_code != 200:
            LOG.warning("xs3lerator fallback returned %s for key=%s", resp.status_code, cache_key)
            return False
        flow.response = http.Response.make(200, resp.content, {
            "Content-Type": resp.headers.get("content-type", "application/octet-stream"),
        })
        flow.response.headers["accept-ranges"] = "bytes"
        flow._save_response = False
        flow._cached = True
        return True
    except Exception as exc:
        LOG.warning("xs3lerator fallback failed for key=%s: %s", cache_key, exc)
        return False

class Proxy:
    def __init__(self):
        self.files = {}
        self.hashes = {}
        self.counter = 0
        if not getattr(ctx, "_executor", None):
            # don't (re)create these on code reloads
            ctx._lock = threading.Lock()
            ctx._executor = ThreadPoolExecutor()
            _start_health_server()

    @_log_hook_errors("load")
    def load(self, loader):
        loader.add_option(
            name="default_policy",
            typespec=str,
            default="Standard",
            help="Default policy when no rule matches (e.g. Standard, StaleIfError)",
        )
        loader.add_option(
            name="refresh_pattern",
            typespec=str,
            default="",
            help="Squid-like refresh_pattern rules (newline separated)",
        )
        loader.add_option(
            name="policy_file",
            typespec=str,
            default=os.environ.get("PASSSAGE_POLICY_FILE", ""),
            help="Path to a Python file defining policy overrides",
        )
        loader.add_option(
            name="allow_policy_header",
            typespec=bool,
            default=bool(os.environ.get("PASSSAGE_ALLOW_POLICY_HEADER")),
            help="Allow client policy override via X-Passsage-Policy",
        )
        loader.add_option(
            name="public_proxy_url",
            typespec=str,
            default=PUBLIC_PROXY_URL,
            help="Externally reachable proxy URL (e.g. http://proxy.example.com:3128). "
                 "Embedded in the mitm.it/proxy-env.sh onboarding script so clients "
                 "export the correct HTTP_PROXY/HTTPS_PROXY address. Required when the "
                 "proxy runs behind a load balancer or in Kubernetes where the internal "
                 "listen address differs from the address clients should connect to. "
                 "Env: PASSSAGE_PUBLIC_PROXY_URL",
        )
        loader.add_option(
            name="access_logs",
            typespec=bool,
            default=ACCESS_LOGS,
            help="Enable S3 access logs in Parquet format",
        )
        loader.add_option(
            name="access_log_prefix",
            typespec=str,
            default=ACCESS_LOG_PREFIX,
            help="S3 prefix for access logs (separate from cached objects)",
        )
        loader.add_option(
            name="access_log_dir",
            typespec=str,
            default=ACCESS_LOG_DIR,
            help="Local spool directory for access logs",
        )
        loader.add_option(
            name="access_log_flush_seconds",
            typespec=str,
            default=ACCESS_LOG_FLUSH_SECONDS,
            help="Flush interval in seconds for access logs",
        )
        loader.add_option(
            name="access_log_flush_bytes",
            typespec=str,
            default=ACCESS_LOG_FLUSH_BYTES,
            help="Flush size threshold for access logs (bytes or 1G/500M/10K)",
        )
        loader.add_option(
            name="access_log_headers",
            typespec=str,
            default=ACCESS_LOG_HEADERS,
            help="Comma-separated headers to include in access logs",
        )
        loader.add_option(
            name="error_logs",
            typespec=bool,
            default=ERROR_LOGS,
            help="Enable Parquet error logs with tracebacks",
        )
        loader.add_option(
            name="error_log_prefix",
            typespec=str,
            default=ERROR_LOG_PREFIX,
            help="S3 prefix for error logs",
        )
        loader.add_option(
            name="error_log_dir",
            typespec=str,
            default=ERROR_LOG_DIR,
            help="Local spool directory for error logs",
        )
        loader.add_option(
            name="error_log_flush_seconds",
            typespec=str,
            default=ERROR_LOG_FLUSH_SECONDS,
            help="Flush interval in seconds for error logs",
        )
        loader.add_option(
            name="error_log_flush_bytes",
            typespec=str,
            default=ERROR_LOG_FLUSH_BYTES,
            help="Flush size threshold for error logs (bytes or 1G/500M/10K)",
        )
        loader.add_option(
            name="xs3lerator_url",
            typespec=str,
            default=os.environ.get("PASSSAGE_XS3LERATOR_URL", ""),
            help="HTTP URL of the xs3lerator service for GET request proxying. "
                 "When set, GET requests are routed through xs3lerator for parallel "
                 "downloads and S3 caching. (env: PASSSAGE_XS3LERATOR_URL)",
        )
        loader.add_option(
            name="s3_hash_prefix_depth",
            typespec=int,
            default=int(os.environ.get("PASSSAGE_S3_HASH_PREFIX_DEPTH", "4")),
            help="Number of hash characters to use as S3 path prefix directories. "
                 "Depth 4 gives 65,536 prefixes to avoid S3 throttling. "
                 "(env: PASSSAGE_S3_HASH_PREFIX_DEPTH)",
        )

    @_log_hook_errors("configure")
    def configure(self, updated):
        if "policy_file" in updated:
            _load_policy_overrides(ctx.options.policy_file)
        if {
            "access_logs",
            "access_log_prefix",
            "access_log_dir",
            "access_log_flush_seconds",
            "access_log_flush_bytes",
            "access_log_headers",
        }.intersection(updated):
            _init_access_logs()
        if {
            "error_logs",
            "error_log_prefix",
            "error_log_dir",
            "error_log_flush_seconds",
            "error_log_flush_bytes",
        }.intersection(updated):
            _init_error_logs()
        _init_elasticsearch()

    @staticmethod
    def cache_expired(cache):
        if (cache_exp := cache.headers.get("Expires")):
            exp_dt = parse_date(cache_exp).astimezone(tz=pytz.utc)
            if datetime.now(tz=pytz.utc) >= exp_dt:
                return True
        return False

    # runs in the threadpool, because we use blocking operations here
    @concurrent
    @_log_hook_errors("requestheaders")
    def requestheaders(self, flow: http.HTTPFlow) -> None:
        flow._cache_head = None
        flow._orig_data = {}
        flow._access_log_start = time.time()
        # we mark this flow as cached only if it returns data from the cache
        flow._cached = False
        flow._cache_hit = False
        flow._cache_fresh = False
        flow._allow_stale_while_revalidate = False
        flow._allow_stale_if_error = False
        flow._save_response = True
        flow._orig_range = None
        flow._range_stripped = False
        flow._serve_reason = "upstream_served"
        flow._cached_method = None
        flow._vary_index_key = None
        flow._cache_saved = False
        flow._object_store_rewrite = False
        flow._xs3lerator_rewrite = False
        flow._xs3lerator_full_size = None
        flow._original_url = None
        flow._proactive_vary_key = None
        flow._forced_swr = False
        policy = flow._policy = get_policy(flow)
        if _is_s3_cache_request(flow):
            flow._save_response = False
            flow._policy = NoCache
            flow._serve_reason = "no_cache_s3_request"
            LOG.debug(
                "Client %s sent cache-object request for %s through proxy "
                "(no_proxy not honored, user-agent: %s)",
                flow.client_conn.peername,
                flow.request.pretty_host,
                flow.request.headers.get("user-agent", "unknown"),
            )
            return
        if flow.request.pretty_host == "mitm.it" and flow.request.path in ("/proxy-env", "/proxy-env.sh"):
            public_url = getattr(ctx.options, "public_proxy_url", "").strip()
            if public_url:
                os.environ["PASSSAGE_PUBLIC_PROXY_URL"] = public_url
            body = load_proxy_env_script()
            if not body:
                flow.response = http.Response.make(
                    500,
                    b"proxy env script unavailable",
                    {"Content-Type": "text/plain"},
                )
            else:
                flow.response = http.Response.make(
                    200,
                    body,
                    {
                        "Content-Type": "text/plain",
                        "Cache-Control": "no-store",
                    },
                )
            flow._save_response = False
            flow._short_circuit = True
            return
        with ctx._lock:
            flow._counter = self.counter
            self.counter += 1
        LOG.debug("Request headers2: %s", flow.request.headers)

        # pass non-GET/HEAD and NoCache request through without caching
        if (flow.request.method not in ("GET", "HEAD")
                or policy == NoCache):
            flow._save_response = False
            flow._serve_reason = "no_cache_policy" if policy == NoCache else "no_cache_method"
            return

        # Look up the object in the cache (Vary-aware)
        flow._cache_key = None
        flow._cache_vary = None
        try:
            normalized_url = _normalize_url(flow)
            depth = _s3_hash_prefix_depth()
            vary_index_key = es_vary_index_id(normalized_url)
            flow._vary_index_key = vary_index_key

            # Speculatively compute cache key using proactive vary so we can
            # batch both ES lookups into a single _mget round-trip.
            speculative_vk = compute_proactive_vary_key(flow.request.headers)
            speculative_cache_key = es_doc_id(normalized_url, speculative_vk)

            LOG.debug("Cache lookup vary index key=%s speculative_cache_key=%s", vary_index_key, speculative_cache_key)
            mget_results = es_mget_docs(_es_meta_index(), [vary_index_key, speculative_cache_key])

            vary_data = mget_results.get(vary_index_key)
            vary_index_head = (
                CacheMeta(200, vary_data, source="elasticsearch")
                if vary_data is not None
                else CacheMeta(404, {}, source="elasticsearch")
            )
            LOG.debug(
                "Vary index status=%s source=%s key=%s",
                vary_index_head.status_code,
                vary_index_head.source,
                vary_index_key,
            )

            need_extra_get = False
            if http_2xx(vary_index_head) and (vary_hdr := vary_index_head.meta.get("vary")):
                flow._cache_vary = vary_hdr
                if "*" not in vary_hdr:
                    vary_key = compute_vary_key(vary_hdr, flow.request.headers)
                    if vary_key is not None:
                        flow._cache_key = es_doc_id(normalized_url, vary_key)
                        LOG.debug(
                            "Cache lookup vary header=%s vary_key=%s cache_key=%s",
                            vary_hdr,
                            vary_key,
                            flow._cache_key,
                        )
                        if flow._cache_key != speculative_cache_key:
                            need_extra_get = True

            if flow._cache_key is None and flow._cache_vary != "*":
                flow._cache_key = speculative_cache_key
                flow._proactive_vary_key = speculative_vk
                if speculative_vk:
                    LOG.debug(
                        "Cache lookup proactive vary key=%s cache_key=%s",
                        speculative_vk,
                        flow._cache_key,
                    )
                else:
                    LOG.debug("Cache lookup default key=%s", flow._cache_key)

            if flow._cache_key:
                if flow._cache_key == speculative_cache_key and not need_extra_get:
                    # Already fetched in the _mget batch
                    spec_data = mget_results.get(speculative_cache_key)
                    if spec_data is not None:
                        flow._cache_head = CacheMeta(200, spec_data, source="elasticsearch")
                        batcher = getattr(ctx, "_last_access_batcher", None)
                        if batcher:
                            batcher.touch(flow._cache_key)
                    else:
                        flow._cache_head = CacheMeta(404, {}, source="elasticsearch")
                else:
                    flow._cache_head = get_cache_metadata(flow._cache_key)
                flow._metadata_source = flow._cache_head.source
                LOG.debug(
                    "Cache HEAD status=%s source=%s key=%s",
                    flow._cache_head.status_code,
                    flow._cache_head.source,
                    flow._cache_key,
                )
        except Exception as e:
            # if S3 is unavailable, go to upstream directly
            LOG.warning(
                "Cache metadata lookup failed for key=%s: %s",
                flow._cache_key,
                e,
            )
            _log_exception(
                e,
                flow=flow,
                context={"hook": "requestheaders", "stage": "cache_head"},
                error_type="s3_head_error",
            )
            return
        if (not (http_2xx(flow._cache_head)
                 or flow._cache_head.status_code == 404)):
            # If we got non-2xx or 404 from the S3, the proxy is not
            # functional, so return this as an error
            flow._save_response = False
            flow._serve_reason = "cache_head_error"
            flow.response = http.Response.make(
                flow._cache_head.status_code,
                f"Got {flow._cache_head.status_code} error while accessing\n{S3_URL}\nplease check its permissions!",
                {"Content-Type": "text/plain"}
            )
            return

        # if we have a GET cached, we can serve both HEAD/GET from the cache. If we have HEAD,
        # only HEAD can be served
        cached_method = False
        if http_2xx(flow._cache_head):
            meta_method = flow._cache_head.meta.get("method")
            if meta_method is None:
                cached_method = True
            elif meta_method == "GET" or meta_method == flow.request.method:
                cached_method = True
        flow._cached_method = cached_method
        if flow._cache_head is not None:
            m = flow._cache_head.meta
            LOG.debug(
                "Cache metadata status=%s method_meta=%s etag_meta=%s lastmod_meta=%s",
                flow._cache_head.status_code,
                m.get("method"),
                m.get("headers", {}).get("etag"),
                (m.get("headers") or {}).get("last-modified"),
            )
            LOG.debug("Cache method match cached_method=%s", cached_method)

        refresh_patterns = get_refresh_patterns()
        cache_hit = http_2xx(flow._cache_head) and cached_method
        freshness = (
            cache_freshness(flow.request.url, flow._cache_head.meta, refresh_patterns)
            if cache_hit
            else None
        )
        cache_fresh = bool(freshness and freshness.is_fresh)
        allow_stale_while_revalidate = bool(freshness and freshness.allow_stale_while_revalidate)
        allow_stale_if_error = bool(freshness and freshness.allow_stale_if_error)
        flow._allow_stale_if_error = allow_stale_if_error
        flow._allow_stale_while_revalidate = allow_stale_while_revalidate
        flow._cache_hit = cache_hit
        flow._cache_fresh = cache_fresh
        if flow.request.headers:
            request_headers = {k.lower(): v for k, v in flow.request.headers.items()}
        else:
            request_headers = {}
        if (
            cache_hit
            and freshness
            and policy in (Standard, StaleIfError)
            and request_requires_revalidation(request_headers, freshness)
        ):
            cache_fresh = False
            allow_stale_while_revalidate = False
            LOG.debug(
                "Client forced revalidation (Cache-Control: no-cache or max-age) -> cache_fresh=False"
            )
        LOG.debug(
            "Cache freshness cached_method=%s cache_fresh=%s policy=%s stale_while_revalidate=%s stale_if_error=%s",
            cached_method,
            cache_fresh,
            policy.__name__,
            allow_stale_while_revalidate,
            allow_stale_if_error,
        )

        # When xs3lerator is configured, serve fresh-cache HEAD requests
        # synthetically from memcached metadata (no xs3lerator involvement).
        xs3lerator_enabled = bool(getattr(ctx.options, "xs3lerator_url", ""))
        if flow.request.method == "HEAD" and xs3lerator_enabled:
            if cache_hit and cache_fresh:
                meta = flow._cache_head.meta
                headers_dict = meta.get("headers", {})
                flow.response = http.Response.make(
                    meta.get("status_code", 200), b"", headers_dict,
                )
                flow._save_response = False
                flow._cached = True
                flow._short_circuit = True
                flow._serve_reason = "head_synthetic_from_cache"
                return
            forced_swr = getattr(flow, "_forced_swr_seconds", None)
            if (cache_hit
                    and forced_swr is not None
                    and freshness is not None
                    and freshness.age_seconds <= forced_swr
                    and policy in (Standard, StaleIfError)):
                meta = flow._cache_head.meta
                headers_dict = meta.get("headers", {})
                flow.response = http.Response.make(
                    meta.get("status_code", 200), b"", headers_dict,
                )
                flow._save_response = False
                flow._cached = True
                flow._short_circuit = True
                flow._forced_swr = True
                flow._serve_reason = "head_forced_swr_synthetic"
                LOG.debug(
                    "HEAD forced SWR (age=%ds <= %ds) -> synthetic from cache",
                    freshness.age_seconds, forced_swr,
                )
                return
            # Stale/miss HEAD: rewrite the flow to route through xs3lerator
            # for upstream connection pooling.
            _rewrite_head_to_xs3lerator(flow)
            return

        if cache_hit:
            cached_status = flow._cache_head.meta.get("status_code", 200) if flow._cache_head else 200
            if xs3lerator_enabled and 300 <= int(cached_status) < 400 and cache_fresh:
                meta = flow._cache_head.meta
                headers_dict = meta.get("headers", {})
                flow.response = http.Response.make(
                    int(cached_status), b"", headers_dict,
                )
                if (reason := meta.get("reason")):
                    flow.response.reason = reason
                flow._save_response = False
                flow._cached = True
                flow._short_circuit = True
                flow._serve_reason = "redirect_synthetic_from_cache"
                LOG.debug("Cache hit: redirect %d -> synthetic response", int(cached_status))
                return

            if (flow.request.method == "GET"
                    and "Range" in flow.request.headers
                    and flow.request.headers.get("if-range")):
                if not _if_range_matches(flow):
                    flow.request.headers.pop("Range", None)
                    flow.request.headers.pop("If-Range", None)
                    LOG.debug("If-Range mismatch on cache hit, stripped Range to serve full response")

            if policy == NoRefresh:
                flow._serve_reason = "cache_hit_norefresh"
                LOG.debug("Cache hit: NoRefresh -> rewrite to object store")
                _rewrite_to_object_store(flow)
                return
            if cache_fresh and policy in (Standard, StaleIfError):
                flow._serve_reason = "cache_hit_fresh"
                LOG.debug("Cache hit: fresh -> rewrite to object store")
                _rewrite_to_object_store(flow)
                return
            if policy in (Standard, StaleIfError) and allow_stale_while_revalidate:
                flow._serve_reason = "cache_hit_stale_while_revalidate"
                LOG.debug("Cache hit: stale-while-revalidate -> rewrite to object store")
                _rewrite_to_object_store(flow)
                return
            forced_swr = getattr(flow, "_forced_swr_seconds", None)
            if (xs3lerator_enabled
                    and flow.request.method == "GET"
                    and policy in (Standard, StaleIfError)
                    and forced_swr is not None
                    and freshness is not None
                    and freshness.age_seconds <= forced_swr):
                cm = flow._cache_head.meta
                has_validators = (
                    cm.get("headers", {}).get("etag")
                    or cm.get("last_modified")
                )
                if has_validators:
                    flow._serve_reason = "cache_hit_forced_swr"
                    flow._forced_swr = True
                    LOG.debug(
                        "Cache hit: forced SWR (age=%ds <= %ds) -> "
                        "serve from cache + background revalidation",
                        freshness.age_seconds, forced_swr,
                    )
                    _rewrite_to_object_store_with_bg_revalidation(flow)
                    return
            if (xs3lerator_enabled
                    and flow.request.method == "GET"
                    and policy in (Standard, StaleIfError)):
                cm = flow._cache_head.meta
                has_validators = (
                    cm.get("headers", {}).get("etag")
                    or (cm.get("headers") or {}).get("last-modified")
                    or cm.get("last_modified")
                )
                if has_validators:
                    flow._serve_reason = "conditional_revalidation_xs3lerator"
                    LOG.debug("Cache hit: stale, conditional revalidation via xs3lerator")
                    _rewrite_to_xs3lerator_revalidate(flow)
                    return
                else:
                    flow._serve_reason = "stale_no_validators"
                    LOG.debug("Cache hit: stale, no validators -> fetch fresh via xs3lerator")
                    _rewrite_to_xs3lerator_miss(flow)
                    return
        else:
            if (flow.request.method == "GET"
                    and flow.request.headers.get("if-modified-since")):
                flow._orig_data["if-modified-since"] = flow.request.headers.pop("if-modified-since", None)
            if (flow.request.method == "GET"
                    and flow.request.headers.get("if-none-match")):
                flow._orig_data["if-none-match"] = flow.request.headers.pop("if-none-match", None)
            if xs3lerator_enabled and flow.request.method == "GET":
                if ("Range" in flow.request.headers
                        and flow.request.headers.get("if-range")):
                    flow.request.headers.pop("Range", None)
                    flow.request.headers.pop("If-Range", None)
                    LOG.debug("If-Range on cache miss (xs3lerator): no validators to compare, serving full response")
                else:
                    flow.request.headers.pop("If-Range", None)
            elif flow.request.method == "GET" and "Range" in flow.request.headers:
                if flow.request.headers.get("if-range"):
                    flow.request.headers.pop("Range", None)
                    flow.request.headers.pop("If-Range", None)
                    LOG.debug("If-Range on cache miss (direct): no validators to compare, serving full response")
                else:
                    flow._orig_range = flow.request.headers.pop("Range")
                    flow._range_stripped = True
                    LOG.debug("Stripped Range header on cache miss: %s", flow._orig_range)

        if xs3lerator_enabled and flow.request.method == "GET" and not getattr(flow, "_xs3lerator_rewrite", False):
            _rewrite_to_xs3lerator_miss(flow)


    @_log_hook_errors("responseheaders")
    def responseheaders(self, flow):
        request_id = getattr(flow, "id", None)
        if request_id is not None:
            flow.response.headers["x-request-id"] = str(request_id)
        if (
            not flow._cached
            and flow._cache_head
            and flow._policy in (Standard, StaleIfError)
            and (
                flow.response.status_code >= 500
                or (flow.response.status_code == 404 and flow._policy == StaleIfError)
            )
            and (
                flow._policy == StaleIfError
                or getattr(flow, "_allow_stale_if_error", False)
                or getattr(flow, "_xs3lerator_rewrite", False)
            )
        ):
            if _fallback_fetch_from_object_store(flow):
                flow._serve_reason = "stale_if_error_fallback"

        flow.response.headers["x-proxy-policy"] = flow._policy.__name__
        if getattr(flow, "_serve_reason", None) == "no_cache_s3_request":
            flow.response.headers["x-passsage-warning"] = (
                "no_proxy-bypass: this request to "
                + flow.request.pretty_host
                + " should bypass the proxy (check client no_proxy setting)"
            )
        # Identify proxy via Via (RFC 7230); do not rewrite Server header
        existing_via = flow.response.headers.get("via", "")
        flow.response.headers["via"] = f"{VIA_HEADER_VALUE}, {existing_via}" if existing_via else VIA_HEADER_VALUE
        if getattr(flow, "_xs3lerator_rewrite", False):
            # Process xs3lerator response headers
            xs3_cache_hit = flow.response.headers.get("x-xs3lerator-cache-hit")
            xs3_full_size = flow.response.headers.get("x-xs3lerator-full-size")
            xs3_revalidated = flow.response.headers.get("x-xs3lerator-revalidated")
            xs3_bg_revalidate = flow.response.headers.get("x-xs3lerator-background-revalidate")
            if xs3_full_size:
                flow._xs3lerator_full_size = int(xs3_full_size)
            if xs3_revalidated in ("true", "stale-error"):
                flow._cached = True
                flow._save_response = False
                if xs3_revalidated == "true":
                    flow._serve_reason = "cache_hit_revalidated"
                else:
                    flow._serve_reason = "stale_if_error_xs3lerator"
                if flow._cache_key:
                    batcher = getattr(ctx, "_last_access_batcher", None)
                    if batcher:
                        batcher.refresh_stored_at(flow._cache_key)
                    else:
                        ctx._executor.submit(
                            refresh_cache_metadata,
                            flow._cache_key,
                            flow._cache_head.meta,
                        )
            elif xs3_cache_hit == "false":
                flow._save_response = True
            if xs3_bg_revalidate == "accepted":
                LOG.debug("xs3lerator accepted background revalidation for key=%s", flow._cache_key)
            # Strip all X-Xs3lerator-* response headers before forwarding to client
            to_remove = [k for k in flow.response.headers if k.lower().startswith("x-xs3lerator-")]
            for k in to_remove:
                del flow.response.headers[k]
            if flow.response.status_code in (200, 206):
                if xs3_full_size:
                    flow.response.stream = True
                else:
                    flow.response.stream = lambda data: save_response(self, flow, data)
            elif flow.response.status_code == 404:
                LOG.warning("xs3lerator returned 404 for key=%s",
                            getattr(flow, "_cache_key", "?"))
        elif getattr(flow, "_object_store_rewrite", False):
            if flow.response.status_code in (200, 206):
                flow.response.stream = True
            elif flow.response.status_code == 404:
                LOG.warning("Object store returned 404 for key=%s (metadata/object inconsistency)",
                            getattr(flow, "_cache_key", "?"))
        elif flow._save_response:
            orig_range = getattr(flow, "_orig_range", None)
            if (
                orig_range
                and flow.response.status_code == 200
                and flow.response.headers.get("content-length")
            ):
                total = int(flow.response.headers["content-length"])
                parsed = _parse_single_range(orig_range, total)
                if parsed is not None:
                    start, end = parsed
                    length = end - start + 1
                    # Preserve original 200 so the cache stores the full object as 200
                    flow._orig_data["status_code"] = 200
                    flow._orig_data["reason"] = flow.response.reason
                    flow._orig_data["content-length"] = str(total)
                    flow.response.status_code = 206
                    flow.response.headers["content-range"] = f"bytes {start}-{end}/{total}"
                    flow.response.headers["content-length"] = str(length)
                    flow.response.stream = _make_range_stream(self, flow, start, end)
                elif _is_unsatisfiable_range(orig_range, total):
                    flow._orig_data["status_code"] = 200
                    flow._orig_data["reason"] = flow.response.reason
                    flow._orig_data["content-length"] = str(total)
                    flow.response.status_code = 416
                    flow.response.headers["content-range"] = f"bytes */{total}"
                    flow.response.headers["content-length"] = "0"
                    # Save full body for caching, but send nothing to client
                    def _discard_stream(data, _self=self, _flow=flow):
                        save_response(_self, _flow, data)
                        return b"" if data else b""
                    flow.response.stream = _discard_stream
                else:
                    # unparseable or multi-range: serve the full 200
                    flow.response.stream = lambda data: save_response(self, flow, data)
            else:
                flow.response.stream = lambda data: save_response(self, flow, data)
        else:
            flow.response.stream = True
        if flow._cached:
            LOG.debug("Cache response: serving from cache")
            flow.response.headers["cache-status"] = f"{SERVER_NAME};hit;detail=object-store"
            is_partial = flow.response.status_code == 206
            saved_framing_headers = {}
            for h in ("content-length", "content-range", "content-encoding", "transfer-encoding"):
                if h in flow.response.headers:
                    saved_framing_headers[h] = flow.response.headers[h]
            apply_cached_metadata(flow)
            for h, v in saved_framing_headers.items():
                flow.response.headers[h] = v
            if is_partial:
                flow.response.status_code = 206
            elif "content-length" not in saved_framing_headers:
                flow.response.headers.pop("content-length", None)
            meta = flow._cache_head.meta if flow._cache_head else {}
            if (stored_dt := cache_stored_at(meta)):
                stored_dt = stored_dt.astimezone(pytz.utc)
                age_seconds = int((datetime.now(tz=pytz.utc) - stored_dt).total_seconds())
                flow.response.headers["age"] = str(max(0, age_seconds))
        else:
            LOG.debug("Cache response: not cached")
        if flow.response.headers.get("last-modified"):
            # if the response has a last-modified header and we got
            # if-modified-since, compare the two and return 304 accordingly
            if (modsince := flow.request.headers.get("if-modified-since")
                    or flow._orig_data.get("if-modified-since")):
                lastmod_dt = parse_date(flow.response.headers["last-modified"])
                modsince_dt = parse_date(modsince)
                if lastmod_dt and modsince_dt and modsince_dt >= lastmod_dt:
                    flow._orig_data["status_code"] = flow.response.status_code
                    flow._orig_data["reason"] = flow.response.reason
                    flow.response.status_code = 304
                    # don't stream/save a 304
                    flow.response.stream = False
                    flow._save_response = False

    @staticmethod
    def _save_to_cache(
        status_code,
        reason,
        method,
        f,
        digest,
        headers,
        url,
        cache_key,
        vary_header,
        vary_request,
        normalized_url,
        xs3lerator_handled=False,
        xs3lerator_full_size=None,
        xs3lerator_stored_key=None,
    ):
        try:
            if f is not None:
                f.flush()
                f.seek(0)

            if xs3lerator_handled and vary_header:
                source_key = xs3lerator_stored_key or es_doc_id(normalized_url)
                if source_key != cache_key:
                    _xs3lerator_link_manifest(source_key, cache_key)

            # Build and write the metadata JSON
            cached_headers = {}
            for k in (
                "etag",
                "last-modified",
                "vary",
                "cache-control",
                "content-type",
                "content-encoding",
                "content-length",
                "content-language",
                "content-disposition",
                "content-location",
                "content-range",
                "accept-ranges",
                "location",
                "link",
                "expires",
            ):
                if k in headers:
                    cached_headers[k] = headers[k]

            content_size = xs3lerator_full_size
            if content_size is None and "content-length" in headers:
                try:
                    content_size = int(headers["content-length"])
                except (ValueError, TypeError):
                    pass

            now_iso = datetime.now(timezone.utc).isoformat()
            meta = {
                "status_code": int(status_code),
                "stored_at": now_iso,
                "last_access": now_iso,
                "url": url[:512],
                "extension": _url_ext(url),
                "headers": cached_headers,
                "doc_type": "metadata",
            }
            if content_size is not None:
                meta["content_size"] = content_size
            if vary_header:
                meta["vary"] = vary_header
            if vary_request:
                meta["vary_request"] = vary_request

            es_index = _es_meta_index()
            es_index_doc(es_index, cache_key, meta)

            # 3. Write the vary index (if applicable)
            if vary_header:
                vary_index_key = es_vary_index_id(normalized_url)
                vary_data = {"vary": vary_header, "doc_type": "vary_index"}
                es_index_doc(es_index, vary_index_key, vary_data)
        except Exception as e:
            LOG.error("Cache save error %s, %s", url, e)
            _log_exception(
                e,
                request_url=url,
                method=method,
                request_headers=headers,
                context={"cache_key": cache_key, "url": url},
                error_type="cache_save_error",
            )
            raise
        finally:
            if f is not None:
                try:
                    f.close()
                except Exception:
                    pass

    @_log_hook_errors("response")
    def response(self, flow):
        if getattr(flow, "_short_circuit", False):
            self.log_response(flow)
            return
        if (flow.request.method not in ("GET", "HEAD")
                or flow._policy == NoCache or flow._cached):
            self.log_response(flow)
            with ctx._lock:
                self.cleanup(flow)
            return

        # Safety net: never cache an *upstream* 206 Partial Content response.
        # When _orig_range is set, we intentionally rewrote 200→206 in
        # responseheaders (the full body is still being saved via the range
        # stream).  Only block caching when the upstream genuinely returned 206
        # (e.g. non-GET, or upstream ignored our Range-stripping).
        if flow.response.status_code == 206 and not getattr(flow, "_orig_range", None):
            flow._save_response = False

        refresh_patterns = get_refresh_patterns()
        if not any(p.pattern.search(flow.request.url) for p in refresh_patterns):
            cc = parse_cache_control(flow.response.headers.get("cache-control", ""))
            if "no-store" in cc or "private" in cc:
                flow._save_response = False
        LOG.debug(
            "Cache save decision save=%s policy=%s cached=%s",
            flow._save_response,
            flow._policy.__name__,
            flow._cached,
        )

        vary_header = flow.response.headers.get("vary")
        if vary_header:
            if "*" in vary_header:
                flow._save_response = False
            else:
                vary_key = compute_vary_key(vary_header, flow.request.headers)
                if vary_key:
                    normalized_url = _normalize_url(flow)
                    flow._cache_key = es_doc_id(normalized_url, vary_key)
                    flow._cache_vary = vary_header
                    flow._cache_vary_request = build_vary_request(
                        vary_header, flow.request.headers
                    )
            LOG.debug(
                "Cache vary header=%s cache_key=%s",
                vary_header,
                getattr(flow, "_cache_key", None),
            )

        if (
            flow._save_response
            and flow._cache_head
            and flow.request.method == "GET"
            and flow._counter in self.hashes
        ):
            if self.hashes[flow._counter].hexdigest() == flow._cache_head.meta.get("sha224"):
                flow._save_response = False
        if flow._cache_head and flow._cache_head.meta:
            if (flow.request.method == "HEAD"
                    and flow._cache_head.meta.get("method") == "GET"):
                flow._save_response = False
        if flow._save_response:
            is_xs3lerator = getattr(flow, "_xs3lerator_rewrite", False)
            xs3_full_size = getattr(flow, "_xs3lerator_full_size", None)

            if is_xs3lerator:
                normalized_url = _normalize_url(flow)
                cache_key = flow._cache_key or es_doc_id(normalized_url)
                flow._cache_saved = True
                save_headers = flow.response.headers.copy()
                if "content-range" in save_headers:
                    del save_headers["content-range"]

                if xs3_full_size:
                    save_headers["content-length"] = str(xs3_full_size)
                    LOG.debug("Cache save enqueue (metadata only, xs3lerator) key=%s", cache_key)
                    ctx._executor.submit(
                        self._save_to_cache,
                        flow._orig_data.get("status_code") or flow.response.status_code,
                        flow._orig_data.get("reason") or flow.response.reason,
                        flow.request.method,
                        None,
                        None,
                        save_headers,
                        getattr(flow, "_original_url", flow.request.url),
                        cache_key,
                        getattr(flow, "_cache_vary", None),
                        getattr(flow, "_cache_vary_request", None),
                        normalized_url,
                        xs3lerator_handled=True,
                        xs3lerator_full_size=xs3_full_size,
                        xs3lerator_stored_key=getattr(flow, "_xs3lerator_stored_key", None),
                    )
                elif flow._counter in self.files and flow._counter in self.hashes:
                    body_file = self.files[flow._counter]
                    body_file.flush()
                    body_size = body_file.tell()
                    save_headers["content-length"] = str(body_size)
                    digest = self.hashes[flow._counter].hexdigest()
                    LOG.debug(
                        "Cache save enqueue (xs3lerator chunked, passsage uploads blob) key=%s size=%d",
                        cache_key, body_size,
                    )
                    ctx._executor.submit(
                        self._save_to_cache,
                        flow._orig_data.get("status_code") or flow.response.status_code,
                        flow._orig_data.get("reason") or flow.response.reason,
                        flow.request.method,
                        body_file,
                        digest,
                        save_headers,
                        getattr(flow, "_original_url", flow.request.url),
                        cache_key,
                        getattr(flow, "_cache_vary", None),
                        getattr(flow, "_cache_vary_request", None),
                        normalized_url,
                    )
                elif 300 <= flow.response.status_code < 400 and flow.response.status_code != 304:
                    LOG.debug(
                        "Cache save enqueue (redirect metadata only, xs3lerator) key=%s status=%d",
                        cache_key, flow.response.status_code,
                    )
                    ctx._executor.submit(
                        self._save_to_cache,
                        flow._orig_data.get("status_code") or flow.response.status_code,
                        flow._orig_data.get("reason") or flow.response.reason,
                        flow.request.method,
                        None,
                        None,
                        save_headers,
                        getattr(flow, "_original_url", flow.request.url),
                        cache_key,
                        getattr(flow, "_cache_vary", None),
                        getattr(flow, "_cache_vary_request", None),
                        normalized_url,
                    )
                else:
                    LOG.warning(
                        "xs3lerator chunked response without captured body, skipping cache save url=%s",
                        flow.request.url[:200],
                    )
                    flow._cache_saved = False
            else:
                # save in the background
                if flow._counter not in self.files or flow._counter not in self.hashes:
                    LOG.debug("Cache save skipped (no body recorded) counter=%s", flow._counter)
                    flow._save_response = False
                    self.log_response(flow)
                    with ctx._lock:
                        self.cleanup(flow)
                    return
                normalized_url = _normalize_url(flow)
                cache_key = flow._cache_key or es_doc_id(normalized_url)
                flow._cache_saved = True
                LOG.debug("Cache save enqueue key=%s", cache_key)
                save_headers = flow.response.headers
                if getattr(flow, "_orig_range", None):
                    save_headers = flow.response.headers.copy()
                    if "content-length" in flow._orig_data:
                        save_headers["content-length"] = flow._orig_data["content-length"]
                    if "content-range" in save_headers:
                        del save_headers["content-range"]
                ctx._executor.submit(
                    self._save_to_cache,
                    flow._orig_data.get("status_code") or flow.response.status_code,
                    flow._orig_data.get("reason") or flow.response.reason,
                    flow.request.method,
                    self.files[flow._counter],
                    self.hashes[flow._counter].hexdigest(),
                    save_headers,
                    flow.request.url,
                    cache_key,
                    getattr(flow, "_cache_vary", None),
                    getattr(flow, "_cache_vary_request", None),
                    normalized_url,
                )

        if flow.response.status_code == 304:
            # empty content on 304 not-modified, even if we have a fully cached
            # GET response, we must not return content
            flow.response.content = None

        self.log_response(flow)
        with ctx._lock:
            self.cleanup(flow)

    @_log_hook_errors("error")
    def error(self, flow):
        writer = _access_log_writer()
        if writer:
            try:
                writer.add(_build_access_log_record(flow, error=str(flow.error)))
            except Exception as exc:
                LOG.debug("Access log write skipped: %s", exc)
        err_writer = _error_log_writer()
        if err_writer:
            try:
                tb_text = ""
                if flow.error and hasattr(flow.error, "__traceback__"):
                    tb_text = "".join(
                        traceback.format_exception(
                            type(flow.error), flow.error, flow.error.__traceback__
                        )
                    )
                if not tb_text:
                    tb_text = traceback.format_exc()
                    err_writer.add(
                        _build_error_log_record(
                            error_type=type(flow.error).__name__ if flow.error else "flow_error",
                            error_message=str(flow.error),
                        traceback_text=tb_text,
                            context={},
                            flow=flow,
                        )
                    )
            except Exception as exc:
                LOG.debug("Error log write skipped: %s", exc)
        with ctx._lock:
            self.cleanup(flow)

    @_log_hook_errors("done")
    def done(self):
        writer = _access_log_writer()
        if writer:
            writer.stop()
        err_writer = _error_log_writer()
        if err_writer:
            err_writer.stop()

    def cleanup(self, flow):
        if not hasattr(flow, "_counter"):
            return
        if flow._counter in self.files:
            del self.files[flow._counter]
        if flow._counter in self.hashes:
            del self.hashes[flow._counter]

    def log_response(self, flow):
        writer = _access_log_writer()
        if writer:
            try:
                writer.add(_build_access_log_record(flow))
            except Exception as exc:
                LOG.debug("Access log write skipped: %s", exc)
        LOG.info(
            "[%s] %s %s %s %s/%s/%s",
            datetime.now().strftime("%m/%d/%Y:%H:%M:%S"),
            flow.request.method,
            flow.request.url,
            flow.response.status_code,
            flow._policy.__name__,
            flow._cached,
            flow._save_response,
        )


addons = [
    Proxy(),
]
