# Copyright (c) 2023-2024 Jan Malakhovski <oxij@oxij.org>
#
# This file is a part of `hoardy-web` project.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import bisect as _bisect
import collections as _c
import dataclasses as _dc
import errno as _errno
import hashlib as _hashlib
import io as _io
import logging as _logging
import os as _os
import re as _re
import shutil as _shutil
import signal as _signal
import stat as _stat
import subprocess as _subprocess
import sys as _sys
import tempfile as _tempfile
import typing as _t

from gettext import gettext, ngettext

from kisstdlib import argparse
from kisstdlib.exceptions import *
from kisstdlib.io import *
from kisstdlib.io.stdio import *
from kisstdlib.logging import *

from .wrr import *
from .output import *
from .filter import *

__prog__ = "hoardy-web"

def issue(pattern : str, *args : _t.Any) -> None:
    message = pattern % args
    if stderr.isatty:
        stderr.write_str_ln("\033[31m" + message + "\033[0m")
    else:
        stderr.write_str_ln(message)
    stderr.flush()

def error(pattern : str, *args : _t.Any) -> None:
    issue(gettext("error") + ": " + pattern, *args)

def die(code : int, pattern : str, *args : _t.Any) -> _t.NoReturn:
    error(pattern, *args)
    _sys.exit(code)

interrupt_msg = "\n" + gettext("Gently finishing up... Press ^C again to forcefully interrupt.")
want_stop = False
should_raise = True
def sig_handler(sig : int, frame : _t.Any) -> None:
    global want_stop
    global should_raise
    want_stop = True
    if should_raise:
        raise KeyboardInterrupt()
    if sig == _signal.SIGINT:
        issue(interrupt_msg)
    should_raise = True

def handle_signals() -> None:
    _signal.signal(_signal.SIGINT, sig_handler)
    _signal.signal(_signal.SIGTERM, sig_handler)

def pred_linst(expr : str, func : LinstFunc, rrexpr : ReqresExpr[_t.Any]) -> bool:
    res = rrexpr.eval_func(func)
    if isinstance(res, bool):
        return res
    else:
        e = CatastrophicFailure(gettext("while evaluating `%s`: expected a value of type `bool`, got `%s`"), expr, repr(res))
        e.elaborate(gettext("while processing `%s`"), rrexpr.source.show_source())
        raise e

def mk_linst_filter(get_attr : _t.Callable[[FilterNameType], list[str]],
                    get_optname : _t.Callable[[FilterNameType], str],
                    name : FilterNameType,
                    yes : bool,
                    matches : PredicateMatchesType[str, LinstFunc, ReqresExpr[_t.Any]]) \
                    -> FilterType[ReqresExpr[_t.Any]]:
    compiled = mk_conditions(str_id, lambda x: linst_compile(x, linst_atom_or_env), get_attr(name))
    num = len(compiled)

    def allows(v : ReqresExpr[_t.Any]) -> bool:
        res = matches(pred_linst, compiled, v)
        return res if yes else not res

    def warn() -> None:
        warn_redundant(get_optname(name), yes, compiled)

    return num, allows, warn

def compile_filters(cargs : _t.Any, attr_prefix : str = "") -> FilterType[ReqresExpr[_t.Any]]:
    opt_prefix = attr_prefix.replace("_", "-")

    def get_attr(x : str) -> _t.Any:
        return getattr(cargs, attr_prefix + x)

    def get_optname(x : str) -> str:
        return f"--{opt_prefix}{x.replace('_', '-')}"

    filters : list[FilterType[ReqresExpr[_t.Any]]] = []

    def add_yn_epoch_filter(name : str, pred : _t.Callable[[str, Epoch, ReqresExpr[_t.Any]], bool]) -> None:
        add_yn_filter(filters, get_attr, get_optname, name,
                      mk_simple_filter, parse_Epoch, lambda c, v: matches_all(pred, c, v))

    def is_before(k : _t.Any, stime : Epoch, rrexpr : ReqresExpr[_t.Any]) -> bool:
        rrstime : Epoch = rrexpr.stime
        return rrstime < stime

    def is_after(k : _t.Any, stime : Epoch, rrexpr : ReqresExpr[_t.Any]) -> bool:
        rrstime : Epoch = rrexpr.stime
        return stime < rrstime

    add_yn_epoch_filter("before", is_before)
    add_yn_epoch_filter("after", is_after)

    def add_yn_field_filter(name : str, field : str | None = None) -> None:
        if field is None:
            field = name

        def get_inputs(rrexpr : ReqresExpr[_t.Any]) -> StrFilterInputType:
            value = rrexpr.get_value(field)
            l = [ value ] if value is not None else []
            return l, l

        add_yn_filter(filters, get_attr, get_optname, name, mk_str_filter, str_id, str_id, get_inputs)

    add_yn_field_filter("protocol")
    add_yn_field_filter("request_method", "request.method")
    add_yn_field_filter("status")

    def neturlify(url : str) -> str:
        return parse_url(url).net_url

    def add_yn_url_filter(name : str) -> None:
        def get_inputs(rrexpr : ReqresExpr[_t.Any]) -> StrFilterInputType:
            net_url = rrexpr.net_url
            lnet_url = [net_url]
            return lnet_url, lambda: [net_url, rrexpr.pretty_net_url]

        add_yn_filter(filters, get_attr, get_optname, name, mk_str_filter, neturlify, neturlify, get_inputs)

    add_yn_url_filter("url")

    def add_yn_headers_grep_filter(name : str, field : str,
                                   matches : PredicateMatchesType[str, PatternSB, IterSB]) -> None:
        def get_inputs(rrexpr : ReqresExpr[_t.Any]) -> IterSB:
            value = rrexpr.get_value(field)
            if value is not None:
                return get_raw_headers(value)
            return []

        add_yn_filter(filters, get_attr, get_optname, name, mk_grep_filter, cargs.ignore_case, matches, get_inputs)

    def add_yn_field_grep_filter(name : str, field : str,
                                 matches : PredicateMatchesType[str, PatternSB, IterSB]) -> None:
        def get_inputs(rrexpr : ReqresExpr[_t.Any]) -> list[str | bytes]:
            value = rrexpr.get_value(field)
            l = [ value ] if value is not None else []
            return l

        add_yn_filter(filters, get_attr, get_optname, name, mk_grep_filter, cargs.ignore_case, matches, get_inputs)

    def add_rr(side : str) -> None:
        add_yn_headers_grep_filter(f"{side}_headers_or_grep", f"{side}.headers", matches_any)
        add_yn_headers_grep_filter(f"{side}_headers_and_grep", f"{side}.headers", matches_all)
        add_yn_field_filter(f"{side}_mime")
        add_yn_field_grep_filter(f"{side}_body_or_grep", f"{side}.body", matches_any)
        add_yn_field_grep_filter(f"{side}_body_and_grep", f"{side}.body", matches_all)

    add_rr("request")
    add_rr("response")

    def add_yn_grep_filter(name : str, matches : PredicateMatchesType[str, PatternSB, IterSB]) -> None:
        def get_inputs(rrexpr : ReqresExpr[_t.Any]) -> list[str | bytes]:
            res : list[str | bytes] = [ rrexpr.raw_url, rrexpr.url, rrexpr.pretty_url ]
            reqres = rrexpr.reqres
            res += get_raw_headers(reqres.request.headers)
            res.append(reqres.request.body)
            if reqres.response is not None:
                res += get_raw_headers(reqres.response.headers)
                res.append(reqres.response.body)
            return res

        add_yn_filter(filters, get_attr, get_optname, name, mk_grep_filter, cargs.ignore_case, matches, get_inputs)

    add_yn_grep_filter("or_grep", matches_any)
    add_yn_grep_filter("and_grep", matches_all)

    filters.append(mk_linst_filter(get_attr, get_optname, "and", True, matches_all))
    filters.append(mk_linst_filter(get_attr, get_optname, "or", False, matches_any))

    return merge_non_empty_filters(filters)

def compile_expr(expr : str) -> tuple[str, LinstFunc]:
    return (expr, linst_compile(expr, ReqresExpr_lookup))

def elaborate_output(kind : str, aliases : dict[str, str], value : str) -> str:
    if value.startswith("format:"):
        return value[7:]
    else:
        try:
            return aliases[value]
        except KeyError:
            raise CatastrophicFailure(gettext('unknown `%s` alias "%s", prepend "format:" if you want it to be interpreted as a Pythonic %%-substutition'), kind, value)

def elaborate_paths(paths : list[str | bytes]) -> None:
    for i in range(0, len(paths)):
        paths[i] = _os.path.expanduser(paths[i])

def handle_paths(cargs : _t.Any) -> None:
    if cargs.stdin0:
        paths = stdin.read_all_bytes().split(b"\0")
        last = paths.pop()
        if last != b"":
            raise Failure(gettext("`--stdin0` input format error"))
        cargs.paths += paths

    elaborate_paths(cargs.paths)

    if cargs.walk_paths is not None:
        cargs.paths.sort(reverse=not cargs.walk_paths)

LoadResult = _t.TypeVar("LoadResult")
LoadFFunc = _t.Callable[[_t.AnyStr], LoadResult]
EmitFunc =  _t.Callable[[LoadResult], None]

def load_map_orderly(load_func : LoadFFunc[_t.AnyStr, LoadResult],
                     emit_func : EmitFunc[LoadResult],
                     dir_or_file_path : _t.AnyStr,
                     *,
                     seen_paths : set[_t.AnyStr] | None = None,
                     follow_symlinks : bool = True,
                     ordering : bool | None = False,
                     errors : str = "fail") -> None:
    if seen_paths is not None:
        abs_dir_or_file_path = _os.path.abspath(dir_or_file_path)
        if abs_dir_or_file_path in seen_paths:
            return
        seen_paths.add(abs_dir_or_file_path)

    for path, _ in walk_orderly(dir_or_file_path,
                                include_files = with_extension_not_in([".part", b".part"]),
                                include_directories = False,
                                ordering = ordering,
                                follow_symlinks = follow_symlinks,
                                handle_error = None if errors == "fail" else _logging.error):
        if want_stop: raise KeyboardInterrupt()

        abs_path = _os.path.abspath(path)
        try:
            if follow_symlinks:
                abs_path = _os.path.realpath(abs_path)

            if seen_paths is not None and \
               abs_path != abs_dir_or_file_path: # do not skip top-level paths added above
                if abs_path in seen_paths:
                    continue
                seen_paths.add(abs_path)

            try:
                data = load_func(abs_path)
            except OSError as exc:
                raise Failure(gettext("failed to open `%s`"), path)

            emit_func(data)
        except Failure as exc:
            if errors == "ignore":
                continue
            exc.elaborate(gettext("while processing `%s`"), path)
            if errors != "fail":
                _logging.error("%s", str(exc))
                continue
            raise exc

def map_wrr_paths(cargs : _t.Any,
                  loadf_func : LoadFFunc[_t.AnyStr, _t.Iterator[ReqresExpr[_t.Any]]],
                  filters_allow : _t.Callable[[ReqresExpr[_t.Any]], bool],
                  emit_func : EmitFunc[ReqresExpr[_t.Any]],
                  paths : list[_t.AnyStr],
                  **kwargs : _t.Any) -> None:
    def emit_many(rrexprs : _t.Iterator[ReqresExpr[_t.Any]]) -> None:
        for rrexpr in rrexprs:
            if want_stop: raise KeyboardInterrupt()

            rrexpr.sniff = cargs.sniff
            if not filters_allow(rrexpr):
                continue
            emit_func(rrexpr)

    global should_raise
    should_raise = False
    for exp_path in paths:
        load_map_orderly(loadf_func, emit_many, exp_path, ordering=cargs.walk_fs, errors=cargs.errors, **kwargs)

def dispatch_rrexprs_load() -> LoadFFunc[_t.AnyStr, _t.Iterator[ReqresExpr[_t.Any]]]:
    import_failed = []
    class Mutable:
        not_warned : bool = True

    have_mitmproxy = False
    try:
        from .mitmproxy import rrexprs_mitmproxy_loadf
    except ImportError as exc:
        import_failed.append(("mitmproxy", str_Exception(exc)))
    else:
        have_mitmproxy = True

    is_wrr : IncludeFilesFunc[_t.AnyStr] = with_extension_in([".wrr", b".wrr"])
    is_wrrb : IncludeFilesFunc[_t.AnyStr] = with_extension_in([".wrrb", b".wrrb"])

    def warn(path : _t.AnyStr, parser : str, exc : Exception) -> None:
        _logging.warn(gettext("while processing `%s`: failed to parse with `%s` parser: %s"), path, parser, str_Exception(exc))

    def rrexprs_load(path : _t.AnyStr) -> _t.Iterator[ReqresExpr[_t.Any]]:
        if is_wrr(path):
            try:
                yield rrexpr_wrr_loadf(path)
                return
            except Exception as exc:
                warn(path, "wrr", exc)
        elif is_wrrb(path):
            try:
                yield from rrexprs_wrr_some_loadf(path)
                return
            except Exception as exc:
                warn(path, "wrrb", exc)
        elif have_mitmproxy:
            try:
                yield from rrexprs_mitmproxy_loadf(path)
                return
            except Exception as exc:
                warn(path, "mitmproxy", exc)

        if Mutable.not_warned:
            Mutable.not_warned = False
            for m, e in import_failed:
                _logging.warn(gettext("failed to import `%s` parser: %s"), m, e)
        raise Failure(gettext("failed to find a suitable parser"))

    return rrexprs_load

def mk_rrexprs_load(cargs : _t.Any) -> LoadFFunc[_t.AnyStr, _t.Iterator[ReqresExpr[_t.Any]]]:
    loader = cargs.loader
    if loader is None:
        return dispatch_rrexprs_load()
    elif loader == "wrr":
        return rrexpr_wrr_loadf
    elif loader == "wrrb":
        return rrexprs_wrr_some_loadf
    elif loader == "mitmproxy":
        from .mitmproxy import rrexprs_mitmproxy_loadf
        return rrexprs_mitmproxy_loadf
    else:
        assert False

def get_bytes(value : _t.Any) -> bytes:
    if value is None or isinstance(value, (bool, int, float, Epoch)):
        value = str(value)

    if isinstance(value, str):
        return value.encode(_sys.getdefaultencoding())
    elif isinstance(value, bytes):
        return value
    else:
        raise Failure(gettext("don't know how to print an expression of type `%s`"), type(value).__name__)

def cmd_pprint(cargs : _t.Any) -> None:
    handle_paths(cargs)

    def emit(rrexpr : ReqresExpr[DeferredSourceType]) -> None:
        wrr_pprint(stdout, rrexpr.reqres, rrexpr.source.show_source(), cargs.abridged, cargs.sniff)
        stdout.flush()

    _num, filters_allow, filters_warn = compile_filters(cargs)
    map_wrr_paths(cargs, mk_rrexprs_load(cargs), filters_allow, emit, cargs.paths)
    filters_warn()

def print_exprs(rrexpr : ReqresExpr[_t.Any], exprs : list[tuple[str, LinstFunc]],
                separator : bytes, fobj : MinimalIOWriter) -> None:
    not_first = False
    for expr, func in exprs:
        try:
            data = get_bytes(rrexpr.eval_func(func))
        except CatastrophicFailure as exc:
            exc.elaborate(gettext("while evaluating `%s`"), expr)
            raise exc

        if not_first:
            fobj.write_bytes(separator)
        not_first = True

        fobj.write_bytes(data)

default_expr = {
    "get": "response.body|eb",
    "run": "response.body|eb",
    "stream": ".",
    "id": "response.body|eb|scrub response +all_refs",
    "void": "response.body|eb|scrub response -all_refs",
    "open": "response.body|eb|scrub response *all_refs",
    "closed": "response.body|eb|scrub response /all_refs",
    "all": "response.body|eb|scrub response &all_refs",
    "semi": "response.body|eb|scrub response *jumps,/actions,/reqs",
}

def cmd_get(cargs : _t.Any) -> None:
    if len(cargs.mexprs) == 0:
        cargs.mexprs = { stdout: [compile_expr(default_expr[cargs.default_expr])] }

    exp_path = _os.path.expanduser(cargs.path)
    rrexpr = rrexpr_wrr_loadf(exp_path)
    rrexpr.sniff = cargs.sniff

    for fobj, exprs in cargs.mexprs.items():
        print_exprs(rrexpr, exprs, cargs.separator, fobj)
        fobj.flush()

def cmd_run(cargs : _t.Any) -> None:
    if len(cargs.exprs) == 0:
        cargs.exprs = [compile_expr(default_expr[cargs.default_expr])]

    if cargs.num_args < 1:
        raise Failure(gettext("`run` sub-command requires at least one PATH"))
    elif cargs.num_args - 1 > len(cargs.args):
        raise Failure(gettext("not enough arguments to satisfy `--num-args`"))

    # move (num_args - 1) arguments from args to paths
    ntail = len(cargs.args) + 1 - cargs.num_args
    args = cargs.args[:ntail]
    cargs.paths = cargs.args[ntail:] + cargs.paths

    elaborate_paths(cargs.paths)

    tmp_paths = []
    try:
        for exp_path in cargs.paths:
            rrexpr = rrexpr_wrr_loadf(exp_path)
            rrexpr.sniff = cargs.sniff

            # TODO: extension guessing
            fileno, tmp_path = _tempfile.mkstemp(prefix = "hoardy_wrr_run_", suffix = ".tmp")
            tmp_paths.append(tmp_path)

            with TIOWrappedWriter(_os.fdopen(fileno, "wb")) as f:
                print_exprs(rrexpr, cargs.exprs, cargs.separator, f)

        retcode = _subprocess.Popen([cargs.command] + args + tmp_paths).wait()
        _sys.exit(retcode)
    finally:
        for path in tmp_paths:
            _os.unlink(path)

def get_StreamEncoder(cargs : _t.Any) -> StreamEncoder:
    stream : StreamEncoder
    if cargs.format == "py":
        stream = PyStreamEncoder(stdout, cargs.abridged)
    elif cargs.format == "cbor":
        stream = CBORStreamEncoder(stdout, cargs.abridged)
    elif cargs.format == "json":
        stream = JSONStreamEncoder(stdout, cargs.abridged)
    elif cargs.format == "raw":
        stream = RawStreamEncoder(stdout, cargs.abridged, cargs.terminator)
    else:
        assert False
    return stream

def cmd_stream(cargs : _t.Any) -> None:
    if len(cargs.exprs) == 0:
        cargs.exprs = [compile_expr(default_expr[cargs.default_expr])]

    handle_paths(cargs)

    stream = get_StreamEncoder(cargs)

    def emit(rrexpr : ReqresExpr[DeferredSourceType]) -> None:
        values : list[_t.Any] = []
        for expr, func in cargs.exprs:
            try:
                values.append(func(rrexpr, None))
            except CatastrophicFailure as exc:
                exc.elaborate(gettext("while evaluating `%s`"), expr)
                raise exc
        stream.emit(rrexpr.source.show_source(), cargs.exprs, values)

    _num, filters_allow, filters_warn = compile_filters(cargs)
    stream.start()
    try:
        map_wrr_paths(cargs, mk_rrexprs_load(cargs), filters_allow, emit, cargs.paths)
    finally:
        stream.finish()
    filters_warn()

def cmd_find(cargs : _t.Any) -> None:
    handle_paths(cargs)

    def emit(rrexpr : ReqresExpr[DeferredSourceType]) -> None:
        stdout.write(rrexpr.source.show_source())
        stdout.write_bytes(cargs.terminator)
        stdout.flush()

    _num, filters_allow, filters_warn = compile_filters(cargs)
    map_wrr_paths(cargs, mk_rrexprs_load(cargs), filters_allow, emit, cargs.paths)
    filters_warn()

example_url = [
    "https://example.org",
    "https://example.org/",
    "https://example.org/index.html",
    "https://example.org/media",
    "https://example.org/media/",
    "https://example.org/view?one=1&two=2&three=&three=3#fragment",
    "https://königsgäßchen.example.org/index.html",
    "https://ジャジェメント.ですの.example.org/испытание/is/",
    "https://xn--hck7aa9d8fj9i.xn--88j1aw.example.org/%D0%B8%D1%81%D0%BF%D1%8B%D1%82%D0%B0%D0%BD%D0%B8%D0%B5/is/",
    "https://xn--hck7aa9d8fj9i.ですの.example.org/исп%D1%8B%D1%82%D0%B0%D0%BD%D0%B8%D0%B5/is/",
]

def make_example(gen : _t.Callable[[str], str], indent : int) -> str:
    rev : dict[str, list[str]] = {}
    for url in example_url:
        current = gen(url)
        try:
            l = rev[current]
        except KeyError:
            l = []
            rev[current] = l
        l.append(url)

    res = []
    for r in rev:
        res.append(" " * indent + "- " + ", ".join(map(lambda x: f"`{x}`", rev[r])) + " -> `" + r + "`")

    return "\n".join(res).replace('%', '%%')

atom_test = [
    "raw_url",
    "net_url",
    "pretty_url",
    "pretty_nurl",
]

def atom_example(name : str, indent : int) -> str:
    return make_example(lambda url: getattr(parse_url(url), name), indent)

ofdsd  = "%(syear)d/%(smonth)02d/%(sday)02d"
ofdms  = "%(shour)02d%(sminute)02d%(ssecond)02d%(stime_msq)03d"
ofdd  = f"%(syear)d-%(smonth)02d-%(sday)02d_{ofdms}"
ofpq   = "%(filepath_parts|abbrev_each 120|pp_to_path)s%(oqm)s%(mq_query|abbrev 120)s"
ofpsq  = "%(filepath_parts|abbrev_each 120|pp_to_path)s%(oqm)s%(mq_query|abbrev 100)s"
ofpnq  = "%(filepath_parts|abbrev_each 120|pp_to_path)s%(oqm)s%(mq_nquery|abbrev 120)s"
ofpsnq = "%(filepath_parts|abbrev_each 120|pp_to_path)s%(oqm)s%(mq_nquery|abbrev 100)s"
ofp_snq = "%(filepath_parts|abbrev_each 120|pp_to_path|replace / __|abbrev 120)s%(oqm)s%(mq_nquery|abbrev 100)s"
ofh4 = "%(net_url|to_ascii|sha256|take_prefix 2|to_hex)s"

output_alias = {
    "default":       f"{ofdsd}/{ofdms}_%(qtime_ms)s_%(method)s_{ofh4}_%(status)s_%(hostname)s_%(num)d",
    "short":         f"{ofdsd}/%(stime_ms)d_%(qtime_ms)s_%(num)d",

    "surl":           "%(scheme)s/%(netloc)s/%(mq_npath)s%(oqm)s%(mq_query)s",
    "surl_msn":       "%(scheme)s/%(netloc)s/%(mq_npath)s%(oqm)s%(mq_query)s__%(method)s_%(status)s_%(num)d",
    "surl_mstn":     f"%(scheme)s/%(netloc)s/%(mq_npath)s%(oqm)s%(mq_query)s__%(method)s_%(status)s_{ofdd}_%(num)d",

    "shupq":         f"%(scheme)s/%(hostname)s/{ofpq}%(filepath_ext)s",
    "shupq_n":       f"%(scheme)s/%(hostname)s/{ofpq}.%(num)d%(filepath_ext)s",
    "shupq_tn":      f"%(scheme)s/%(hostname)s/{ofpq}.{ofdd}_%(num)d%(filepath_ext)s",
    "shupq_msn":     f"%(scheme)s/%(hostname)s/{ofpsq}.%(method)s_%(status)s_%(num)d%(filepath_ext)s",
    "shupq_mstn":    f"%(scheme)s/%(hostname)s/{ofpsq}.%(method)s_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",

    "shupnq":        f"%(scheme)s/%(hostname)s/{ofpnq}%(filepath_ext)s",
    "shupnq_n":      f"%(scheme)s/%(hostname)s/{ofpnq}.%(num)d%(filepath_ext)s",
    "shupnq_tn":     f"%(scheme)s/%(hostname)s/{ofpnq}.{ofdd}_%(num)d%(filepath_ext)s",
    "shupnq_msn":    f"%(scheme)s/%(hostname)s/{ofpsnq}.%(method)s_%(status)s_%(num)d%(filepath_ext)s",
    "shupnq_mstn":   f"%(scheme)s/%(hostname)s/{ofpsnq}.%(method)s_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",
    "shupnq_mhs":    f"%(scheme)s/%(hostname)s/{ofpnq}.%(method)s_{ofh4}_%(status)s%(filepath_ext)s",
    "shupnq_mhsn":   f"%(scheme)s/%(hostname)s/{ofpsnq}.%(method)s_{ofh4}_%(status)s_%(num)d%(filepath_ext)s",
    "shupnq_mhstn":  f"%(scheme)s/%(hostname)s/{ofpsnq}.%(method)s_{ofh4}_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",

    "srhupq":        f"%(scheme)s/%(rhostname)s/{ofpq}%(filepath_ext)s",
    "srhupq_n":      f"%(scheme)s/%(rhostname)s/{ofpq}.%(num)d%(filepath_ext)s",
    "srhupq_tn":     f"%(scheme)s/%(rhostname)s/{ofpq}.{ofdd}_%(num)d%(filepath_ext)s",
    "srhupq_msn":    f"%(scheme)s/%(rhostname)s/{ofpsq}.%(method)s_%(status)s_%(num)d%(filepath_ext)s",
    "srhupq_mstn":   f"%(scheme)s/%(rhostname)s/{ofpsq}.%(method)s_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",

    "srhupnq":       f"%(scheme)s/%(rhostname)s/{ofpnq}%(filepath_ext)s",
    "srhupnq_n":     f"%(scheme)s/%(rhostname)s/{ofpnq}.%(num)d%(filepath_ext)s",
    "srhupnq_tn":    f"%(scheme)s/%(rhostname)s/{ofpnq}.{ofdd}_%(num)d%(filepath_ext)s",
    "srhupnq_msn":   f"%(scheme)s/%(rhostname)s/{ofpsnq}.%(method)s_%(status)s_%(num)d%(filepath_ext)s",
    "srhupnq_mstn":  f"%(scheme)s/%(rhostname)s/{ofpsnq}.%(method)s_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",
    "srhupnq_mhs":   f"%(scheme)s/%(rhostname)s/{ofpnq}.%(method)s_{ofh4}_%(status)s%(filepath_ext)s",
    "srhupnq_mhsn":  f"%(scheme)s/%(rhostname)s/{ofpsnq}.%(method)s_{ofh4}_%(status)s_%(num)d%(filepath_ext)s",
    "srhupnq_mhstn": f"%(scheme)s/%(rhostname)s/{ofpsnq}.%(method)s_{ofh4}_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",

    "url":            "%(netloc)s/%(mq_npath)s%(oqm)s%(mq_query)s",
    "url_msn":        "%(netloc)s/%(mq_npath)s%(oqm)s%(mq_query)s__%(method)s_%(status)s_%(num)d",
    "url_mstn":      f"%(netloc)s/%(mq_npath)s%(oqm)s%(mq_query)s__%(method)s_%(status)s_{ofdd}_%(num)d",

    "hupq":          f"%(hostname)s/{ofpq}%(filepath_ext)s",
    "hupq_n":        f"%(hostname)s/{ofpq}.%(num)d%(filepath_ext)s",
    "hupq_tn":       f"%(hostname)s/{ofpq}.{ofdd}_%(num)d%(filepath_ext)s",
    "hupq_msn":      f"%(hostname)s/{ofpsq}.%(method)s_%(status)s_%(num)d%(filepath_ext)s",
    "hupq_mstn":     f"%(hostname)s/{ofpsq}.%(method)s_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",

    "hupnq":         f"%(hostname)s/{ofpnq}%(filepath_ext)s",
    "hupnq_n":       f"%(hostname)s/{ofpnq}.%(num)d%(filepath_ext)s",
    "hupnq_tn":      f"%(hostname)s/{ofpnq}.{ofdd}_%(num)d%(filepath_ext)s",
    "hupnq_msn":     f"%(hostname)s/{ofpsnq}.%(method)s_%(status)s_%(num)d%(filepath_ext)s",
    "hupnq_mstn":    f"%(hostname)s/{ofpsnq}.%(method)s_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",
    "hupnq_mhs":     f"%(hostname)s/{ofpnq}.%(method)s_{ofh4}_%(status)s%(filepath_ext)s",
    "hupnq_mhsn":    f"%(hostname)s/{ofpsnq}.%(method)s_{ofh4}_%(status)s_%(num)d%(filepath_ext)s",
    "hupnq_mhstn":   f"%(hostname)s/{ofpsnq}.%(method)s_{ofh4}_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",

    "rhupq":         f"%(rhostname)s/{ofpq}%(filepath_ext)s",
    "rhupq_n":       f"%(rhostname)s/{ofpq}.%(num)d%(filepath_ext)s",
    "rhupq_tn":      f"%(rhostname)s/{ofpq}.{ofdd}_%(num)d%(filepath_ext)s",
    "rhupq_msn":     f"%(rhostname)s/{ofpsq}.%(method)s_%(status)s_%(num)d%(filepath_ext)s",
    "rhupq_mstn":    f"%(rhostname)s/{ofpsq}.%(method)s_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",

    "rhupnq":        f"%(rhostname)s/{ofpnq}%(filepath_ext)s",
    "rhupnq_n":      f"%(rhostname)s/{ofpnq}.%(num)d%(filepath_ext)s",
    "rhupnq_tn":     f"%(rhostname)s/{ofpnq}.{ofdd}_%(num)d%(filepath_ext)s",
    "rhupnq_msn":    f"%(rhostname)s/{ofpsnq}.%(method)s_%(status)s_%(num)d%(filepath_ext)s",
    "rhupnq_mstn":   f"%(rhostname)s/{ofpsnq}.%(method)s_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",
    "rhupnq_mhs":    f"%(rhostname)s/{ofpnq}.%(method)s_{ofh4}_%(status)s%(filepath_ext)s",
    "rhupnq_mhsn":   f"%(rhostname)s/{ofpsnq}.%(method)s_{ofh4}_%(status)s_%(num)d%(filepath_ext)s",
    "rhupnq_mhstn":  f"%(rhostname)s/{ofpsnq}.%(method)s_{ofh4}_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",

    "flat":          f"%(hostname)s/{ofp_snq}%(filepath_ext)s",
    "flat_n":        f"%(hostname)s/{ofp_snq}.%(num)d%(filepath_ext)s",
    "flat_tn":       f"%(hostname)s/{ofp_snq}.{ofdd}_%(num)d%(filepath_ext)s",
    "flat_ms":       f"%(hostname)s/{ofp_snq}.%(method)s_%(status)s%(filepath_ext)s",
    "flat_msn":      f"%(hostname)s/{ofp_snq}.%(method)s_%(status)s_%(num)d%(filepath_ext)s",
    "flat_mstn":     f"%(hostname)s/{ofp_snq}.%(method)s_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",
    "flat_mhs":      f"%(hostname)s/{ofp_snq}.%(method)s_{ofh4}_%(status)s%(filepath_ext)s",
    "flat_mhsn":     f"%(hostname)s/{ofp_snq}.%(method)s_{ofh4}_%(status)s_%(num)d%(filepath_ext)s",
    "flat_mhstn":    f"%(hostname)s/{ofp_snq}.%(method)s_{ofh4}_%(status)s_{ofdd}_%(num)d%(filepath_ext)s",
}

content_output_alias = {
    "default": "_content/sha256/%(content_sha256|take_prefix 1|to_hex)s/%(content_sha256|to_hex)s%(filepath_ext)s"
}

def output_example(name : str, indent : int) -> str:
    def gen(url : str) -> str:
        x = ReqresExpr(UnknownSource(), trivial_Reqres(parse_url(url)))
        x.values["num"] = 0
        return output_alias[name] % x
    return make_example(gen, indent)

def test_outputs_aliases() -> None:
    def mk(url : str) -> ReqresExpr[UnknownSource]:
        return ReqresExpr(UnknownSource(), trivial_Reqres(parse_url(url)))

    res = []
    prev = ""
    for name in output_alias:
        for url in example_url:
            x = mk(url)
            x.values["num"] = 0
            prefix = name + ":" + " " * (12 - len(name)) + " "
            current = output_alias[name] % x
            if prev != current:
                res.append(prefix + current)
            else:
                res.append(prefix + "==")
            prev = current

    pristine = """
default:      1970/01/01/001640000_0_GET_8198_C200C_example.org_0
default:      ==
default:      1970/01/01/001640000_0_GET_f0dc_C200C_example.org_0
default:      1970/01/01/001640000_0_GET_086d_C200C_example.org_0
default:      1970/01/01/001640000_0_GET_3fbb_C200C_example.org_0
default:      1970/01/01/001640000_0_GET_5658_C200C_example.org_0
default:      1970/01/01/001640000_0_GET_4f11_C200C_königsgäßchen.example.org_0
default:      1970/01/01/001640000_0_GET_c4ae_C200C_ジャジェメント.ですの.example.org_0
default:      ==
default:      ==
short:        1970/01/01/1000000_0_0
short:        ==
short:        ==
short:        ==
short:        ==
short:        ==
short:        ==
short:        ==
short:        ==
short:        ==
surl:         https/example.org/
surl:         ==
surl:         https/example.org/index.html
surl:         https/example.org/media
surl:         ==
surl:         https/example.org/view?one=1&two=2&three&three=3
surl:         https/königsgäßchen.example.org/index.html
surl:         https/ジャジェメント.ですの.example.org/испытание/is
surl:         ==
surl:         ==
surl_msn:     https/example.org/__GET_C200C_0
surl_msn:     ==
surl_msn:     https/example.org/index.html__GET_C200C_0
surl_msn:     https/example.org/media__GET_C200C_0
surl_msn:     ==
surl_msn:     https/example.org/view?one=1&two=2&three&three=3__GET_C200C_0
surl_msn:     https/königsgäßchen.example.org/index.html__GET_C200C_0
surl_msn:     https/ジャジェメント.ですの.example.org/испытание/is__GET_C200C_0
surl_msn:     ==
surl_msn:     ==
surl_mstn:    https/example.org/__GET_C200C_1970-01-01_001640000_0
surl_mstn:    ==
surl_mstn:    https/example.org/index.html__GET_C200C_1970-01-01_001640000_0
surl_mstn:    https/example.org/media__GET_C200C_1970-01-01_001640000_0
surl_mstn:    ==
surl_mstn:    https/example.org/view?one=1&two=2&three&three=3__GET_C200C_1970-01-01_001640000_0
surl_mstn:    https/königsgäßchen.example.org/index.html__GET_C200C_1970-01-01_001640000_0
surl_mstn:    https/ジャジェメント.ですの.example.org/испытание/is__GET_C200C_1970-01-01_001640000_0
surl_mstn:    ==
surl_mstn:    ==
shupq:        https/example.org/index.htm
shupq:        ==
shupq:        https/example.org/index.html
shupq:        https/example.org/media/index.htm
shupq:        ==
shupq:        https/example.org/view/index?one=1&two=2&three&three=3.htm
shupq:        https/königsgäßchen.example.org/index.html
shupq:        https/ジャジェメント.ですの.example.org/испытание/is/index.htm
shupq:        ==
shupq:        ==
shupq_n:      https/example.org/index.0.htm
shupq_n:      ==
shupq_n:      https/example.org/index.0.html
shupq_n:      https/example.org/media/index.0.htm
shupq_n:      ==
shupq_n:      https/example.org/view/index?one=1&two=2&three&three=3.0.htm
shupq_n:      https/königsgäßchen.example.org/index.0.html
shupq_n:      https/ジャジェメント.ですの.example.org/испытание/is/index.0.htm
shupq_n:      ==
shupq_n:      ==
shupq_tn:     https/example.org/index.1970-01-01_001640000_0.htm
shupq_tn:     ==
shupq_tn:     https/example.org/index.1970-01-01_001640000_0.html
shupq_tn:     https/example.org/media/index.1970-01-01_001640000_0.htm
shupq_tn:     ==
shupq_tn:     https/example.org/view/index?one=1&two=2&three&three=3.1970-01-01_001640000_0.htm
shupq_tn:     https/königsgäßchen.example.org/index.1970-01-01_001640000_0.html
shupq_tn:     https/ジャジェメント.ですの.example.org/испытание/is/index.1970-01-01_001640000_0.htm
shupq_tn:     ==
shupq_tn:     ==
shupq_msn:    https/example.org/index.GET_C200C_0.htm
shupq_msn:    ==
shupq_msn:    https/example.org/index.GET_C200C_0.html
shupq_msn:    https/example.org/media/index.GET_C200C_0.htm
shupq_msn:    ==
shupq_msn:    https/example.org/view/index?one=1&two=2&three&three=3.GET_C200C_0.htm
shupq_msn:    https/königsgäßchen.example.org/index.GET_C200C_0.html
shupq_msn:    https/ジャジェメント.ですの.example.org/испытание/is/index.GET_C200C_0.htm
shupq_msn:    ==
shupq_msn:    ==
shupq_mstn:   https/example.org/index.GET_C200C_1970-01-01_001640000_0.htm
shupq_mstn:   ==
shupq_mstn:   https/example.org/index.GET_C200C_1970-01-01_001640000_0.html
shupq_mstn:   https/example.org/media/index.GET_C200C_1970-01-01_001640000_0.htm
shupq_mstn:   ==
shupq_mstn:   https/example.org/view/index?one=1&two=2&three&three=3.GET_C200C_1970-01-01_001640000_0.htm
shupq_mstn:   https/königsgäßchen.example.org/index.GET_C200C_1970-01-01_001640000_0.html
shupq_mstn:   https/ジャジェメント.ですの.example.org/испытание/is/index.GET_C200C_1970-01-01_001640000_0.htm
shupq_mstn:   ==
shupq_mstn:   ==
shupnq:       https/example.org/index.htm
shupnq:       ==
shupnq:       https/example.org/index.html
shupnq:       https/example.org/media/index.htm
shupnq:       ==
shupnq:       https/example.org/view/index?one=1&two=2&three=3.htm
shupnq:       https/königsgäßchen.example.org/index.html
shupnq:       https/ジャジェメント.ですの.example.org/испытание/is/index.htm
shupnq:       ==
shupnq:       ==
shupnq_n:     https/example.org/index.0.htm
shupnq_n:     ==
shupnq_n:     https/example.org/index.0.html
shupnq_n:     https/example.org/media/index.0.htm
shupnq_n:     ==
shupnq_n:     https/example.org/view/index?one=1&two=2&three=3.0.htm
shupnq_n:     https/königsgäßchen.example.org/index.0.html
shupnq_n:     https/ジャジェメント.ですの.example.org/испытание/is/index.0.htm
shupnq_n:     ==
shupnq_n:     ==
shupnq_tn:    https/example.org/index.1970-01-01_001640000_0.htm
shupnq_tn:    ==
shupnq_tn:    https/example.org/index.1970-01-01_001640000_0.html
shupnq_tn:    https/example.org/media/index.1970-01-01_001640000_0.htm
shupnq_tn:    ==
shupnq_tn:    https/example.org/view/index?one=1&two=2&three=3.1970-01-01_001640000_0.htm
shupnq_tn:    https/königsgäßchen.example.org/index.1970-01-01_001640000_0.html
shupnq_tn:    https/ジャジェメント.ですの.example.org/испытание/is/index.1970-01-01_001640000_0.htm
shupnq_tn:    ==
shupnq_tn:    ==
shupnq_msn:   https/example.org/index.GET_C200C_0.htm
shupnq_msn:   ==
shupnq_msn:   https/example.org/index.GET_C200C_0.html
shupnq_msn:   https/example.org/media/index.GET_C200C_0.htm
shupnq_msn:   ==
shupnq_msn:   https/example.org/view/index?one=1&two=2&three=3.GET_C200C_0.htm
shupnq_msn:   https/königsgäßchen.example.org/index.GET_C200C_0.html
shupnq_msn:   https/ジャジェメント.ですの.example.org/испытание/is/index.GET_C200C_0.htm
shupnq_msn:   ==
shupnq_msn:   ==
shupnq_mstn:  https/example.org/index.GET_C200C_1970-01-01_001640000_0.htm
shupnq_mstn:  ==
shupnq_mstn:  https/example.org/index.GET_C200C_1970-01-01_001640000_0.html
shupnq_mstn:  https/example.org/media/index.GET_C200C_1970-01-01_001640000_0.htm
shupnq_mstn:  ==
shupnq_mstn:  https/example.org/view/index?one=1&two=2&three=3.GET_C200C_1970-01-01_001640000_0.htm
shupnq_mstn:  https/königsgäßchen.example.org/index.GET_C200C_1970-01-01_001640000_0.html
shupnq_mstn:  https/ジャジェメント.ですの.example.org/испытание/is/index.GET_C200C_1970-01-01_001640000_0.htm
shupnq_mstn:  ==
shupnq_mstn:  ==
shupnq_mhs:   https/example.org/index.GET_8198_C200C.htm
shupnq_mhs:   ==
shupnq_mhs:   https/example.org/index.GET_f0dc_C200C.html
shupnq_mhs:   https/example.org/media/index.GET_086d_C200C.htm
shupnq_mhs:   https/example.org/media/index.GET_3fbb_C200C.htm
shupnq_mhs:   https/example.org/view/index?one=1&two=2&three=3.GET_5658_C200C.htm
shupnq_mhs:   https/königsgäßchen.example.org/index.GET_4f11_C200C.html
shupnq_mhs:   https/ジャジェメント.ですの.example.org/испытание/is/index.GET_c4ae_C200C.htm
shupnq_mhs:   ==
shupnq_mhs:   ==
shupnq_mhsn:  https/example.org/index.GET_8198_C200C_0.htm
shupnq_mhsn:  ==
shupnq_mhsn:  https/example.org/index.GET_f0dc_C200C_0.html
shupnq_mhsn:  https/example.org/media/index.GET_086d_C200C_0.htm
shupnq_mhsn:  https/example.org/media/index.GET_3fbb_C200C_0.htm
shupnq_mhsn:  https/example.org/view/index?one=1&two=2&three=3.GET_5658_C200C_0.htm
shupnq_mhsn:  https/königsgäßchen.example.org/index.GET_4f11_C200C_0.html
shupnq_mhsn:  https/ジャジェメント.ですの.example.org/испытание/is/index.GET_c4ae_C200C_0.htm
shupnq_mhsn:  ==
shupnq_mhsn:  ==
shupnq_mhstn: https/example.org/index.GET_8198_C200C_1970-01-01_001640000_0.htm
shupnq_mhstn: ==
shupnq_mhstn: https/example.org/index.GET_f0dc_C200C_1970-01-01_001640000_0.html
shupnq_mhstn: https/example.org/media/index.GET_086d_C200C_1970-01-01_001640000_0.htm
shupnq_mhstn: https/example.org/media/index.GET_3fbb_C200C_1970-01-01_001640000_0.htm
shupnq_mhstn: https/example.org/view/index?one=1&two=2&three=3.GET_5658_C200C_1970-01-01_001640000_0.htm
shupnq_mhstn: https/königsgäßchen.example.org/index.GET_4f11_C200C_1970-01-01_001640000_0.html
shupnq_mhstn: https/ジャジェメント.ですの.example.org/испытание/is/index.GET_c4ae_C200C_1970-01-01_001640000_0.htm
shupnq_mhstn: ==
shupnq_mhstn: ==
srhupq:       https/org.example/index.htm
srhupq:       ==
srhupq:       https/org.example/index.html
srhupq:       https/org.example/media/index.htm
srhupq:       ==
srhupq:       https/org.example/view/index?one=1&two=2&three&three=3.htm
srhupq:       https/org.example.königsgäßchen/index.html
srhupq:       https/org.example.ですの.ジャジェメント/испытание/is/index.htm
srhupq:       ==
srhupq:       ==
srhupq_n:     https/org.example/index.0.htm
srhupq_n:     ==
srhupq_n:     https/org.example/index.0.html
srhupq_n:     https/org.example/media/index.0.htm
srhupq_n:     ==
srhupq_n:     https/org.example/view/index?one=1&two=2&three&three=3.0.htm
srhupq_n:     https/org.example.königsgäßchen/index.0.html
srhupq_n:     https/org.example.ですの.ジャジェメント/испытание/is/index.0.htm
srhupq_n:     ==
srhupq_n:     ==
srhupq_tn:    https/org.example/index.1970-01-01_001640000_0.htm
srhupq_tn:    ==
srhupq_tn:    https/org.example/index.1970-01-01_001640000_0.html
srhupq_tn:    https/org.example/media/index.1970-01-01_001640000_0.htm
srhupq_tn:    ==
srhupq_tn:    https/org.example/view/index?one=1&two=2&three&three=3.1970-01-01_001640000_0.htm
srhupq_tn:    https/org.example.königsgäßchen/index.1970-01-01_001640000_0.html
srhupq_tn:    https/org.example.ですの.ジャジェメント/испытание/is/index.1970-01-01_001640000_0.htm
srhupq_tn:    ==
srhupq_tn:    ==
srhupq_msn:   https/org.example/index.GET_C200C_0.htm
srhupq_msn:   ==
srhupq_msn:   https/org.example/index.GET_C200C_0.html
srhupq_msn:   https/org.example/media/index.GET_C200C_0.htm
srhupq_msn:   ==
srhupq_msn:   https/org.example/view/index?one=1&two=2&three&three=3.GET_C200C_0.htm
srhupq_msn:   https/org.example.königsgäßchen/index.GET_C200C_0.html
srhupq_msn:   https/org.example.ですの.ジャジェメント/испытание/is/index.GET_C200C_0.htm
srhupq_msn:   ==
srhupq_msn:   ==
srhupq_mstn:  https/org.example/index.GET_C200C_1970-01-01_001640000_0.htm
srhupq_mstn:  ==
srhupq_mstn:  https/org.example/index.GET_C200C_1970-01-01_001640000_0.html
srhupq_mstn:  https/org.example/media/index.GET_C200C_1970-01-01_001640000_0.htm
srhupq_mstn:  ==
srhupq_mstn:  https/org.example/view/index?one=1&two=2&three&three=3.GET_C200C_1970-01-01_001640000_0.htm
srhupq_mstn:  https/org.example.königsgäßchen/index.GET_C200C_1970-01-01_001640000_0.html
srhupq_mstn:  https/org.example.ですの.ジャジェメント/испытание/is/index.GET_C200C_1970-01-01_001640000_0.htm
srhupq_mstn:  ==
srhupq_mstn:  ==
srhupnq:      https/org.example/index.htm
srhupnq:      ==
srhupnq:      https/org.example/index.html
srhupnq:      https/org.example/media/index.htm
srhupnq:      ==
srhupnq:      https/org.example/view/index?one=1&two=2&three=3.htm
srhupnq:      https/org.example.königsgäßchen/index.html
srhupnq:      https/org.example.ですの.ジャジェメント/испытание/is/index.htm
srhupnq:      ==
srhupnq:      ==
srhupnq_n:    https/org.example/index.0.htm
srhupnq_n:    ==
srhupnq_n:    https/org.example/index.0.html
srhupnq_n:    https/org.example/media/index.0.htm
srhupnq_n:    ==
srhupnq_n:    https/org.example/view/index?one=1&two=2&three=3.0.htm
srhupnq_n:    https/org.example.königsgäßchen/index.0.html
srhupnq_n:    https/org.example.ですの.ジャジェメント/испытание/is/index.0.htm
srhupnq_n:    ==
srhupnq_n:    ==
srhupnq_tn:   https/org.example/index.1970-01-01_001640000_0.htm
srhupnq_tn:   ==
srhupnq_tn:   https/org.example/index.1970-01-01_001640000_0.html
srhupnq_tn:   https/org.example/media/index.1970-01-01_001640000_0.htm
srhupnq_tn:   ==
srhupnq_tn:   https/org.example/view/index?one=1&two=2&three=3.1970-01-01_001640000_0.htm
srhupnq_tn:   https/org.example.königsgäßchen/index.1970-01-01_001640000_0.html
srhupnq_tn:   https/org.example.ですの.ジャジェメント/испытание/is/index.1970-01-01_001640000_0.htm
srhupnq_tn:   ==
srhupnq_tn:   ==
srhupnq_msn:  https/org.example/index.GET_C200C_0.htm
srhupnq_msn:  ==
srhupnq_msn:  https/org.example/index.GET_C200C_0.html
srhupnq_msn:  https/org.example/media/index.GET_C200C_0.htm
srhupnq_msn:  ==
srhupnq_msn:  https/org.example/view/index?one=1&two=2&three=3.GET_C200C_0.htm
srhupnq_msn:  https/org.example.königsgäßchen/index.GET_C200C_0.html
srhupnq_msn:  https/org.example.ですの.ジャジェメント/испытание/is/index.GET_C200C_0.htm
srhupnq_msn:  ==
srhupnq_msn:  ==
srhupnq_mstn: https/org.example/index.GET_C200C_1970-01-01_001640000_0.htm
srhupnq_mstn: ==
srhupnq_mstn: https/org.example/index.GET_C200C_1970-01-01_001640000_0.html
srhupnq_mstn: https/org.example/media/index.GET_C200C_1970-01-01_001640000_0.htm
srhupnq_mstn: ==
srhupnq_mstn: https/org.example/view/index?one=1&two=2&three=3.GET_C200C_1970-01-01_001640000_0.htm
srhupnq_mstn: https/org.example.königsgäßchen/index.GET_C200C_1970-01-01_001640000_0.html
srhupnq_mstn: https/org.example.ですの.ジャジェメント/испытание/is/index.GET_C200C_1970-01-01_001640000_0.htm
srhupnq_mstn: ==
srhupnq_mstn: ==
srhupnq_mhs:  https/org.example/index.GET_8198_C200C.htm
srhupnq_mhs:  ==
srhupnq_mhs:  https/org.example/index.GET_f0dc_C200C.html
srhupnq_mhs:  https/org.example/media/index.GET_086d_C200C.htm
srhupnq_mhs:  https/org.example/media/index.GET_3fbb_C200C.htm
srhupnq_mhs:  https/org.example/view/index?one=1&two=2&three=3.GET_5658_C200C.htm
srhupnq_mhs:  https/org.example.königsgäßchen/index.GET_4f11_C200C.html
srhupnq_mhs:  https/org.example.ですの.ジャジェメント/испытание/is/index.GET_c4ae_C200C.htm
srhupnq_mhs:  ==
srhupnq_mhs:  ==
srhupnq_mhsn: https/org.example/index.GET_8198_C200C_0.htm
srhupnq_mhsn: ==
srhupnq_mhsn: https/org.example/index.GET_f0dc_C200C_0.html
srhupnq_mhsn: https/org.example/media/index.GET_086d_C200C_0.htm
srhupnq_mhsn: https/org.example/media/index.GET_3fbb_C200C_0.htm
srhupnq_mhsn: https/org.example/view/index?one=1&two=2&three=3.GET_5658_C200C_0.htm
srhupnq_mhsn: https/org.example.königsgäßchen/index.GET_4f11_C200C_0.html
srhupnq_mhsn: https/org.example.ですの.ジャジェメント/испытание/is/index.GET_c4ae_C200C_0.htm
srhupnq_mhsn: ==
srhupnq_mhsn: ==
srhupnq_mhstn: https/org.example/index.GET_8198_C200C_1970-01-01_001640000_0.htm
srhupnq_mhstn: ==
srhupnq_mhstn: https/org.example/index.GET_f0dc_C200C_1970-01-01_001640000_0.html
srhupnq_mhstn: https/org.example/media/index.GET_086d_C200C_1970-01-01_001640000_0.htm
srhupnq_mhstn: https/org.example/media/index.GET_3fbb_C200C_1970-01-01_001640000_0.htm
srhupnq_mhstn: https/org.example/view/index?one=1&two=2&three=3.GET_5658_C200C_1970-01-01_001640000_0.htm
srhupnq_mhstn: https/org.example.königsgäßchen/index.GET_4f11_C200C_1970-01-01_001640000_0.html
srhupnq_mhstn: https/org.example.ですの.ジャジェメント/испытание/is/index.GET_c4ae_C200C_1970-01-01_001640000_0.htm
srhupnq_mhstn: ==
srhupnq_mhstn: ==
url:          example.org/
url:          ==
url:          example.org/index.html
url:          example.org/media
url:          ==
url:          example.org/view?one=1&two=2&three&three=3
url:          königsgäßchen.example.org/index.html
url:          ジャジェメント.ですの.example.org/испытание/is
url:          ==
url:          ==
url_msn:      example.org/__GET_C200C_0
url_msn:      ==
url_msn:      example.org/index.html__GET_C200C_0
url_msn:      example.org/media__GET_C200C_0
url_msn:      ==
url_msn:      example.org/view?one=1&two=2&three&three=3__GET_C200C_0
url_msn:      königsgäßchen.example.org/index.html__GET_C200C_0
url_msn:      ジャジェメント.ですの.example.org/испытание/is__GET_C200C_0
url_msn:      ==
url_msn:      ==
url_mstn:     example.org/__GET_C200C_1970-01-01_001640000_0
url_mstn:     ==
url_mstn:     example.org/index.html__GET_C200C_1970-01-01_001640000_0
url_mstn:     example.org/media__GET_C200C_1970-01-01_001640000_0
url_mstn:     ==
url_mstn:     example.org/view?one=1&two=2&three&three=3__GET_C200C_1970-01-01_001640000_0
url_mstn:     königsgäßchen.example.org/index.html__GET_C200C_1970-01-01_001640000_0
url_mstn:     ジャジェメント.ですの.example.org/испытание/is__GET_C200C_1970-01-01_001640000_0
url_mstn:     ==
url_mstn:     ==
hupq:         example.org/index.htm
hupq:         ==
hupq:         example.org/index.html
hupq:         example.org/media/index.htm
hupq:         ==
hupq:         example.org/view/index?one=1&two=2&three&three=3.htm
hupq:         königsgäßchen.example.org/index.html
hupq:         ジャジェメント.ですの.example.org/испытание/is/index.htm
hupq:         ==
hupq:         ==
hupq_n:       example.org/index.0.htm
hupq_n:       ==
hupq_n:       example.org/index.0.html
hupq_n:       example.org/media/index.0.htm
hupq_n:       ==
hupq_n:       example.org/view/index?one=1&two=2&three&three=3.0.htm
hupq_n:       königsgäßchen.example.org/index.0.html
hupq_n:       ジャジェメント.ですの.example.org/испытание/is/index.0.htm
hupq_n:       ==
hupq_n:       ==
hupq_tn:      example.org/index.1970-01-01_001640000_0.htm
hupq_tn:      ==
hupq_tn:      example.org/index.1970-01-01_001640000_0.html
hupq_tn:      example.org/media/index.1970-01-01_001640000_0.htm
hupq_tn:      ==
hupq_tn:      example.org/view/index?one=1&two=2&three&three=3.1970-01-01_001640000_0.htm
hupq_tn:      königsgäßchen.example.org/index.1970-01-01_001640000_0.html
hupq_tn:      ジャジェメント.ですの.example.org/испытание/is/index.1970-01-01_001640000_0.htm
hupq_tn:      ==
hupq_tn:      ==
hupq_msn:     example.org/index.GET_C200C_0.htm
hupq_msn:     ==
hupq_msn:     example.org/index.GET_C200C_0.html
hupq_msn:     example.org/media/index.GET_C200C_0.htm
hupq_msn:     ==
hupq_msn:     example.org/view/index?one=1&two=2&three&three=3.GET_C200C_0.htm
hupq_msn:     königsgäßchen.example.org/index.GET_C200C_0.html
hupq_msn:     ジャジェメント.ですの.example.org/испытание/is/index.GET_C200C_0.htm
hupq_msn:     ==
hupq_msn:     ==
hupq_mstn:    example.org/index.GET_C200C_1970-01-01_001640000_0.htm
hupq_mstn:    ==
hupq_mstn:    example.org/index.GET_C200C_1970-01-01_001640000_0.html
hupq_mstn:    example.org/media/index.GET_C200C_1970-01-01_001640000_0.htm
hupq_mstn:    ==
hupq_mstn:    example.org/view/index?one=1&two=2&three&three=3.GET_C200C_1970-01-01_001640000_0.htm
hupq_mstn:    königsgäßchen.example.org/index.GET_C200C_1970-01-01_001640000_0.html
hupq_mstn:    ジャジェメント.ですの.example.org/испытание/is/index.GET_C200C_1970-01-01_001640000_0.htm
hupq_mstn:    ==
hupq_mstn:    ==
hupnq:        example.org/index.htm
hupnq:        ==
hupnq:        example.org/index.html
hupnq:        example.org/media/index.htm
hupnq:        ==
hupnq:        example.org/view/index?one=1&two=2&three=3.htm
hupnq:        königsgäßchen.example.org/index.html
hupnq:        ジャジェメント.ですの.example.org/испытание/is/index.htm
hupnq:        ==
hupnq:        ==
hupnq_n:      example.org/index.0.htm
hupnq_n:      ==
hupnq_n:      example.org/index.0.html
hupnq_n:      example.org/media/index.0.htm
hupnq_n:      ==
hupnq_n:      example.org/view/index?one=1&two=2&three=3.0.htm
hupnq_n:      königsgäßchen.example.org/index.0.html
hupnq_n:      ジャジェメント.ですの.example.org/испытание/is/index.0.htm
hupnq_n:      ==
hupnq_n:      ==
hupnq_tn:     example.org/index.1970-01-01_001640000_0.htm
hupnq_tn:     ==
hupnq_tn:     example.org/index.1970-01-01_001640000_0.html
hupnq_tn:     example.org/media/index.1970-01-01_001640000_0.htm
hupnq_tn:     ==
hupnq_tn:     example.org/view/index?one=1&two=2&three=3.1970-01-01_001640000_0.htm
hupnq_tn:     königsgäßchen.example.org/index.1970-01-01_001640000_0.html
hupnq_tn:     ジャジェメント.ですの.example.org/испытание/is/index.1970-01-01_001640000_0.htm
hupnq_tn:     ==
hupnq_tn:     ==
hupnq_msn:    example.org/index.GET_C200C_0.htm
hupnq_msn:    ==
hupnq_msn:    example.org/index.GET_C200C_0.html
hupnq_msn:    example.org/media/index.GET_C200C_0.htm
hupnq_msn:    ==
hupnq_msn:    example.org/view/index?one=1&two=2&three=3.GET_C200C_0.htm
hupnq_msn:    königsgäßchen.example.org/index.GET_C200C_0.html
hupnq_msn:    ジャジェメント.ですの.example.org/испытание/is/index.GET_C200C_0.htm
hupnq_msn:    ==
hupnq_msn:    ==
hupnq_mstn:   example.org/index.GET_C200C_1970-01-01_001640000_0.htm
hupnq_mstn:   ==
hupnq_mstn:   example.org/index.GET_C200C_1970-01-01_001640000_0.html
hupnq_mstn:   example.org/media/index.GET_C200C_1970-01-01_001640000_0.htm
hupnq_mstn:   ==
hupnq_mstn:   example.org/view/index?one=1&two=2&three=3.GET_C200C_1970-01-01_001640000_0.htm
hupnq_mstn:   königsgäßchen.example.org/index.GET_C200C_1970-01-01_001640000_0.html
hupnq_mstn:   ジャジェメント.ですの.example.org/испытание/is/index.GET_C200C_1970-01-01_001640000_0.htm
hupnq_mstn:   ==
hupnq_mstn:   ==
hupnq_mhs:    example.org/index.GET_8198_C200C.htm
hupnq_mhs:    ==
hupnq_mhs:    example.org/index.GET_f0dc_C200C.html
hupnq_mhs:    example.org/media/index.GET_086d_C200C.htm
hupnq_mhs:    example.org/media/index.GET_3fbb_C200C.htm
hupnq_mhs:    example.org/view/index?one=1&two=2&three=3.GET_5658_C200C.htm
hupnq_mhs:    königsgäßchen.example.org/index.GET_4f11_C200C.html
hupnq_mhs:    ジャジェメント.ですの.example.org/испытание/is/index.GET_c4ae_C200C.htm
hupnq_mhs:    ==
hupnq_mhs:    ==
hupnq_mhsn:   example.org/index.GET_8198_C200C_0.htm
hupnq_mhsn:   ==
hupnq_mhsn:   example.org/index.GET_f0dc_C200C_0.html
hupnq_mhsn:   example.org/media/index.GET_086d_C200C_0.htm
hupnq_mhsn:   example.org/media/index.GET_3fbb_C200C_0.htm
hupnq_mhsn:   example.org/view/index?one=1&two=2&three=3.GET_5658_C200C_0.htm
hupnq_mhsn:   königsgäßchen.example.org/index.GET_4f11_C200C_0.html
hupnq_mhsn:   ジャジェメント.ですの.example.org/испытание/is/index.GET_c4ae_C200C_0.htm
hupnq_mhsn:   ==
hupnq_mhsn:   ==
hupnq_mhstn:  example.org/index.GET_8198_C200C_1970-01-01_001640000_0.htm
hupnq_mhstn:  ==
hupnq_mhstn:  example.org/index.GET_f0dc_C200C_1970-01-01_001640000_0.html
hupnq_mhstn:  example.org/media/index.GET_086d_C200C_1970-01-01_001640000_0.htm
hupnq_mhstn:  example.org/media/index.GET_3fbb_C200C_1970-01-01_001640000_0.htm
hupnq_mhstn:  example.org/view/index?one=1&two=2&three=3.GET_5658_C200C_1970-01-01_001640000_0.htm
hupnq_mhstn:  königsgäßchen.example.org/index.GET_4f11_C200C_1970-01-01_001640000_0.html
hupnq_mhstn:  ジャジェメント.ですの.example.org/испытание/is/index.GET_c4ae_C200C_1970-01-01_001640000_0.htm
hupnq_mhstn:  ==
hupnq_mhstn:  ==
rhupq:        org.example/index.htm
rhupq:        ==
rhupq:        org.example/index.html
rhupq:        org.example/media/index.htm
rhupq:        ==
rhupq:        org.example/view/index?one=1&two=2&three&three=3.htm
rhupq:        org.example.königsgäßchen/index.html
rhupq:        org.example.ですの.ジャジェメント/испытание/is/index.htm
rhupq:        ==
rhupq:        ==
rhupq_n:      org.example/index.0.htm
rhupq_n:      ==
rhupq_n:      org.example/index.0.html
rhupq_n:      org.example/media/index.0.htm
rhupq_n:      ==
rhupq_n:      org.example/view/index?one=1&two=2&three&three=3.0.htm
rhupq_n:      org.example.königsgäßchen/index.0.html
rhupq_n:      org.example.ですの.ジャジェメント/испытание/is/index.0.htm
rhupq_n:      ==
rhupq_n:      ==
rhupq_tn:     org.example/index.1970-01-01_001640000_0.htm
rhupq_tn:     ==
rhupq_tn:     org.example/index.1970-01-01_001640000_0.html
rhupq_tn:     org.example/media/index.1970-01-01_001640000_0.htm
rhupq_tn:     ==
rhupq_tn:     org.example/view/index?one=1&two=2&three&three=3.1970-01-01_001640000_0.htm
rhupq_tn:     org.example.königsgäßchen/index.1970-01-01_001640000_0.html
rhupq_tn:     org.example.ですの.ジャジェメント/испытание/is/index.1970-01-01_001640000_0.htm
rhupq_tn:     ==
rhupq_tn:     ==
rhupq_msn:    org.example/index.GET_C200C_0.htm
rhupq_msn:    ==
rhupq_msn:    org.example/index.GET_C200C_0.html
rhupq_msn:    org.example/media/index.GET_C200C_0.htm
rhupq_msn:    ==
rhupq_msn:    org.example/view/index?one=1&two=2&three&three=3.GET_C200C_0.htm
rhupq_msn:    org.example.königsgäßchen/index.GET_C200C_0.html
rhupq_msn:    org.example.ですの.ジャジェメント/испытание/is/index.GET_C200C_0.htm
rhupq_msn:    ==
rhupq_msn:    ==
rhupq_mstn:   org.example/index.GET_C200C_1970-01-01_001640000_0.htm
rhupq_mstn:   ==
rhupq_mstn:   org.example/index.GET_C200C_1970-01-01_001640000_0.html
rhupq_mstn:   org.example/media/index.GET_C200C_1970-01-01_001640000_0.htm
rhupq_mstn:   ==
rhupq_mstn:   org.example/view/index?one=1&two=2&three&three=3.GET_C200C_1970-01-01_001640000_0.htm
rhupq_mstn:   org.example.königsgäßchen/index.GET_C200C_1970-01-01_001640000_0.html
rhupq_mstn:   org.example.ですの.ジャジェメント/испытание/is/index.GET_C200C_1970-01-01_001640000_0.htm
rhupq_mstn:   ==
rhupq_mstn:   ==
rhupnq:       org.example/index.htm
rhupnq:       ==
rhupnq:       org.example/index.html
rhupnq:       org.example/media/index.htm
rhupnq:       ==
rhupnq:       org.example/view/index?one=1&two=2&three=3.htm
rhupnq:       org.example.königsgäßchen/index.html
rhupnq:       org.example.ですの.ジャジェメント/испытание/is/index.htm
rhupnq:       ==
rhupnq:       ==
rhupnq_n:     org.example/index.0.htm
rhupnq_n:     ==
rhupnq_n:     org.example/index.0.html
rhupnq_n:     org.example/media/index.0.htm
rhupnq_n:     ==
rhupnq_n:     org.example/view/index?one=1&two=2&three=3.0.htm
rhupnq_n:     org.example.königsgäßchen/index.0.html
rhupnq_n:     org.example.ですの.ジャジェメント/испытание/is/index.0.htm
rhupnq_n:     ==
rhupnq_n:     ==
rhupnq_tn:    org.example/index.1970-01-01_001640000_0.htm
rhupnq_tn:    ==
rhupnq_tn:    org.example/index.1970-01-01_001640000_0.html
rhupnq_tn:    org.example/media/index.1970-01-01_001640000_0.htm
rhupnq_tn:    ==
rhupnq_tn:    org.example/view/index?one=1&two=2&three=3.1970-01-01_001640000_0.htm
rhupnq_tn:    org.example.königsgäßchen/index.1970-01-01_001640000_0.html
rhupnq_tn:    org.example.ですの.ジャジェメント/испытание/is/index.1970-01-01_001640000_0.htm
rhupnq_tn:    ==
rhupnq_tn:    ==
rhupnq_msn:   org.example/index.GET_C200C_0.htm
rhupnq_msn:   ==
rhupnq_msn:   org.example/index.GET_C200C_0.html
rhupnq_msn:   org.example/media/index.GET_C200C_0.htm
rhupnq_msn:   ==
rhupnq_msn:   org.example/view/index?one=1&two=2&three=3.GET_C200C_0.htm
rhupnq_msn:   org.example.königsgäßchen/index.GET_C200C_0.html
rhupnq_msn:   org.example.ですの.ジャジェメント/испытание/is/index.GET_C200C_0.htm
rhupnq_msn:   ==
rhupnq_msn:   ==
rhupnq_mstn:  org.example/index.GET_C200C_1970-01-01_001640000_0.htm
rhupnq_mstn:  ==
rhupnq_mstn:  org.example/index.GET_C200C_1970-01-01_001640000_0.html
rhupnq_mstn:  org.example/media/index.GET_C200C_1970-01-01_001640000_0.htm
rhupnq_mstn:  ==
rhupnq_mstn:  org.example/view/index?one=1&two=2&three=3.GET_C200C_1970-01-01_001640000_0.htm
rhupnq_mstn:  org.example.königsgäßchen/index.GET_C200C_1970-01-01_001640000_0.html
rhupnq_mstn:  org.example.ですの.ジャジェメント/испытание/is/index.GET_C200C_1970-01-01_001640000_0.htm
rhupnq_mstn:  ==
rhupnq_mstn:  ==
rhupnq_mhs:   org.example/index.GET_8198_C200C.htm
rhupnq_mhs:   ==
rhupnq_mhs:   org.example/index.GET_f0dc_C200C.html
rhupnq_mhs:   org.example/media/index.GET_086d_C200C.htm
rhupnq_mhs:   org.example/media/index.GET_3fbb_C200C.htm
rhupnq_mhs:   org.example/view/index?one=1&two=2&three=3.GET_5658_C200C.htm
rhupnq_mhs:   org.example.königsgäßchen/index.GET_4f11_C200C.html
rhupnq_mhs:   org.example.ですの.ジャジェメント/испытание/is/index.GET_c4ae_C200C.htm
rhupnq_mhs:   ==
rhupnq_mhs:   ==
rhupnq_mhsn:  org.example/index.GET_8198_C200C_0.htm
rhupnq_mhsn:  ==
rhupnq_mhsn:  org.example/index.GET_f0dc_C200C_0.html
rhupnq_mhsn:  org.example/media/index.GET_086d_C200C_0.htm
rhupnq_mhsn:  org.example/media/index.GET_3fbb_C200C_0.htm
rhupnq_mhsn:  org.example/view/index?one=1&two=2&three=3.GET_5658_C200C_0.htm
rhupnq_mhsn:  org.example.königsgäßchen/index.GET_4f11_C200C_0.html
rhupnq_mhsn:  org.example.ですの.ジャジェメント/испытание/is/index.GET_c4ae_C200C_0.htm
rhupnq_mhsn:  ==
rhupnq_mhsn:  ==
rhupnq_mhstn: org.example/index.GET_8198_C200C_1970-01-01_001640000_0.htm
rhupnq_mhstn: ==
rhupnq_mhstn: org.example/index.GET_f0dc_C200C_1970-01-01_001640000_0.html
rhupnq_mhstn: org.example/media/index.GET_086d_C200C_1970-01-01_001640000_0.htm
rhupnq_mhstn: org.example/media/index.GET_3fbb_C200C_1970-01-01_001640000_0.htm
rhupnq_mhstn: org.example/view/index?one=1&two=2&three=3.GET_5658_C200C_1970-01-01_001640000_0.htm
rhupnq_mhstn: org.example.königsgäßchen/index.GET_4f11_C200C_1970-01-01_001640000_0.html
rhupnq_mhstn: org.example.ですの.ジャジェメント/испытание/is/index.GET_c4ae_C200C_1970-01-01_001640000_0.htm
rhupnq_mhstn: ==
rhupnq_mhstn: ==
flat:         example.org/index.htm
flat:         ==
flat:         example.org/index.html
flat:         example.org/media__index.htm
flat:         ==
flat:         example.org/view__index?one=1&two=2&three=3.htm
flat:         königsgäßchen.example.org/index.html
flat:         ジャジェメント.ですの.example.org/испытание__is__index.htm
flat:         ==
flat:         ==
flat_n:       example.org/index.0.htm
flat_n:       ==
flat_n:       example.org/index.0.html
flat_n:       example.org/media__index.0.htm
flat_n:       ==
flat_n:       example.org/view__index?one=1&two=2&three=3.0.htm
flat_n:       königsgäßchen.example.org/index.0.html
flat_n:       ジャジェメント.ですの.example.org/испытание__is__index.0.htm
flat_n:       ==
flat_n:       ==
flat_tn:      example.org/index.1970-01-01_001640000_0.htm
flat_tn:      ==
flat_tn:      example.org/index.1970-01-01_001640000_0.html
flat_tn:      example.org/media__index.1970-01-01_001640000_0.htm
flat_tn:      ==
flat_tn:      example.org/view__index?one=1&two=2&three=3.1970-01-01_001640000_0.htm
flat_tn:      königsgäßchen.example.org/index.1970-01-01_001640000_0.html
flat_tn:      ジャジェメント.ですの.example.org/испытание__is__index.1970-01-01_001640000_0.htm
flat_tn:      ==
flat_tn:      ==
flat_ms:      example.org/index.GET_C200C.htm
flat_ms:      ==
flat_ms:      example.org/index.GET_C200C.html
flat_ms:      example.org/media__index.GET_C200C.htm
flat_ms:      ==
flat_ms:      example.org/view__index?one=1&two=2&three=3.GET_C200C.htm
flat_ms:      königsgäßchen.example.org/index.GET_C200C.html
flat_ms:      ジャジェメント.ですの.example.org/испытание__is__index.GET_C200C.htm
flat_ms:      ==
flat_ms:      ==
flat_msn:     example.org/index.GET_C200C_0.htm
flat_msn:     ==
flat_msn:     example.org/index.GET_C200C_0.html
flat_msn:     example.org/media__index.GET_C200C_0.htm
flat_msn:     ==
flat_msn:     example.org/view__index?one=1&two=2&three=3.GET_C200C_0.htm
flat_msn:     königsgäßchen.example.org/index.GET_C200C_0.html
flat_msn:     ジャジェメント.ですの.example.org/испытание__is__index.GET_C200C_0.htm
flat_msn:     ==
flat_msn:     ==
flat_mstn:    example.org/index.GET_C200C_1970-01-01_001640000_0.htm
flat_mstn:    ==
flat_mstn:    example.org/index.GET_C200C_1970-01-01_001640000_0.html
flat_mstn:    example.org/media__index.GET_C200C_1970-01-01_001640000_0.htm
flat_mstn:    ==
flat_mstn:    example.org/view__index?one=1&two=2&three=3.GET_C200C_1970-01-01_001640000_0.htm
flat_mstn:    königsgäßchen.example.org/index.GET_C200C_1970-01-01_001640000_0.html
flat_mstn:    ジャジェメント.ですの.example.org/испытание__is__index.GET_C200C_1970-01-01_001640000_0.htm
flat_mstn:    ==
flat_mstn:    ==
flat_mhs:     example.org/index.GET_8198_C200C.htm
flat_mhs:     ==
flat_mhs:     example.org/index.GET_f0dc_C200C.html
flat_mhs:     example.org/media__index.GET_086d_C200C.htm
flat_mhs:     example.org/media__index.GET_3fbb_C200C.htm
flat_mhs:     example.org/view__index?one=1&two=2&three=3.GET_5658_C200C.htm
flat_mhs:     königsgäßchen.example.org/index.GET_4f11_C200C.html
flat_mhs:     ジャジェメント.ですの.example.org/испытание__is__index.GET_c4ae_C200C.htm
flat_mhs:     ==
flat_mhs:     ==
flat_mhsn:    example.org/index.GET_8198_C200C_0.htm
flat_mhsn:    ==
flat_mhsn:    example.org/index.GET_f0dc_C200C_0.html
flat_mhsn:    example.org/media__index.GET_086d_C200C_0.htm
flat_mhsn:    example.org/media__index.GET_3fbb_C200C_0.htm
flat_mhsn:    example.org/view__index?one=1&two=2&three=3.GET_5658_C200C_0.htm
flat_mhsn:    königsgäßchen.example.org/index.GET_4f11_C200C_0.html
flat_mhsn:    ジャジェメント.ですの.example.org/испытание__is__index.GET_c4ae_C200C_0.htm
flat_mhsn:    ==
flat_mhsn:    ==
flat_mhstn:   example.org/index.GET_8198_C200C_1970-01-01_001640000_0.htm
flat_mhstn:   ==
flat_mhstn:   example.org/index.GET_f0dc_C200C_1970-01-01_001640000_0.html
flat_mhstn:   example.org/media__index.GET_086d_C200C_1970-01-01_001640000_0.htm
flat_mhstn:   example.org/media__index.GET_3fbb_C200C_1970-01-01_001640000_0.htm
flat_mhstn:   example.org/view__index?one=1&two=2&three=3.GET_5658_C200C_1970-01-01_001640000_0.htm
flat_mhstn:   königsgäßchen.example.org/index.GET_4f11_C200C_1970-01-01_001640000_0.html
flat_mhstn:   ジャジェメント.ですの.example.org/испытание__is__index.GET_c4ae_C200C_1970-01-01_001640000_0.htm
flat_mhstn:   ==
flat_mhstn:   ==
"""

    #print("\n" + "\n".join(res))

    pl = pristine.strip().split("\n")
    assert len(pl) == len(res)

    for i in range(0, len(res)):
        a = pl[i]
        b = res[i]
        if a != b:
            raise CatastrophicFailure("expected %s, got %s", a, b)

not_allowed = gettext("; this is not allowed to prevent accidental data loss")
variance_help = gettext("; your `--output` format fails to provide enough variance to solve this problem automatically (did your forget to place a `%%(num)d` substitution in there?)") + not_allowed

@_dc.dataclass
class Memory:
    consumption : int = 0

@_dc.dataclass
class SeenCounter(_t.Generic[_t.AnyStr]):
    mem : Memory
    state : _c.OrderedDict[_t.AnyStr, int] = _dc.field(default_factory=_c.OrderedDict)

    def __len__(self) -> int:
        return len(self.state)

    def count(self, value : _t.AnyStr) -> int:
        try:
            count = self.state[value]
        except KeyError:
            self.state[value] = 0
            self.mem.consumption += len(value)
            return 0
        else:
            count += 1
            self.state[value] = count
            return count

    def pop(self) -> tuple[_t.AnyStr, int]:
        res = self.state.popitem(False)
        abs_out_path, _ = res
        self.mem.consumption -= len(abs_out_path)
        return res

DeferredDestinationType = _t.TypeVar("DeferredDestinationType")
class DeferredOperation(_t.Generic[DeferredSourceType, DeferredDestinationType]):
    """A deferred `source` -> `destination` operation with updatable `source`.

       This exists to help you to eliminatate away repeated `os.rename`,
       `os.symlink`, etc calls to the same `destination` and to help
       implementing disk writes batching.
    """
    source : DeferredSourceType
    destination : DeferredDestinationType

    def __init__(self, source : DeferredSourceType, destination : DeferredDestinationType, overwrite : bool, allow_updates : bool) -> None:
        self.source = source
        self.destination = destination
        self.overwrite = overwrite
        self.allow_updates = allow_updates
        self.updated = False

    def approx_size(self) -> int:
        return 48 + self.source.approx_size()

    def switch_source(self, new_source : DeferredSourceType, force : bool = False) -> bool:
        """Switch source of this operation.

           Returns `True` if this change was permitted.
        """
        if not force:
            old_source = self.source
            if not new_source.replaces(old_source) or \
               new_source.same_as(old_source) or \
               new_source.get_bytes() == old_source.get_bytes():
                return True

        if not self.allow_updates:
            # updates are not allowed
            return False

        # update
        self.source = new_source
        self.updated = True
        return True

    def run(self, dsync : DeferredSync | None = None) -> None:
        """Write the `source` to `destination`."""
        raise NotImplementedError()

class DeferredFileWrite(DeferredOperation[DeferredSourceType, _t.AnyStr], _t.Generic[DeferredSourceType, _t.AnyStr]):
    def approx_size(self) -> int:
        return super().approx_size() + len(self.destination)

    def run(self, dsync : DeferredSync | None = None) -> None:
        data = self.source.get_bytes()
        if self.updated and file_content_equals(self.destination, data):
            # nothing to do
            return
        atomic_write(data, self.destination, self.overwrite or self.allow_updates, dsync = dsync)

def make_deferred_emit(cargs : _t.Any,
                       destination : _t.AnyStr,
                       output_format : _t.AnyStr,
                       action : str,
                       actioning : str,
                       defer : _t.Callable[[ReqresExpr[DeferredSourceType], _t.AnyStr, bool, bool],
                                           DeferredOperation[ReqresExpr[DeferredSourceType], _t.AnyStr]],
                       allow_updates : bool,
                       moving : bool = False,
                       symlinking : bool = False) \
        -> tuple[_t.Callable[[ReqresExpr[DeferredSourceType]], None],
                 _t.Callable[[], None]]:
    terminator = cargs.terminator if cargs.dry_run is not None else None

    # current memory consumption
    mem = Memory()
    # for each `--output` value, how many times it was seen
    seen_counter : SeenCounter[_t.AnyStr] = SeenCounter(mem)

    # ReqresExpr cache indexed by destination path, this exists mainly
    # to minimize the number of calls to `stat`.
    rrexpr_cache : _c.OrderedDict[_t.AnyStr, ReqresExpr[DeferredSourceType]] = _c.OrderedDict()

    # Deferred IO operations (aka "intents") that are yet to be executed,
    # indexed by filesystem paths. This is used both as a queue and as an
    # LRU-cache so that, e.g. repeated updates to the same output file would be
    # computed in memory.
    deferred : _c.OrderedDict[_t.AnyStr, DeferredOperation[ReqresExpr[DeferredSourceType], _t.AnyStr]] = _c.OrderedDict()

    # Deferred file system updates. This collects references to everything
    # that should be fsynced to disk before proceeding to make flush_updates
    # below both atomic and efficient.
    dsync = DeferredSync()

    max_memory_mib = cargs.max_memory * 1024 * 1024
    def flush_updates(final : bool) -> None:
        """Flush some of the queue."""
        max_deferred : int = cargs.max_deferred if not final else 0
        max_memory : int = max_memory_mib if not final else 0
        max_seen : int = cargs.max_seen
        max_cached : int = cargs.max_cached
        max_batched : int = cargs.max_batched

        num_deferred = len(deferred)
        num_cached = len(rrexpr_cache)
        num_seen = len(seen_counter)
        if num_deferred <= max_deferred and \
           num_cached <= max_cached and \
           num_seen <= max_seen and \
           mem.consumption <= max_memory:
            return

        done_files : list[_t.AnyStr] | None = None
        if terminator is not None:
            done_files = []

        def run_intent(abs_out_path : _t.AnyStr, intent : DeferredOperation[ReqresExpr[DeferredSourceType], _t.AnyStr]) -> None:
            mem.consumption -= intent.approx_size() + len(abs_out_path)
            rrexpr = intent.source

            if not cargs.quiet:
                if cargs.dry_run:
                    ing = gettext(f"dry-run: (not) {actioning}")
                else:
                    ing = gettext(actioning)

                stderr.write_str(ing)
                stderr.write_str(": `")
                stderr.write(rrexpr.show_source())
                stderr.write_str("` -> `")
                stderr.write(abs_out_path)
                stderr.write_str_ln("`")
                stderr.flush()

            try:
                if not cargs.dry_run:
                    intent.run(dsync)
            except Failure as exc:
                if cargs.errors == "ignore":
                    return
                exc.elaborate(gettext(f"while {actioning} `%s` -> `%s`"),
                              rrexpr.show_source(),
                              fsdecode_maybe(abs_out_path))
                if cargs.errors != "fail":
                    _logging.error("%s", str(exc))
                    return
                # raise CatastrophicFailure so that load_map_orderly wouldn't try handling it
                raise CatastrophicFailure("%s", str(exc))

            if done_files is not None:
                done_files.append(abs_out_path)

            old_rrexpr = rrexpr_cache.pop(abs_out_path, None)
            if old_rrexpr is not None:
                mem.consumption -= old_rrexpr.approx_size() + len(abs_out_path)
            rrexpr_cache[abs_out_path] = rrexpr
            mem.consumption += rrexpr.approx_size() + len(abs_out_path)

        # flush seen cache
        while num_seen > 0 and \
              (num_seen > max_seen or mem.consumption > max_memory):
            abs_out_path, _ = seen_counter.pop()
            num_seen -= 1

            # if we are over the `--seen-number` not only we must forget
            # about older files, we must also run all operations on the paths
            # we are eliminating so that later deferredIO could pick up newly
            # created files and number them properly
            intent = deferred.pop(abs_out_path, None)
            if intent is None:
                continue
            run_intent(abs_out_path, intent)
            num_deferred -= 1

        if not final and \
           num_deferred <= max_deferred + max_batched and \
           mem.consumption <= max_memory:
            # we have enough resources to delay some more deferredIO, let's do
            # so, so that when we finally hit our resource limits, we would
            # execute max_batched or more deferred actions at once. this
            # improves IO performance by a lot
            max_deferred += max_batched

        # flush deferred
        while num_deferred > 0 and \
              (num_deferred > max_deferred or mem.consumption > max_memory):
            abs_out_path, intent = deferred.popitem(False)
            run_intent(abs_out_path, intent)
            num_deferred -= 1

        # fsync
        dsync.sync()

        # report to stdout
        if done_files is not None:
            for abs_out_path in done_files:
                stdout.write(abs_out_path)
                stdout.write_bytes(terminator)

            stdout.flush()
            fsync_maybe(stdout.fobj.fileno())

        # delete source files when doing --move, etc
        dsync.finish()

        # flush rrexpr_cache
        while num_cached > 0 and \
              (num_cached > max_cached or mem.consumption > max_memory):
            abs_out_path, source = rrexpr_cache.popitem(False)
            num_cached -= 1
            mem.consumption -= source.approx_size() + len(abs_out_path)

    def finish_updates() -> None:
        """Flush all of the queue."""
        flush_updates(True)
        assert mem.consumption == 0

    def load_defer(prev_rrexpr : ReqresExpr[DeferredSourceType] | None,
                   new_rrexpr : ReqresExpr[DeferredSourceType],
                   abs_out_path : _t.AnyStr) \
                   -> tuple[bool, ReqresExpr[DeferredSourceType] | None, DeferredOperation[ReqresExpr[DeferredSourceType], _t.AnyStr] | None]:
        new_source = new_rrexpr.source
        if isinstance(new_source, FileSource) and new_source.path == abs_out_path:
            # hot evaluation path: moving, copying, hardlinking, symlinking, etc to itself
            # this is a noop
            return True, prev_rrexpr, None

        old_rrexpr : ReqresExpr[_t.Any]
        if prev_rrexpr is None:
            try:
                out_lstat = _os.lstat(abs_out_path)
            except FileNotFoundError:
                # target does not exists
                return True, new_rrexpr, defer(new_rrexpr, abs_out_path, False, allow_updates)
            except OSError as exc:
                handle_ENAMETOOLONG(exc, abs_out_path)
                raise exc
            else:
                if _stat.S_ISLNK(out_lstat.st_mode):
                    # abs_out_path is a symlink
                    try:
                        # check symlink target is reachable
                        out_stat = _os.stat(abs_out_path)
                    except FileNotFoundError:
                        # target is a broken symlink
                        return True, new_rrexpr, defer(new_rrexpr, abs_out_path, True, allow_updates)
                    else:
                        if not allow_updates and not symlinking:
                            raise Failure("destination exists and is a symlink" + not_allowed)

                        # get symlink target and use it as abs_out_path, thus
                        # (SETSRC) below will re-create the original source
                        abs_in_path = _os.path.realpath(abs_out_path)

                        if isinstance(new_source, FileSource) and new_source.path == abs_out_path:
                            # similarly to the above
                            return True, prev_rrexpr, None
                elif not allow_updates and symlinking:
                    raise Failure("destination exists and is not a symlink" + not_allowed)
                else:
                    out_stat = out_lstat
                    abs_in_path = abs_out_path

            # (SETSRC)
            old_rrexpr = rrexpr_wrr_loadf(abs_in_path, out_stat)
            old_rrexpr.sniff = cargs.sniff
        else:
            old_rrexpr = prev_rrexpr

        intent = defer(old_rrexpr, abs_out_path, False, allow_updates)
        permitted = intent.switch_source(new_rrexpr, moving)
        return permitted, intent.source, (intent if intent.source is not old_rrexpr else None)

    def emit(new_rrexpr : ReqresExpr[DeferredSourceType]) -> None:
        if want_stop: raise KeyboardInterrupt()

        new_rrexpr.values["num"] = 0
        def_out_path = output_format % new_rrexpr
        prev_rel_out_path = None
        while True:
            new_rrexpr.values["num"] = seen_counter.count(def_out_path)
            if isinstance(destination, str):
                rel_out_path = _os.path.join(destination, output_format % new_rrexpr)
            else:
                rel_out_path = _os.path.join(destination, _os.fsencode(output_format % new_rrexpr))
            abs_out_path : _t.AnyStr = _os.path.abspath(rel_out_path)

            old_rrexpr : ReqresExpr[DeferredSourceType] | None
            old_rrexpr = rrexpr_cache.pop(abs_out_path, None)
            if old_rrexpr is not None:
                mem.consumption -= old_rrexpr.approx_size() + len(abs_out_path)

            updated_rrexpr : ReqresExpr[DeferredSourceType] | None
            intent : DeferredOperation[ReqresExpr[DeferredSourceType], _t.AnyStr] | None
            intent = deferred.pop(abs_out_path, None)
            if intent is None:
                try:
                    permitted, updated_rrexpr, intent = load_defer(old_rrexpr, new_rrexpr, abs_out_path)
                except Failure as exc:
                    exc.elaborate(gettext(f"while {actioning} `%s` -> `%s`"),
                                  new_rrexpr.show_source(),
                                  fsdecode_maybe(abs_out_path))
                    raise exc

                if updated_rrexpr is not None:
                    updated_rrexpr.unload()
            else:
                mem.consumption -= intent.approx_size() + len(abs_out_path)
                permitted = intent.switch_source(new_rrexpr, moving) # (switchSource)
                updated_rrexpr = intent.source

            del old_rrexpr

            if intent is not None:
                deferred[abs_out_path] = intent
                mem.consumption += intent.approx_size() + len(abs_out_path)

            if updated_rrexpr is not None:
                rrexpr_cache[abs_out_path] = updated_rrexpr
                mem.consumption += updated_rrexpr.approx_size() + len(abs_out_path)

            if not permitted:
                if prev_rel_out_path == rel_out_path:
                    exc2 = Failure(gettext("destination already exists") + variance_help)
                    exc2.elaborate(gettext(f"while {actioning} `%s` -> `%s`"),
                                   new_rrexpr.show_source(),
                                   fsdecode_maybe(abs_out_path))
                    raise exc2
                prev_rel_out_path = rel_out_path
                continue

            if intent is None:
                # noop
                if terminator is not None:
                    stdout.write(abs_out_path)
                    stdout.write_bytes(terminator)

            break

        if not cargs.lazy:
            flush_updates(False)

    return emit, finish_updates

def make_organize_emit(cargs : _t.Any, destination : _t.AnyStr, output_format : _t.AnyStr, allow_updates : bool) \
        -> tuple[_t.Callable[[ReqresExpr[DeferredSourceType]], None],
                 _t.Callable[[], None]]:
    action_op : _t.Any
    action = cargs.action
    if allow_updates:
        actioning = "updating " + action
    else:
        actioning = action + "ing"
    moving = False
    copying = False
    symlinking = False
    check_data = True
    if action == "move":
        if allow_updates:
            raise Failure(gettext("`--move` and `--latest` are not allowed together, it will lose your data"))
        actioning = "moving"
        action_op = atomic_move
        moving = True
    elif action == "copy":
        if allow_updates:
            raise Failure(gettext("`--copy` and `--latest` are not allowed together at the moment, it could lose your data"))
        action_op = atomic_copy2
        copying = True
    elif action == "hardlink":
        if allow_updates:
            raise Failure(gettext("`--hardlink` and `--latest` are not allowed together at the moment, it could lose your data"))
        action_op = atomic_link
    elif action == "symlink":
        action_op = atomic_symlink
        check_data = False
        symlinking = True
    else:
        assert False

    # becase we can't explicitly reuse the type variables bound by the whole function above
    DeferredSourceType2 = _t.TypeVar("DeferredSourceType2", bound=DeferredSource)
    AnyStr2 = _t.TypeVar("AnyStr2", str, bytes)

    class DeferredOrganize(DeferredFileWrite[ReqresExpr[DeferredSourceType2], AnyStr2], _t.Generic[DeferredSourceType2, AnyStr2]):
        def switch_source(self, new_rrexpr : ReqresExpr[DeferredSourceType2], force : bool = False) -> bool:
            old_rrexpr = self.source
            old_source = old_rrexpr.source
            new_source = new_rrexpr.source

            if not force:
                if not new_source.replaces(old_source) or \
                   new_source.same_as(old_source) or \
                   check_data and new_rrexpr.get_bytes() == old_rrexpr.get_bytes():
                    # noop
                    return True

            if not self.allow_updates:
                # updates are not allowed
                return False

            if old_rrexpr.stime > new_rrexpr.stime:
                # ours is newer
                return True

            # update
            self.source = new_rrexpr
            self.updated = True
            return True

        def run(self, dsync : DeferredSync | None = None) -> None:
            rrexpr = self.source
            source = rrexpr.source
            if isinstance(source, FileSource) and source.path == self.destination:
                # noop, this could happen after (switchSource)
                return

            try:
                if isinstance(source, FileSource):
                    action_op(source.path, self.destination, self.overwrite or self.allow_updates, dsync = dsync)
                elif copying:
                    # fallback to DeferredFileWrite in this case
                    super().run(dsync)
                else:
                    raise Failure(gettext(f"can't {action} the source to the destination because the source is not stored as a separate WRR file; did you mean to run with `--copy` intead of `--{action}`?"))
            except FileExistsError:
                raise Failure(gettext(f"`%s` already exists"), self.destination)
            except OSError as exc:
                handle_ENAMETOOLONG(exc, self.destination)
                if exc.errno == _errno.EXDEV:
                    raise Failure(gettext(f"can't {action} across file systems"))
                raise exc

    return make_deferred_emit(cargs, destination, output_format, action, actioning, DeferredOrganize, allow_updates, moving, symlinking)

def cmd_organize(cargs : _t.Any) -> None:
    if cargs.walk_paths == "unset":
        cargs.walk_paths = None if not cargs.allow_updates else False
    if cargs.walk_fs == "unset":
        cargs.walk_fs = True if not cargs.allow_updates else False

    output_format = elaborate_output("--output", output_alias, cargs.output) + ".wrr"
    handle_paths(cargs)

    rrexprs_load = mk_rrexprs_load(cargs)
    _num, filters_allow, filters_warn = compile_filters(cargs)

    emit : EmitFunc[ReqresExpr[FileSource]]
    if cargs.destination is not None:
        # destination is set explicitly
        emit, finish = make_organize_emit(cargs, _os.path.expanduser(cargs.destination), output_format, cargs.allow_updates)
        try:
            map_wrr_paths(cargs, rrexprs_load, filters_allow, emit, cargs.paths)
        finally:
            finish()
    else:
        if cargs.allow_updates:
            raise Failure(gettext("`--latest` without `--to` is not allowed"))

        # each path is its own destination
        for exp_path in cargs.paths:
            try:
                path_stat = _os.stat(exp_path)
            except FileNotFoundError:
                raise Failure(gettext("`%s` does not exist"), exp_path)

            if not _stat.S_ISDIR(path_stat.st_mode):
                raise Failure(gettext("`%s` is not a directory but no `--to` is specified"), exp_path)

        for exp_path in cargs.paths:
            emit, finish = make_organize_emit(cargs, exp_path, output_format, False)
            try:
                map_wrr_paths(cargs, rrexprs_load, filters_allow, emit, [exp_path])
            finally:
                finish()

    filters_warn()

def cmd_import_generic(cargs : _t.Any,
                       rrexprs_loadf : _t.Callable[[str | bytes], _t.Iterator[ReqresExpr[DeferredSourceType]]]) -> None:
    output_format = elaborate_output("--output", output_alias, cargs.output) + ".wrr"
    handle_paths(cargs)

    _num, filters_allow, filters_warn = compile_filters(cargs)

    emit : EmitFunc[ReqresExpr[DeferredSourceType]]
    emit, finish = make_deferred_emit(cargs, cargs.destination, output_format, "import", "importing", DeferredFileWrite, cargs.allow_updates)
    try:
        map_wrr_paths(cargs, rrexprs_loadf, filters_allow, emit, cargs.paths)
    finally:
        finish()

    filters_warn()

def cmd_import_bundle(cargs : _t.Any) -> None:
    cmd_import_generic(cargs, rrexprs_wrr_bundle_loadf)

def cmd_import_mitmproxy(cargs : _t.Any) -> None:
    from .mitmproxy import rrexprs_mitmproxy_loadf
    cmd_import_generic(cargs, rrexprs_mitmproxy_loadf)

def path_to_url(x : str) -> str:
    return x.replace("?", "%3F")

def cmd_export_mirror(cargs : _t.Any) -> None:
    if len(cargs.exprs) == 0:
        cargs.exprs = [compile_expr(default_expr[cargs.default_expr])]

    handle_paths(cargs)
    elaborate_paths(cargs.boring)

    destination = _os.path.expanduser(cargs.destination)
    output_format = elaborate_output("--output", output_alias, cargs.output)

    if cargs.content_destination is not None:
        content_destination = _os.path.expanduser(cargs.content_destination)
    else:
        content_destination = destination
    content_output_format = elaborate_output("--content-output", content_output_alias, cargs.content_output)

    action_op : _t.Any
    action = cargs.content_action
    copying = False
    if action == "copy":
        copying = True
    elif action == "hardlink":
        action_op = atomic_link
    elif action == "symlink":
        action_op = atomic_symlink
    else:
        assert False

    if cargs.absolute is None:
        relative = action != "symlink"
    else:
        relative = not cargs.absolute

    max_depth : int = cargs.depth
    sniff = cargs.sniff
    allow_updates = cargs.allow_updates == True
    skip_existing = cargs.allow_updates == "partial"

    mem = Memory()
    seen_counter : SeenCounter[str] = SeenCounter(mem)

    max_memory_mib = cargs.max_memory * 1024 * 1024

    @_dc.dataclass
    class Indexed:
        stime : Epoch
        rrexpr : ReqresExpr[_t.Any]
        _abs_out_path : str | None = _dc.field(default = None)
        _rrapprox_size : int = 0

        def __post_init__(self) -> None:
            self._rrapprox_size = res = self.rrexpr.approx_size()
            mem.consumption += 40 + res

        def __del__(self) -> None:
            mem.consumption -= 40 + self._rrapprox_size + \
                (len(self._abs_out_path) if self._abs_out_path is not None else 0)

        @property
        def abs_out_path(self) -> str:
            abs_out_path = self._abs_out_path
            if abs_out_path is not None:
                return abs_out_path

            rrexpr = self.rrexpr
            rrexpr.values["num"] = 0
            def_out_path = output_format % rrexpr
            rrexpr.values["num"] = seen_counter.count(def_out_path)
            rel_out_path = _os.path.join(destination, output_format % rrexpr)
            self._abs_out_path = abs_out_path = _os.path.abspath(rel_out_path)
            mem.consumption += len(abs_out_path)
            return abs_out_path

        def unload(self) -> None:
            rrexpr = self.rrexpr
            rrexpr.unload()
            res = rrexpr.approx_size()
            mem.consumption += res - self._rrapprox_size
            self._rrapprox_size = res

    NetURLType : _t.TypeAlias = str
    PathType : _t.TypeAlias = str
    PageIDType = tuple[Epoch, NetURLType]
    NetURLOrPageIDType = NetURLType | PageIDType

    # `Indexed` objects migrate from `index` to `queue` or `new_queue`, from
    # `new_queue` to `queue`.
    index : _c.defaultdict[NetURLType, list[Indexed]] = _c.defaultdict(list)
    Queue = _c.OrderedDict[NetURLOrPageIDType, Indexed]
    queue : Queue = _c.OrderedDict()
    done : dict[PageIDType, PathType | None] = dict()

    class Stats:
        indexed = 0
        n = 0
        doc_n = 0

    multiples : bool
    which : bool | Epoch | None
    multiples, which = cargs.mode

    def is_replaced_by(oobj : Indexed, nobj : Indexed) -> bool:
        if isinstance(which, bool):
            if not which:
                return oobj.stime > nobj.stime
            else:
                return oobj.stime <= nobj.stime
        elif which is not None:
            return abs(which - nobj.stime) < abs(which - oobj.stime)
        else:
            return False

    def from_index(stime : Epoch, net_url : NetURLType, precise : bool) -> Indexed | None:
        iobjs = index.get(net_url, None)
        if iobjs is None:
            # unavailable
            return None

        ilen = len(iobjs)
        if ilen == 1 or not multiples:
            return iobjs[0]
        elif which is not None and not precise:
            if isinstance(which, bool):
                if not which:
                    # `--oldest*`
                    return iobjs[0]
                else:
                    # `--latest*`
                    return iobjs[-1]
            else:
                # `--nearest-*`
                stime = which

        pos = _bisect.bisect_right(iobjs, stime, key=lambda x: x.stime)
        if pos == 0:
            return iobjs[0]
        elif pos >= ilen:
            return iobjs[-1]

        iprev = iobjs[pos - 1]
        inext = iobjs[pos]
        if abs(stime - iprev.stime) < abs(stime - inext.stime):
            return iprev
        else:
            return inext

    def report_queued(stime : Epoch, net_url : NetURLType, pretty_net_url : NetURLType, source : DeferredSourceType, level : int, old_stime : Epoch | None = None) -> None:
        if stdout.isatty:
            stdout.write_bytes(b"\033[33m")
        durl = net_url if pretty_net_url == net_url else f"{net_url} ({pretty_net_url})"
        ispace = " " * (2 * level)
        if old_stime is None:
            stdout.write_str_ln(ispace + gettext(f"queued [%s] %s from %s") % (stime.format(), durl, source.show_source()))
        else:
            stdout.write_str_ln(ispace + gettext(f"requeued [%s] -> [%s] %s from %s") % (old_stime.format(), stime.format(), durl, source.show_source()))
        if stdout.isatty:
            stdout.write_bytes(b"\033[0m")
        stdout.flush()

    if cargs.default_filters:
        cargs.request_method += ["GET", "DOM"]
        cargs.status_re += [".200C"]

    _num, filters_allow, filters_warn = compile_filters(cargs)
    rrexprs_load = mk_rrexprs_load(cargs)

    root_filters_num, root_filters_allow, root_filters_warn = compile_filters(cargs, "root_")
    have_root_filters = root_filters_num > 0

    if stdout.isatty:
        stdout.write_bytes(b"\033[32m")
    stdout.write_str_ln(gettext("loading input `PATH`s..."))
    if stdout.isatty:
        stdout.write_bytes(b"\033[0m")
    stdout.flush()

    def collect(enqueue_all : bool) -> EmitFunc[ReqresExpr[DeferredSourceType]]:
        def emit(rrexpr : ReqresExpr[DeferredSourceType]) -> None:
            stime = rrexpr.stime
            net_url = rrexpr.net_url

            nobj = Indexed(stime, rrexpr)

            iobjs = index.get(net_url, None)
            if iobjs is None:
                # first time seeing this `net_url`
                index[net_url] = [nobj]
            elif not multiples:
                if is_replaced_by(iobjs[0], nobj):
                    iobjs[0] = nobj
            else:
                pos = _bisect.bisect_right(iobjs, stime, key=lambda x: x.stime)
                # NB: technically, this is O(N), but in reality this is usually O(1)
                # for us, since most inserts are to the end of the list
                iobjs.insert(pos, nobj)
                Stats.indexed += 1

            unqueued = True
            page_id = (stime, net_url)
            for pid in [net_url, page_id]:
                qobj = queue.get(pid, None)
                if qobj is not None:
                    if is_replaced_by(qobj, nobj):
                        queue[pid] = nobj
                        report_queued(stime, net_url, rrexpr.pretty_net_url, rrexpr.source, 1, qobj.stime)
                    unqueued = False

            if unqueued:
                if enqueue_all or have_root_filters and root_filters_allow(rrexpr):
                    if multiples and which is None:
                        # `--all`
                        pid = page_id
                    else:
                        pid = net_url
                    queue[pid] = nobj
                    report_queued(stime, net_url, rrexpr.pretty_net_url, rrexpr.source, 1)
                    unqueued = False

            if unqueued or mem.consumption > max_memory_mib:
                nobj.unload()
        return emit

    seen_paths : set[PathType] = set()
    map_wrr_paths(cargs, rrexprs_load, filters_allow, collect(not have_root_filters), cargs.paths, seen_paths=seen_paths)
    map_wrr_paths(cargs, rrexprs_load, filters_allow, collect(False), cargs.boring, seen_paths=seen_paths)

    if multiples:
        total = Stats.indexed
    else:
        total = len(index)

    depth : int = 0

    def remap_url_fallback(stime : Epoch, expected_content_types : list[str], purl : ParsedURL) -> PathType:
        trrexpr = ReqresExpr(UnknownSource(), fallback_Reqres(purl, expected_content_types, stime, stime, stime))
        trrexpr.values["num"] = 0
        rel_out_path : PathType = _os.path.join(destination, output_format % trrexpr)
        return _os.path.abspath(rel_out_path)

    def render(stime : Epoch,
               net_url : NetURLType,
               rrexpr : ReqresExpr[DeferredSourceType],
               abs_out_path : PathType,
               enqueue : bool,
               new_queue : Queue,
               level : int) -> PathType | None:
        page_id = (stime, net_url)
        source = rrexpr.source

        level0 = level == 0
        Stats.n += 1
        if level0:
            Stats.doc_n += 1
        n = Stats.n
        doc_n = Stats.doc_n
        n100 = 100 * n
        n_total = n + len(new_queue) + len(queue)

        if stdout.isatty:
            if level0:
                stdout.write_bytes(b"\033[32m")
            else:
                stdout.write_bytes(b"\033[34m")
        ispace = " " * (2 * level)
        if level0:
            stdout.write_str_ln(gettext(f"exporting input #%d, %.2f%% of %d queued (%.2f%% of %d indexed), document #%d, depth %d") % (n, n100 / n_total, n_total, n100 / total, total, doc_n, depth))
        else:
            stdout.write_str_ln(ispace + gettext(f"exporting requisite input #%d, %.2f%% of %d queued (%.2f%% of %d indexed)") % (n, n100 / n_total, n_total, n100 / total, total))
        stdout.write_str_ln(ispace + gettext("stime [%s]") % (stime.format(),))
        stdout.write_str_ln(ispace + gettext("net_url %s") % (net_url,))
        stdout.write_str_ln(ispace + gettext("src %s") % (source.show_source(),))
        if stdout.isatty:
            stdout.write_bytes(b"\033[0m")
        stdout.flush()

        try:
            done[page_id] = None # (breakCycles)

            document_dir = _os.path.dirname(abs_out_path)
            remap_cache : dict[tuple[str, bool], str] = dict()

            def remap_url(link_type : LinkType, fallbacks : list[str] | None, url : NetURLType) -> NetURLType | None:
                if want_stop: raise KeyboardInterrupt()

                is_requisite = link_type == LinkType.REQ
                cache_id = (url, is_requisite)
                try:
                    return remap_cache[cache_id]
                except KeyError:
                    pass

                try:
                    purl = parse_url(url)
                except URLParsingError:
                    ispace = " " * (2 * (level + 1))
                    issue(ispace + gettext("malformed URL `%s`"), url)
                    remap_cache[cache_id] = res = get_void_url(link_type)
                    return res

                unet_url = purl.net_url

                if unet_url == net_url:
                    # this is a reference to an inter-page `id`
                    remap_cache[cache_id] = res = purl.ofm + purl.fragment
                    return res
                elif purl.scheme not in Reqres_url_schemes:
                    ispace = " " * (2 * (level + 1))
                    issue(ispace + gettext("not remapping `%s`"), url)
                    remap_cache[cache_id] = url
                    return url

                uobj = from_index(stime, unet_url, is_requisite)
                if uobj is None:
                    # unavailable
                    uabs_out_path = None
                else:
                    ustime = uobj.stime
                    upage_id = (ustime, unet_url)

                    if is_requisite:
                        # use content_destination path here
                        try:
                            uabs_out_path = done[upage_id]
                        except KeyError:
                            # unqueue it
                            for q in [new_queue, queue]:
                                try:
                                    del q[upage_id]
                                except KeyError: pass
                                try:
                                    del q[unet_url]
                                except KeyError: pass

                            # render it immediately
                            # NB: (breakCycles) breaks dependency cycles that can make this loop infinitely
                            uabs_out_path = render(ustime, unet_url, uobj.rrexpr, uobj.abs_out_path, enqueue, new_queue, level + 1)
                            uobj.unload()
                    elif upage_id in done:
                        # nothing to do
                        uabs_out_path = uobj.abs_out_path
                        # NB: will be unloaded already
                    elif upage_id in new_queue or upage_id in queue or \
                         unet_url in new_queue or unet_url in queue:
                        # nothing to do
                        uabs_out_path = uobj.abs_out_path
                        if mem.consumption > max_memory_mib:
                            uobj.unload()
                    elif enqueue:
                        uabs_out_path = uobj.abs_out_path
                        new_queue[upage_id] = uobj
                        report_queued(ustime, unet_url, purl.pretty_net_url, uobj.rrexpr.source, level + 1)
                        if mem.consumption > max_memory_mib:
                            uobj.unload()
                    else:
                        # this will not be exported
                        uabs_out_path = None
                        # NB: Not setting `uobj._abs_out_path = None` here
                        # because it might be a requisite for another
                        # page. In which case, when not running with
                        # `--remap-all`, this page will void this
                        # `unet_url` unnecessarily, yes.

                if uabs_out_path is None:
                    if fallbacks is not None:
                        uabs_out_path = remap_url_fallback(stime, fallbacks, purl)
                    else:
                        return None

                if relative:
                    uabs_out_path = _os.path.relpath(uabs_out_path, document_dir)
                remap_cache[cache_id] = res = path_to_url(uabs_out_path) + purl.ofm + purl.fragment
                return res

            rrexpr.remap_url = remap_url

            old_data = read_whole_file_maybe(abs_out_path)

            data : bytes
            if skip_existing and old_data is not None:
                data = old_data

                if stdout.isatty:
                    stdout.write_bytes(b"\033[33m")
                stdout.write_str_ln(ispace + gettext("reusing file content: destination exists and `--skip-existing` is set"))
                if stdout.isatty:
                    stdout.write_bytes(b"\033[0m")
            else:
                with TIOWrappedWriter(_io.BytesIO()) as f:
                    print_exprs(rrexpr, cargs.exprs, cargs.separator, f)
                    data = f.fobj.getvalue()

            if copying:
                real_out_path = abs_out_path
                if old_data != data:
                    atomic_write(data, abs_out_path, allow_updates)
            else:
                rrexpr.values["content"] = data
                rrexpr.values["content_sha256"] = sha256_raw = _hashlib.sha256(data).digest()
                sha256_hex = sha256_raw.hex()

                real_out_path = _os.path.join(content_destination, content_output_format % rrexpr)

                old_content = read_whole_file_maybe(real_out_path)

                if old_content is None:
                    atomic_write(data, real_out_path, False)
                elif old_content != data:
                    raise Failure(gettext("wrong file content in `%s`: expected sha256 `%s`, got sha256 `%s`"), real_out_path, sha256_hex, _hashlib.sha256(old_content).hexdigest())

                if old_data != data:
                    action_op(real_out_path, abs_out_path, allow_updates)

                stdout.write_str_ln(ispace + gettext("content_dst %s") % (real_out_path,))

            stdout.write_str_ln(ispace + gettext("dst %s") % (abs_out_path,))
            stdout.flush()

            done[page_id] = real_out_path
            return real_out_path
        except Failure as exc:
            if cargs.errors == "ignore":
                return abs_out_path
            exc.elaborate(gettext(f"while processing `%s`"), source.show_source())
            if cargs.errors != "fail":
                _logging.error("%s", str(exc))
                return abs_out_path
            raise CatastrophicFailure("%s", str(exc))
        except Exception:
            error(gettext("while processing `%s`"), source.show_source())
            raise

    while len(queue) > 0:
        if want_stop: raise KeyboardInterrupt()

        new_queue : Queue = _c.OrderedDict()
        enqueue = depth < max_depth

        while len(queue) > 0:
            if want_stop: raise KeyboardInterrupt()

            pid, iobj = queue.popitem(False)
            if isinstance(pid, NetURLType):
                net_url = pid
            else:
                net_url = pid[1]
            render(iobj.stime, net_url, iobj.rrexpr, iobj.abs_out_path, enqueue, new_queue, 0)
            iobj.unload()

        queue = new_queue
        depth += 1

    filters_warn()
    root_filters_warn()

def add_doc(fmt : argparse.BetterHelpFormatter) -> None:
    _ : _t.Callable[[str], str] = gettext

    fmt.add_text(_("# Examples"))

    fmt.start_section(_("Pretty-print all reqres in `../simple_server/pwebarc-dump` using an abridged (for ease of reading and rendering) verbose textual representation"))
    fmt.add_code(f"{__prog__} pprint ../simple_server/pwebarc-dump")
    fmt.end_section()

    fmt.start_section(_("Pipe raw response body from a given `WRR` file to stdout"))
    fmt.add_code(f'{__prog__} get ../simple_server/pwebarc-dump/path/to/file.wrr')
    fmt.end_section()

    fmt.start_section(_(f"Pipe response body scrubbed of dynamic content from a given `WRR` file to stdout"))
    fmt.add_code(f'{__prog__} get -e "response.body|eb|scrub response defaults" ../simple_server/pwebarc-dump/path/to/file.wrr')
    fmt.end_section()

    fmt.start_section(_("Get first 2 bytes (4 characters) of a hex digest of sha256 hash computed on the URL without the fragment/hash part"))
    fmt.add_code(f'{__prog__} get -e "net_url|to_ascii|sha256|take_prefix 2|to_hex" ../simple_server/pwebarc-dump/path/to/file.wrr')
    fmt.end_section()

    fmt.start_section(_("Pipe response body from a given `WRR` file to stdout, but less efficiently, by generating a temporary file and giving it to `cat`"))
    fmt.add_code(f"{__prog__} run cat ../simple_server/pwebarc-dump/path/to/file.wrr")
    fmt.add_text(_(f"Thus `{__prog__} run` can be used to do almost anything you want, e.g."))
    fmt.add_code(f"{__prog__} run less ../simple_server/pwebarc-dump/path/to/file.wrr")
    fmt.add_code(f"{__prog__} run -- sort -R ../simple_server/pwebarc-dump/path/to/file.wrr")
    fmt.add_code(f"{__prog__} run -n 2 -- diff -u ../simple_server/pwebarc-dump/path/to/file-v1.wrr ../simple_server/pwebarc-dump/path/to/file-v2.wrr")
    fmt.end_section()

    fmt.start_section(_(f"""List paths of all `WRR` files from `../simple_server/pwebarc-dump` that contain complete `200 OK` responses with `text/html` bodies larger than 1K"""))
    fmt.add_code(f"""{__prog__} find --status-re .200C --response-mime text/html --and "response.body|len|> 1024" ../simple_server/pwebarc-dump""")
    fmt.end_section()

    fmt.start_section(_(f"Rename all `WRR` files in `../simple_server/pwebarc-dump/default` according to their metadata using `--output default` (see the `{__prog__} organize` section for its definition, the `default` format is designed to be human-readable while causing almost no collisions, thus making `num` substitution parameter to almost always stay equal to `0`, making things nice and deterministic)"))
    fmt.add_code(f"{__prog__} organize ../simple_server/pwebarc-dump/default")
    fmt.add_text(_("alternatively, just show what would be done"))
    fmt.add_code(f"{__prog__} organize --dry-run ../simple_server/pwebarc-dump/default")
    fmt.end_section()

    fmt.add_text(_("# Advanced examples"))

    fmt.start_section(_("Pretty-print all reqres in `../simple_server/pwebarc-dump` by dumping their whole structure into an abridged Pythonic Object Representation (repr)"))
    fmt.add_code(f"{__prog__} stream --expr . ../simple_server/pwebarc-dump")
    fmt.add_code(f"{__prog__} stream -e . ../simple_server/pwebarc-dump")
    fmt.end_section()

    fmt.start_section(_("Pretty-print all reqres in `../simple_server/pwebarc-dump` using the unabridged verbose textual representation"))
    fmt.add_code(f"{__prog__} pprint --unabridged ../simple_server/pwebarc-dump")
    fmt.add_code(f"{__prog__} pprint -u ../simple_server/pwebarc-dump")
    fmt.end_section()

    fmt.start_section(_("Pretty-print all reqres in `../simple_server/pwebarc-dump` by dumping their whole structure into the unabridged Pythonic Object Representation (repr) format"))
    fmt.add_code(f"{__prog__} stream --unabridged --expr . ../simple_server/pwebarc-dump")
    fmt.add_code(f"{__prog__} stream -ue . ../simple_server/pwebarc-dump")
    fmt.end_section()

    fmt.start_section(_("Produce a `JSON` list of `[<file path>, <time it finished loading in seconds since UNIX epoch>, <URL>]` tuples (one per reqres) and pipe it into `jq` for indented and colored output"))
    fmt.add_code(f"{__prog__} stream --format=json -ue fs_path -e finished_at -e request.url ../simple_server/pwebarc-dump | jq .")
    fmt.end_section()

    fmt.start_section(_("Similarly, but produce a `CBOR` output"))
    fmt.add_code(f"{__prog__} stream --format=cbor -ue fs_path -e finished_at -e request.url ../simple_server/pwebarc-dump | less")
    fmt.end_section()

    fmt.start_section(_("Concatenate all response bodies of all the requests in `../simple_server/pwebarc-dump`"))
    fmt.add_code(f'{__prog__} stream --format=raw --not-terminated -ue "response.body|eb" ../simple_server/pwebarc-dump | less')
    fmt.end_section()

    fmt.start_section(_("Print all unique visited URLs, one per line"))
    fmt.add_code(f"{__prog__} stream --format=raw --lf-terminated -ue request.url ../simple_server/pwebarc-dump | sort | uniq")
    fmt.end_section()

    fmt.start_section(_("Same idea, but using NUL bytes, with some post-processing, and two URLs per line"))
    fmt.add_code(f"{__prog__} stream --format=raw --zero-terminated -ue request.url ../simple_server/pwebarc-dump | sort -z | uniq -z | xargs -0 -n2 echo")
    fmt.end_section()

    fmt.add_text(_("## How to handle binary data"))

    fmt.add_text(_(f"Trying to use response bodies produced by `{__prog__} stream --format=json` is likely to result garbled data as `JSON` can't represent raw sequences of bytes, thus binary data will have to be encoded into UNICODE using replacement characters:"))
    fmt.add_code(f"{__prog__} stream --format=json -ue . ../simple_server/pwebarc-dump/path/to/file.wrr | jq .")
    fmt.add_text(_("The most generic solution to this is to use `--format=cbor` instead, which would produce a verbose `CBOR` representation equivalent to the one used by `--format=json` but with binary data preserved as-is:"))
    fmt.add_code(f"{__prog__} stream --format=cbor -ue . ../simple_server/pwebarc-dump/path/to/file.wrr | less")
    fmt.add_text(_("Or you could just dump raw response bodies separately:"))
    fmt.add_code(f"{__prog__} stream --format=raw -ue response.body ../simple_server/pwebarc-dump/path/to/file.wrr | less")
    fmt.add_code(f"{__prog__} get ../simple_server/pwebarc-dump/path/to/file.wrr | less")

class ArgumentParser(argparse.BetterArgumentParser):
    def error(self, message : str) -> _t.NoReturn:
        self.print_usage(_sys.stderr)
        die(2, "%s", message)

def main() -> None:
    _ : _t.Callable[[str], str] = gettext

    parser = ArgumentParser(
        prog=__prog__,
        description=_("A tool to display, search, programmatically extract values from, organize, manipulate, import, and export Web Request+Response (`WRR`) archive files produced by the `Hoardy-Web` Web Extension browser add-on.") + "\n\n" +
_("Glossary: a `reqres` (`Reqres` when a Python type) is an instance of a structure representing `HTTP` request+response pair with some additional metadata."),
        additional_sections = [add_doc],
        allow_abbrev = False,
        add_version = True,
        add_help = False)
    parser.add_argument("-h", "--help", action="store_true", help=_("show this help message and exit"))
    parser.add_argument("--markdown", action="store_true", help=_("show help messages formatted in Markdown"))

    subparsers = parser.add_subparsers(title="subcommands")

    def no_cmd(cargs : _t.Any) -> None:
        parser.print_help(stderr) # type: ignore
        _sys.exit(2)
    parser.set_defaults(func=no_cmd)

    def add_errors(cmd : _t.Any) -> None:
        grp = cmd.add_argument_group("error handling")
        grp.add_argument("--errors", choices=["fail", "skip", "ignore"], default="fail", help=_("""when an error occurs:
- `fail`: report failure and stop the execution; default
- `skip`: report failure but skip the reqres that produced it from the output and continue
- `ignore`: `skip`, but don't report the failure"""))

    date_spec = _("; the `DATE` can be specified either as a number of seconds since UNIX epoch using `@<number>` format where `<number>` can be a floating point, or using one of the following formats:`YYYY-mm-DD HH:MM:SS[.NN*] (+|-)HHMM`, `YYYY-mm-DD HH:MM:SS[.NN*]`, `YYYY-mm-DD HH:MM:SS`, `YYYY-mm-DD HH:MM`, `YYYY-mm-DD`, `YYYY-mm`, `YYYY`; if no `(+|-)HHMM` part is specified, the `DATE` is assumed to be in local time; if other parts are unspecified they are inherited from `<year>-01-01 00:00:00.0`")
    date_spec_id = _("; the `DATE` format is the same as above")
    fullmatch_re = _("; this option matches the given regular expression against the whole input value; to match against any part of the input value, use `.*<re>.*` or `^.*<re>.*$`")
    ie_whitelist = _("; in short, this option defines a whitelisted element rule")
    ie_blacklist = _("; in short, this option defines a blacklisted element rule")

    def add_filter_options(cmd : _t.Any) -> None:
        agrp = cmd.add_argument_group("filtering options")
        grp = agrp.add_mutually_exclusive_group()
        grp.add_argument(f"--ignore-case", dest="ignore_case", action="store_const", const=True, help=_(f"when filtering with `--*grep*`, match case-insensitively"))
        grp.add_argument(f"--case-sensitive", dest="ignore_case", action="store_const", const=False, help=_(f"when filtering with `--*grep*`, match case-sensitively"))
        grp.add_argument(f"--smart-case", dest="ignore_case", action="store_const", const=None, help=_(f"when filtering with `--*grep*`, match case-insensitively if there are no uppercase letters in the corresponding `*PATTERN*` option argument and case-sensitively otherwise; default"))

    def add_filters(cmd : _t.Any, kind : str, do_what : str, root : bool = False) -> None:
        if not root:
            intro = gettext("input filters; if none are specified, then all reqres from input `PATH`s will be taken")
            opt_prefix = ""
            attr_prefix = ""
            root_short = []
        else:
            intro = gettext("recursion root filters; if none are specified, then all URLs available from input `PATH`s will be treated as roots (except for those given via `--boring`)")
            opt_prefix = "root-"
            attr_prefix = "root_"
            root_short = ["--root", "-r"]

        agrp = cmd.add_argument_group(intro + gettext(f"""; can be specified multiple times in arbitrary combinations; the resulting logical expression that will be checked is `all_of(before) and all_of(not_before) and all_of(after) and all_of(not_after) and any_of(protocol) and not any_of(not_protcol) and any_of(request_method) and not any_of(not_request_method) ... and any_of(grep) and not any_of(not_grep) and all_of(and_grep) and not all_of(not_and_grep) and all_of(ands) and any_of(ors)`"""))

        def add_time_filter(opt : str, what : str, sub : str) -> None:
            agrp.add_argument(f"--{opt_prefix}{opt}", metavar="DATE", action="append", type=str, default = [], help=_(f"{do_what} its `stime` is {what} than this") + sub)

        add_time_filter("before", "smaller", date_spec)
        add_time_filter("not-before", "larger or equal", date_spec_id)
        add_time_filter("after", "larger", date_spec_id)
        add_time_filter("not-after", "smaller or equal", date_spec_id)

        def map_opts(suffix : str, *opts : str) -> list[str]:
            return list(map(lambda opt: f"--{opt_prefix}{opt}{suffix}", opts))

        def add_str_filter(opt : str, yes : bool,
                           what : str, what_p : str, what_re : str,
                           *,
                           short : list[str] = [],
                           abs_short : list[str] = [], abs_short_p : list[str] = [], abs_short_re : list[str] = [],
                           sub : str = "", sub_p : str = "", sub_re : str = "") -> None:
            metavar = opt.upper().replace("-", "_")
            if yes:
                one_of, is_equal_to, is_a_prefix, matches, wb = "one of", "is equal to", "is a prefix", "matches", ie_whitelist
            else:
                one_of, is_equal_to, is_a_prefix, matches, wb = "none of", "are equal to", "are a prefix", "match", ie_blacklist

            agrp.add_argument(*map_opts("", opt, *short), *abs_short, metavar=metavar, action="append", type=str, default = [], help=_(f"{do_what} {one_of} the given `{metavar}` option arguments {is_equal_to} its {what}") + sub + wb)
            agrp.add_argument(*map_opts("-prefix", opt, *short), *abs_short_p, metavar=f"{metavar}_PREFIX", action="append", type=str, default = [], help=_(f"{do_what} {one_of} the given `{metavar}_PREFIX` option arguments {is_a_prefix} of its {what_p}") + sub_p + wb)
            agrp.add_argument(*map_opts("-re", opt, *short), *abs_short_re, metavar=f"{metavar}_RE", action="append", type=str, default = [], help=_(f"{do_what} {one_of} the given `{metavar}_RE` regular expressions {matches} its {what_re}") + sub_re + wb)

        def add_yn_str_filter(opt : str, *args : _t.Any, **kwargs : _t.Any) -> None:
            add_str_filter(opt, True, *args, **kwargs)
            kwargs["short"] = map(lambda opt: "not-" + opt, kwargs.get("short", []))
            add_str_filter("not-" + opt, False, *args, **kwargs)

        def add_field_filter(opt : str, field : str, *args : _t.Any, **kwargs : _t.Any) -> None:
            what = f"`{field}` (of `{__prog__} get --expr`, which see)"
            kwargs["sub_re"] = kwargs["sub_re"] + " " + fullmatch_re if "sub_re" in kwargs else fullmatch_re
            add_yn_str_filter(opt, what, what, what, *args, **kwargs)

        add_field_filter("protocol", "protocol")
        add_field_filter("request-method", "request.method", short = ["method"])
        add_field_filter("status", "status")

        mixed_url_allowed = (_("; Punycode UTS46 IDNAs, plain UNICODE IDNAs, percent-encoded URL components, and UNICODE URL components in arbitrary mixes and combinations are allowed; e.g. `%s` will be silently normalized into its Punycode UTS46 and percent-encoded version of `%s`, which will then be matched against") % (example_url[-1], example_url[-2])).replace('%', '%%')
        similarly_allowed = _("; similarly to the previous option, arbitrary mixes of URL encodinds are allowed")
        unmixed_reurl_allowed = _("; only Punycode UTS46 IDNAs with percent-encoded URL components or plain UNICODE IDNAs with UNICODE URL components are allowed; regular expressions that use mixes of differently encoded parts will fail to match properly")
        same_allowed = _("; option argument format and caveats are idential to the `not-`less option above")

        def add_url_filter(opt : str, field : str, yes : bool, **kwargs : _t.Any) -> None:
            what = f"`{field}` (of `{__prog__} get --expr`, which see)"
            add_str_filter(opt, yes,
                           what, what,
                           f"`{field}` or `pretty_{field}` (of `{__prog__} get --expr`, which see)",
                           sub = mixed_url_allowed if yes else same_allowed,
                           sub_p = similarly_allowed if yes else same_allowed,
                           sub_re = unmixed_reurl_allowed + fullmatch_re if yes else same_allowed,
                           **kwargs)

        add_url_filter("url", "net_url", True, abs_short_p = root_short)
        add_url_filter("not-url", "net_url", False)

        def add_grep_filter(opt : str, metavar : str,
                            what: str, what_re : str,
                            short : list[str] = [], short_re : list[str] = [],
                            sub : str = "", sub_re : str = "") -> None:
            agrp.add_argument(*map_opts("", opt, *short), metavar=metavar, action="append", type=str, default = [], help=_(f"{do_what} {what}") + sub)
            agrp.add_argument(*map_opts("-re", opt, *short), metavar=f"{metavar}_RE", action="append", type=str, default = [], help=_(f"{do_what} {what_re}") + sub_re)

        grep_headers = _("; each `HTTP` header of `*.headers` is matched as a single `<header_name>: <header_value>` value")
        grep_binary =  _("; at the moment, binary values are matched against given option arguments by encoding the latter into `UTF-8` first, which means that `*.headers` and `*.body` values that use encodings other than `UTF-8` are not guaranteed to match properly")
        def add_greps(prefix : str, opt : str, multi : bool, what : str, what_id : str, sub : str, sub_id : str) -> None:
            if not multi:
                mall, many, msome, mel = "", "", "", ""
            else:
                mall = "at least one of the elements of "
                many = "any of the elements of "
                msome = "some element of "
                mel = "the elements of "

            add_grep_filter(f"{prefix}or-{opt}", "OR_PATTERN",
                            f"at least one of the given `OR_PATTERN` option arguments is a substring of {mall}{what}",
                            f"at least one of the given `OR_PATTERN_RE` regular expressions matches a substring of {mall}{what_id}",
                            short = [f"{prefix}{opt}"],
                            sub = sub + ie_whitelist, sub_re = sub_id + ie_whitelist)
            add_grep_filter(f"not-{prefix}or-{opt}", "NOT_OR_PATTERN",
                            f"none of the given `NOT_OR_PATTERN` option arguments are substrings of {many}{what_id}",
                            f"none of the given `NOT_OR_PATTERN_RE` regular expressions match any substrings of {many}{what_id}",
                            short = [f"not-{prefix}{opt}"],
                            sub = sub_id + ie_blacklist, sub_re = sub_id + ie_blacklist)

            add_grep_filter(f"{prefix}and-{opt}", "AND_PATTERN",
                            f"each of the given `AND_PATTERN` option arguments is a substring of {msome}{what_id}",
                            f"each of the given `AND_PATTERN_RE` regular expressions matches a substring of {msome}{what_id}",
                            sub = sub_id, sub_re = sub_id)
            add_grep_filter(f"not-{prefix}and-{opt}", "NOT_AND_PATTERN",
                            f"one or more of the given `NOT_AND_PATTERN` option arguments is not a substring of {mel}{what_id}",
                            f"one or more of the given `NOT_AND_PATTERN_RE` regular expressions fails to match any substrings of {mel}{what_id}",
                            sub = sub_id, sub_re = sub_id)

        example_mime = list(canonical_mime_of.items())[0]
        mixed_mime_allowed = _("; both canonical and non-canonical MIME types are allowed; e.g., giving `%s` or `%s` will produce the same predicate") % (example_mime[0], example_mime[1])
        unmixed_pmime_allowed = _("; given prefixes will only ever be matched against canonicalized MIME types")
        unmixed_remime_allowed = _("; given regular expressions will only ever be matched against canonicalized MIME types")

        def add_mime_filter(opt : str, side : str, yes : bool) -> None:
            what = f"`{side}_mime` (of `{__prog__} get --expr`, which see)"
            add_str_filter(opt, yes,
                           what, what, what,
                           sub = mixed_mime_allowed if yes else same_allowed,
                           sub_p = unmixed_pmime_allowed if yes else same_allowed,
                           sub_re = unmixed_remime_allowed + fullmatch_re if yes else same_allowed)

        def add_rr_filter(side : str) -> None:
            add_greps(f"{side}-headers-", "grep", True,
                      f"the list containing all `{side}.headers` (of `{__prog__} get --expr`, which see)",
                      "the above list",
                      grep_headers + grep_binary, _("; matching caveats are the same as above"))

            add_greps(f"{side}-body-", "grep", False,
                      f"`{side}.body` (of `{__prog__} get --expr`, which see)",
                      f"`{side}.body`",
                      grep_binary, _("; matching caveats are the same as above"))

            add_mime_filter(f"{side}-mime", side, True)
            add_mime_filter(f"not-{side}-mime", side, False)

        add_rr_filter("request")
        add_rr_filter("response")

        add_greps("", "grep", True,
                  f"the list containing `raw_url`, `url`, `pretty_url`, all `request.headers`, `request.body`, all `response.headers`, and `response.body` (of `{__prog__} get --expr`, which see)",
                  "the above list",
                  grep_headers + grep_binary, _("; matching caveats are the same as above"))

        agrp.add_argument(f"--{opt_prefix}and", metavar="EXPR", action="append", type=str, default = [],
                         help=_(f"{do_what} all of the given expressions of the same format as `{__prog__} get --expr` (which see) evaluate to `true`"))
        agrp.add_argument(f"--{opt_prefix}or", metavar="EXPR", action="append", type=str, default = [],
                         help=_(f"{do_what} some of the given expressions of the same format as `{__prog__} get --expr` (which see) evaluate to `true`"))

        if kind == "export mirror" and not root:
            agrp = cmd.add_argument_group("default input filters")
            default_filters = '--method "GET" --method "DOM" --status-re ".200C"'
            ddest = f"default_filters"
            agrp.add_argument(f"--ignore-bad-inputs", dest=ddest, action="store_const", const=True, help=_("initialize input filters to `%s`; default") % (default_filters,))
            agrp.add_argument(f"--index-all-inputs", dest=ddest, action="store_const", const=False, help=_("do not set any input filters by default; note, however, that this sub-command expects input filters to be at least as restrictive as the above default and is likely to produce broken outputs if those filters are unset; use this option at your own risk"))
            cmd.set_defaults(**{ddest: True})

    def add_pure(cmd : _t.Any) -> None:
        cmd.add_argument("-q", "--quiet", action="store_true", help=_("don't print end-of-program warnings to stderr"))

    def add_impure(cmd : _t.Any) -> None:
        grp = cmd.add_mutually_exclusive_group()
        grp.add_argument("--dry-run", action="store_true", help=_("perform a trial run without actually performing any changes"))
        grp.add_argument("-q", "--quiet", action="store_true", help=_("don't log computed updates and don't print end-of-program warnings to stderr"))

    def add_common(cmd : _t.Any, kind : str, filter_what : str) -> None:
        add_errors(cmd)
        add_paths(cmd, kind)
        add_sniff(cmd, kind)
        add_filter_options(cmd)
        add_filters(cmd, kind, filter_what)

    def add_abridged(cmd : _t.Any) -> None:
        grp = cmd.add_mutually_exclusive_group()
        grp.add_argument("-u", "--unabridged", dest="abridged", action="store_false", help=_("print all data in full"))
        grp.add_argument("--abridged", action="store_true", help=_("shorten long strings for brevity, useful when you want to visually scan through batch data dumps; default"))
        cmd.set_defaults(abridged = True)

    def add_termsep(cmd : _t.Any, name : str, what : str = "printing of `--expr` values", whatval : str = "print `--expr` values", allow_not : bool = True, allow_none : bool = False, short : bool = True) -> None:
        agrp = cmd.add_argument_group(what)
        grp = agrp.add_mutually_exclusive_group()

        def_def = "; " + _("default")
        def_val : bytes | None
        if allow_none:
            grp.add_argument("--no-print", dest=f"{name}ator", action="store_const", const = None, help=_("don't print anything") + def_def)
            def_lf = ""
            def_val = None
        else:
            def_lf = def_def
            def_val = b"\n"

        if allow_not:
            grp.add_argument(f"--not-{name}ated", dest=f"{name}ator", action="store_const", const = b"", help=_(f"{whatval} without {name}ating them with anything, just concatenate them"))

        if short:
            grp.add_argument("-l", f"--lf-{name}ated", dest=f"{name}ator", action="store_const", const = b"\n", help=_(f"{whatval} {name}ated with `\\n` (LF) newline characters") + def_lf)
            grp.add_argument("-z", f"--zero-{name}ated", dest=f"{name}ator", action="store_const", const = b"\0", help=_(f"{whatval} {name}ated with `\\0` (NUL) bytes"))
        else:
            grp.add_argument(f"--lf-{name}ated", dest=f"{name}ator", action="store_const", const = b"\n", help=_(f"{whatval} {name}ated with `\\n` (LF) newline characters") + def_lf)
            grp.add_argument(f"--zero-{name}ated", dest=f"{name}ator", action="store_const", const = b"\0", help=_(f"{whatval} {name}ated with `\\0` (NUL) bytes"))

        if name == "termin":
            cmd.set_defaults(terminator = def_val)
        elif name == "separ":
            cmd.set_defaults(separator = def_val)

    def add_terminator(cmd : _t.Any, *args : _t.Any, **kwargs : _t.Any) -> None:
        add_termsep(cmd, "termin", *args, **kwargs)

    def add_separator(cmd : _t.Any, *args : _t.Any, **kwargs : _t.Any) -> None:
        add_termsep(cmd, "separ", *args, **kwargs)

    def add_paths(cmd : _t.Any, kind : str) -> None:
        def_paths : bool | str | None
        def_walk : bool | str | None
        if kind == "organize":
            def_def = "; " + _("default when `--no-overwrites`")
            def_sup = "; " + _("default when `--latest`")
            def_paths = "unset"
            def_walk = "unset"
        else:
            def_def = "; " + _("default")
            def_sup = ""
            def_paths = None
            def_walk = True

        agrp = cmd.add_argument_group("path ordering")
        grp = agrp.add_mutually_exclusive_group()
        grp.add_argument("--paths-given-order", dest="walk_paths", action="store_const", const = None, help=_("`argv` and `--stdin0` `PATH`s are processed in the order they are given") + def_def)
        grp.add_argument("--paths-sorted", dest="walk_paths", action="store_const", const = True, help=_("`argv` and `--stdin0` `PATH`s are processed in lexicographic order"))
        grp.add_argument("--paths-reversed", dest="walk_paths", action="store_const", const = False, help=_("`argv` and `--stdin0` `PATH`s are processed in reverse lexicographic order") + def_sup)
        cmd.set_defaults(walk_paths = def_paths)

        grp = agrp.add_mutually_exclusive_group()
        grp.add_argument("--walk-fs-order", dest="walk_fs", action="store_const", const = None, help=_("recursive file system walk is done in the order `readdir(2)` gives results"))
        grp.add_argument("--walk-sorted", dest="walk_fs", action="store_const", const = True, help=_("recursive file system walk is done in lexicographic order") + def_def)
        grp.add_argument("--walk-reversed", dest="walk_fs", action="store_const", const = False, help=_("recursive file system walk is done in reverse lexicographic order") + def_sup)
        cmd.set_defaults(walk_fs = def_walk)

        agrp = cmd.add_argument_group("input loading")
        grp = agrp.add_mutually_exclusive_group()
        grp.add_argument("--load-any", dest="loader", action="store_const", const=None, help=_("for each given input `PATH`, decide which loader to use based on its file extension; default"))
        grp.add_argument("--load-wrr", dest="loader", action="store_const", const="wrr", help=_("load all inputs using the single-`WRR` per-file loader"))
        grp.add_argument("--load-wrrb", dest="loader", action="store_const", const="wrr", help=_("load all inputs using the `WRR` bundle loader, this will load separate `WRR` files as single-`WRR` bundles too"))
        grp.add_argument("--load-mitmproxy", dest="loader", action="store_const", const="mitmproxy", help=_("load inputs using the `mitmproxy` dump loader"))
        grp.set_defaults(loader = None)

        agrp.add_argument("--stdin0", action="store_true", help=_("read zero-terminated `PATH`s from stdin, these will be processed after `PATH`s specified as command-line arguments"))

        if kind == "export mirror":
            agrp.add_argument("--boring", metavar="PATH", action="append", type=str, default = [], help=_("low-priority input `PATH`; boring `PATH`s will be processed after all `PATH`s specified as positional command-line arguments and those given via `--stdin0` and will not be queued as roots even when no `--root-*` options are specified"))

        agrp.add_argument("paths", metavar="PATH", nargs="*", type=str, help=_("inputs, can be a mix of files and directories (which will be traversed recursively)"))

    def add_sniff(cmd : _t.Any, kind : str) -> None:
        oscrub = f"this influeences generated file names because `filepath_parts` and `filepath_ext` of `{__prog__} get --expr` (which see) depend on both the original file extension present in the URL and the detected `MIME` type of its content"
        wscrub = "higher values make the `scrub` function (which see) censor out more things when `-unknown`, `-styles`, or `-scripts` options are set; in particular, at the moment, with `--sniff-paranoid` and `-scripts` most plain text files will be censored out as potential `JavaScript`"
        if kind == "pprint":
            what = "this simply populates the `potentially` lists in the output in various ways"
        elif kind == "organize" or kind == "import":
            what = oscrub
        elif kind != "export mirror":
            what = wscrub
        else:
            what = f"{oscrub}; also, {wscrub}"

        agrp = cmd.add_argument_group(_("`MIME` type sniffing; this controls the use of [the `mimesniff` algorithm](https://mimesniff.spec.whatwg.org/); for this sub-command " + what))
        grp = agrp.add_mutually_exclusive_group()
        grp.add_argument("--sniff-default", dest="sniff", action="store_const", const=SniffContentType.NONE, help=_("run `mimesniff` when the spec says it should be run; i.e. trust `Content-Type` `HTTP` headers most of the time; default"))
        grp.add_argument("--sniff-force", dest="sniff", action="store_const", const=SniffContentType.FORCE, help=_("run `mimesniff` regardless of what `Content-Type`  and `X-Content-Type-Options` `HTTP` headers say; i.e. for each reqres, run `mimesniff` algorithm on the `Content-Type` `HTTP` header and the actual contents of `(request|response).body` (depending on the first argument of `scrub`) to determine what the body actually contains, then interpret the data as intersection of what `Content-Type` and `mimesniff` claim it to be; e.g. if `Content-Type` says `text/plain` but `mimesniff` says `text/plain or text/javascript`, interpret it as `text/plain`"))
        grp.add_argument("--sniff-paranoid", dest="sniff", action="store_const", const=SniffContentType.PARANOID, help=_(f"do what `--sniff-force` does, but interpret the results in the most paranoid way possible; e.g. if `Content-Type` says `text/plain` but `mimesniff` says `text/plain or text/javascript`, interpret it as `text/plain or text/javascript`; which, for instance, will then make `scrub` with `-scripts` censor it out, since it can be interpreted as a script"))
        grp.set_defaults(sniff = SniffContentType.NONE)

    # pprint
    cmd = subparsers.add_parser("pprint", help=_("pretty-print given `WRR` files"),
                                description = _("""Pretty-print given `WRR` files to stdout."""))
    add_pure(cmd)
    add_common(cmd, "pprint", "pretty-print reqres when")
    add_abridged(cmd)
    cmd.set_defaults(func=cmd_pprint)

    class AddExpr(argparse.Action):
        def __call__(self, parser : _t.Any, cfg : argparse.Namespace, value : _t.Any, option_string : _t.Optional[str] = None) -> None:
            cfg.exprs.append(compile_expr(value))

    fd_fileobj = {0: stdin, 1: stdout, 2: stderr}

    class AddExprFd(argparse.Action):
        def __call__(self, parser : _t.Any, cfg : argparse.Namespace, value : _t.Any, option_string : _t.Optional[str] = None) -> None:
            fileno = cfg.expr_fd
            try:
                fobj = fd_fileobj[fileno]
            except KeyError:
                fobj = TIOWrappedWriter(_os.fdopen(fileno, "wb"))
                fd_fileobj[fileno] = fobj

            try:
                els = cfg.mexprs[fobj]
            except KeyError:
                els = []
                cfg.mexprs[fobj] = els

            els.append(compile_expr(value))

    def add_expr(cmd : _t.Any, kind : str) -> None:
        def __(value : str, indent : int = 6) -> str:
            prefix = " " * indent
            lines = value.split("\n")
            return f"\n{prefix}".join([_(line) for line in lines]).replace('%', '%%')

        agrp = cmd.add_argument_group("expression evaluation")

        if kind == "get":
            agrp.add_argument("--expr-fd", metavar="INT", type=int, default = 1, help=_(f"file descriptor to which the results of evaluations of the following `--expr`s computations should be written; can be specified multiple times, thus separating different `--expr`s into different output streams; default: `%(default)s`, i.e. `stdout`"))

            def_expr = f"`{default_expr[kind]}`, which will dump the `HTTP` response body"
            agrp.add_argument("-e", "--expr", dest="mexprs", metavar="EXPR", action=AddExprFd, type=str, default = {}, help=_(f'an expression to compute; can be specified multiple times in which case computed outputs will be printed sequentially (see also "printing" options below); the default depends on `--remap-*` options below') + \
                _("; each `EXPR` describes a state-transformer (pipeline) which starts from value `None` and evaluates a script built from the following") + ":\n" + \
                "- " + _("constants and functions:") + "\n" + \
                "".join([f"  - `{name}`: {__(value[0])}\n" for name, value in ReqresExpr_atoms.items()]) + \
                "- " + _("reqres fields, these work the same way as constants above, i.e. they replace current value of `None` with field's value, if reqres is missing the field in question, which could happen for `response*` fields, the result is `None`:") + "\n" + \
                "".join([f"  - `{name}`: {__(value)}\n" for name, value in Reqres_fields.items()]) + \
                "- " + _("derived attributes:") + "\n" + \
                "".join([f"  - `{name}`: {__(value)}\n" for name, value in ReqresExpr_derived_attrs.items()]) + \
                "- " + _("a compound expression built by piping (`|`) the above, for example") + __(f""":
- `{default_expr["get"]}` (the default for `get` and `run`) will print raw `response.body` or an empty byte string, if there was no response;
- `{default_expr["get"]}|scrub response defaults` will take the above value, `scrub` it using default content scrubbing settings which will censor out all actions and references to page requisites;
- `response.complete` will print the value of `response.complete` or `None`, if there was no response;
- `response.complete|false` will print `response.complete` or `False`;
- `net_url|to_ascii|sha256|to_hex` will print a hexadecimal representation of the `sha256` hash of the URL that was actually sent over the network;
- `net_url|to_ascii|sha256|take_prefix 2|to_hex` will print the first 2 bytes (4 characters) of the above;
- `path_parts|take_prefix 3|pp_to_path` will print first 3 path components of the URL, minimally quoted to be used as a path;
- `query_ne_parts|take_prefix 3|qsl_to_path|abbrev 128` will print first 3 non-empty query parameters of the URL, abbreviated to 128 characters or less, minimally quoted to be used as a path;""", 2) + \
                "\n\nExample URL mappings:\n" + \
                "".join([f"  - `{name}`:\n" + atom_example(name, 4) + "\n" for name in atom_test])
            )
        else:
            agrp.add_argument("-e", "--expr", dest="exprs", metavar="EXPR", action=AddExpr, type=str, default = [], help=_(f"an expression to compute, same expression format and semantics as `{__prog__} get --expr` (which see); can be specified multiple times; the default depends on `--remap-*` options below"))

        if kind == "stream":
            add_terminator(cmd, "`--format=raw` `--expr` printing", "print `--format=raw` `--expr` output values")
        elif kind != "export mirror":
            add_separator(cmd)
        else:
            add_separator(cmd, "exporting of `--expr`", "export `--expr` values", short = False)

        def alias(what : str) -> str:
            return _("set the default value of `--expr` to `%s`") % (default_expr[what],)

        agrp = cmd.add_argument_group("default value of `--expr`")
        grp = agrp.add_mutually_exclusive_group()

        if kind != "export mirror":
            grp.add_argument("--no-remap", dest="default_expr", action="store_const", const=kind, help=alias(kind) + _("; i.e. produce the raw response body; default"))

        grp.add_argument("--remap-id", dest="default_expr", action="store_const", const="id", help=alias("id") + _("; i.e. remap all URLs of response body with an identity function (which, as a whole, is NOT an identity function, it will transform all relative URLs into absolute ones) and will censor out all dynamic content (e.g. `JavaScript`); results will NOT be self-contained"))
        grp.add_argument("--remap-void", dest="default_expr", action="store_const", const="void", help=alias("void") + _("; i.e. remap all URLs of response body into `javascript:void(0)` and empty `data:` URLs and censor out all dynamic content; results will be self-contained"))

        if kind != "export mirror":
            cmd.set_defaults(default_expr = kind)
        else:
            grp.add_argument("--remap-open", "-k", "--convert-links", dest="default_expr", action="store_const", const="open", help=alias("open") + _("; i.e. remap all URLs of response body present in input `PATH`s and reachable from `--root-*`s in no more that `--depth` steps to their corresponding `--output` paths, remap all other URLs like `--remap-id` does, and censor out all dynamic content; results almost certainly will NOT be self-contained"))
            grp.add_argument("--remap-closed", dest="default_expr", action="store_const", const="open", help=alias("closed") + _("; i.e. remap all URLs of response body present in input `PATH`s and reachable from `--root-*`s in no more that `--depth` steps to their corresponding `--output` paths, remap all other URLs like `--remap-void` does, and censor out all dynamic content; results will be self-contained"))
            grp.add_argument("--remap-semi", dest="default_expr", action="store_const", const="semi", help=alias("semi") + _("; i.e. remap all jump links of response body like `--remap-open` does, remap action links and references to page requisites like `--remap-closed` does, and censor out all dynamic content; this is a better version of `--remap-open` which keeps the `export`ed `mirror`s self-contained with respect to page requisites, i.e. generated pages can be opened in a web browser without it trying to access the Internet, but all navigations to missing and unreachable URLs will still point to the original URLs; results will be semi-self-contained"))
            grp.add_argument("--remap-all", dest="default_expr", action="store_const", const="all", help=alias("all") + _(f"""; i.e. remap all links and references of response body like `--remap-closed` does, except, instead of voiding missing and unreachable URLs, replace them with fallback URLs whenever possble, and censor out all dynamic content; results will be self-contained; default

`{__prog__} export mirror` uses `--output` paths of trivial `GET <URL> -> 200 OK` as fallbacks for `&(jumps|actions|reqs)` options of `scrub`.
This will remap links pointing to missing and unreachable URLs to missing files.
However, for simple `--output` formats (like the default `hupq`), those files can later be generated by running `{__prog__} export mirror` with `WRR` files containing those missing or unreachable URLs as inputs.
I.e. this behaviour allows you to add new data to an already `export`ed mirror without regenerating old files that reference newly added URLs.
I.e. this allows `{__prog__} export mirror` to be used incrementally.

Note however, that using fallbacks when the `--output` format depends on anything but the URL itself (e.g. if it mentions timestamps) will produce a mirror with unrecoverably broken links.
"""))
            cmd.set_defaults(default_expr = "all")

            agrp = cmd.add_argument_group("link conversions")
            grp = agrp.add_mutually_exclusive_group()

            grp.add_argument("--relative", dest="absolute", action="store_const", const=False, help=_("when remapping URLs to local files, produce links and references with relative URLs (relative to the `--output` files under `OUTPUT_DESTINATION`); default when `--copy` or `--hardlink`"))
            grp.add_argument("--absolute", dest="absolute", action="store_const", const=True, help=_("when remapping URLs to local files, produce links and references with absolute URLs; default when `--symlink`"))
            cmd.set_defaults(absolute = None)

    # get
    cmd = subparsers.add_parser("get", help=_("print values produced by computing given expressions on a given `WRR` file"),
                                description = _(f"""Compute output values by evaluating expressions `EXPR`s on a given reqres stored at `PATH`, then print them to stdout terminating each value as specified."""))

    add_sniff(cmd, "get")
    add_expr(cmd, "get")

    cmd.add_argument("path", metavar="PATH", type=str, help=_("input `WRR` file path"))
    cmd.set_defaults(func=cmd_get)

    # run
    cmd = subparsers.add_parser("run", help=_("spawn a process with generated temporary files produced by given expressions computed on given `WRR` files as arguments"),
                                description = _("""Compute output values by evaluating expressions `EXPR`s for each of `NUM` reqres stored at `PATH`s, dump the results into into newly generated temporary files terminating each value as specified, spawn a given `COMMAND` with given arguments `ARG`s and the resulting temporary file paths appended as the last `NUM` arguments, wait for it to finish, delete the temporary files, exit with the return code of the spawned process."""))

    add_sniff(cmd, "run")
    add_expr(cmd, "run")

    cmd.add_argument("-n", "--num-args", metavar="NUM", type=int, default = 1, help=_("number of `PATH`s; default: `%(default)s`"))
    cmd.add_argument("command", metavar="COMMAND", type=str, help=_("command to spawn"))
    cmd.add_argument("args", metavar="ARG", nargs="*", type=str, help=_("additional arguments to give to the `COMMAND`"))
    cmd.add_argument("paths", metavar="PATH", nargs="+", type=str, help=_("input `WRR` file paths to be mapped into new temporary files"))
    cmd.set_defaults(func=cmd_run)

    # stream
    cmd = subparsers.add_parser("stream", help=_(f"produce a stream of structured lists containing values produced by computing given expressions on given `WRR` files, a generalized `{__prog__} get`"),
                                description = _("""Compute given expressions for each of given `WRR` files, encode them into a requested format, and print the result to stdout."""))
    add_pure(cmd)
    add_common(cmd, "stream", "stream-print reqres when")
    add_abridged(cmd)
    cmd.add_argument("--format", choices=["py", "cbor", "json", "raw"], default="py", help=_("""generate output in:
- py: Pythonic Object Representation aka `repr`; default
- cbor: Concise Binary Object Representation aka `CBOR` (RFC8949)
- json: JavaScript Object Notation aka `JSON`; **binary data can't be represented, UNICODE replacement characters will be used**
- raw: concatenate raw values; termination is controlled by `*-terminated` options
"""))
    add_expr(cmd, "stream")
    cmd.set_defaults(func=cmd_stream)

    # find
    cmd = subparsers.add_parser("find", help=_("print paths of `WRR` files matching specified criteria"),
                                description = _(f"""Print paths of `WRR` files matching specified criteria."""))
    add_pure(cmd)
    add_common(cmd, "find", "print path of reqres when")
    add_terminator(cmd, "found files printing", "print absolute paths of matching `WRR` files", allow_not=False)
    cmd.set_defaults(sniff = SniffContentType.NONE)
    cmd.set_defaults(func=cmd_find)

    def add_organize_memory(cmd : _t.Any, max_deferred : int = 1024, max_batch : int = 128) -> None:
        agrp = cmd.add_argument_group("caching, deferring, and batching")
        agrp.add_argument("--seen-number", metavar = "INT", dest="max_seen", type=int, default=16384, help=_(f"""track at most this many distinct generated `--output` values; default: `%(default)s`;
making this larger improves disk performance at the cost of increased memory consumption;
setting it to zero will force force `{__prog__}` to constantly re-check existence of `--output` files and force `{__prog__}` to execute  all IO actions immediately, disregarding `--defer-number` setting"""))
        agrp.add_argument("--cache-number", metavar = "INT", dest="max_cached", type=int, default=8192, help=_(f"""cache `stat(2)` information about this many files in memory; default: `%(default)s`;
making this larger improves performance at the cost of increased memory consumption;
setting this to a too small number will likely force `{__prog__}` into repeatedly performing lots of `stat(2)` system calls on the same files;
setting this to a value smaller than `--defer-number` will not improve memory consumption very much since deferred IO actions also cache information about their own files
"""))
        agrp.add_argument("--defer-number", metavar = "INT", dest="max_deferred", type=int, default=max_deferred, help=_("""defer at most this many IO actions; default: `%(default)s`;
making this larger improves performance at the cost of increased memory consumption;
setting it to zero will force all IO actions to be applied immediately"""))
        agrp.add_argument("--batch-number", metavar = "INT", dest="max_batched", type=int, default=max_batch, help=_(f"""queue at most this many deferred IO actions to be applied together in a batch; this queue will only be used if all other resource constraints are met; default: `%(default)s`"""))
        agrp.add_argument("--max-memory", metavar = "INT", dest="max_memory", type=int, default=1024, help=_("""the caches, the deferred actions queue, and the batch queue, all taken together, must not take more than this much memory in MiB; default: `%(default)s`;
making this larger improves performance;
the actual maximum whole-program memory consumption is `O(<size of the largest reqres> + <--seen-number> + <sum of lengths of the last --seen-number generated --output paths> + <--cache-number> + <--defer-number> + <--batch-number> + <--max-memory>)`"""))
        agrp.add_argument("--lazy", action="store_true", help=_(f"""sets all of the above options to positive infinity;
most useful when doing `{__prog__} organize --symlink --latest --output flat` or similar, where the number of distinct generated `--output` values and the amount of other data `{__prog__}` needs to keep in memory is small, in which case it will force `{__prog__}` to compute the desired file system state first and then perform all disk writes in a single batch"""))

    def add_fileout(cmd : _t.Any, kind : str) -> None:
        agrp = cmd.add_argument_group("file outputs")

        if kind == "organize":
            agrp.add_argument("-t", "--to", dest="destination", metavar="OUTPUT_DESTINATION", type=str, help=_("destination directory; when unset each source `PATH` must be a directory which will be treated as its own `OUTPUT_DESTINATION`"))
            agrp.add_argument("-o", "--output", metavar="OUTPUT_FORMAT", default="default", type=str, help=_("""format describing generated output paths, an alias name or "format:" followed by a custom pythonic %%-substitution string:""") + "\n" + \
                         "- " + _("available aliases and corresponding %%-substitutions:") + "\n" + \
                         "".join([f"  - `{name}`{' ' * (12 - len(name))}: `{value.replace('%', '%%')}`" + ("; the default" if name == "default" else "") + "\n" + output_example(name, 8) + "\n" for name, value in output_alias.items()]) + \
                         "- " + _("available substitutions:") + "\n" + \
                         "  - " + _(f"all expressions of `{__prog__} get --expr` (which see)") + ";\n" + \
                         "  - `num`: " + _("number of times the resulting output path was encountered before; adding this parameter to your `--output` format will ensure all generated file names will be unique"))
        elif kind == "import" or kind == "export mirror":
            if kind != "export mirror":
                def_def = "default"
            else:
                def_def = "hupq_n"

            agrp.add_argument("-t", "--to", dest="destination", metavar="OUTPUT_DESTINATION", type=str, required=True, help=_("destination directory"))
            agrp.add_argument("-o", "--output", metavar="OUTPUT_FORMAT", default=def_def, type=str, help=_(f"""format describing generated output paths, an alias name or "format:" followed by a custom pythonic %%-substitution string; same expression format as `{__prog__} organize --output` (which see); default: %(default)s"""))
        else:
            assert False

        add_terminator(cmd, "new `--output`s printing", "print absolute paths of newly produced or replaced files", allow_not=False, allow_none=True)

        agrp = cmd.add_argument_group("updates to `--output`s")
        grp = agrp.add_mutually_exclusive_group()

        def_disallow = _("disallow overwrites and replacements of any existing files under `OUTPUT_DESTINATION`, i.e. only ever create new files under `OUTPUT_DESTINATION`, producing errors instead of attempting any other updates; default")

        def def_dangerous(what : str) -> str:
            return _(f"DANGEROUS! not recommended, {what} to a new `OUTPUT_DESTINATION` with the default `--no-overwrites` and then `rsync`ing some of the files over to the old `OUTPUT_DESTINATION` is a safer way to do this")

        if kind == "organize":
            grp.add_argument("--no-overwrites", dest="allow_updates", action="store_const", const=False, help=def_disallow + ";\n" + \
                _("""`--output` targets that are broken symlinks will be considered to be non-existent and will be replaced;
when the operation's source is binary-eqivalent to the `--output` target, the operation will be permitted, but the disk write will be reduced to a noop, i.e. the results will be deduplicated;
the `dirname` of a source file and the `--to` target directories can be the same, in that case the source file will be renamed to use new `--output` name, though renames that attempt to swap files will still fail
"""))
            grp.add_argument("--latest", dest="allow_updates", action="store_const", const=True, help=_("""replace files under `OUTPUT_DESTINATION` with their latest version;
this is only allowed in combination with `--symlink` at the moment;
for each source `PATH` file, the destination `--output` file will be replaced with a symlink to the source if and only if `stime_ms` of the source reqres is newer than `stime_ms` of the reqres stored at the destination file
"""))
        elif kind == "import":
            grp.add_argument("--no-overwrites", dest="allow_updates", action="store_const", const=False, help=def_disallow)
            grp.add_argument("--overwrite-dangerously", dest="allow_updates", action="store_const", const=True, help=_("permit overwrites to files under `OUTPUT_DESTINATION`") + ";\n" + def_dangerous("importing"))
        elif kind == "export mirror":
            grp.add_argument("--no-overwrites", dest="allow_updates", action="store_const", const=False, help=def_disallow + ";\n" + \
                _("""repeated exports of the same export targets with the same parameters (which, therefore, will produce the same `--output` data) are allowed and will be reduced to noops;
however, trying to overwrite existing files under `OUTPUT_DESTINATION` with any new data will produce errors;
this allows reusing the `OUTPUT_DESTINATION` between unrelated exports and between exports that produce the same data on disk in their common parts
"""))
            grp.add_argument("--skip-existing", "--partial", dest="allow_updates", action="store_const", const="partial", help=_("""skip rendering of targets which have a corresponding file under `OUTPUT_DESTINATION`, use the contents of such files instead;
using this together with `--depth` is likely to produce a partially broken result, since skipping an export target will also skip all the documents it references;
on the other hand, this is quite useful when growing a partial mirror generated with `--remap-all`
"""))
            grp.add_argument("--overwrite-dangerously", dest="allow_updates", action="store_const", const=True, help=_("export all targets while permitting overwriting of old `--output` files under `OUTPUT_DESTINATION`") + ";\n" + def_dangerous("exporting"))

        cmd.set_defaults(allow_updates = False)

    # organize
    cmd = subparsers.add_parser("organize", help=_("programmatically rename/move/hardlink/symlink `WRR` files based on their contents"),
                                description = _(f"""Parse given `WRR` files into their respective reqres and then rename/move/hardlink/symlink each file to `OUTPUT_DESTINATION` with the new path derived from each reqres' metadata.

Operations that could lead to accidental data loss are not permitted.
E.g. `{__prog__} organize --move` will not overwrite any files, which is why the default `--output` contains `%(num)d`."""))
    add_organize_memory(cmd)
    add_impure(cmd)
    add_common(cmd, "organize", "organize reqres when")

    agrp = cmd.add_argument_group("action")
    grp = agrp.add_mutually_exclusive_group()
    grp.add_argument("--move", dest="action", action="store_const", const="move", help=_("move source files under `OUTPUT_DESTINATION`; default"))
    grp.add_argument("--copy", dest="action", action="store_const", const="copy", help=_("copy source files to files under `OUTPUT_DESTINATION`"))
    grp.add_argument("--hardlink", dest="action", action="store_const", const="hardlink", help=_("create hardlinks from source files to paths under `OUTPUT_DESTINATION`"))
    grp.add_argument("--symlink", dest="action", action="store_const", const="symlink", help=_("create symlinks from source files to paths under `OUTPUT_DESTINATION`"))
    cmd.set_defaults(action = "move")

    add_fileout(cmd, "organize")

    cmd.set_defaults(func=cmd_organize)

    def add_import_args(cmd : _t.Any) -> None:
        add_organize_memory(cmd, 0, 1024)
        add_impure(cmd)
        add_common(cmd, "import", "import reqres when")
        add_fileout(cmd, "import")

    # import
    supcmd = subparsers.add_parser("import", help=_("convert other `HTTP` archive formats into `WRR`"),
                                   description = _(f"""Use specified parser to parse data in each `INPUT` `PATH` into (a sequence of) reqres and then generate and place their `WRR` dumps into separate `WRR` files under `OUTPUT_DESTINATION` with paths derived from their metadata.
In short, this is `{__prog__} organize --copy` for `INPUT` files that use different files formats."""))
    supsub = supcmd.add_subparsers(title="file formats")

    cmd = supsub.add_parser("bundle", help=_("convert `WRR` bundles into separate `WRR` files"),
                            description = _(f"""Parse each `INPUT` `PATH` as a `WRR` bundle (an optionally compressed sequence of `WRR` dumps) and then generate and place their `WRR` dumps into separate `WRR` files under `OUTPUT_DESTINATION` with paths derived from their metadata."""))
    add_import_args(cmd)
    cmd.set_defaults(func=cmd_import_bundle)

    cmd = supsub.add_parser("mitmproxy", help=_("convert `mitmproxy` stream dumps into `WRR` files"),
                            description = _(f"""Parse each `INPUT` `PATH` as `mitmproxy` stream dump (by using `mitmproxy`'s own parser) into a sequence of reqres and then generate and place their `WRR` dumps into separate `WRR` files under `OUTPUT_DESTINATION` with paths derived from their metadata."""))
    add_import_args(cmd)
    cmd.set_defaults(func=cmd_import_mitmproxy)

    def add_export_memory(cmd : _t.Any) -> None:
        agrp = cmd.add_argument_group("caching")
        agrp.add_argument("--max-memory", metavar = "INT", dest="max_memory", type=int, default=1024, help=_("""the caches, all taken together, must not take more than this much memory in MiB; default: `%(default)s`;
making this larger improves performance;
the actual maximum whole-program memory consumption is `O(<size of the largest reqres> + <numer of indexed files> + <sum of lengths of all their --output paths> + <--max-memory>)`"""))

    # export
    supcmd = subparsers.add_parser("export", help=_(f"convert `WRR` archives into other formats"),
                                   description = _(f"""Parse given `WRR` files into their respective reqres, convert to another file format, and then dump the result under `OUTPUT_DESTINATION` with the new path derived from each reqres' metadata.
"""))
    supsub = supcmd.add_subparsers(title="file formats")

    cmd = supsub.add_parser("mirror", help=_("convert given `WRR` files into a local website mirror stored in interlinked plain files"),
                            description = _(f"""Parse given `WRR` files, filter out those that have no responses, transform and then dump their response bodies into separate files under `OUTPUT_DESTINATION` with the new path derived from each reqres' metadata.
Essentially, this is a combination of `{__prog__} organize --copy` followed by in-place `{__prog__} get` which has the advanced URL remapping capabilities of `(*|/|&)(jumps|actions|reqs)` options available in its `scrub` function.

In short, this sub-command generates static offline website mirrors, producing results similar to those of `wget -mpk`.
"""))
    add_export_memory(cmd)
    add_impure(cmd)
    add_common(cmd, "export mirror", "consider reqres for export when")
    add_expr(cmd, "export mirror")

    agrp = cmd.add_argument_group("what gets exported")
    grp = agrp.add_mutually_exclusive_group()

    def which(x : str) -> str:
        return _(f"for each URL, export {x}")

    oldest = which("its oldest available version")
    near = which("an available version that is closest to the given `DATE` value")
    latest = which("its latest available version")
    hybrid = _(", except, for each URL that is a requisite resource, export a version that is time-closest to the referencing document")
    hybrid_long = hybrid + _("; i.e., this will make each exported page refer to requisites (images, media, `CSS`, fonts, etc) that were archived around the time the page itself was archived, even if those requisite resources changed in time; this produces results that are as close to the original web page as possible at the cost of much more memory to `export`")
    hybrid_short = hybrid +  _("; see `--oldest-hybrid` above for more info")

    class EmitNear(argparse.Action):
        def __call__(self, parser : _t.Any, cfg : argparse.Namespace, value : _t.Any, option_string : _t.Optional[str] = None) -> None:
            setattr(cfg, self.dest, self.const(parse_Epoch(value)))

    grp.add_argument("--oldest", dest="mode", action="store_const", const=(False, False), help=oldest)
    grp.add_argument("--oldest-hybrid", dest="mode", action="store_const", const=(True, False), help=oldest + hybrid_long)
    grp.add_argument("--nearest", dest="mode", metavar="DATE", action=EmitNear, const=lambda x: (False, x), help=near + date_spec)
    grp.add_argument("--nearest-hybrid", dest="mode", metavar="DATE", action=EmitNear, const=lambda x: (True, x), help=near + hybrid_short + date_spec_id)
    grp.add_argument("--latest", dest="mode", action="store_const", const=(False, True), help=latest + _("; default"))
    grp.add_argument("--latest-hybrid", dest="mode", action="store_const", const=(True, True), help=latest + hybrid_short)
    grp.add_argument("--all", dest="mode", action="store_const", const=(True, None), help=_("export all available versions of all available URLs; this is likely to take a lot of time and eat a lot of memory!"))
    cmd.set_defaults(mode=(False, True)) # `--latest`

    add_fileout(cmd, "export mirror")

    agrp = cmd.add_argument_group("content-addressed file output mode")

    agrp.add_argument("--copy", dest="content_action", action="store_const", const="copy", help=_("do not use content-addressed outputs, simply write rendered output data to files under `OUTPUT_DESTINATION`"))
    agrp.add_argument("--hardlink", dest="content_action", action="store_const", const="hardlink", help=_("write rendered output data to files under `CONTENT_DESTINATION`, then hardlink them to paths under `OUTPUT_DESTINATION`; default"))
    agrp.add_argument("--symlink", dest="content_action", action="store_const", const="symlink", help=_("write rendered output data to files under `CONTENT_DESTINATION`, then symlink them to paths under `OUTPUT_DESTINATION`"))
    cmd.set_defaults(content_action = "hardlink")

    agrp = cmd.add_argument_group("content-addressed file output settings")

    agrp.add_argument("--content-to", dest="content_destination", metavar="CONTENT_DESTINATION", type=str, help=_("content-addressed destination directory; if not specified, reuses `OUTPUT_DESTINATION`"))
    agrp.add_argument("--content-output", metavar="CONTENT_FORMAT", default="default", type=str, help=_("""format describing generated content-addressed output paths, an alias name or "format:" followed by a custom pythonic %%-substitution string:""") + "\n" + \
                      "- " + _("available aliases and corresponding %%-substitutions:") + "\n" + \
                      "".join([f"  - `{name}`{' ' * (12 - len(name))}: `{value.replace('%', '%%')}`" + ("; the default" if name == "default" else "") + "\n" for name, value in content_output_alias.items()]) + \
                      "- " + _("available substitutions:") + "\n" + \
                      "  - " + _(f"all expressions of `{__prog__} get --expr` (which see)") + ";\n" + \
                      "  - `content`: " + _("rendered content") + "\n" + \
                      "  - `content_sha256`: " + _("alias for `content|sha256`"))

    add_filters(cmd, "export mirror", "take reqres as export root when", True)

    agrp = cmd.add_argument_group("recursion depth")
    agrp.add_argument("-d", "--depth", metavar="DEPTH", type=int, default=0, help=_('maximum recursion depth level; the default is `0`, which means "`--root-*` documents and their requisite resources only"; setting this to `1` will also export one level of documents referenced via jump and action links, if those are being remapped to local files with `--remap-*`; higher values will mean even more recursion'))

    cmd.set_defaults(func=cmd_export_mirror)

    cargs = parser.parse_args(_sys.argv[1:])

    if cargs.help:
        if cargs.markdown:
            parser.set_formatter_class(argparse.MarkdownBetterHelpFormatter)
            print(parser.format_help(8192))
        else:
            print(parser.format_help())
        _sys.exit(0)

    _logging.basicConfig(level=_logging.WARNING,
                         stream = stderr)
    errorcnt = CounterHandler()
    logger = _logging.getLogger()
    logger.addHandler(errorcnt)
    #logger.setLevel(_logging.DEBUG)

    handle_signals()

    try:
        cargs.func(cargs)
    except KeyboardInterrupt:
        error("%s", _("Interrupted!"))
        errorcnt.errors += 1
    except CatastrophicFailure as exc:
        error("%s", str(exc))
        errorcnt.errors += 1
    except Exception as exc:
        stderr.write_str(str_Exception(exc))
        errorcnt.errors += 1

    stdout.flush()
    stderr.flush()

    if errorcnt.warnings > 0:
        stderr.write_str_ln(ngettext("There was %d warning!", "There were %d warnings!", errorcnt.warnings) % (errorcnt.warnings,))
    if errorcnt.errors > 0:
        stderr.write_str_ln(ngettext("There was %d error!", "There were %d errors!", errorcnt.errors) % (errorcnt.errors,))
        _sys.exit(1)
    _sys.exit(0)

if __name__ == "__main__":
    main()
