import logging
import os

logger = logging.getLogger(__name__)


def generate_robots_txt(user_agent: str = "*") -> str:
    """
    Return a robots.txt string that ALSO reserves text-and-data-mining (TDM) rights under the W3C TDM
    Reservation Protocol.
    See `https://www.edrlab.org/open-standards/tdmrep/` for more details.
    Parameters
    ----------
    user_agent : str
        Which bot the rule applies to (“*” = every bot).

    Notes
    -----
    * The extra `tdm: no` line is an **informal** shorthand understood by early
        TDM-aware crawlers (e.g. Stability AI via Spawning).
    * Official compliance still requires a `/.well-known/tdmrep.json` file or
        the `tdm-reservation: 1` HTTP response header.
    """
    rules = [f"User-agent: {user_agent}"]
    rules.append("Disallow: /")
    rules.append("tdm: no")  # machine-readable TDM opt-out
    return "\n".join(rules)


def save_robots_txt(path: str = "robots.txt", verbose: bool = False) -> None:
    # if path is a directory, create a robots.txt file in that directory
    if os.path.isdir(path):
        path = os.path.join(path, "robots.txt")
    content = generate_robots_txt()
    with open(path, "w") as f:
        f.write(content)
    if verbose:
        logger.info(f"'robots.txt' has been written to {path}")
