import logging
import os

logger = logging.getLogger(__name__)


def generate_robots_txt(disallow_all: bool = True, user_agent: str = "*") -> str:
    """
    Generate contents of a robots.txt file.

    :param disallow_all: If True, disallow all crawling.
    :param user_agent: The user-agent string to target.
    :return: String content of robots.txt.
    """
    if disallow_all:
        rules = [f"User-agent: {user_agent}", "Disallow: /"]
    else:
        rules = [f"User-agent: {user_agent}", "Disallow:"]

    return "\n".join(rules)


def save_robots_txt(path: str = "robots.txt", verbose: bool = False) -> None:
    # if path is a directory, create a robots.txt file in that directory
    if os.path.isdir(path):
        path = os.path.join(path, "robots.txt")
    content = generate_robots_txt()
    with open(path, "w") as f:
        f.write(content)
    if verbose:
        logger.info(f"'robots.txt' has been written to {path}")
