"""Tokenizer analysis."""


def get_tokenizer_info(model_name: str) -> dict | None:
    """Get tokenizer information for a model. Lazy imports transformers."""
    try:
        from transformers import AutoTokenizer

        tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
        return {
            "vocab_size": tokenizer.vocab_size,
            "model_max_length": getattr(tokenizer, "model_max_length", None),
            "pad_token": tokenizer.pad_token,
            "eos_token": tokenizer.eos_token,
            "bos_token": tokenizer.bos_token,
        }
    except ImportError:
        return None
    except Exception:
        return None
