"""Token counter using HuggingFace tokenizers library."""

from __future__ import annotations

from context_manager.budget.base import TokenCounter


class HFTokenCounter(TokenCounter):
    """Count tokens for HuggingFace-hosted models (Llama, Mistral, etc.)."""

    def __init__(self, model: str = "meta-llama/Meta-Llama-3-8B") -> None:
        try:
            from tokenizers import Tokenizer
        except ImportError as exc:
            raise ImportError(
                "tokenizers is required for HuggingFace models. "
                "Install with: pip install llm-ctx-mgr[huggingface]"
            ) from exc

        self._tokenizer = Tokenizer.from_pretrained(model)
        self._model = model

    def count(self, text: str) -> int:
        """Return token count using HuggingFace tokenizer."""
        return len(self._tokenizer.encode(text).ids)

    def model_name(self) -> str:
        return self._model
