import tiktoken

ENCODING = "cl100k_base"
_FALLBACK_CHARS_PER_TOKEN = 4

_encoder_cache: tiktoken.Encoding | None = None


def get_encoder() -> tiktoken.Encoding | None:
    global _encoder_cache
    if _encoder_cache is not None:
        return _encoder_cache
    try:
        _encoder_cache = tiktoken.get_encoding(ENCODING)
        return _encoder_cache
    except Exception:
        return None


def count_tokens(text: str) -> int:
    enc = get_encoder()
    if enc is not None:
        return len(enc.encode(text))
    return max(1, len(text) // _FALLBACK_CHARS_PER_TOKEN)


def segment_content(
    content: str,
    token_limit: int,
    *,
    overlap: int = 0,
) -> list[str]:
    if not content.strip():
        return []

    encoder = get_encoder()
    if encoder is None:
        char_limit = token_limit * _FALLBACK_CHARS_PER_TOKEN
        return [content[i:i + char_limit] for i in range(0, len(content), char_limit)]

    lines = content.splitlines(keepends=True)
    chunks: list[str] = []
    current: list[str] = []
    current_tokens = 0

    for line in lines:
        line_tokens = len(encoder.encode(line))

        if current_tokens + line_tokens > token_limit and current:
            chunks.append("".join(current))
            if overlap > 0 and current:
                keep_tokens = min(overlap, current_tokens)
                kept = []
                kept_tokens = 0
                for ln in reversed(current):
                    lt = len(encoder.encode(ln))
                    if kept_tokens + lt <= keep_tokens:
                        kept.insert(0, ln)
                        kept_tokens += lt
                    else:
                        break
                current = kept
                current_tokens = kept_tokens
            else:
                current = []
                current_tokens = 0

        if line_tokens > token_limit:
            encoded = encoder.encode(line)
            token_chars = encoder.decode(encoded[:token_limit])
            current.append(token_chars)
            chunks.append("".join(current))
            remainder = line[len(token_chars):]
            if remainder.strip():
                sub_chunks = segment_content(remainder, token_limit, overlap=overlap)
                chunks.extend(sub_chunks[:-1] if sub_chunks else [])
                if sub_chunks:
                    current = [sub_chunks[-1]]
                    current_tokens = count_tokens(sub_chunks[-1])
                else:
                    current = []
                    current_tokens = 0
            else:
                current = []
                current_tokens = 0
        else:
            current.append(line)
            current_tokens += line_tokens

    if current:
        chunks.append("".join(current))

    return chunks
