from keble_helpers.ai.tokens import count_tokens, limit_tokens


def test_get_string_tokens():
    """compare result of count_tokens and https://platform.openai.com/tokenizer"""
    batches = [
        {
            "text": """包子开花
新品的话否词还是要谨慎一些吧，一些大词就算暂时不出单也可以不否，先否掉相关性弱的词
2023-04-27 0 0 0 感谢 0

EkkoWo - 这个人很狗啥都不会

最近的广告是悄悄的改变了，而且根据劳伦斯大佬最近的帖子，按照这个逻辑，最有可能的就是虽然否定了不相关的词，但是这部分不相关流量会影响整体流量
2023-04-27 0 0 0 感谢 0

龙岗彭于晏分晏 - 3年精品运营 只会白帽

个人观点，否词注意单复数，否了一个单数，复数也会否定，如果这词不相关就可以否定，单独拉出来打精准的词不要在自动否定了，竞价高一点或者低一点都行，慢慢测.
2023-04-27 0 0 0 感谢 0

cssssb
否词确实影响流量，不过我这边的经历是只要广告持续开着后面数据都会跑回来。我这有个产品的自动广告开了半年多了，刚开始否词流量就腰斩，后来慢慢就没事儿了，否词也没啥大影响。不过我没有精准否大词，因为我自动广告竞价低，我觉得不可能跟手动抢位置，放着权当是去捡漏了，毕竟自动的本意还是扩大流量找新词。
2023-04-27 0 0 0 感谢 0
匿名用户
匿名用户

首先你得确定不相关的词和高点击不出单的词，你是 否精准还是否词组，这两种否定方式是不一样的！你是整个词连词根都否？还是只是否不相关的修饰词？要知道词根否多了，系统会认为这词根跟你的产品不相关了，自然也不会给你展示的机会。还有就是你来差评，虽然中间也来几个好评，但是不知道有没有稀释掉原来的差评，也就是得看同行竞对已经整个类目的评分，如果你评分没优势，人家不点击你的产品也很正常~""",
            "token": 745,
        },
        {
            "text": """Learn about language model tokenization
OpenAI's large language models (sometimes referred to as GPT's) process text using tokens, which are common sequences of characters found in a set of text. The models learn to understand the statistical relationships between these tokens, and excel at producing the next token in a sequence of tokens.

You can use the tool below to understand how a piece of text might be tokenized by a language model, and the total count of tokens in that piece of text.

It's important to note that the exact tokenization process varies between models. Newer models like GPT-3.5 and GPT-4 use a different tokenizer than previous models, and will produce different tokens for the same input text.""",
            "token": 148,
        },
    ]
    for batch in batches:
        cal_token = count_tokens(batch["text"])
        assert abs(cal_token - batch["token"]) < 10, (
            f"Differences between cal_token {cal_token} and expectation {batch['token']} larger than 10 tokens."
        )


def test_limit_tokens():
    content = """Learn about language model tokenization
OpenAI's large language models (sometimes referred to as GPT's) process text using tokens, which are common sequences of characters found in a set of text. The models learn to understand the statistical relationships between these tokens, and excel at producing the next token in a sequence of tokens.

You can use the tool below to understand how a piece of text might be tokenized by a language model, and the total count of tokens in that piece of text.

It's important to note that the exact tokenization process varies between models. Newer models like GPT-3.5 and GPT-4 use a different tokenizer than previous models, and will produce different tokens for the same input text."""
    assert count_tokens(content) > 100
    after_limit = limit_tokens(content, max_token=100)
    assert count_tokens(after_limit) <= 100
    assert content.index(after_limit) == 0
