import os

import pytest

from auditok import DataValidator, StreamTokenizer, StringDataSource


class AValidator(DataValidator):
    def is_valid(self, frame):
        return frame == "A"


@pytest.fixture
def validator():
    return AValidator()


def test_init_min_0_init_max_silence_0(validator):
    tokenizer = StreamTokenizer(
        validator,
        min_length=5,
        max_length=20,
        max_continuous_silence=4,
        init_min=0,
        init_max_silence=0,
        mode=0,
    )

    data_source = StringDataSource("aAaaaAaAaaAaAaaaaaaaAAAAAAAA")
    #                                ^              ^   ^      ^
    #                                2              16  20     27
    tokens = tokenizer.tokenize(data_source)

    assert (
        len(tokens) == 2
    ), "wrong number of tokens, expected: 2, found: {}".format(len(tokens))
    tok1, tok2 = tokens[0], tokens[1]

    data = "".join(tok1[0])
    start = tok1[1]
    end = tok1[2]
    assert (
        data == "AaaaAaAaaAaAaaaa"
    ), "wrong data for token 1, expected: 'AaaaAaAaaAaAaaaa', found: {}".format(
        data
    )
    assert (
        start == 1
    ), "wrong start frame for token 1, expected: 1, found: {}".format(start)
    assert (
        end == 16
    ), "wrong end frame for token 1, expected: 16, found: {}".format(end)

    data = "".join(tok2[0])
    start = tok2[1]
    end = tok2[2]
    assert (
        data == "AAAAAAAA"
    ), "wrong data for token 2, expected: 'AAAAAAAA', found: {}".format(data)
    assert (
        start == 20
    ), "wrong start frame for token 2, expected: 20, found: {}".format(start)
    assert (
        end == 27
    ), "wrong end frame for token 2, expected: 27, found: {}".format(end)


def test_init_min_3_init_max_silence_0(validator):
    tokenizer = StreamTokenizer(
        validator,
        min_length=5,
        max_length=20,
        max_continuous_silence=4,
        init_min=3,
        init_max_silence=0,
        mode=0,
    )

    data_source = StringDataSource("aAaaaAaAaaAaAaaaaaAAAAAAAAAaaaaaaAAAAA")
    #                                                 ^           ^  ^   ^
    #                                                 18          30 33  37

    tokens = tokenizer.tokenize(data_source)

    assert (
        len(tokens) == 2
    ), "wrong number of tokens, expected: 2, found: {}".format(len(tokens))
    tok1, tok2 = tokens[0], tokens[1]

    data = "".join(tok1[0])
    start = tok1[1]
    end = tok1[2]
    assert (
        data == "AAAAAAAAAaaaa"
    ), "wrong data for token 1, expected: 'AAAAAAAAAaaaa', found: {}".format(
        data
    )
    assert (
        start == 18
    ), "wrong start frame for token 1, expected: 18, found: {}".format(start)
    assert (
        end == 30
    ), "wrong end frame for token 1, expected: 30, found: {}".format(end)

    data = "".join(tok2[0])
    start = tok2[1]
    end = tok2[2]
    assert (
        data == "AAAAA"
    ), "wrong data for token 2, expected: 'AAAAA', found: {}".format(data)
    assert (
        start == 33
    ), "wrong start frame for token 2, expected: 33, found: {}".format(start)
    assert (
        end == 37
    ), "wrong end frame for token 2, expected: 37, found: {}".format(end)


def test_init_min_3_init_max_silence_2(validator):
    tokenizer = StreamTokenizer(
        validator,
        min_length=5,
        max_length=20,
        max_continuous_silence=4,
        init_min=3,
        init_max_silence=2,
        mode=0,
    )

    data_source = StringDataSource("aAaaaAaAaaAaAaaaaaaAAAAAAAAAaaaaaaaAAAAA")
    #                                    ^          ^  ^           ^   ^   ^
    #                                    5          16 19          31  35  39
    tokens = tokenizer.tokenize(data_source)

    assert (
        len(tokens) == 3
    ), "wrong number of tokens, expected: 3, found: {}".format(len(tokens))
    tok1, tok2, tok3 = tokens[0], tokens[1], tokens[2]

    data = "".join(tok1[0])
    start = tok1[1]
    end = tok1[2]
    assert (
        data == "AaAaaAaAaaaa"
    ), "wrong data for token 1, expected: 'AaAaaAaA', found: {}".format(data)
    assert (
        start == 5
    ), "wrong start frame for token 1, expected: 5, found: {}".format(start)
    assert (
        end == 16
    ), "wrong end frame for token 1, expected: 16, found: {}".format(end)

    data = "".join(tok2[0])
    start = tok2[1]
    end = tok2[2]
    assert (
        data == "AAAAAAAAAaaaa"
    ), "wrong data for token 2, expected: 'AAAAAAAAAaaaa', found: {}".format(
        data
    )
    assert (
        start == 19
    ), "wrong start frame for token 2, expected: 19, found: {}".format(start)
    assert (
        end == 31
    ), "wrong end frame for token 2, expected: 31, found: {}".format(end)

    data = "".join(tok3[0])
    start = tok3[1]
    end = tok3[2]
    assert (
        data == "AAAAA"
    ), "wrong data for token 3, expected: 'AAAAA', found: {}".format(data)
    assert (
        start == 35
    ), "wrong start frame for token 3, expected: 35, found: {}".format(start)
    assert (
        end == 39
    ), "wrong end frame for token 3, expected: 39, found: {}".format(end)


@pytest.fixture
def tokenizer_min_max_length(validator):
    return StreamTokenizer(
        validator,
        min_length=6,
        max_length=20,
        max_continuous_silence=2,
        init_min=3,
        init_max_silence=3,
        mode=0,
    )


def test_min_length_6_init_max_length_20(tokenizer_min_max_length):
    data_source = StringDataSource("aAaaaAaAaaAaAaaaaaAAAAAAAAAaaaaaAAAAA")
    #                                ^            ^   ^         ^
    #                                1            14  18        28

    tokens = tokenizer_min_max_length.tokenize(data_source)

    assert (
        len(tokens) == 2
    ), "wrong number of tokens, expected: 2, found: {}".format(len(tokens))
    tok1, tok2 = tokens[0], tokens[1]

    data = "".join(tok1[0])
    start = tok1[1]
    end = tok1[2]
    assert (
        data == "AaaaAaAaaAaAaa"
    ), "wrong data for token 1, expected: 'AaaaAaAaaAaAaa', found: {}".format(
        data
    )
    assert (
        start == 1
    ), "wrong start frame for token 1, expected: 1, found: {}".format(start)
    assert (
        end == 14
    ), "wrong end frame for token 1, expected: 14, found: {}".format(end)

    data = "".join(tok2[0])
    start = tok2[1]
    end = tok2[2]
    assert (
        data == "AAAAAAAAAaa"
    ), "wrong data for token 2, expected: 'AAAAAAAAAaa', found: {}".format(data)
    assert (
        start == 18
    ), "wrong start frame for token 2, expected: 18, found: {}".format(start)
    assert (
        end == 28
    ), "wrong end frame for token 2, expected: 28, found: {}".format(end)


@pytest.fixture
def tokenizer_min_max_length_1_1(validator):
    return StreamTokenizer(
        validator,
        min_length=1,
        max_length=1,
        max_continuous_silence=0,
        init_min=0,
        init_max_silence=0,
        mode=0,
    )


def test_min_length_1_init_max_length_1(tokenizer_min_max_length_1_1):
    data_source = StringDataSource("AAaaaAaaaAaAaaAaAaaaaaAAAAAAAAAaaaaaAAAAA")

    tokens = tokenizer_min_max_length_1_1.tokenize(data_source)

    assert (
        len(tokens) == 21
    ), "wrong number of tokens, expected: 21, found: {}".format(len(tokens))


@pytest.fixture
def tokenizer_min_max_length_10_20(validator):
    return StreamTokenizer(
        validator,
        min_length=10,
        max_length=20,
        max_continuous_silence=4,
        init_min=3,
        init_max_silence=3,
        mode=0,
    )


def test_min_length_10_init_max_length_20(tokenizer_min_max_length_10_20):
    data_source = StringDataSource(
        "aAaaaAaAaaAaAaaaaaaAAAAAaaaaaaAAAAAaaAAaaAAA"
    )
    #     ^              ^             ^            ^
    #     1              16            30           45

    tokens = tokenizer_min_max_length_10_20.tokenize(data_source)

    assert (
        len(tokens) == 2
    ), "wrong number of tokens, expected: 2, found: {}".format(len(tokens))
    tok1, tok2 = tokens[0], tokens[1]

    data = "".join(tok1[0])
    start = tok1[1]
    end = tok1[2]
    assert (
        data == "AaaaAaAaaAaAaaaa"
    ), "wrong data for token 1, expected: 'AaaaAaAaaAaAaaaa', found: {}".format(
        data
    )
    assert (
        start == 1
    ), "wrong start frame for token 1, expected: 1, found: {}".format(start)
    assert (
        end == 16
    ), "wrong end frame for token 1, expected: 16, found: {}".format(end)

    data = "".join(tok2[0])
    start = tok2[1]
    end = tok2[2]
    assert (
        data == "AAAAAaaAAaaAAA"
    ), "wrong data for token 2, expected: 'AAAAAaaAAaaAAA', found: {}".format(
        data
    )
    assert (
        start == 30
    ), "wrong start frame for token 2, expected: 30, found: {}".format(start)
    assert (
        end == 43
    ), "wrong end frame for token 2, expected: 43, found: {}".format(end)


@pytest.fixture
def tokenizer_min_max_length_4_5(validator):
    return StreamTokenizer(
        validator,
        min_length=4,
        max_length=5,
        max_continuous_silence=4,
        init_min=3,
        init_max_silence=3,
        mode=0,
    )


def test_min_length_4_init_max_length_5(tokenizer_min_max_length_4_5):
    data_source = StringDataSource(
        "aAaaaAaAaaAaAaaaaaAAAAAAAAaaaaaaAAAAAaaaaaAAaaAaa"
    )
    #                      ^   ^^   ^    ^   ^     ^   ^
    #                      18 2223  27   32  36    42  46

    tokens = tokenizer_min_max_length_4_5.tokenize(data_source)

    assert (
        len(tokens) == 4
    ), "wrong number of tokens, expected: 4, found: {}".format(len(tokens))
    tok1, tok2, tok3, tok4 = tokens[0], tokens[1], tokens[2], tokens[3]

    data = "".join(tok1[0])
    start = tok1[1]
    end = tok1[2]
    assert (
        data == "AAAAA"
    ), "wrong data for token 1, expected: 'AAAAA', found: {}".format(data)
    assert (
        start == 18
    ), "wrong start frame for token 1, expected: 18, found: {}".format(start)
    assert (
        end == 22
    ), "wrong end frame for token 1, expected: 22, found: {}".format(end)

    data = "".join(tok2[0])
    start = tok2[1]
    end = tok2[2]
    assert (
        data == "AAAaa"
    ), "wrong data for token 2, expected: 'AAAaa', found: {}".format(data)
    assert (
        start == 23
    ), "wrong start frame for token 2, expected: 23, found: {}".format(start)
    assert (
        end == 27
    ), "wrong end frame for token 2, expected: 27, found: {}".format(end)

    data = "".join(tok3[0])
    start = tok3[1]
    end = tok3[2]
    assert (
        data == "AAAAA"
    ), "wrong data for token 3, expected: 'AAAAA', found: {}".format(data)
    assert (
        start == 32
    ), "wrong start frame for token 3, expected: 32, found: {}".format(start)
    assert (
        end == 36
    ), "wrong end frame for token 3, expected: 36, found: {}".format(end)

    data = "".join(tok4[0])
    start = tok4[1]
    end = tok4[2]
    assert (
        data == "AAaaA"
    ), "wrong data for token 4, expected: 'AAaaA', found: {}".format(data)
    assert (
        start == 42
    ), "wrong start frame for token 4, expected: 42, found: {}".format(start)
    assert (
        end == 46
    ), "wrong end frame for token 4, expected: 46, found: {}".format(end)


@pytest.fixture
def tokenizer_max_continuous_silence_0(validator):
    return StreamTokenizer(
        validator,
        min_length=5,
        max_length=10,
        max_continuous_silence=0,
        init_min=3,
        init_max_silence=3,
        mode=0,
    )


def test_min_5_max_10_max_continuous_silence_0(
    tokenizer_max_continuous_silence_0,
):
    data_source = StringDataSource("aaaAAAAAaAAAAAAaaAAAAAAAAAa")
    #                                  ^   ^ ^    ^  ^       ^
    #                                  3   7 9   14 17      25

    tokens = tokenizer_max_continuous_silence_0.tokenize(data_source)

    assert (
        len(tokens) == 3
    ), "wrong number of tokens, expected: 3, found: {}".format(len(tokens))
    tok1, tok2, tok3 = tokens[0], tokens[1], tokens[2]

    data = "".join(tok1[0])
    start = tok1[1]
    end = tok1[2]
    assert (
        data == "AAAAA"
    ), "wrong data for token 1, expected: 'AAAAA', found: {}".format(data)
    assert (
        start == 3
    ), "wrong start frame for token 1, expected: 3, found: {}".format(start)
    assert (
        end == 7
    ), "wrong end frame for token 1, expected: 7, found: {}".format(end)

    data = "".join(tok2[0])
    start = tok2[1]
    end = tok2[2]
    assert (
        data == "AAAAAA"
    ), "wrong data for token 2, expected: 'AAAAAA', found: {}".format(data)
    assert (
        start == 9
    ), "wrong start frame for token 2, expected: 9, found: {}".format(start)
    assert (
        end == 14
    ), "wrong end frame for token 2, expected: 14, found: {}".format(end)

    data = "".join(tok3[0])
    start = tok3[1]
    end = tok3[2]
    assert (
        data == "AAAAAAAAA"
    ), "wrong data for token 3, expected: 'AAAAAAAAA', found: {}".format(data)
    assert (
        start == 17
    ), "wrong start frame for token 3, expected: 17, found: {}".format(start)
    assert (
        end == 25
    ), "wrong end frame for token 3, expected: 25, found: {}".format(end)


@pytest.fixture
def tokenizer_max_continuous_silence_1(validator):
    return StreamTokenizer(
        validator,
        min_length=5,
        max_length=10,
        max_continuous_silence=1,
        init_min=3,
        init_max_silence=3,
        mode=0,
    )


def test_min_5_max_10_max_continuous_silence_1(
    tokenizer_max_continuous_silence_1,
):
    data_source = StringDataSource("aaaAAAAAaAAAAAAaaAAAAAAAAAa")
    #                                  ^        ^^ ^ ^        ^
    #                                  3       12131517      26
    #                                         (12 13 15 17)

    tokens = tokenizer_max_continuous_silence_1.tokenize(data_source)

    assert (
        len(tokens) == 3
    ), "wrong number of tokens, expected: 3, found: {}".format(len(tokens))
    tok1, tok2, tok3 = tokens[0], tokens[1], tokens[2]

    data = "".join(tok1[0])
    start = tok1[1]
    end = tok1[2]
    assert (
        data == "AAAAAaAAAA"
    ), "wrong data for token 1, expected: 'AAAAAaAAAA', found: {}".format(data)
    assert (
        start == 3
    ), "wrong start frame for token 1, expected: 3, found: {}".format(start)
    assert (
        end == 12
    ), "wrong end frame for token 1, expected: 12, found: {}".format(end)

    data = "".join(tok2[0])
    start = tok2[1]
    end = tok2[2]
    assert (
        data == "AAa"
    ), "wrong data for token 2, expected: 'AAa', found: {}".format(data)
    assert (
        start == 13
    ), "wrong start frame for token 2, expected: 13, found: {}".format(start)
    assert (
        end == 15
    ), "wrong end frame for token 2, expected: 15, found: {}".format(end)

    data = "".join(tok3[0])
    start = tok3[1]
    end = tok3[2]
    assert (
        data == "AAAAAAAAAa"
    ), "wrong data for token 3, expected: 'AAAAAAAAAa', found: {}".format(data)
    assert (
        start == 17
    ), "wrong start frame for token 3, expected: 17, found: {}".format(start)
    assert (
        end == 26
    ), "wrong end frame for token 3, expected: 26, found: {}".format(end)


@pytest.fixture
def tokenizer_strict_min_length(validator):
    return StreamTokenizer(
        validator,
        min_length=5,
        max_length=8,
        max_continuous_silence=3,
        init_min=3,
        init_max_silence=3,
        mode=StreamTokenizer.STRICT_MIN_LENGTH,
    )


def test_STRICT_MIN_LENGTH(tokenizer_strict_min_length):
    data_source = StringDataSource("aaAAAAAAAAAAAA")
    #                                 ^      ^
    #                                 2      9

    tokens = tokenizer_strict_min_length.tokenize(data_source)

    assert (
        len(tokens) == 1
    ), "wrong number of tokens, expected: 1, found: {}".format(len(tokens))
    tok1 = tokens[0]

    data = "".join(tok1[0])
    start = tok1[1]
    end = tok1[2]
    assert (
        data == "AAAAAAAA"
    ), "wrong data for token 1, expected: 'AAAAAAAA', found: {}".format(data)
    assert (
        start == 2
    ), "wrong start frame for token 1, expected: 2, found: {}".format(start)
    assert (
        end == 9
    ), "wrong end frame for token 1, expected: 9, found: {}".format(end)


@pytest.fixture
def tokenizer_drop_trailing_silence(validator):
    return StreamTokenizer(
        validator,
        min_length=5,
        max_length=10,
        max_continuous_silence=2,
        init_min=3,
        init_max_silence=3,
        mode=StreamTokenizer.DROP_TRAILING_SILENCE,
    )


def test_DROP_TAILING_SILENCE(tokenizer_drop_trailing_silence):
    data_source = StringDataSource("aaAAAAAaaaaa")
    #                                 ^   ^
    #                                 2   6

    tokens = tokenizer_drop_trailing_silence.tokenize(data_source)

    assert (
        len(tokens) == 1
    ), "wrong number of tokens, expected: 1, found: {}".format(len(tokens))
    tok1 = tokens[0]

    data = "".join(tok1[0])
    start = tok1[1]
    end = tok1[2]
    assert (
        data == "AAAAA"
    ), "wrong data for token 1, expected: 'AAAAA', found: {}".format(data)
    assert (
        start == 2
    ), "wrong start frame for token 1, expected: 2, found: {}".format(start)
    assert (
        end == 6
    ), "wrong end frame for token 1, expected: 6, found: {}".format(end)


@pytest.fixture
def tokenizer_strict_min_and_drop_trailing_silence(validator):
    return StreamTokenizer(
        validator,
        min_length=5,
        max_length=8,
        max_continuous_silence=3,
        init_min=3,
        init_max_silence=3,
        mode=StreamTokenizer.STRICT_MIN_LENGTH
        | StreamTokenizer.DROP_TRAILING_SILENCE,
    )


def test_STRICT_MIN_LENGTH_and_DROP_TAILING_SILENCE(
    tokenizer_strict_min_and_drop_trailing_silence,
):
    data_source = StringDataSource("aaAAAAAAAAAAAAaa")
    #                                 ^      ^
    #                                 2      8

    tokens = tokenizer_strict_min_and_drop_trailing_silence.tokenize(
        data_source
    )

    assert (
        len(tokens) == 1
    ), "wrong number of tokens, expected: 1, found: {}".format(len(tokens))
    tok1 = tokens[0]

    data = "".join(tok1[0])
    start = tok1[1]
    end = tok1[2]
    assert (
        data == "AAAAAAAA"
    ), "wrong data for token 1, expected: 'AAAAAAAA', found: {}".format(data)
    assert (
        start == 2
    ), "wrong start frame for token 1, expected: 2, found: {}".format(start)
    assert (
        end == 9
    ), "wrong end frame for token 1, expected: 9, found: {}".format(end)


@pytest.fixture
def tokenizer_callback(validator):
    return StreamTokenizer(
        validator,
        min_length=5,
        max_length=8,
        max_continuous_silence=3,
        init_min=3,
        init_max_silence=3,
        mode=0,
    )


def test_callback(tokenizer_callback):
    tokens = []

    def callback(data, start, end):
        tokens.append((data, start, end))

    data_source = StringDataSource("aaAAAAAAAAAAAAa")
    #                                 ^      ^^   ^
    #                                 2      910  14

    tokenizer_callback.tokenize(data_source, callback=callback)

    assert (
        len(tokens) == 2
    ), "wrong number of tokens, expected: 2, found: {}".format(len(tokens))


def test_max_length_inf_single_token(validator):
    """With max_length=inf, the entire valid sequence is one token."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=float("inf"),
        max_continuous_silence=3,
    )
    data_source = StringDataSource("aAAAaaAAAAAAAAAaaAAa")
    #                                ^                 ^
    #                                1                 18
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    data = "".join(tokens[0][0])
    assert data == "AAAaaAAAAAAAAAaaAAa"
    assert tokens[0][1] == 1
    assert tokens[0][2] == 19


def test_max_length_inf_splits_on_silence(validator):
    """With max_length=inf, tokens still split on max_continuous_silence."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=float("inf"),
        max_continuous_silence=2,
    )
    data_source = StringDataSource("AAAAAaaaAAAAA")
    #                               ^   ^   ^   ^
    #                               0   4   8   12
    # silence gap of 3 ('aaa') > max_continuous_silence (2) -> two tokens
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 2
    assert "".join(tokens[0][0]) == "AAAAAaa"
    assert tokens[0][1] == 0
    assert tokens[0][2] == 6
    assert "".join(tokens[1][0]) == "AAAAA"
    assert tokens[1][1] == 8
    assert tokens[1][2] == 12


def test_max_length_inf_respects_min_length(validator):
    """With max_length=inf, min_length is still enforced."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=5,
        max_length=float("inf"),
        max_continuous_silence=1,
        mode=StreamTokenizer.STRICT_MIN_LENGTH,
    )
    data_source = StringDataSource("AAaaAAAAAAAA")
    #                                   ^^^^^^^^
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    assert "".join(tokens[0][0]) == "AAAAAAAA"
    assert tokens[0][1] == 4
    assert tokens[0][2] == 11


def test_max_length_inf_with_drop_trailing_silence(validator):
    """With max_length=inf and DROP_TRAILING_SILENCE, trailing silence
    is removed."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=float("inf"),
        max_continuous_silence=3,
        mode=StreamTokenizer.DROP_TRAILING_SILENCE,
    )
    data_source = StringDataSource("AAAAAaaa")
    #                               ^   ^
    #                               0   4
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    assert "".join(tokens[0][0]) == "AAAAA"
    assert tokens[0][1] == 0
    assert tokens[0][2] == 4


# ── max_leading_silence tests ──


def test_max_leading_silence_basic(validator):
    """Leading silence frames are prepended to the token."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=1,
        max_leading_silence=2,
    )
    data_source = StringDataSource("aaaAAAAAa")
    #                               012345678
    # buffer fills with frames 1,2 (maxlen=2), token starts at 1
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    assert "".join(tokens[0][0]) == "aaAAAAAa"
    assert tokens[0][1] == 1
    assert tokens[0][2] == 8


def test_max_leading_silence_less_silence_than_requested(validator):
    """When fewer silent frames exist than max_leading_silence, use what's
    available."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=1,
        max_leading_silence=5,
    )
    data_source = StringDataSource("aAAAAAa")
    #                               0123456
    # only 1 silent frame available before first A
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    assert "".join(tokens[0][0]) == "aAAAAAa"
    assert tokens[0][1] == 0
    assert tokens[0][2] == 6


def test_max_leading_silence_no_silence_before_token(validator):
    """Token at start of stream with no preceding silence."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=1,
        max_leading_silence=3,
    )
    data_source = StringDataSource("AAAAAaaa")
    #                               01234567
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    assert "".join(tokens[0][0]) == "AAAAAa"
    assert tokens[0][1] == 0
    assert tokens[0][2] == 5


def test_max_leading_silence_two_tokens_no_overlap(validator):
    """Two tokens with leading silence; starts never overlap previous ends."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=1,
        max_leading_silence=2,
    )
    data_source = StringDataSource("aaAAAAAaaaAAAAAaa")
    #                               01234567890123456
    # Token 1: buffer=[a,a] from frames 0,1 → start=0
    # Token 2: buffer=[a] from frame 9 (1 frame in SILENCE after frame 8
    #          was consumed) → start=9

    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 2

    assert "".join(tokens[0][0]) == "aaAAAAAa"
    assert tokens[0][1] == 0
    assert tokens[0][2] == 7

    assert "".join(tokens[1][0]) == "aAAAAAa"
    assert tokens[1][1] == 9
    assert tokens[1][2] == 15

    # verify no overlap
    assert tokens[1][1] > tokens[0][2]


def test_max_leading_silence_wide_gap_between_tokens(validator):
    """With a wide silence gap, buffer captures the last N frames."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=1,
        max_leading_silence=2,
    )
    data_source = StringDataSource("aAAAaaaaaaAAAa")
    #                               01234567890123
    # Token 1: buffer=[a] from frame 0 → start=0
    # After token 1, SILENCE entered. buffer starts collecting.
    #   Frame 5 was consumed at the POSSIBLE_SILENCE→SILENCE transition.
    #   Frame 6: buffer=[a]
    #   Frame 7: buffer=[a,a]
    #   Frame 8: buffer=[a,a] (evicts 6, keeps 7,8)
    #   Frame 9: buffer=[a,a] (evicts 7, keeps 8,9)
    # Frame 10 (A): leading=[a,a] from frames 8,9 → start=10-2=8
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 2

    assert "".join(tokens[0][0]) == "aAAAa"
    assert tokens[0][1] == 0
    assert tokens[0][2] == 4

    assert "".join(tokens[1][0]) == "aaAAAa"
    assert tokens[1][1] == 8
    assert tokens[1][2] == 13

    assert tokens[1][1] > tokens[0][2]


def test_max_leading_silence_with_drop_trailing_silence(validator):
    """Leading silence is kept even when trailing silence is dropped."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=2,
        max_leading_silence=2,
        mode=StreamTokenizer.DROP_TRAILING_SILENCE,
    )
    data_source = StringDataSource("aaaAAAAAaaa")
    #                               01234567890
    # buffer=[a,a] from frames 1,2. Token starts at 1.
    # DROP_TRAILING removes the trailing 'aa' (silence_length=2 from
    # frames 8,9; frame 10 triggers SILENCE)
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    assert "".join(tokens[0][0]) == "aaAAAAA"
    assert tokens[0][1] == 1
    assert tokens[0][2] == 7


def test_max_leading_silence_with_init_min_rejected_then_accepted(validator):
    """init_min rejects first attempt; leading silence is discarded and
    recollected for the successful detection."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=1,
        init_min=3,
        init_max_silence=0,
        max_leading_silence=2,
    )
    data_source = StringDataSource("aaAaaAAAAAa")
    #                               01234567890
    # Frame 0,1: SILENCE, buffer=[a,a]
    # Frame 2 (A): prepend buffer → data=[a,a,A], start=0. init_count=1 < 3
    #              → POSSIBLE_NOISE. buffer cleared.
    # Frame 3 (a): POSSIBLE_NOISE, sl=1 > init_max_silent=0 → SILENCE.
    #              data=[]. buffer is empty.
    # Frame 4 (a): SILENCE, buffer=[a]
    # Frame 5 (A): prepend buffer=[a] → data=[a,A], start=4. init_count=1 < 3
    #              → POSSIBLE_NOISE. buffer cleared.
    # Frame 6 (A): POSSIBLE_NOISE, valid. init_count=2.
    # Frame 7 (A): POSSIBLE_NOISE, valid. init_count=3 >= 3 → NOISE.
    # Frame 8 (A): NOISE.
    # Frame 9 (A): NOISE. data=[a,A,A,A,A,A]
    # Frame 10(a): POSSIBLE_SILENCE. sl=1>=mcs(1) would need another frame...
    #   Actually mcs=1, so sl=1 is OK. data=[a,A,A,A,A,A,a]
    #   Wait, this needs a second silent frame to trigger SILENCE.
    #   End of data → _post_process → deliver.
    # Token: data=[a,A,A,A,A,A,a], start=4, end=4+7-1=10
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    assert "".join(tokens[0][0]) == "aAAAAAa"
    assert tokens[0][1] == 4
    assert tokens[0][2] == 10


def test_max_leading_silence_with_init_min_accepted(validator):
    """init_min and max_leading_silence compose: leading silence is kept
    when init_min is satisfied."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=1,
        init_min=3,
        init_max_silence=1,
        max_leading_silence=2,
    )
    data_source = StringDataSource("aaaAaAAAa")
    #                               012345678
    # Frame 0,1: SILENCE, buffer=[a,a]
    # Frame 2 (a): SILENCE, buffer=[a,a] (evicts 0, keeps 1,2)
    # Frame 3 (A): prepend [a,a] → data=[a,a,A], start=1. init=1 < 3
    #              → POSSIBLE_NOISE. buffer cleared.
    # Frame 4 (a): POSSIBLE_NOISE, sl=1 <= init_max_silent(1). data=[a,a,A,a]
    # Frame 5 (A): POSSIBLE_NOISE, valid. sl=0. init=2 < 3. data=[a,a,A,a,A]
    # Frame 6 (A): POSSIBLE_NOISE, valid. init=3 >= 3 → NOISE. data=[a,a,A,a,A,A]
    # Frame 7 (A): NOISE. data=[a,a,A,a,A,A,A]
    # Frame 8 (a): POSSIBLE_SILENCE. sl=1>=mcs(1)... no wait, sl=1 and mcs=1.
    #   Actually: sl is set to 1. 1 >= 1 is checked on the NEXT silent frame.
    #   End of data → _post_process. state=POSSIBLE_SILENCE, len(data)=8 > sl=1.
    #   → deliver.
    # Token: data=[a,a,A,a,A,A,A,a], start=1, end=1+8-1=8
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    assert "".join(tokens[0][0]) == "aaAaAAAa"
    assert tokens[0][1] == 1
    assert tokens[0][2] == 8


def test_max_leading_silence_0_is_backward_compatible(validator):
    """max_leading_silence=0 produces identical results to omitting it."""
    tokenizer_default = StreamTokenizer(
        validator,
        min_length=5,
        max_length=20,
        max_continuous_silence=4,
        init_min=3,
        init_max_silence=2,
    )
    tokenizer_explicit = StreamTokenizer(
        validator,
        min_length=5,
        max_length=20,
        max_continuous_silence=4,
        init_min=3,
        init_max_silence=2,
        max_leading_silence=0,
    )
    data = "aAaaaAaAaaAaAaaaaaaAAAAAAAAAaaaaaaaAAAAA"
    tokens_default = tokenizer_default.tokenize(StringDataSource(data))
    tokens_explicit = tokenizer_explicit.tokenize(StringDataSource(data))
    assert len(tokens_default) == len(tokens_explicit)
    for t1, t2 in zip(tokens_default, tokens_explicit):
        assert "".join(t1[0]) == "".join(t2[0])
        assert t1[1] == t2[1]
        assert t1[2] == t2[2]


# ── max_trailing_silence tests ──


def test_max_trailing_silence_none_keeps_all(validator):
    """max_trailing_silence=None (default) keeps all trailing silence."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=3,
    )
    data_source = StringDataSource("AAAAAaaaa")
    #                               012345678
    # mcs=3: frames 5,6,7 tolerated, frame 8 triggers SILENCE
    # Token includes trailing aaa (3 frames)
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    assert "".join(tokens[0][0]) == "AAAAAaaa"
    assert tokens[0][1] == 0
    assert tokens[0][2] == 7


def test_max_trailing_silence_0_drops_all(validator):
    """max_trailing_silence=0 drops all trailing silence."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=3,
        max_trailing_silence=0,
    )
    data_source = StringDataSource("AAAAAaaaa")
    #                               012345678
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    assert "".join(tokens[0][0]) == "AAAAA"
    assert tokens[0][1] == 0
    assert tokens[0][2] == 4


def test_max_trailing_silence_partial(validator):
    """max_trailing_silence=1 keeps only 1 frame of trailing silence."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=3,
        max_trailing_silence=1,
    )
    data_source = StringDataSource("AAAAAaaaa")
    #                               012345678
    # 3 trailing frames, keep only 1 → "AAAAAa"
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    assert "".join(tokens[0][0]) == "AAAAAa"
    assert tokens[0][1] == 0
    assert tokens[0][2] == 5


def test_max_trailing_silence_larger_than_actual(validator):
    """When max_trailing_silence > actual trailing silence, keep all."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=2,
        max_trailing_silence=5,
    )
    data_source = StringDataSource("AAAAAaaa")
    #                               01234567
    # mcs=2: frames 5,6 tolerated, frame 7 triggers SILENCE
    # 2 trailing frames < max_trailing_silence(5), keep all
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    assert "".join(tokens[0][0]) == "AAAAAaa"
    assert tokens[0][1] == 0
    assert tokens[0][2] == 6


def test_max_trailing_silence_0_matches_drop_trailing_mode(validator):
    """max_trailing_silence=0 gives the same result as DROP_TRAILING_SILENCE
    mode flag."""
    tokenizer_mode = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=3,
        mode=StreamTokenizer.DROP_TRAILING_SILENCE,
    )
    tokenizer_param = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=3,
        max_trailing_silence=0,
    )
    data = "aaAAAAAaaaAAAa"
    tokens_mode = tokenizer_mode.tokenize(StringDataSource(data))
    tokens_param = tokenizer_param.tokenize(StringDataSource(data))
    assert len(tokens_mode) == len(tokens_param)
    for t1, t2 in zip(tokens_mode, tokens_param):
        assert "".join(t1[0]) == "".join(t2[0])
        assert t1[1] == t2[1]
        assert t1[2] == t2[2]


def test_max_trailing_silence_with_max_leading_silence(validator):
    """Both leading and trailing silence control work together."""
    tokenizer = StreamTokenizer(
        validator,
        min_length=1,
        max_length=20,
        max_continuous_silence=3,
        max_leading_silence=2,
        max_trailing_silence=1,
    )
    data_source = StringDataSource("aaaAAAAAaaaa")
    #                               0123456789 10 11
    # buffer=[a,a] from frames 1,2. Leading prepended → start=1
    # mcs=3: frames 8,9,10 tolerated, frame 11 triggers SILENCE
    # 3 trailing frames, keep 1 → trim 2
    tokens = tokenizer.tokenize(data_source)
    assert len(tokens) == 1
    assert "".join(tokens[0][0]) == "aaAAAAAa"
    assert tokens[0][1] == 1
    assert tokens[0][2] == 8
